统计ES性能的python脚本

提示：扫一扫查出行
【扫一扫了解最新限行尾号】
复制提示

思路：通过http请求获取es集群中某一index的索引docs数目变化来进行ES性能统计


import time

from datetime import datetime

import urllib2

def get_docs(data_type, today):

    # curl '192.168.3.153:9200/_cat/indices/metadata-dis-2017.05.16-*-93001?v'

    url = 'http://192.168.3.153:9200/_cat/indices/%s-dis-%s-*' % (data_type, today)

    out = urllib2.urlopen(urllib2.Request(url))

    data = out.read()

    docs_cnt = 0

    for line in data.split('\n'):

        if line:

            docs_cnt += int(line.split()[5])

    return docs_cnt

def main():

    today = datetime.now().strftime("%Y.%m.%d")

    init_data = {"event": 0, "metadata": 0}

    docs_cnt, docs_cnt2, total_speed = dict(init_data), dict(init_data), dict(init_data)

    data_types = docs_cnt.keys()

    cnt = 0

    sleepy_time = 20

    for data_type in data_types:

        try:

            got_doc_cnt = get_docs(data_type, today)

        except:

            got_doc_cnt = docs_cnt[data_type]

        docs_cnt[data_type] = got_doc_cnt

    while True:

        time.sleep(sleepy_time)

        for data_type in data_types:

            try:

                got_doc_cnt = get_docs(data_type, today)

            except:

                got_doc_cnt = docs_cnt[data_type]

            docs_cnt2[data_type] = got_doc_cnt

        cnt += 1

        for data_type in data_types:

            speed = (docs_cnt2[data_type]-docs_cnt[data_type])/(sleepy_time+0.0)

            total_speed[data_type] += speed

            print "cnt=%d %s speed = %.1f total_speed=%.1f" % (cnt, data_type, speed, total_speed[data_type]/cnt)

        docs_cnt = dict(docs_cnt2)

if __name__ == "__main__":

    main()