Recording performance parameters of Elasticsearch in Bigdesk

Collect Elasticsearch performance parameters in Bigdesk and save to database or elk for long-term monitoring. Based on the Python script implementation, the script is as follows:

#coding =GBK

Import Httplib
Import JSON
Import time
Import Es_savelog
Import ConfigHelper
Import Mqhelper


def main ():

#变量初始化
#上一次统计数据
dictlastnodeinfo={}
#本次统计当前节点
dictnodeinfo={}

Print "Start ..."
While 1==1:
Flag=confighelper.getintconfig ("Flag")
If flag <> 1:
#判断是否满足退出条件
Print "terminating" +STR (flag)
Break

Urlarray = Confighelper.getstringconfig ("Esurl"). Split (' | ')
#取出每次执行完成后的休眠时长: Seconds
Sleeptime=confighelper.getfloatconfig ("Sleeptime")

For Urlindex in range (0,len (Urlarray)):
Url=urlarray[urlindex]
conn = Httplib. Httpconnection (URL)

#取出ES版本号
Conn.request ("GET", "" ")
Serverinfo=conn.getresponse ()
Objserverjson=json.loads (Serverinfo.read ())
Esversion=str (objserverjson["Version" ["Number"])

#取出集群健康状况
Conn.request ("GET", "/_cluster/health")
Healthinfo=conn.getresponse ()
Objhealthjson=json.loads (Healthinfo.read ())
Health=str (objhealthjson["status"])

#取出各ES节点统计数据
Conn.request ("GET", "/_nodes/stats?human=true")
Nodesread = Conn.getresponse ()
Objnodesjson=json.loads (Nodesread.read ())

For I in range (0,len (objnodesjson["Nodes"].values ())):
Try
esnode=objnodesjson["Nodes"].values () [i]
Nodename=str (esnode["name"])
dictnodeinfo["Esversion"]=esversion
dictnodeinfo["Health"]=health

#记录ES节点名称
dictnodeinfo["NodeName"]=nodename
dictnodeinfo["Interval"]=sleeptime

#记录CPU信息
dictnodeinfo["OSUSERCPU"]=esnode["OS" ["CPU"] ["User"]

#记录ThreadpoolCount
dictnodeinfo["Threadpoolcount"]=esnode["Thread_pool" ["Search"] ["active"]

#记录JVM堆内存
dictnodeinfo["Heapmem"]=float (esnode["JVM" ["Mem"] ["heap_used"].replace ("GB", ""). Replace ("MB", "")
Curgcyoungcount=int (esnode["JVM" ["GC"] ["Collectors"] ["Young"] ["Collection_count"])
Curgcoldcount=int (esnode["JVM" ["GC"] ["Collectors"] ["old"] ["Collection_count"])
Curgcyoungtime=int (esnode["JVM" ["GC"] ["Collectors"] ["Young"] ["Collection_time_in_millis"])
Curgncoldtime=int (esnode["JVM" ["GC"] ["Collectors"] ["old"] ["Collection_time_in_millis"])
Lastgcyoungcount=int (Dictlastnodeinfo.get (nodename+ "_gcyoungcount",-1))
Lastgcoldcount=int (Dictlastnodeinfo.get (nodename+ "_gcoldcount",-1))
Lastgcyoungtime=int (Dictlastnodeinfo.get (nodename+ "_gcyoungtime",-1))
Lastgcoldtime=int (Dictlastnodeinfo.get (nodename+ "_gcoldtime",-1))
If Lastgcyoungcount>=0 and Lastgcoldcount>=0 and Lastgcyoungtime>=0 and lastgcyoungtime>=0:
dictnodeinfo["Gcyoungcount"]=curgcyoungcount-lastgcyoungcount
dictnodeinfo["Gcoldcount"]=curgcoldcount-lastgcoldcount
dictnodeinfo["Gcyoungtime"]=curgcyoungtime-lastgcyoungtime
dictnodeinfo["Gcoldtime"]=curgncoldtime-lastgcoldtime
If lastgcoldcount>0:
dictnodeinfo["Gcyocountrate"]=lastgcyoungcount/lastgcoldcount
Dictlastnodeinfo[nodename+ "_gcyoungcount"]=curgcyoungcount
Dictlastnodeinfo[nodename+ "_gcoldcount"]=curgcoldcount
Dictlastnodeinfo[nodename+ "_gcyoungtime"]=curgcyoungtime
Dictlastnodeinfo[nodename+ "_gcoldtime"]=curgncoldtime

#记录连接数信息
dictnodeinfo["Channeltransport"]=esnode["Transport" ["Server_open"]
dictnodeinfo["channelhttp"]=esnode["http" ["Current_open"]

#记录当前节点Indices-query Information
objsearch=esnode["Indices" ["Search"]
curquerytotal=objsearch["Query_total"]
curfetchtotal=objsearch["Fetch_total"]
curtimestamp=esnode["Timestamp"]
Lastquerytotal=dictlastnodeinfo.get (nodename+ "_querytotal",-1)
Lastfetchtotal=dictlastnodeinfo.get (nodename+ "_fetchtotal",-1)
Lasttimestamp=dictlastnodeinfo.get (nodename+ "_timestamp",-1)

If Lastquerytotal>0 and curquerytotal>0:
Curquerycount=curquerytotal-lastquerytotal
Curfetchcount=curfetchtotal-lastfetchtotal
Curquerytime= (Curtimestamp-lasttimestamp)/1000
dictnodeinfo["Interval"]=curquerytime
#print curquerytotal,lastquerytotal,curquerycount,curtimestamp,lasttimestamp,curquerytime,curquerycount/ Curquerytime
#记录QPS
If curquerytime>0:
dictnodeinfo["Indicesqueryps"]=curquerycount/curquerytime
dictnodeinfo["Indicesfetchps"]=curfetchcount/curquerytime
#print Curquerycount,curquerytime,curquerycount/curquerytime

#更新上次节点数据对象
Dictlastnodeinfo[nodename+ "_querytotal"]=curquerytotal
Dictlastnodeinfo[nodename+ "_fetchtotal"]=curfetchtotal
Dictlastnodeinfo[nodename+ "_timestamp"]=curtimestamp

#取出cache信息
dictnodeinfo["Filtercache"] = float (esnode["indices" ["Filter_cache"] ["Memory_size"].replace ("MB", ""). Replace (" KB "," "))
dictnodeinfo["Fieldcache"] = float (esnode["indices" ["Fielddata"] ["Memory_size"].replace ("MB", ""). Replace ("KB", " "))

#保存数据到数据库
if (Dictnodeinfo.get ("Indicesqueryps", -1) < 0 or dictnodeinfo.get ("Gcyoungcount", -1) < 0):
Continue
Es_savelog. Savelog (Dictnodeinfo)

#推送ELK消息
dictnodeinfo["IndexName"] = "Esbigdesk"
dictnodeinfo["LogTime"] = Time.strftime ("%y-%m-%d%h:%m:%s.000", Time.localtime ())
Print Json.dumps (dictnodeinfo)
Mqhelper.sendmessage (Json.dumps (Dictnodeinfo))
Dictnodeinfo.clear ()
Except Exception,ex:
Print Exception, ":", ex


#休眠
Time.sleep (Sleeptime)


#启动
If __name__== "__main__":
Main ()
Print "Over"

Recording performance parameters of Elasticsearch in Bigdesk