Automatic statistics Kafka cluster logs

Writing a Python script statistic.py

#!/usr/bin/python "" "Pip install kazoo" "" "" "Pip install kafka-python" "" import  Timeimport threadingfrom kazoo.client import kazooclientfrom  kafka.consumer  import kafkaconsumerimport elasticsearchearlyres={}def _get_partitions_logsize (  topic,zookeepers,broker_list ):     zk=kazooclient ( hosts = zookeepers, read_only = true )     try:         Zk.start ()        # res={}    path =  "/ brokers/topics/"+topic+"/partitions "    if zk.exists ( path ):         partitions = zk.get_children ( path )          eachsave = {}        consumer  = kafkaconsumer (  topic,group_id= "Kafka_monitor", Metadata_broker_list=broker_list.split (",")  )          fetch = consumer._offsets.fetch         for partition in partitions:             logsize = fetch[  ( topic, int (partition )  )  ]             eachsave[ int ( partition )  ]  = logsize    else:        return { }    return eachsave    except exception as e:      #   print e    return {}     finally:    zk.stop () def analyze_logsize ( zookeepers,broker_ list )  :  &NBsp; zk = kazooclient (hosts=zookeepers,read_only=true)     try:         zk.start ()         path =   "/brokers/topics/"         sum=0         if zk.exists (PATH):             topics = zk.get_children ( path )              for topic in topics:                 add_dict = {}                 nowpartitions = _get_partitions_logsize ( Topic,zookeepers,broker_list)                  if nowpartitions !=  {}:                     for partition in nowpartitions:                          sum += nowpartitions [ partition ]             return sum             else:            pass     except Exception as e:         pass        print e    finally:         zk.stop () if __name__ ==  ' __main__ ':     A = analyze_logsize ( "127.0.0.1:2181/kafka/1001",  "127.0.0.1"  )     utc = time.localtime (  time.time ()  )     File = open  (  "/usr/home/shixi_kaiwen/ Script/develop/logsizecheck/2016/increment_day. " +time.strftime (  '%m-%d  ', UTC), "W"  )     File2 = open  (  " /usr/home/shixi_kaiwen/script/develop/logsizecheck/2016/lastlogsize "," r+ ")     last =  int  ( file2.read ()  )     increment = a - last     increment = str  ( increment )      File.write ( increment )     file.close ()     file2.close ()     file3 = open (  "/usr/home/shixi_kaiwen/script/develop/logsizecheck/2016/ Lastlogsize ", " W ")     File3.write  ( str ( a )  ) &Nbsp;   file3.close ()     print  "last = ",last     print  "now_logsize = ",a    print  "increment = " , increment

Create a lastlogsize file under its directory

echo "" >./lastlogsize

Add Crontab Task

Crontab-e

1 * * * python/usr/home/shixi_kaiwen/script/develop/logsizecheck/2016/statistic.py >> Python/usr/home/shixi_ kaiwen/script/develop/logsizecheck/2016/statistic.py >>/usr/home/shixi_kaiwen/script/develop/logsizecheck /2016/output.log 2>&1

This article is from the "humble" blog, please be sure to keep this source http://openex.blog.51cto.com/6280069/1731406

Automatic statistics Kafka cluster logs