Python parsing nignx access log script sharing

#!/usr/bin/env python # coding=utf-8 #------------------------------------------------------# Name:nginx Log Parsing script # P Urpose: This script is only used to parse the Nginx access log # version:1.0 # author:leo # created:2013-05-07 # modified:2013-05-07 # Copyrigh T: (c) LEO #------------------------------------------------------Import sys import time #该类是用来打印格式 class display    Format (object): Def format_size (self,size): "Format traffic unit '" KB = 1024x768 #KB-b b is byte MB = 1048576 #MB-B GB = 1073741824 #GB, b TB = 1099511627776 #TB, b if size >= tb:size = str ( SIZE/TB) + ' T ' elif size < kb:size = str (size) + ' B ' elif size >= GB and size < Tb:size  = str (SIZE/GB) + ' G ' elif size >= MB and size < Gb:size = str (SIZE/MB) + ' M ' else:size = STR (SIZE/KB) + ' K ' return size #定义字符串格式化 formatstring = '%-15s%-10s%-12s%8s%10s%10s%10s%10s%10s%10s% 10s ' Def transverse_lINE (self): ' Output horizontal ' ' Print self.formatstring% '-'-' *15, '-' *10, '-' *12, '-' *12, '-' *10, '-' *10, '-' *10, '-' *10, '-' *10 , '-' *10, '-' *10 ' def head (self): ' ' Output header info ' ' Print self.formatstring% (' IP ', ' traffic ', ' Times ', ' times% ', ' 200 ') , ' 404 ', ' $ ', ' 403 ', ' 302 ', ' 304 ', ' 503 ') def error_print (self): ' Output error message ' ' Print print ' Usage: ' + SYS.A     Rgv[0] + ' Nginxlogfilepath [number] ' Print sys.exit (1) def execut_time (self): ' Output script execution time ' ' Print Print "Script Execution time:%.3f second"% Time.clock () Print #该类是用来生成主机信息的字典 class Hostinfo (object): host_in FO = [' $ ', ' 404 ', ' $ ', ' 302 ', ' 304 ', ' 503 ', ' 403 ', ' Times ', ' size '] def __init__ (self,host): Self.host = host = {}.from Keys (self.host_info,0) def increment (self,status_times_size,is_size): "This method is used to add 1" if status to each value in the Host_info _times_size = = ' Times ': self.host[' times ' + = 1 elif is_size:self.host[' size '] = self.host[' size ' + stat Us_times_size Else:      Self.host[status_times_size] + = 1 def get_value (self,value): "The method is to take the corresponding value of each host information" "Return Self.host[v Alue] #该类是用来分析文件 class Fileanalysis (object): Def __init__ (self): "Initialize an empty dictionary" ' self.report_dict = {} sel f.total_request_times,self.total_traffic,self.total_200, self.total_404,self.total_500,self.total_403, self.total_302, self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0 def split_eachline_todict (self,line): ' ' ' Split            Each line in the file and returns a dictionary "Split_line = line.split () split_dict = {' Remote_host ': split_line[0], ' status ': Split_line[8], ' Bytes_sent ': split_line[9],} return split_dict def generate_log_report (self,logfile): "' Read file, analyze split         The _eachline_todict method generates a dictionary "for line in logfile:try:line_dict = Self.split_eachline_todict (line) Host = line_dict[' remote_host ' status = line_dict[' status '] except Valueerror:continue ex      Cept indexerror:continue  If host not in self.report_dict:host_info_obj = Hostinfo (host) self.report_dict[host] = Host_info_obj Else:host_info_obj = Self.report_dict[host] Host_info_obj.increment (' Times ', False) if status In Host_info_obj.host_info:host_info_obj.increment (status,false) try:bytes_sent = Int (line_dict[' Bytes_sent ']) except valueerror:bytes_sent = 0 host_info_obj.increment (bytes_sent,true) return SE     Lf.report_dict def return_sorted_list (self,true_dict): "' Calculates the number of States, total traffic, number of requests, and calculates the total amount of each State and generates a true dictionary to facilitate sorting '                  For host_key in true_dict:host_value = True_dict[host_key] times = Host_value.get_value (' Times ')                  Self.total_request_times = Self.total_request_times + times size = Host_value.get_value (' size ') Self.total_traffic = self.total_traffic + Size o200 = Host_value.get_value (' + ') o404 = host_value.get_ Value (' 404 ') o500= Host_value.get_value (' + ') o403 = Host_value.get_value (' 403 ') o302 = Host_value.get_value (' 302 ') o304 = Host_value.get_value (' 304 ') o503 = Host_value.get_value (' 503 ') true_dict[host_key] = {' $ ': o200, ' 404 ': O40        4, ' $ ': o500, ' 403 ': o403, ' 302 ': o302, ' 304 ': o304, ' 503 ': o503, ' times ': times, ' Size ': size} self.total_200 = self.total_200 + o200 self.total_404 = self.total_404 + o404 self.total_500 = Self.tot al_500 + o500 self.total_302 = self.total_302 + o302 self.total_304 = self.total_304 + o304 self.total_5                               self.total_503 + o503 sorted_list = sorted (True_dict.items (), Key=lambda t: (t[1][' times '), t[1][' size '), reverse=true) return Sorted_list class Main (object): Def Main (self): "" "      Isplay_format = DisplayFormat () arg_length = Len (sys.argv) If arg_length = = 1:display_format.error_print () elif Arg_length = =2 or Arg_length = = 3:infile_name = sys.argv[1] try:infile = open (Infile_name, ' R ') if Arg_le         Ngth = = 3:lines = Int (sys.argv[2]) else:lines = 0 except Ioerror,e:print Print e display_format.error_print () except valueerror:print print "please Enter A         Volid number!! " Display_format.error_print () else:display_format.error_print () Fileanalysis_obj = Fileanalysis () not_ True_dict = Fileanalysis_obj.generate_log_report (infile) Log_report = Fileanalysis_obj.return_sorted_list (not_true_     dict) total_ip = Len (log_report) If Lines:log_report = Log_report[0:lines] Infile.close () print Total_traffic = Display_format.format_size (fileanalysis_obj.total_traffic) total_request_times = fileAnalysis_obj.t Otal_request_times print ' Total IP:%s All traffic:%s Total Request times:%d '% (total_ip,total_traffic,t OtAl_request_times) Print display_format.head () Display_format.transverse_line () for host in Log_report:       Times = host[1][' times ' times_percent = (float (times)/float (fileanalysis_obj.total_request_times)) * 100                         Print display_format.formatstring% (host[0], display_format.format_size (host[1][' size '),                         TIMES,STR (times_percent) [0:5], host[1]['],host[1][' 404 '],                              host[1]['],host[1][' 403 ', host[1][' 302 '],host[1][' 304 '],host[1][' 503 ']) if (not lines) or total_ip = = Lines:display_format.transverse_line () print Display_format.fo Rmatstring% (total_ip,total_traffic, total_request_times, ' 100% ', Fileana lysis_obj.total_200, fileanalysis_obj.total_404, fileanalysis_obj.total_50                      0,    fileanalysis_obj.total_403, fileanalysis_obj.total_302, Fileanalysis_ obj.total_304, fileanalysis_obj.total_503) display_format.execut_time () if __name__ = = ' __ma In__ ': main_obj = Main () Main_obj.main ()

