Python statistics nginx log domain download volume

Statistics Nginx access log, Access.log form:

1xx.xx.xxx.xxx--[09/oct/2017:10:48:21 +0800] "get/images/yyy/2044/974435412336/cover/9787503434r.jpg HTTP/1.1" 304 0 "http://www.xxx.net/" "mozilla/5.0 (Windows NT 6.1; Win64; x64) applewebkit/537.36 (khtml, like Gecko) chrome/xx.0.xx2.xx safari/537.36 "0.001" "-" "-" "-" xxx.xxx.xxx.xxx:xxxx "" 304 "" 0 "" 0.001 "

(1) The domain name and number of visits will be stored in the database

(2) List the top 10 downloads

Method: Read the Access.log log to intercept its required information to generate a dictionary, using the uniqueness of the dictionary keys to count the number of downloads, and to sort the dictionary, to intercept the top 10 domain names and download times

#!/usr/bin/env python3#__*__coding:utf8__*__import mysqldbimport re#db =  MySQLdb.connect (host= ' localhost ', user= ' root ', passwd= ' 123456 ', db= ' TestDB ') #access. Log :log style  is ===> ' 114.250.90.86  - - [09/oct/2017:10:48:21 +0800]  ' GET  /images///book/201709/9787503541216/cover/9787503541216.jpg http/1.1 " 304 0 "/http www.dyz100.net/" " mozilla/5.0  (windows nt 6.1; win64; x64)  applewebkit/ 537.36  (Khtml, like gecko)  chrome/60.0.3112.113 safari/537.36 " " 0.001 " "-"   "-"   "-" "10.1.12.201:9092"   "304"   "0"   "0.001" "Def domainname ():     dn_dict = {}    patten =  ' (HTTP|HTTPS)://[a-z]+. [A-z0-9]+. [a-za-z]+ '     f = open ('/tmp/access.log ', ' R ')     for  Line in f:   &nbsP;    domain_name = re.search (Patten,line.split () [ten])          if Domain_Name:             dn = domain_name.group (0)              if dn not in dn_dict:                 dn_dict[dn] = 1             else:                 dn_dict[dn] += 1     #使用内建函数sorted () sort     # Sorted (Dict.items (),  key=lambda e:e[1], reverse=true)     dn_ranking =  sorted (Dn_dict.items (), key=lambda item:item[1], reverse = true)      return dn_rankiNgdef dispalydomainname ():     "print top 10 ranking for  download  '     dn_ranking = domainname ()     print (" Download volume Top 10 Domain Name list is: ")     for item in dn_ranking[:11]:         print ("Domain: %s ===>  downloaded:  %d"  %  (Item[0],item[1]) def  insert ():     db = mysqldb.connect (host= ' localhost ', user= ' root ', passwd= ' 123456 ', db= ' TestDB ')     cursor = db.cursor ()      Cursor.execute (' drop table if exists ranking ')     sql =  ' create table ranking  (Id int)  not null primary key auto_ Increment, domainname varchar (+),  download_times int (+)) '      Cursor.execute (SQL)     dn_rAnking = domainname ()     for item in dn_ranking:         sql =  ' insert into ranking  (domainname,download _times)  values  ('%s ',%d) '  % (item[0],item[1])          Cursor.execute (SQL)     # sql =  "insert into ranking " ( Domainname,download_times)  values  (' WPT ', 2) '     try:          #执行sql语句         cursor.execute (SQL)          db.commit ()     except:         db.rollback ()     db.close () Def select ():     db = mysqldb.connect (host= ' localhost ', user= ' root ', passwd= ' 123456 ', db= ' TestDB ')      cursor = db.curSor ()     sql =  ' select * from ranking '      try:        cursor.execute (SQL)          results = cursor.fetchall ()         for  row in results:            print (" Domain: %s ===>  download times:  %d " % (row[1],row[2)))     except:         print (' Error: unable to fetch data ')      db.close () if __name__ ==  ' __main__ ':     dispalydomainname ()       #insert ()      #select ()

650) this.width=650; "src=" Https://s4.51cto.com/wyfs02/M01/A6/DB/wKioL1ncmeGSYWIqAAdxRctKAjI233.png "title=" Pictures _ 20171010180908.png "alt=" Wkiol1ncmegsywiqaadxrctkaji233.png "/>

650) this.width=650; "src=" Https://s2.51cto.com/wyfs02/M02/A6/DB/wKioL1ncmfmzmv0HAAnf2N8ETA8887.png "title=" Pictures _ 20171010180914.png "alt=" Wkiol1ncmfmzmv0haanf2n8eta8887.png "/>

This article is from the "LINUX" blog, so be sure to keep this source http://wangpengtai.blog.51cto.com/3882831/1971249

Python statistics nginx log domain download volume