Python uses inotify to write Nginx logs to the database in real time

Using the Pyinotify library, I'm using this here, https://github.com/seb-m/pyinotify.
In fact, on-line Yum also has pyinotify Library can be installed.
Write database is pymysql here to make a record,
The first-service pyinotify implements a tail-f function:

#!/opt/python3/bin/python3#import pyinotifyimport timeimport osimport sysclass processtransientfile (pyinotify. processevent):    def process_in_modify (self,event): Line        = File.readline ()        if line:            print (line, end= ' ' If __name__ = = ' __main__ ':    filename = sys.argv[1]    file = open (filename, ' R ')    st_results = os.stat (filename )    st_size = st_results[6]    file.seek (st_size)    wm = Pyinotify. Watchmanager ()    notifier = pyinotify. Notifier (WM)    wm.watch_transient_file (filename, pyinotify. In_modify, Processtransientfile)    Notifier.loop ()

The log can then be viewed in real time via Pytaif/usr/local/nginx/logs/www.tbbpay.com.access.log.

This is real-time viewing, just like the tail-f feature. Prints only one row,

Now is the definition of a nginxloganalyzer function for log analysis, is the default Nginx log, this does not use the regular, using the soil method to find specific characters.

def nginxloganalyzer: #print (line) G1 = Line.find (' [') g2 = Line.find ('] ') h1 = Line.find (' "') H2 = L Ine.find (' "', h1+1) h3 = Line.find ('" ', h2+1) h4 = Line.find (' "', h3+1) h5 = Line.find ('" ', h4+1) h6 = line.fi nd (' "', h5+1) #print (" g1:%d "%g1) #print (" g2:%d "%g2) #print (" h1:%d "%h1) #print (" h2:%d "%h2) #print (" h3:%d "%h3     ) #print ("h4:%d"%h4) #print ("h5:%d"%h5) #print ("h6:%d"%h6) remote_addr = "Remote_user =" "Time=" " time_local = "" Time_zone = "request =" "Status =" "Body_bytes_sent =" "Http_referer =" "Http_ User_agent = "" Http_x_forwarded_for = "" time = line[g1+1:g2] time_local = time.split () [0] Time_zone = time.s Plit () [1] request = Line[h1+1:h2] Http_referer = line[h3+1:h4] http_user_agent = Line[h5+1:h6] Remote_addr = l Ine.split () [0] Remote_user = line.split () [1] status = Line.split () [8] body_bytes_sent = Line.split () [9] Re Quest = Urllib.parse.unqUote (Request) print ("time:%s"% (time)) print ("time_local:%s"% (time_local)) print ("time_zone:%s"% (Time_zone)) Print ("request:%s"% (request)) print ("http_referer:%s"% (http_referer)) print ("http_user_agent:%s"% (Http_user_agen T) print ("status:%s"% (status)) print ("body_bytes_sent:%s"% (body_bytes_sent)) print ("Request--------:%s"% ( Urllib.parse.unquote (Request))) L = [] L.append (REMOTE_ADDR) l.append (Remote_user) l.append (time) l.append (time_local) l.append (time_zone) l.append (Request) L.append (status) l.append (body_bytes_sent) l.append (http _referer) l.append (http_user_agent) l.append (http_x_forwarded_for) print (L) return L

Parse a row log data to get a list to spare, and then write a intodb function to insert the database

Here, write the database first.

CREATE DATABASE ' Nginxlog ' charset=utf8;use nginxlog; CREATE TABLE ' Nginxlog ' (  ' id ' int (one) not null auto_increment,  ' remote_add ' varchar () DEFAULT NULL,  ' Remote_user ' varchar ' default NULL, ' time  ' varchar default NULL,  ' time_local ' varchar (default null,<) c5/> ' time_zone ' varchar default NULL,  ' request ' varchar (1024x768) default null,  ' status ' varchar (default) NULL,  ' body_bytes_sent ' varchar (TEN) default null,  ' http_referer ' varchar (1024x768) default null,  ' Http_user _agent ' varchar (1024x768) default null,  ' http_x_forwarded_for ' varchar (1024x768) default NULL,  PRIMARY KEY (' id '),) Engine=innodb auto_increment=1001 DEFAULT Charset=utf8; GRANT all privileges the nginxlog.* to ' nginxlog ' @ ' 192.168.1.112 ' identified by ' nginxlog ';

This creates the libraries and tables for the table Nginxlog. and create a connection user.

Here are the functions defined by the Insert database

def intodb (line):    l = nginxloganalyzer [line]        s = "INSERT into ' nginxlog '. ' Nginxlog ' (' id ', ' remote_add ', ' remote _user ', ' time ', ' time_local ', ' time_zone ', ' request ', ' status ', ' body_bytes_sent ', ' http_referer ', ' http_user_agent ', ' Http_x_forwarded_for ') VALUES (' null ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s '); "% ( L[0],L[1],L[2],L[3],L[4],L[5],L[6],L[7],L[8],L[9],L[10])    print (s)    Cur.execute (s)    Conn.commit ()

Here are all source code nginxlogtomysql.py

[[email protected] ~]# cat nginxlogtomysql.py #!/opt/python3/bin/python3#import pyinotifyimport timeimport Osimport sysimport urllibimport urllib3import pymysqlclass processtransientfile (pyinotify. processevent): Def process_in_modify (self,event): line = File.readline () If line: #nginxLogAna Lyzer (line) Intodb (line) def Nginxloganalyzer (line): Print (line,end= ") G1 = Line.find (' [') g2 = Line.f    IND ('] ') h1 = Line.find (' "') H2 = Line.find ('" ', h1+1) h3 = Line.find (' "', h2+1) h4 = Line.find ('" ', h3+1) h5 = Line.find (' "', h4+1) h6 = Line.find ('" ', h5+1) remote_addr = "" Remote_user = "" Time= "" time_local =    "Time_zone =" "Request =" "Status =" "Body_bytes_sent =" "Http_referer =" "Http_user_agent =" " Http_x_forwarded_for = "" time = line[g1+1:g2] time_local = time.split () [0] Time_zone = time.split () [1] requ EST = line[h1+1:h2] http_referer = Line[h3+1:h4] Http_user_aGent = Line[h5+1:h6] remote_addr = line.split () [0] Remote_user = line.split () [1] status = Line.split () [8] Body _bytes_sent = Line.split () [9] request = Urllib.parse.unquote (Request) #print ("time:%s"% (time)) #print ("Time_ local:%s "% (time_local)) #print (" time_zone:%s "% (Time_zone)) #print (" request:%s "% (Request)) #print (" Http_referer :%s "% (Http_referer)) #print (" http_user_agent:%s "% (http_user_agent)) #print (" status:%s "% (status) #print (" Body_ bytes_sent:%s "% (body_bytes_sent)) #print (" Request--------:%s "% (urllib.parse.unquote (request)) L = [] L.append ( REMOTE_ADDR) l.append (Remote_user) l.append (time) l.append (time_local) l.append (time_zone) l.append (Request ) l.append (status) l.append (body_bytes_sent) l.append (http_referer) l.append (http_user_agent) l.append (http _x_forwarded_for) #print (l) return ldef Intodb (line): L = Nginxloganalyzer [line] s = "INSERT into ' nginxlog '." Nginxlog ' (' id ', ' remote_add ', ' remote_user ', ' time ', ' time_local ', ' time_zone ', ' request ', ' status ', ' body_bytes_sent ', ' http_referer ', ' http_ ' User_agent ', ' http_x_forwarded_for ') VALUES (' null ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s '); % (l[0],l[1],l[2],l[3],l[4],l[5],l[6],l[7],l[8],l[9],l[10]) #print (s) cur.execute (s) conn.commit () if __name__ = = ' __MAIN__ ': conn = pymysql.connect (host= ' 192.168.1.112 ', port=3306, user= ' Nginxlog ', passwd= ' Nginxlog ', db= ' Nginxlog ', charset= "UTF8") cur = conn.cursor () cur.execute ("SET NAMES UTF8") filename = sys.argv[1] File = open (filename, ' r ') st_results = os.stat (filename) st_size = st_results[6] File.seek (st_size) wm = Pyinotify. Watchmanager () notifier = pyinotify. Notifier (WM) Wm.watch_transient_file (filename, pyinotify. In_modify, Processtransientfile) Notifier.loop () [[email protected] ~]#

Run as follows:

./nginxlogtomysql.py/usr/local/nginx/logs/www.tbbpay.com.access.log

The effect is as follows

Python uses inotify to write Nginx logs to the database in real time