Python Log Collection Server

Cause: The Log collection service for Python is thread-safe (writes to the same file, locks are used), but it cannot be processed for multi-process scenarios. The Python official documentation recommends using a TCP server dedicated to the collection of logs to ensure that the writes to the files are secure. The twisted-based implementation of the Log collection server is available for reference, and the program is tested on CentOS and can be used in production environments

6,000 log records per second when client and server are on a single machine

#!/usr/bin/env python#-*-coding:utf-8-*-################################### logserver# receive and store log information via socket # Note: Test program for test_logserver.py# modification time: 2013-11-22#################################### standard libimport OS, sysimport Timeimport twistedimport loggingimport tracebackimport structimport cpickle # use Cpickle for Speedfrom Lo Gging.handlers Import timedrotatingfilehandlerfrom twisted.internet.protocol Import protocol, factory# our own Codeimport settingsfrom Log_record Import initlog# generates a def logger (name) based on Gen_log_name name: name = Name.replace ('. ', ' _  ') return name + '. Log ' class Receiveprotocol (Protocol): Header_length = 4 def __init__ (self): self.buf        = ' Self.state = ' init ' self.require_length = receiveprotocol.header_length # Record the logger name of the current completion initialization Self.logger_dict = {} def connectionmade (self): logger = Logging.getlogger (' log_server ') log Ger.info (' [makeconnection] establishes a socket connection with logger_client.      ')          def datareceived (self, data): Logger = Logging.getlogger (' log_server ') Try:if len (self                . BUF + data) >= Self.require_length:self.buf = self.buf + data # ready for action data = Self.buf[0:self.require_length] # Take out the buffer self.buf = Self.buf[self.require_leng Th:] Self.solve (data) # may read multiple log records at a time if Self.buf:self.da            Tareceived (") else:self.buf + = data except Baseexception, E: Logger.error (' Processing SMS record failed ' + str (e) + ' \ n ' + traceback.format_exc ()) def solve (self, data): Logger = l Ogging.getlogger (' log_server ') Statehandler = None Try:pto = ' proto_ ' + self.state St            Atehandler = GetAttr (self,pto) except AttributeError:logger.error (' callback ', self.state, ' not found ') Self.transport.Loseconnection () Statehandler (data) if self.state = = ' Done ': self.transport.loseConnectio N () def connectionlost (self, reason): Logger = Logging.getlogger (' log_server ') logger.info (' [conn Ectionlost] The socket connection to the logger_client is closed.        ') # Log def proto_record (self, data): LogRecord = Logging.makelogrecord (cpickle.loads (data)) If Logrecord.name not in Self.logger_dict:logger = Initlog (logrecord.name, settings. PATH, Gen_log_name (logrecord.name), loglevel=logging. DEBUG) Self.logger_dict[logrecord.name] = Logger Self.logger_dict[logrecord.name].handle (Logreco RD) # Modify the next action and the desired length self.state = ' init ' self.require_length = Receiveprotocol.header_leng TH # processing header information def proto_init (self, data): Length = Struct.unpack ('! I ', data[0:receiveprotocol.header_length]) [0] # Modify the next action and the desired length self.state = ' record ' Self. require_length = Length If Len (self.buf) >= self.require_length:data = self.buf[0:self.requ        Ire_length] # Take the data out of the buffer self.buf = self.buf[self.require_length: self.solve (data) Class Receivefactory (Factory): Def buildprotocol (self, addr): Return Receiveprotocol () def main (): print ' Lo    Gserver has started. '         Logger = Logging.getlogger (' log_server ') logger.info (' Logserver has started. ') From twisted.internet import Epollreactor epollreactor.install () reactor = Twisted.internet.reactor Reactor.liste NTCP (settings. PORT, Receivefactory ()) Reactor.run () if __name__ = = ' __main__ ': Main ()

Complete the program:

Https://github.com/vearne/log_server/tree/master

Resources:

Https://docs.python.org/2/howto/logging-cookbook.html#logging-cookbook

Python Log Collection Server