Detailed Python log printing and writing concurrency implementation code

We generally use logging log printing, but logging is thread-safe, multi-process also has a lot of introduction, introduce some file lock, to logging well configuration, can support.

However, by testing, it is easy to find multiple processes that are prone to duplicate file writes or to print a normal file.

My log requirements are relatively simple, able to distinguish between files, the correct writing log file.

The file lock is introduced, the log write function is encapsulated into an operation _logger class, and the log name and write level are encapsulated into a business class logger.

This example is based on the Python3 implementation. This example 20 processes concurrently, write 3 files each, each file writes more than 100 rows of data, there is no data redundancy in the log file, there is no data missing.

#-*-coding:utf-8-*-"" "Author:yinshunyaodate:2017/3/5 0005 PM 10:50" "# import Loggingimport osimport time# Use a third-party system lock for file locking and unlocking if os.name = = ' nt ': Import Win32con, win32file, pywintypes lock_ex = Win32con. Lockfile_exclusive_lock lock_sh = 0 # The default value LOCK_NB = Win32con. lockfile_fail_immediately __overlapped = pywintypes. OVERLAPPED () def lock (file, flags): hfile = Win32file._get_osfhandle (File.fileno ()) Win32file. LockFileEx (hfile, flags, 0, 0xffff0000, __overlapped) def unlock (file): hfile = Win32file._get_osfhandle (file.fi Leno ()) Win32file. Unlockfileex (hfile, 0, 0xffff0000, __overlapped) elif os.name = ' posix ': from fcntl import lock_ex def LOCK (file, FL AGS): Fcntl.flock (File.fileno (), flags) def unlock (file): Fcntl.flock (File.fileno (), Fcntl.    Lock_un) else:raise runtimeerror ("File Locker only support NT and Posix platforms!") Class _logger:file_path = '  #初始化日志路径 @staticmethod def init ():      If not _logger.file_path: _logger.file_path = '%s/log '% Os.path.abspath (Os.path.dirname (__file__)) return True @staticmethod def _write (Messge, file_name): If not Messge:return True MESSG E = Messge.replace (' \ t ', ', ') file = ' {}/{} '. Format (_logger.file_path, file_name) while True:try                : F = open (file, ' A + ') lock (F, LOCK_EX) break except: Time.sleep (0.01) Continue # Make sure the buffer contents are written to the file while True:try:f.writ                E (messge + ' \ n ') F.flush () Break Except:time.sleep (0.01)  Continue while True:try:unlock (f) f.close () return True Except:time.sleep (0.01) Continue @staticmethod def write (message , file_name, only_print=False): If not _logger.init (): Return print (message) If not only_print: _logger._write (mes        Sage, file_name) class Logger:def __init__ (self, logger_name, file_name= "): Self.logger_name = Logger_name Self.file_name = file_name # Generates messages based on message level, custom format, Def _build_message (self, message, level): Try:r Eturn ' [%s]\t[%5s]\t[%8s]\t%s '% (time.strftime ('%y-%m-%d%h:%m:%s '), level, self.logger_name, message ) except Exception as E:print (' Parse log message exception: {} '. Format (e)) return ' Def warning (self, messag E): _logger.write (self._build_message (message, ' WARN '), Self.file_name) def WARN (self, message): _logger. Write (self._build_message (message, ' WARN '), self.file_name) def error (self, message): _logger.write (self._build_ Message (Message, ' ERROR '), self.file_name) def info (self, message): _logger.write (self._build_message (Message, ' INFO '), Self.file_name,TRUE) def debug (self, message): _logger.write (self._build_message (Message, ' Debug '), Self.file_name) # cyclic print log test letter Number Def _print_test (count): Logger = Logger (logger_name= ' test{} '. Format (count), File_name= ' test{} '. Format (count% 3)) K EY = 0 while True:key + = 1 # print (' {}-{} '. Format (logger, key)) Logger.debug ('%d '% key) L    Ogger.error ('%d '% key) if __name__ = = ' __main__ ': From multiprocessing import Pool, Freeze_support Freeze_support () # Process pool to test pool = Pool (processes=20) Count = 0 while count < 20:count + = 1 Pool.apply_async (f Unc=_print_test, args= (count,)) Else:pool.close () Pool.join ()