Detailed description of python log printing and writing concurrency implementation code

This article describes how to print python logs and write concurrent implementation code. generally, logging logs are used for printing. However, logging is thread-safe and many processes are introduced. some file locks are introduced, configure logging to support logging.

However, through testing, it is found that multiple processes are prone to repeated file writing or printing normal file writing failures.

My log requirements are relatively simple. I can differentiate files and write log files correctly.

The file lock is introduced. the log writing function is encapsulated into an action_logger class. the log name and write level are encapsulated into a business Logger class.

This example is implemented based on python3. In this example, 20 processes are concurrently written into three files, each of which writes more than 100 lines of data per second. there is no data redundancy in the log file and no data omission.

#-*-Coding: UTF-8-*-"Author: yinshunyaoDate: 0005 "# import loggingimport osimport time # use a third-party system lock to lock files and unlock if OS. name = 'nt ': import win32con, win32file, pywintypes LOCK_EX = win32con. LOCKFILE_EXCLUSIVE_LOCK LOCK_SH = 0 # The default value LOCK_NB = win32con. LOCKFILE_FAIL_IMMEDIATELY _ overlapped = pywintypes. OVERLAPPED () def lock (file, flags): hfile = win32file. _ get_osfhandle (fi Le. fileno () win32file. lockFileEx (hfile, flags, 0, 0xffff0000, _ overlapped) def unlock (file): hfile = win32file. _ get_osfhandle (file. fileno () win32file. unlockFileEx (hfile, 0, 0xffff0000, _ overlapped) elif OS. name = 'posix': from fcntl import LOCK_EX def lock (file, flags): fcntl. flock (file. fileno (), flags) def unlock (file): fcntl. flock (file. fileno (), fcntl. LOCK_UN) else: raise RuntimeError ("File L Ocker only support NT and Posix platforms! ") Class _ Logger: file_path ='' # Initialize the log path @ staticmethod def init (): if not _ Logger. file_path: _ Logger. file_path = '% s/log' % OS. path. abspath (OS. path. dirname (_ file _) return True @ staticmethod def _ write (messge, file_name): if not messge: return True messge = messge. replace ('\ t',', ') file = '{}/{}'. format (_ Logger. file_path, file_name) while True: try: f = open (file, 'A + ') lock (f, LOCK_EX) break failed t: time. sleep (0.01) continue # ensure that the buffer content is written to the file while True: try: f. write (messge + '\ n') f. flush () break failed t: time. sleep (0.01) continue while True: try: unlock (f) f. close () return True before T: time. sleep (0.01) continue @ staticmethod def write (message, file_name, only_print = False): if not _ Logger. init (): return print (message) if not only_print: _ Logger. _ write (message, file_name) class Logger: def _ init _ (self, logger_name, file_name = ''): self. logger_name = logger_name self. file_name = file_name # Generate the message def _ build_message (self, message, level) based on the message level and custom format: try: return '[% s] \ t [% 5 s] \ t [% 8 s] \ t % s' \ % (time. strftime ('% Y-% m-% d % H: % M: % S'), level, self. logger_name, message) failed t Exception as e: print ('log parsing message Exception :{}'. format (e) return ''def warning (self, message): _ Logger. write (self. _ build_message (message, 'warn'), self. file_name) def warn (self, message): _ Logger. write (self. _ build_message (message, 'warn'), self. file_name) def error (self, message): _ Logger. write (self. _ build_message (message, 'error'), self. file_name) def info (self, message): _ Logger. write (self. _ build_message (message, 'info'), self. file_name, True) def debug (self, message): _ Logger. write (self. _ build_message (message, 'debug'), self. file_name) # loop print log test function def _ print_test (count): logger = Logger (logger_name = 'Test {}'. format (count), file_name = 'Test {}'. format (count % 3) key = 0 while True: key + = 1 # print ('{}-{}'. format (logger, key) logger. debug ('% d' % key) logger. error ('% d' % key) if _ name _ =' _ main _ ': from multiprocessing import Pool, freeze_support () # test pool = Pool (processes = 20) count = 0 while count <20: count + = 1 pool. apply_async (func = _ print_test, args = (count,) else: pool. close () pool. join ()

The above is a detailed description of the python log printing and writing concurrency implementation code. For more details, please follow other articles in the first PHP community!