Python-implemented file synchronization server instance

The example in this article describes the file synchronization server implemented by Python. Share to everyone for your reference. The implementation method is as follows:

The server uses Asyncore, and the file is saved to local after it is received.

The client uses Pyinotify to monitor the directory changes and send the changed files to the server.

Focus:

1. Use the structs package to send the file information, after the server received, according to the file information to receive the client sent over the file.

2. The client uses multi-threading, Pyinotify monitors the file changes, puts it in a queue, and is sent by another thread.

On the code:

Service side:

# receive file from client and store them to file use asyncore.# #/usr/bin/python #coding: utf-8 import asyncore Import Socket from socket Import errno Import logging import time import sys import struct import OS import fcntl import Threadin G from rrd_graph import makegraph try:import rrdtool except (Importerror, importwarnning): print "Hope this Informati On can help you: ' print ' Can not ' find pyinotify module in Sys path, just run [apt-get install Python-rrdtool] in Ubuntu. "Sys.exit (1) class RequestHandler (Asyncore.dispatcher): Def __init__ (self, sock, Map=none, chunk_size=1024): Self      . Logger = Logging.getlogger ('%s-%s '% (self.__class__.__name__, str (sock.getsockname ()))) Self.chunk_size = Chunk_size asyncore.dispatcher.__init__ (self,sock,map) self.data_to_write = List () def readable (self): #self. logger.de     Bug ("readable () called.") Return True def writable (self): response = (not self.connected) or Len (self.data_to_write) #self. logger.deBug (' writable ()%s data length,%s '% (response, Len (self.data_to_write))) return response def Handle_wri Te (self): data = Self.data_to_write.pop () #self. Logger.debug ("Handle_write ()->%s size:%s", Data.rstrip (' \ r \ n '), Len (data)) sent = Self.send (Data[:self.chunk_size]) if sent < Len (data): remaining = data[sent:] Se Lf.data_to_write.append (remaining) def handle_read (self): self.writen_size = 0 Nagios_perfdata = '. /perfdata ' Head_packet_format = '! LL128S128SL "head_packet_size = struct.calcsize (head_packet_format) data = SELF.RECV (head_packet_size) if not     Data:return Filepath_len, Filename_len, filepath,filename, filesize = Struct.unpack (head_packet_format,data) filepath = Os.path.join (Nagios_perfdata, filepath[:filepath_len]) filename = Filename[:filename_len] Self.logger . Debug ("Update file:%s"% filepath + '/' + filename) try:if not os.path.exists (filepath): Os.makedirs(filepath) except oserror:pass self.fd = open (Os.path.join (filepath,filename), ' W ') #self. FD = Open (fil ename, ' W ') if filesize > self.chunk_size:times = Filesize/self.chunk_size first_part_size = times * Self.chunk_size second_part_size = filesize% self.chunk_size while 1:try:data = Self.recv           (self.chunk_size) #self. Logger.debug ("Handle_read ()->%s size.", Len (data)) except Socket.error,e: If e.args[0] = = errno. Ewouldblock:print "Ewouldblock" time.sleep (1) Else: #self. Logger.debug ("Err or happend while receive data:%s "% e", break Else:self.fd.write (data) SELF.FD.FL Ush () Self.writen_size + = len (data) if self.writen_size = = First_part_size:break #r Eceive the packet at last while 1:try:data = Self.recv (second_part_size) #self. LogGer.debug ("Handle_read ()->%s size.", Len (data)) except socket.error,e:if e.args[0] = = errno. Ewouldblock:print "Ewouldblock" time.sleep (1) Else: #self. Logger.debug ("Err or happend while receive data:%s "% e", break Else:self.fd.write (data) SELF.FD.FL Ush () Self.writen_size + = len (data) If Len (data) = = Second_part_size:break elif files ize <= Self.chunk_size:while 1:try:data = self.recv (filesize) #self. Logger.debug (" Handle_read ()->%s size. ", Len (data)) except socket.error,e:if e.args[0] = = errno. Ewouldblock:print "Ewouldblock" time.sleep (1) Else: #self. Logger.debug ("Err or happend while receive data:%s "% e", break Else:self.fd.write (data) SELF.FD.FL        Ush () Self.writen_size + = len (data)   If Len (data) = = Filesize:break Self.logger.debug ("File size:%s"% Self.writen_size) class SyncServer ( Asyncore.dispatcher): Def __init__ (self,host,port): asyncore.dispatcher.__init__ (self) self.debug = True sel F.logger = Logging.getlogger (self.__class__.__name__) self.create_socket (socket.af_inet,socket. SOCK_STREAM) self.set_reuse_addr () Self.bind ((Host,port)) Self.listen (+) def handle_accept (self): CLI Ent_socket = Self.accept () If Client_socket is None:pass else:sock, addr = Client_socket #self . Logger.debug ("Incoming connection from%s"% repr (addr)) handler = RequestHandler (Sock=sock) class Runserver (thread Ing. Thread): def __init__ (self): Super (Runserver,self). __init__ () Self.daemon = False def run (self): Server = SyncServer (", 9999) Asyncore.loop (use_poll=true) def startserver (): Logging.basicconfig (level=logging.       DEBUG, format= '% (name) s:% (message) s ',      ) Runserver (). Start () #MakeGraph (). Start () if __name__ = = ' __main__ ': StartServer () 

Client:

# Monitor path with inotify (Python module), and send them to remote server.# # use Sendfile (2) Instead of the Send function in Socket, if we have python-sendfile installed.# import socket import time import OS import sys import struct import thread ing import Queue try:import pyinotify except (Importerror, importwarnning): print "Hope this information can help Y    OU: "Print" Can not find pyinotify module in Sys path, just run [apt-get install python-pyinotify] in Ubuntu. " Sys.exit (1) Try:from sendfile import sendfile except (importerror,importwarnning): Pass Filetype_filter = [". Rrd", " . xml "] def check_filetype (pathname): For suffix_name in filetype_filter:if pathname[-4:] = = Suffix_name:r Eturn True try:end_string = Pathname.rsplit ('. ') [-1:] [0] end_int = Int (end_string) Except:pass else: # means pathname endwith digit return False CLA SS Sync_file (Threading. Thread): Def __init__ (self, addr, events_queue): Super (SYNc_file,self). __init__ () Self.daemon = False Self.queue = events_queue self.addr = addr Self.chunk_size          = 1024x768 def run (self): while 1:event = Self.queue.get () if Check_filetype (event.pathname):  Print Time.asctime (), event.maskname, event.pathname filepath = event.path.split ('/') [ -1:][0] filename = Event.name filesize = Os.stat (Os.path.join (event.path, filename)). st_size sock = Socket.socket (socket.a F_inet, Socket.          SOCK_STREAM) Filepath_len = Len (filepath) Filename_len = len (filename) sock.connect (SELF.ADDR) Offset = 0 data = Struct.pack ("! Ll128s128sl ", Filepath_len, Filename_len, filepath,filename,filesize) fd = open (Event.pathname, ' RB ') sock              . Sendall (data) if "Sendfile" in Sys.modules: # print ' Use Sendfile (2) ' while 1:           Sent = Sendfile (Sock.fileno (), Fd.fileno (), offset, self.chunk_size)   If sent = = 0:break Offset + = Sent else: # print "Use original send fun Ction "While 1:data = Fd.read (self.chunk_size) if is not Data:break sock. Send (data) sock.close () Fd.close () class EventHandler (pyinotify. processevent): Def __init__ (self, Events_queue): Super (Eventhandler,self). __init__ () Self.events_queue = even Ts_queue def my_init (self): Pass def process_in_close_write (self,event): Self.events_queue.put (event) d EF process_in_moved_to (self,event): Self.events_queue.put (event) def start_notify (path, Mask, sync_server): Events _queue = Queue.queue () Sync_thread_pool = list () for I in range: Sync_thread_pool.append (Sync_file (sync_se RVer, Events_queue))) for I in Sync_thread_pool:i.start () WM = Pyinotify. Watchmanager () notifier = pyinotify. Notifier (Wm,eventhandler (events_queue)) Wdd = Wm.add_watch (Path,mask,rec=true) Notifier.loop () def do_notify (): Perfdata_path = '/var/lib/pnp4nagios/perfdata ' mask = Pyinoti Fy. In_close_write|pyinotify.     In_moved_to sync_server = (' 127.0.0.1 ', 9999) start_notify (perfdata_path,mask,sync_server) if __name__ = = ' __main__ ': Do_notify ()

Python Monitoring thread pool

#!/usr/bin/python Import Threading Import Time Class Monitor (threading. Thread): Def __init__ (self, *args,**kwargs): Super (Monitor,self). __init__ () Self.daemon = False Self.args = Args Self.kwargs = Kwargs self.pool_list = [] def run: print Self.args print Self.kwargs for n Ame,value in Self.kwargs.items (): obj = value[0] temp = {} Temp[name] = obj self.pool_list.append (         temp) while 1:print self.pool_list for Name,value in Self.kwargs.items (): obj = value[0] Parameters = value[1:] died_threads = Self.cal_died_thread (self.pool_list,name) print "Died_threads", Died_ Threads if Died_threads >0:for i in Range (died_threads): print "Start%s thread ..."% NA Me t = obj[0].__class__ (*parameters) T.start () self.add_to_pool_list (T,name) E Lse:break Time.sleep (0.5) def cal_died_thread (Self,pool_List,name): i = 0 for item in Self.pool_list:for k,v in Item.items (): if name = = K:lists = V for T in Lists:if not t.isalive (): self.remove_from_pool_list (t) I +=1 return i def           Add_to_pool_list (self,obj,name): For item in Self.pool_list:for k,v in Item.items (): if name = = k:          V.append (obj) def remove_from_pool_list (self, obj): For item in Self.pool_list:for k,v in Item.items (): Try:v.remove (obj) except:pass Else:return

How to use:

Rrds_queue = Queue.queue ()   make_rrds_pool = [] for   I in range (5):     make_rrds_pool.append (Makerrds (Rrds_ Queue)) for   i in Make_rrds_pool:     i.start ()   make_graph_pool = [] for   I in range (5):     make_graph_ Pool.append (Makegraph (rrds_queue)) for   i in Make_graph_pool:     i.start ()   monitor = Monitor (Make_rrds_ Pool= (Make_rrds_pool, rrds_queue), \            make_graph_pool= (Make_graph_pool, Rrds_queue))   Monitor.start ()

Analytical:

1. Accept the dictionary parameter, value is a tuple, the first element is the thread pool, followed by parameters.
2. Monitor the number of threads in the thread pool every 0.5 seconds, if the thread dies, log the number of dead threads, and then start the same number of threads.
3. If no thread dies, do nothing.

Calling the Django module externally

Import OS import sys sys.path.insert (0, '/data/cloud_manage ') from django.core.management import Setup_environ Import Settings Setup_environ (settings) from Common.monitor Import Monitor from django.db import connection, transaction

The premise is, to create a new Django project, here we create a new cloud_manage.
This will not only invoke Django's own modules, but also invoke project itself.

Hopefully this article will help you with Python programming.

