Paramiko multi-Process Synchronization directory

Recently, the Lsync synchronization discovery is slow, so the files are synchronized through the pipeline after the script.

The contents are as follows:

#!/usr/bin/python   import paramiko  import osimport sysimport  TIMEIMPORT MULTIPROCESSINGIMPORT DATETIME IP_LIST = []ROOM_ID = SYS.ARGV[1] Cur_time = datetime.datetime.now ()  def upfile (Host_ip,local_path,remote_path):     privatekey = os.path.expanduser ('/root/.ssh/id_rsa ')     key=paramiko . Rsakey.from_private_key_file (Privatekey)     scp = paramiko. Transport ((host_ip, 22))     scp.connect (username= ' root ',  pkey=key)      sftp = paramiko. Sftpclient.from_transport (SCP)     src = local_path    des  = remote_path    os.chdir (Os.path.split (Local_path) [0])      Parent=os.path.split (Local_path) [0]    for walker in os.walk (parent):         try:             sftp.mkdir (walker[0])             for  file in walker[2]:                 sftp.put (Os.path.join (Walker[0],file), Os.path.join (walker[0],file))                  print  "Execute time  is %s and room is %s " % (cur_time,room_id)                           except:            for file in  walker[2]:                 sftp.put (Os.path.join (walker[0],file), Os.path.join (Walker[0],file))     scp.close () def all_ip ():     f = file ('/opt/scripts/static_host.txt ', ' R ')     c = f.readlines ( )     for x in c:        ip =  x.split (' \ n ') [0]        ip_list.append (IP)      f.close ()   if __name__== ' __main__ ':    print  "-" *50     print room_id    if not room_id:         sys.exit ()     all_ip ()     pool =  Multiprocessing. Pool (processes=5)     threads = []    print  "Begin ... "    for i in ip_list:        room_ Path =&nBSP; ' /app/data/ckl/' + room_id +  '/'         pool.apply_async ( Upfile, (I,room_path,room_path))     pool.close ()     pool.join ()       for res in threads:         Print res.get ()

Add a pipeline to invoke this script after PHP execution is complete:

PHP/APP/DATA/CKL/YIIC JSON notify | Sed-n ' S/^done: \ (. *\)/\1/p ' | Xargs-i {}/opt/scripts/sync_rooms.py {}

The following section is posted online:

Import socketimport osfrom stat import s_isdirclass sshsession (object):     # usage:    # detects dsa or rsa from  key_file, either as a string filename or a    #  file object.  Password auth is possible, but I will  judge you for     # using it. so:    #  ssh=sshsession (' targetserver.com ', ' Root ', Key_file=open (' Mykey.pem ', ' R '))     #  Ssh=sshsession (' targetserver.com ', ' root ', key_file= '/home/me/mykey.pem ')     # ssh= Sshsession (' targetserver.com ', ' root ', ' MyPassword ')     # ssh.put (' filename ', '/remote/ File/destination/path ')     # ssh.put_all ('/path/to/local/source/dir ', '/path/to/remote /destination ')   &nbsP; # ssh.get_all ('/path/to/remote/source/dir ', '/path/to/local/destination ')     #  ssh.command (' echo  ' Command to execute "')     def __init__ (self, Hostname,username= ' root ', Key_file=none,password=none):        #         #  Accepts a file-like object  ( Anything with a readlines ()  function)            #  in either dss_key or rsa_key with a private  Key.  since i don ' T         #  ever  intend to leave a server open to a password auth.         #        self.sock =  socket.socket (socket.af_ineT, socket. Sock_stream)         self.sock.connect ((hostname,22))          self.t = paramiko. Transport (Self.sock)         self.t.start_client ()          keys = paramiko.util.load_host_keys (Os.path.expanduser (' ~/.ssh/known_ Hosts '))         key = self.t.get_remote_server_key ()          # supposed to check for key in  Keys, but i don ' T much care right now to find the right  notation        if key_file is not none:             if isinstance (KEY,STR):            &nbSp;    key_file=open (Key, ' R ')              key_head=key_file.readline ()              key_file.seek (0)             if  ' DSA '  in key_head:                 keytype=paramiko. dsskey            elif  ' RSA '  in key _head:                keytype= Paramiko. rsakey            else:                 raise exception ("Can ' t  Identify key type ")             Pkey=keytype.from_private_key (Key_file)              self.t.auth_publickey (Username, pkey)         else:             if password is not none:                 self.t.auth_ Password (username,password,fallback=false)              else: raise exception (' Must supply either key_file or password ')         self.sftp=paramiko. Sftpclient.from_transport (self.t)     def command (self,cmd):         #  breaks the command by lines, sends and  receives         #  each line and its output separately         #        #  returns the server response  text as a string        chan =  Self.t.open_session ()         chan.get_pty ()          chan.invoke_shell ()         chan.settimeout ( 20.0)         ret= "        try :             ret+=chan.recv (1024x768)          except:             chan.send (' \ n ')             ret+=chan.recv ( 1024x768)             RET+=CHAN.RECV (1024x768)              RET+=CHAN.RECV (1024x768)         for line  in cmd.split (' \ n '):             Chan.send (Line.strip ()  +  ' \ n ')              RET+=CHAN.RECV (1024x768)         return ret     def put (Self,localfile,remotefile):        #   copy localfile to remotefile, overwriting or creating as needed.         self.sftp.put (Localfile,remotefile)     def  Put_all (Self,localpath,remotepath):         #  recursively  upload a full diRectory        os.chdir (Os.path.split (LocalPath) [0])          parent=os.path.split (LocalPath) [1]         for walker in os.walk (parent):             try:                 self.sftp.mkdir (Os.path.join (remotepath,walker[0))              except:                 pass            for  file in walker[2]:                 self.put (Os.path.join (Walker[0],file), Os.path.join (remotepath,walker[0],file))      def get (self,remoteFile,localfile):         #  copy remotefile to  localfile, overwriting or creating as needed.         self.sftp.get (Remotefile,localfile)     def sftp_walk (self, RemotePath):        # kindof a stripped down   version of os.walk, implemented for          # sftp.  tried running it flat without the yields, but  it really        # chokes on big  directories.        path=remotepath         files=[]        folders=[]         for f in self.sftp.listdir_attr (RemotePath):             if s_isdir (F.st_mode):                 folders.append (F.filename)              else:                 files.append (F.filename)         print  (path,folders,files)         yield path,folders,files         for folder in folders:             new_path=os.path.join (Remotepath,folder)              for x in self.sftp_walk (New_path):            &Nbsp;    yield x    def get_all (Self,remotepath,localpath):         #  recursively download a full  directory        #  harder than it sounded  at first, since paramiko won ' t walk         #        # For the record, something  like this would gennerally be faster:         # ssh [email protected]  ' Tar -cz /source/folder '  | tar -xz         self.sftp.chdir (Os.path.split (RemotePath) [0])          parent=os.path.split (RemotePath) [1]         try:            os.mkdir (LocalPath)          except:            pass         for walker in self.sftp_walk (parent):             try:                 os.mkdir (Os.path.join (localpath,walker[0))              except:                 pass             for file in walker[2]:                 self.get (Os.path.join (Walker[0],file), Os.path.join (localpath,walker[ 0],file))    &NBsp;def write_command (Self,text,remotefile):        #   writes text to remotefile, and makes remotefile executable.         #  This is perhaps a bit niche,  but i was thinking i needed it.        #   For the record, I was incorrect.         self.sftp.open (RemoteFile, ' W '). Write (text)          Self.sftp.chmod (remotefile,755)

This article is from the "take a deep Breath again" blog, make sure to keep this source http://ckl893.blog.51cto.com/8827818/1785297

Paramiko multi-Process Synchronization directory