Recently, the Lsync synchronization discovery is slow, so the files are synchronized through the pipeline after the script.
The contents are as follows:
#!/usr/bin/python import paramiko import osimport sysimport TIMEIMPORT MULTIPROCESSINGIMPORT DATETIME IP_LIST = []ROOM_ID = SYS.ARGV[1] Cur_time = datetime.datetime.now () def upfile (Host_ip,local_path,remote_path): privatekey = os.path.expanduser ('/root/.ssh/id_rsa ') key=paramiko . Rsakey.from_private_key_file (Privatekey) scp = paramiko. Transport ((host_ip, 22)) scp.connect (username= ' root ', pkey=key) sftp = paramiko. Sftpclient.from_transport (SCP) src = local_path des = remote_path os.chdir (Os.path.split (Local_path) [0]) Parent=os.path.split (Local_path) [0] for walker in os.walk (parent): try: sftp.mkdir (walker[0]) for file in walker[2]: sftp.put (Os.path.join (Walker[0],file), Os.path.join (walker[0],file)) print "Execute time is %s and room is %s " % (cur_time,room_id) except: for file in walker[2]: sftp.put (Os.path.join (walker[0],file), Os.path.join (Walker[0],file)) scp.close () def all_ip (): f = file ('/opt/scripts/static_host.txt ', ' R ') c = f.readlines ( ) for x in c: ip = x.split (' \ n ') [0] ip_list.append (IP) f.close () if __name__== ' __main__ ': print "-" *50 print room_id if not room_id: sys.exit () all_ip () pool = Multiprocessing. Pool (processes=5) threads = [] print "Begin ... " for i in ip_list: room_ Path =&nBSP; ' /app/data/ckl/' + room_id + '/' pool.apply_async ( Upfile, (I,room_path,room_path)) pool.close () pool.join () for res in threads: Print res.get ()
Add a pipeline to invoke this script after PHP execution is complete:
PHP/APP/DATA/CKL/YIIC JSON notify | Sed-n ' S/^done: \ (. *\)/\1/p ' | Xargs-i {}/opt/scripts/sync_rooms.py {}
The following section is posted online:
Import socketimport osfrom stat import s_isdirclass sshsession (object): # usage: # detects dsa or rsa from key_file, either as a string filename or a # file object. Password auth is possible, but I will judge you for # using it. so: # ssh=sshsession (' targetserver.com ', ' Root ', Key_file=open (' Mykey.pem ', ' R ')) # Ssh=sshsession (' targetserver.com ', ' root ', key_file= '/home/me/mykey.pem ') # ssh= Sshsession (' targetserver.com ', ' root ', ' MyPassword ') # ssh.put (' filename ', '/remote/ File/destination/path ') # ssh.put_all ('/path/to/local/source/dir ', '/path/to/remote /destination ') &nbsP; # ssh.get_all ('/path/to/remote/source/dir ', '/path/to/local/destination ') # ssh.command (' echo ' Command to execute "') def __init__ (self, Hostname,username= ' root ', Key_file=none,password=none): # # Accepts a file-like object ( Anything with a readlines () function) # in either dss_key or rsa_key with a private Key. since i don ' T # ever intend to leave a server open to a password auth. # self.sock = socket.socket (socket.af_ineT, socket. Sock_stream) self.sock.connect ((hostname,22)) self.t = paramiko. Transport (Self.sock) self.t.start_client () keys = paramiko.util.load_host_keys (Os.path.expanduser (' ~/.ssh/known_ Hosts ')) key = self.t.get_remote_server_key () # supposed to check for key in Keys, but i don ' T much care right now to find the right notation if key_file is not none: if isinstance (KEY,STR): &nbSp; key_file=open (Key, ' R ') key_head=key_file.readline () key_file.seek (0) if ' DSA ' in key_head: keytype=paramiko. dsskey elif ' RSA ' in key _head: keytype= Paramiko. rsakey else: raise exception ("Can ' t Identify key type ") Pkey=keytype.from_private_key (Key_file) self.t.auth_publickey (Username, pkey) else: if password is not none: self.t.auth_ Password (username,password,fallback=false) else: raise exception (' Must supply either key_file or password ') self.sftp=paramiko. Sftpclient.from_transport (self.t) def command (self,cmd): # breaks the command by lines, sends and receives # each line and its output separately # # returns the server response text as a string chan = Self.t.open_session () chan.get_pty () chan.invoke_shell () chan.settimeout ( 20.0) ret= " try : ret+=chan.recv (1024x768) except: chan.send (' \ n ') ret+=chan.recv ( 1024x768)        RET+=CHAN.RECV (1024x768)     RET+=CHAN.RECV (1024x768) for line in cmd.split (' \ n '): Chan.send (Line.strip () + ' \ n ')  RET+=CHAN.RECV (1024x768) return ret def put (Self,localfile,remotefile): # copy localfile to remotefile, overwriting or creating as needed. self.sftp.put (Localfile,remotefile) def Put_all (Self,localpath,remotepath): # recursively upload a full diRectory os.chdir (Os.path.split (LocalPath) [0]) parent=os.path.split (LocalPath) [1] for walker in os.walk (parent): try: self.sftp.mkdir (Os.path.join (remotepath,walker[0)) except: pass for file in walker[2]: self.put (Os.path.join (Walker[0],file), Os.path.join (remotepath,walker[0],file)) def get (self,remoteFile,localfile): # copy remotefile to localfile, overwriting or creating as needed. self.sftp.get (Remotefile,localfile) def sftp_walk (self, RemotePath): # kindof a stripped down version of os.walk, implemented for # sftp. tried running it flat without the yields, but it really # chokes on big directories. path=remotepath files=[] folders=[] for f in self.sftp.listdir_attr (RemotePath): if s_isdir (F.st_mode): folders.append (F.filename) else: files.append (F.filename) print (path,folders,files) yield path,folders,files for folder in folders: new_path=os.path.join (Remotepath,folder) for x in self.sftp_walk (New_path): &Nbsp; yield x def get_all (Self,remotepath,localpath): # recursively download a full directory # harder than it sounded at first, since paramiko won ' t walk # # For the record, something like this would gennerally be faster: # ssh [email protected] ' Tar -cz /source/folder ' | tar -xz self.sftp.chdir (Os.path.split (RemotePath) [0]) parent=os.path.split (RemotePath) [1] try: os.mkdir (LocalPath) except: pass for walker in self.sftp_walk (parent): try: os.mkdir (Os.path.join (localpath,walker[0)) except: pass for file in walker[2]: self.get (Os.path.join (Walker[0],file), Os.path.join (localpath,walker[ 0],file))    &NBsp;def write_command (Self,text,remotefile): # writes text to remotefile, and makes remotefile executable. # This is perhaps a bit niche, but i was thinking i needed it. # For the record, I was incorrect. self.sftp.open (RemoteFile, ' W '). Write (text) Self.sftp.chmod (remotefile,755)
This article is from the "take a deep Breath again" blog, make sure to keep this source http://ckl893.blog.51cto.com/8827818/1785297
Paramiko multi-Process Synchronization directory