Python Incremental Sync Remote folder

Because of the time, some sensitive information is not removed, so when you send a message, remember to Filter. Otherwise the crawl is very powerful.

The purpose of this script is to synchronize a small number of logs on a remote machine to a machine and then filter the display via Logstash

Made several mistakes.

2. Local write error locals,locals is a unique python-defined variable, and cannot have duplicate names

4. SFTP old error, IOError: [Errno 2] No such file, this is because the local directory is not created

6. The first Full-volume synchronization, the second synchronization within five minutes of the modified files

8. Special handling of the list, remove \ n with Strip

10. The difference between read () and readlines, in fact, the Python Basic programming class books have talked about
    

Demonstrate

Cat FileList

X.x.x.x::p assword::tomcat::/home/python/::/opt/logs/::

#!/usr/bin/python   import pexpectimport paramiko  import osimport  sysimport timeimport multiprocessingimport datetimeimport crash_on_ipyfrom stat  import s_isdir  ip_list = [] #room_id  = sys.argv[1]class run_cmd ( ):       def __init__ (self,hostname=none,password=none,username=none,port= none,echo_cmd=none):           #threading. thread.__init__ (self)           self.hostname=hostname           self.password=password           self.username=username           self.port=port          self.echo_cmd=echo_cmd            #self. Thread_stop=false      def run (self):           paramiko.util.log_to_file (' paramiko.log ')            s=paramiko. Sshclient ()           s.set_missing_host_key_policy (paramiko. Autoaddpolicy ())           s.connect (hostname =  Self.hostname,username=self.username, password=self.password)            stdin,stdout,stderr=s.exec_command (self.echo_cmd)            return stdout.readlines ()            S.close ()       def stop (self):            self.thread_stop=trueclass get_thread ():    def __init__ (self,hostname,password,username,port=none):    #def  __init__ (self, Hostname,username= ' root ', key_file=none,password=none):     #def  __init__ (self,hostname =none,password=none,username=none,port=none,local_dir=none,remote_dir=none):         self.hostname = hostname         self.username = username        self.password =  Password        self.scp = paramiko. Sshclient ()         self.scp.set_missing_host_key_policy (paramiko. Autoaddpolicy ())         self.scp = paramiko. Transport ((hostname, 22))         self.scp.connect (username= Username, password=password)         selF.sftp = paramiko. Sftpclient.from_transport (self.scp)     def _walk_remote (self, dirpath):         dirnames = []         filenames = []        for fd in  Self.sftp.listdir_attr (dirpath):             if  s_isdir (fd.st_mode):                 dirnames.append (fd.filename)              else:                 Filenames.append (fd.filename)         yield dirpath, dirnames , filenames        for dirname in dirnames:   &Nbsp;         new_dirpath = os.path.join (dirpath,  Dirname)             # yield from  Self._walk_remote (new_dirpath)             for  walk in self._walk_remote (new_dirpath):                 yield walk    def getall (self,local, Remote):                 st_ Mode = self.sftp.stat (remote). St_mode        if not  s_isdir (st_mode):             filename =  os.path.basename (remote)              Self.sftp.get (remote, os.path.Join (local, filename))         else:             parent, child = os.path.split (remote)              for dirpath, dirnames, filenames  in self._walk_remote (remote):                 dirpath = dirpath.replace (parent,  '. ')                 parentc =  os.path.join (local,dirpath)                  if not os.path.exists (parentc):                   os.makedirs (parentc)                  for dirname in dirnames:                     try:                          Os.makedirs (os.path.join (local, dirpath, dirname))                      except:                          pass                for  filename in filenames:                     localpath = os.path.join (local, dirpath,  Filename)   &NBsp;                  Remotepath = os.path.join (parent, dirpath, filename)                      self.sftp.get (remotepath,  localpath)         self.scp.close () if __name__== ' __main__ ':     port = 22    now = datetime.datetime.now ()     strdatetime = now.strftime ("%y-%m-%d")     year= Strdatetime.split ('-') [0]    mon=strdatetime.split ('-') [1]    day= Strdatetime.split ('-') [2]    datenow1= year +  "/"  + mon +   "/"  + day +  "/"     Datenow= year +  "/"  +  mon    print " -"*50    f = file ('/home/python/filelist ', ' r ')     c =  f.readlines ()     for x in c:         hostname = x.split (':: ') [0]        password  = x.split (':: ') [1]        username = x.split (':: ') ) [2]        local= x.split (':: ') [3].strip (' \ n ')          remotes = x.split (':: ') [4].strip (' \ n ')          localz=local +  "/"  + mon +  "/"  + day         if remotes.endswith ('/'):             remote1 = remotes + Datenow             remote2 = remotes + datenow1         else:            remote3 = remotes         if not os.path.exists (localz):             remote = remote1             getthread=get_thread (hostname,password,username)              getthread.getall (local,remote)          else:            remote =  remote2            echo_cmd= '/bin/find  %s -maxdepth 1 -type d -mmin -5 '  %  (remote)          &nBsp;   cmd_thread=run_cmd (hostname,password,username,port,echo_cmd)              result=cmd_thread.run ()              del result[0]             for item in result:                 print str (item)                  items = item.strip (' \ n ')                  getthread=get_thread (hostname,password , Username)                  Getthread.getall (localz,items)     f.close ()

Execute time Python test.py

The most important can be modified in increments of five minutes, save Time.

This article from "people, to have their own ideas" blog, declined to reprint!

Python Incremental Sync Remote folder