Python Incremental Sync Remote folder

Source: Internet
Author: User

Because of the time, some sensitive information is not removed, so when you send a message, remember to Filter. Otherwise the crawl is very powerful.

The purpose of this script is to synchronize a small number of logs on a remote machine to a machine and then filter the display via Logstash

Made several mistakes.

  1. Local write error locals,locals is a unique python-defined variable, and cannot have duplicate names

  2. SFTP old error, IOError: [Errno 2] No such file, this is because the local directory is not created

  3. The first Full-volume synchronization, the second synchronization within five minutes of the modified files

  4. Special handling of the list, remove \ n with Strip

  5. The difference between read () and readlines, in fact, the Python Basic programming class books have talked about

Demonstrate

Cat FileList

X.x.x.x::p assword::tomcat::/home/python/::/opt/logs/::

#!/usr/bin/python   import pexpectimport paramiko  import osimport  sysimport timeimport multiprocessingimport datetimeimport crash_on_ipyfrom stat  import s_isdir  ip_list = [] #room_id  = sys.argv[1]class run_cmd ( ):       def __init__ (self,hostname=none,password=none,username=none,port= none,echo_cmd=none):           #threading. thread.__init__ (self)           self.hostname=hostname           self.password=password           self.username=username           self.port=port          self.echo_cmd=echo_cmd            #self. Thread_stop=false      def run (self):           paramiko.util.log_to_file (' paramiko.log ')            s=paramiko. Sshclient ()           s.set_missing_host_key_policy (paramiko. Autoaddpolicy ())           s.connect (hostname =  Self.hostname,username=self.username, password=self.password)            stdin,stdout,stderr=s.exec_command (self.echo_cmd)            return stdout.readlines ()            S.close ()       def stop (self):            self.thread_stop=trueclass get_thread ():    def __init__ (self,hostname,password,username,port=none):    #def  __init__ (self, Hostname,username= ' root ', key_file=none,password=none):     #def  __init__ (self,hostname =none,password=none,username=none,port=none,local_dir=none,remote_dir=none):         self.hostname = hostname         self.username = username        self.password =  Password        self.scp = paramiko. Sshclient ()         self.scp.set_missing_host_key_policy (paramiko. Autoaddpolicy ())         self.scp = paramiko. Transport ((hostname, 22))         self.scp.connect (username= Username, password=password)         selF.sftp = paramiko. Sftpclient.from_transport (self.scp)     def _walk_remote (self, dirpath):         dirnames = []         filenames = []        for fd in  Self.sftp.listdir_attr (dirpath):             if  s_isdir (fd.st_mode):                 dirnames.append (fd.filename)              else:                 Filenames.append (fd.filename)         yield dirpath, dirnames , filenames        for dirname in dirnames:   &Nbsp;         new_dirpath = os.path.join (dirpath,  Dirname)             # yield from  Self._walk_remote (new_dirpath)             for  walk in self._walk_remote (new_dirpath):                 yield walk    def getall (self,local, Remote):                 st_ Mode = self.sftp.stat (remote). St_mode        if not  s_isdir (st_mode):             filename =  os.path.basename (remote)              Self.sftp.get (remote, os.path.Join (local, filename))         else:             parent, child = os.path.split (remote)              for dirpath, dirnames, filenames  in self._walk_remote (remote):                 dirpath = dirpath.replace (parent,  '. ')                 parentc =  os.path.join (local,dirpath)                  if not os.path.exists (parentc):                   os.makedirs (parentc)                  for dirname in dirnames:                     try:                          Os.makedirs (os.path.join (local, dirpath, dirname))                      except:                          pass                for  filename in filenames:                     localpath = os.path.join (local, dirpath,  Filename)   &NBsp;                  Remotepath = os.path.join (parent, dirpath, filename)                      self.sftp.get (remotepath,  localpath)         self.scp.close () if __name__== ' __main__ ':     port = 22    now = datetime.datetime.now ()     strdatetime = now.strftime ("%y-%m-%d")     year= Strdatetime.split ('-') [0]    mon=strdatetime.split ('-') [1]    day= Strdatetime.split ('-') [2]    datenow1= year +  "/"  + mon +   "/"  + day +  "/"     Datenow= year +  "/"  +  mon    print " -"*50    f = file ('/home/python/filelist ', ' r ')     c =  f.readlines ()     for x in c:         hostname = x.split (':: ') [0]        password  = x.split (':: ') [1]        username = x.split (':: ') ) [2]        local= x.split (':: ') [3].strip (' \ n ')          remotes = x.split (':: ') [4].strip (' \ n ')          localz=local +  "/"  + mon +  "/"  + day         if remotes.endswith ('/'):             remote1 = remotes + Datenow             remote2 = remotes + datenow1         else:            remote3 = remotes         if not os.path.exists (localz):             remote = remote1             getthread=get_thread (hostname,password,username)              getthread.getall (local,remote)          else:            remote =  remote2            echo_cmd= '/bin/find  %s -maxdepth 1 -type d -mmin -5 '  %  (remote)          &nBsp;   cmd_thread=run_cmd (hostname,password,username,port,echo_cmd)              result=cmd_thread.run ()              del result[0]             for item in result:                 print str (item)                  items = item.strip (' \ n ')                  getthread=get_thread (hostname,password , Username)                  Getthread.getall (localz,items)     f.close ()

Execute time Python test.py

The most important can be modified in increments of five minutes, save Time.

This article from "people, to have their own ideas" blog, declined to reprint!

Python Incremental Sync Remote folder

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.