Use multiple threads to download files from ftp!

Use multiple threads to download files from ftp! [Copy link]

Information

Direct elevator 1# Posted on 16:57:08| View only the author | view in reverse order At the requirement of hongfu yesterday, a simple example of multi‑thread multipart download of files from ftp. #-*-Encoding: gb18030 -*- Import ftplib, string Import OS, sys Import threading Class myftp: Def _ init _ (self, host = '', user ='', passwd = ''): Self. Host = Host Self. User = user Self. passwd = passwd Self. filename ='' Self. FTP = ftplib. FTP (host, user, passwd) Def download_by_thread (self, filename, threadnum = 1, blocksize = 8192 ): Self. filename = filename # Getting file names Onlyname = OS. Path. basename (filename) Cmd = "size" + filename # Obtain the size of the file to be downloaded Ret = self. FTP. sendcmd (CMD) Self. FTP. Quit () # Calculate the download size of each thread when multiple threads are used for download. Fsize = int (string. Split (RET) [1]) Print 'file', filename, 'size: ', fsize Rest = none Bsize = fsize/threadnum # Creating a thread Threads = [] For I in range (0, threadnum-1 ): Begin = bsize * I Print I, begin, bsize TP = threading. Thread (target = self. download_file, argS = (I, filename, begin, bsize, blocksize, rest ,)) Threads. append (TP) Have1 = bsize * threadnum Have2 = fsize-have1 Lastsize = bsize + have2 Begin = bsize * (threadnum-1) Print threadnum-1, maid, lastsize TP = threading. Thread (target = self. download_file, argS = (threadnum-1, filename, begin, lastsize, blocksize, rest ,)) Threads. append (TP) Print 'threads: ', Len (threads) For T in threads: T. Start () For T in threads: T. Join () # Download is complete for each thread. Merge the temporary file into a file. FW = open (onlyname, "WB ") For I in range (0, threadnum ): Fname = onlyname + '. Part.' + STR (I) Print fname If not OS. Path. isfile (fname ): Print 'not found ', fname Continue F1 = open (fname, 'rb ') While 1: Data = f1.read (8192) If not Len (data ): Break FW. Write (data) F1.close () OS. Remove (fname) FW. Close () Print 'all OK' Def download_file (self, counter, filename, begin = 0, size = 0, blocksize = 8192, rest = none ): Onlyname = OS. Path. basename (filename) Tname = threading. currentthread (). getname () # Splits = string. Split (tname, '-') [-1] # Create a connection to download the connection. Each thread has a connection. Note that some FTP servers have no restrictions on how many connections a single IP Address can have. Myftp = ftplib. FTP (self. Host, self. User, self. passwd) # Creating temporary files Fp = open (onlyname + '. Part.' + STR (parts), 'wb ') # FP. Seek (BEGIN) Callback = FP. Write Haveread = 0 Myftp. voidcmd ('Type I ') # Tell the server where the file is to be downloaded Cmd1 = "rest" + STR (BEGIN) Print tname, cmd1 Ret = myftp. sendcmd (cmd1) # Start download Cmd = "retr" + filename Conn = myftp. transfercmd (CMD, rest) Readsize = blocksize While 1: If size> 0: Last = size-haveread If last> blocksize: Readsize = blocksize Else: Readsize = last Data = conn. Recv (readsize) If not data: Break # Callback (FP, data) # Downloaded data length Haveread = haveread + Len (data) # Only data of the specified length can be downloaded. Exit upon download If haveread> size: Print tname, 'haveread: ', haveread, 'size:', size HS = haveread-size Callback (data [: Hs]) Break Elif haveread = size: Callback (data) Print tname, 'haveread: ', haveread Break Callback (data) Conn. Close () FP. Close () Try: Ret = myftp. getresp () Except t exception, E: Print tname, E Myftp. Quit () Return ret FTP = myftp ("127.0.0.1", 'anonus us', '127.0.0.1 ') Filename = '/incoming/cygwin.zip' FTP. download_by_thread (filename, 10)