Extracted from: LOMOOO "crawling ts video streams from webpages", lomooots

Extracted from: LOMOOO "crawling ts video streams from webpages", lomooots

import requests
import os
from multiprocessing.dummy import Pool

# In rare cases, if you want to obtain the original socket response from the server, you can access r. raw. If you really want to do this, make sure that stream = True is set in the initial request. You can do this:
#
#>>> R = requests. get ('https: // github.com/timeline.json', stream = True)
#>>> R. raw
# <Requests. packages. urllib3.response. HTTPResponse object at 0x101194810>
#>>> R. raw. read (10)
# '\ X1f \ x8b \ x08 \ x00 \ x00 \ x00 \ x00 \ x00 \ x00 \ x00 \ x03'
# However, you should save the text stream to a file in the following mode:
#
# With open (filename, 'wb ') as fd:
# For chunk in r. iter_content (chunk_size ):
# Fd. write (chunk)
# Using Response. iter_content will process a large number of things you have to deal with directly using Response. raw. When downloading a stream, the above is the preferred way to obtain the content. Note that chunk_size can be freely adjusted to a number that may better fit your use cases.

# Chunk_size: You can set the size you specified to specify the maximum value of data retrieved each time. Note: not all content blocks returned by each request are chunk_size values.

Def download (j ):
Global picpath
Url = 'HTTP: // xxx5 % s' % str (j) + '. Ts'
For l in range (20): # A maximum of 20 download failures
Try:
Retu = requests. get (url, stream = True)
Print (str (j). zfill (3) + 'Download completed ')
Break
Except t:
Print (U' % s file, retry % d Times '% (str (j). zfill (3), l + 1 ))
Picpath = r 'C: \ Users \ bin \ PycharmProjects \ untitled \ % s' % str (j). zfill (3) + '. ts'
File = open (picpath, 'wb ')
For chunk in retu. iter_content (chunk_size = 1024*8 ):
If chunk:
File. write (chunk)
File. flush ()
File. close ()

If _ name _ = "_ main __":
List = [I for I in range (2250)] # is the length of the video I crawled
Pool = Pool (4)
Pool. map (download, list)
Pool. close ()
Pool. join ()

Lst = []
For I in range (0, 22 50 ):
Test = r 'C: \ Users \ bin \ PycharmProjects \ untitled \ abc \ % s. Ts' % str (I). zfill (3)

    　　if not os.path.exists(test):
        　　print(str(i).zfill(3)+'.ts')
        　　lst.append(i)
　　print(lst)

　　if  len(lst):
　 　　　pool = Pool(4)
　　
 　　　　pool.map(download, lst)

 　　pool.close()
 　　pool.join()

　　else:

 　　print

U'download all completed, merging'
OS. system (r'copy/B C: \ Users \ bin \ PycharmProjects \ untitled \ *. ts C: \ Users \ bin \ PycharmProjects \ untitled \ new. ts ')
Print
U'merging complete'

# In Python, the thread multiprocessing module is the same as the process module. The usage is also basically the same. The only difference is that the import Pool such as from multiprocessing import Pool represents the process Pool;
# The import Pool from multiprocessing. dummy import Pool indicates the thread Pool. In this way, the concurrency in the thread can be realized.