Sitecopy, a script for the Shanzhai Web site UI

http://fayaa.com/code/view/15377/

Python language: sitecopy, a script for Shanzhai Web site UI #coding: Utf-8
Import Re, OS, Shutil, sys
Import URLLIB2, Socket, cookielib
From threading import Thread, Stack_size, Lock
From queue import queue
Import time
From gzip import gzipfile
From Stringio import Stringio

Class Contentencodingprocessor (Urllib2. Basehandler):
"" "A handler to add GZIP capabilities to URLLIB2 requests" "

# Add headers to requests
def http_request (self, req):
Req. Add_header ("accept-encoding", "gzip, deflate")
return req

# decode
def http_response (self, req, resp):
Old_resp = Resp
# gzip
If resp. Headers. Get ("content-encoding") = = "gzip":
GZ = Gzipfile (
Fileobj = Stringio (resp. read ()),
mode = "R"
)
RESP = urllib2. Addinfourl (GZ, Old_resp. Headers, old_resp. URL, old_resp. Code)
Resp. msg = Old_resp. Msg
# deflate
If resp. Headers. Get ("content-encoding") = = "Deflate":
GZ = Stringio (Deflate (resp. read ()))
RESP = urllib2. Addinfourl (GZ, Old_resp. Headers, old_resp. URL, old_resp. Code) # ' class to add info ()
Resp. msg = Old_resp. Msg
Return RESP

# deflate Support
Import zlib
def deflate (data): # zlib only provides the zlib compress format, not the deflate format;
Try: # so on top of the all there ' s This workaround:
Return zlib. Decompress (data,-zlib). Max_wbits)
Except Zlib. Error:
Return zlib. Decompress (data)

Class Fetcher:
'''
HTML fetcher

Basic usage
-----------
From Fetcher Import Fetcher
f = fetcher ()
F.get (URL)

Post
----
req = Urllib2. Request (...)
F.post (req)

Multi-thread
------------
f = fetcher (threads=10)
For URL in URLs:
F.push (URL)
While F.taskleft ()
url,html = F.pop ()
Deal_with (url,html)
'''
def __init__ (self, timeout = ten, threads = None, stacksize = 32768 *, Loginfunc = none):
#proxy_support = Urllib2. Proxyhandler ({' http ': ' http://localhost:3128 '})
Cookie_support = Urllib2. Httpcookieprocessor (Cookielib. Cookiejar ())
Encoding_support = Contentencodingprocessor ()
#self. opener = Urllib2.build_opener (cookie_support,encoding_support,proxy_support,urllib2. HttpHandler)
Self. Opener = Urllib2. Build_opener (Cookie_support, Encoding_support, Urllib2. HttpHandler)
Self. req = Urllib2. Request (' http://www.hsbc.com ')
Socket. Setdefaulttimeout (Timeout)
Self. Q_req = Queue ()
Self. Q_ans = Queue ()
Self. Lock = Lock ()
Self. Running = 0
If Loginfunc:
Self. Opener = Loginfunc (self. Opener)
If Threads:
Self. Threads = Threads
Stack_size (stacksize)
For I in range (threads):
t = Thread (target = self. Threadget)
T. Setdaemon (True)
T. Start ()

def __del__ (self):
Time. Sleep (0.5)
Self. Q_req. Join ()
Self. Q_ans. Join ()

def taskleft (self):
return self. Q_req. Qsize () + self. Q_ans. Qsize () + self. Running

def push (self, req, repeat = 3):
If not self. Threads:
print ' no thread, return get instead '
Return get (req, repeat)
Self. Q_req. Put (req)

def pop (self):
Try:
data = self. Q_ans. Get (block = True, timeout = 10)
Self. Q_ans. Task_done ()
Except:
data = [', ']
Return data

def threadget (self):
While True:
req = self. Q_req. Get ()
With self. Lock:
Self. Running + 1
ans = self. Get (req)
print ' Got ', req
Self. Q_ans. Put ((req, ans))
Try:
Self. Q_req. Task_done ()
Except:
Pass
With self. Lock:
Self. Running-= 1
Time. Sleep (0.1) # don ' t spam

def proxyisworking (self):
Try:
Self. Opener. Open (' http://www.hsbc.com '). Read (1024)
Return True
Except Exception, what:
Print What
Return False
def get (self, req, repeat = 3):
'''
HTTP GET req and repeat 3 times if failed
HTML text is returned when succeeded
' is returned when failed
'''
Try:
Response = self. Opener. Open (req)
Data = Response. Read ()
Except Exception, what:
Print What, req
            if repeat > 0