Python implementation crawler Download Comics example

The code is as follows:

#!/usr/bin/python3.2
Import Os,socket
Import Urllib
Import Urllib.request,threading,time
Import Re,sys
Global MANHUAWEB,WEBURL,FLODER,CHAPTERBEGIN,CURRENTTHREADNUM,THREADCOUNT,MUTEX,MUTEX2

Weburl= "
Floder= "
Chapterbegin=0
Currentthreadnum=0
Threadcount=6

If Len (SYS.ARGV) >=3:
WEBURL=SYS.ARGV[1]
FLODER=SYS.ARGV[2]
Else
Print ("Usag:downloadmanhua weburl floder chapterbegin=0 threadnnum=6")
Sys.exit (0)
If Len (SYS.ARGV) >=4:
Chapterbegin=int (Sys.argv[3])
If Len (SYS.ARGV) >=5:
threadcount= (int) (sys.argv[4])

Def jin (I,jinzhi):
Finalans= ""
Answer=i%jinzhi
I=int (I/jinzhi)
If answer>9:
FINALANS=FINALANS+CHR (Ord (' a ') + (ANSWER-10))
Else
Finalans=finalans+str (Answer)
If i!=0:
Finalans=jin (I,jinzhi) +finalans
Return Finalans
def urlparse (p,a,c,k):
d={}
E=lambda C:jin (c,36)
If 1:
While C:
C=c-1
If not K[c]:
D[jin (c,36)]=jin (c,36)
Else
D[jin (c,36)]=k[c]
K=[lambda E:d[e]]
E=lambda c: ' \\w+ '
C=1
Newstr= ""
While C:
C=c-1
If K[C]:
For I in Range (0,len (p)):
Tempi=p[i]
Tempi=ord (TEMPI)
If Tempi>=ord (' a ') and Tempi<=ord (' F '):
NEWSTR+=D[CHR (TEMPI)]
Elif Tempi>=ord (' 0 ') and Tempi<=ord (' 9 '):
NEWSTR+=D[CHR (TEMPI)]
Else
NEWSTR+=CHR (TEMPI)
Return NEWSTR
def meispower (s):
P=re.compile (R "(? =\}\ (). *", Re. IGNORECASE)
S=p.findall (s)
S=s[0]
S=s[0: (Len (s)-19)]
Par=s.split (', ')
Par[3]=par[3][1:len (Par[3])
Answer=par[3].split (' | ')
Chapterpath=urlparse (Par[0],int (par[1]), int (par[2]), answer)
Allurl=re.findall (' imgpath=[^;] * ', Chapterpath) [0]
ALLURL=ALLURL[10: (Len (Allurl)-2)]
Return Allurl
def pictofile (weburl,filename,loop=100):
If loop<0:
Print (' can\ ' t download the picture%s '%weburl)
Return
Loop=loop-1
If os.path.exists (filename):
Return
Try
Url=urllib.request.urlopen (Weburl)
Data=url.read ()
If Len (data) <2048:
Url.close ()
Pictofile (Weburl,filename,loop)
Else
Print (' Download from%s name is%s\n '% (weburl,filename))
Myfile=open ('%s '%filename, ' WB ')
Myfile.write (data)
Myfile.close ()
Url.close ();
Except Socket.timeout:
Print (' timeout ')
Pictofile (Weburl,filename,loop)
Except Exception as E:
Print (' Error ', E)
Pictofile (Weburl,filename,loop)
Finally
Pass
def downloadpic (Url,loadpicdir,num):
#download the all URL picture to Loadpicdir
Global CURRENTTHREADNUM,MUTEX,MUTEX2
Mymode=re.compile (R ' [0-9a-z.] *\z ')
Try
Mutex2.acquire ()
Os.chdir (Loadpicdir)
Mutex2.release ()
Except
Print ("Can ' t open the Floder%s'll be create"%loadpicdir)
Try
if (mutex2.locked ()):
Os.mkdir (Loadpicdir)
Os.chdir (Loadpicdir)
Mutex2.release ()
Print (' Create Floder succeed ')
Except
Print ("Can ' t create floder%s"%loadpicdir)
if (Mutex.acquire ()):
Mutex.release ()
Quit (0)
Name=mymode.findall (URL)
Filename= ' Manhua ' +name[0]
Pictofile (url,loadpicdir+ '//' +str (num) + '-' +filename)
Mutex.acquire ()
Currentthreadnum=currentthreadnum-1
Mutex.release ()
def downloadchapter (url,loadpicdir,num,begin=0):
Global Manhuaweb,threadcount,currentthreadnum,mutex
Print (Manhuaweb+url)
Webdata=urllib.request.urlopen (Manhuaweb+url). Read ()
Webdata=webdata.decode (' UTF-8 ')
Chaptername=re.findall (R '<title>[^_]* ', WebData] [0]</title>
Chaptername=chaptername[7:len (Chaptername)]
Webscrip=re.findall (R ' Eval.*[^<>] ', WebData)
Chapterurl=meispower (Webscrip[0]);
Chapterurl= ' Http://mhimg.ali213.net ' +chapterurl
For I in Range (Begin,num):
Try
while (Currentthreadnum>=threadcount):
Time.sleep (0.5)
Mutex.acquire ()
Currentthreadnum=currentthreadnum+1
Mutex.release ()
Threading. Thread (target=downloadpic,args= (R '%s%d.jpg '% (chapterurl,i), Loadpicdir+chaptername,num)). Start ()
Except Socket.error:
Mutex.acquire ()
I=i-1
Currentthreadnum=currentthreadnum-1
Mutex.release ()
Except Exception as Error:
Print (Error, ' Break ')
Print (' download chapter%d of picture make a error '%i)
Break
If __name__== ' __main__ ':
Manhuaweb=r ' Http://manhua.ali213.net '
Socket.setdefaulttimeout (60.0)
Mutex=threading. Lock ()
Mutex2=threading. Lock ()

Webfile=urllib.request.urlopen (Weburl)
Webdata=webfile.read ();
Webdata=webdata.decode (' UTF-8 ')
Meshmode=re.compile (R '. * ')
Meshdata=meshmode.findall (WebData) [0]
Indexmode=re.compile (R ' ([0-9]* page) ')
Indexdata=indexmode.findall (Meshdata)

Picurlmode=re.compile (R '/comic/[0-9/]*.html ')
Picurldata=picurlmode.findall (Meshdata)

Chapterlength=len (Picurldata)
Nummode=re.compile (R ' [\d]+ ')

I=chapterbegin
While I<>
MANHUACHAPTER=PICURLDATA[CHAPTERLENGTH-I-1]
Downloadchapter (Manhuachapter,floder,int (Nummode.findall (indexdata[chapterlength-i-1]) [0])
I=i+1

