Python crawler0723.py

#! /Usr/ENV Python
#-*-Coding: UTF-8 -*-
Import urllib
Import urllib2
Import random
Import requests
Import OS, sys
Import mysqldb
From sgmllib import sgmlparser
Import re
Num = 0
Def main ():
Try:
Conn = mysqldb. Connect (host = 'localhost', user = 'root', passwd = '000000', DB = 'ssssbookdb', charset = "utf8 ")
Conn. Query ("set names utf8 ")
Except t exception, E:
Print E
SYS. Exit ()
Cursor = conn. cursor ()
For k in range (1,2574 ):
Url = "http://apk.gfan.com/apps_7_1_" + STR (k) + ". html"
Html = requests. Get (URL)
Result = html. Content
Pattern = Re. Compile ('<a href = "([http://apk.gfan.com]? /Product/APP \ d00001, 80000.html )"')
Dataresult = Re. findall (pattern, result)
Dataresult = List (SET (dataresult ))
For I in dataresult:
T = "http://apk.gfan.com" + I
Print t
Html = requests. Get (t)
Result = html. Content
Pattern = Re. Compile ('<Div class = "appdiscrib"> [\ s] *? <H4> (. + ?) </H4> ')
Data0 = Re. findall (pattern, result)
# Print data0 [0]
Pattern = Re. Compile ('version number (. + ?) </LI> ')
Data1 = Re. findall (pattern, result)
Pattern = Re. Compile ('initiator (. ++ ?) </LI> ')
Data2 = Re. findall (pattern, result)
Pattern = Re. Compile ('release time (. + ?) </LI> ')
Data3 = Re. findall (pattern, result)
Pattern = Re. Compile ('file size (. + ?) </LI> ')
Data4 = Re. findall (pattern, result)
Pattern = Re. Compile ('supports firmware (. ++ ?) </LI> ')
Data5 = Re. findall (pattern, result)
Pattern = Re. Compile ('application introduction Data6 = Re. findall (pattern, result)
For items in data6:
Pass # print re. sub ('<br/>', '', items)
SQL = "insert into address (name, version, developer, pubtime, filesize, support, introduction) values (% s, % s, % s, % s )"
For items in data6:

If (data5 ):
Values = (data0 [0], data1 [0], data2 [0], data3 [0], data4 [0], data5 [0], re. sub ('<br/>', '', items ))
Else:
Values = (data0 [0], data1 [0], data2 [0], data3 [0], data4 [0], 'null', re. sub ('<br/>', '', items ))
# Print values
# Print SQL % values
Cursor.exe cute (SQL, values)
Conn. Commit ()
Pattern = Re. Compile ('<Div class = "apptitle Clearfix"> [\ s] *? ')
Data = Re. findall (pattern, result)
For J in data:
Print J
# Temp = urllib2.urlopen (I [10:])
# This is the Save function. The first parameter is the address, and the second parameter is the saved file name. Let the last eight bits of the address be used as the file name.
# Urllib. urlretrieve (J [1:-2], J [-40:])
Temp = requests. Get (J [1:-2])
Global num
F = file ("picture/" + STR (Num), "W + ")
Num = num + 1
Print num
F. Write (temp. Content)

# SQL = "select * From Address"
#Cursor.exe cute (SQL)
# Conn. Commit ()
# Finalresult = cursor. fetchall ()
# If finalresult:
# For X in finalresult:
# Pass # print X [0:]
Cursor. Close ()
Conn. Close ()
F. Close ()

If _ name __= = "_ main __":
Main ()