Python common function functions summary 1. write string to file in line
Importsys, OS, time, JSONdefSavecontext (filename,*name): Format='^'Context=Name[0] forIinchName[1:]: Context= context + format +Str (i) Context= str (context). replace ('(','('). Replace (')',')'). Replace (',',','). Replace (':',':') #Remove First spacefilename =Filename.strip ()#Read Directory namePath =os.path.dirname (filename)#Create a directory if the directory does not exist if notos.path.exists (path): os.makedirs (path)#Read file nameName =os.path.basename (filename) FP= open (filename,'a') Fp.write (context+'\ n') Fp.close ()2. Create an initialization browser
#Coding:utf-8Importsys, OS, time, JSONImportUrllib2 fromPyqueryImportPyquery as PQ fromlxmlImportetree fromSeleniumImportWebdriver fromUrlparseImportUrljoin#Set Utf-8 modeReload (SYS) sys.setdefaultencoding ("Utf-8" )#Initialize Browser creationdefinit_drive (): UA="mozilla/5.0 (Windows NT 6.3; WOW64) applewebkit/537.36 (khtml, like Gecko) chrome/56.0.2924.3 safari/537.36"Cap=Webdriver. Desiredcapabilities.phantomjs cap["Phantomjs.page.settings.resourceTimeout"] = 20000cap["phantomjs.page.settings.loadImages"] =True cap["Phantomjs.page.settings.disk-cache"] =True cap["phantomjs.page.settings.userAgent"] =UA cap["phantomjs.page.customheaders.user-agent"] =UA cap["Phantomjs.page.customHeaders.Referer"] ="http://tj.ac.10086.cn/login/" #Driver = Webdriver. Phantomjs (executable_path= '/home/shutong/phantomjs/bin/phantomjs ', Desired_capabilities=cap, service_args=['-- Ignore-ssl-errors=true '])Driver = Webdriver. Phantomjs (Desired_capabilities=cap, service_args=['--ignore-ssl-errors=true']) driver.set_page_load_timeout (60) Driver.set_script_timeout (60) returnDriver
Where to get Web page html
# Initialize Create browser driver= = driver.page_source# Exit browser driver.quit ()
3. Get Web page HTML function based on URL
#Coding:utf-8Importrequests, JSON, time, RE, OS, sys, timeImportUrllib2ImportRandomImportNumPy as NP#set to Utf-8 modeReload (SYS) sys.setdefaultencoding ("Utf-8" )#finally get the data for the URLdefgethtml (URL): Ua_list= ["mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv2.0.1) gecko/20100101 firefox/4.0.1","mozilla/5.0 (Windows NT 6.1; rv2.0.1) gecko/20100101 firefox/4.0.1","opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U EN) presto/2.8.131 version/11.11","opera/9.80 (Windows NT 6.1; U EN) presto/2.8.131 version/11.11","mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) applewebkit/535.11 (khtml, like Gecko) chrome/17.0.963.56 safari/535.11"] User_agent=Random.choice (ua_list) Request=Urllib2. Request (URL) request.add_header ("user-agent", user_agent) Response= Urllib2.urlopen (request,data=none,timeout=60) HTML=Response.read ()#can be encoded according to the encoding format #HTML = unicode (HTML, ' Utf-8 ') returnHtml
4. Get different formats for time
Import TimeImportSYSImportOSImportShutilImportMySQLdbImportUrllib2 fromPyqueryImportPyquery as PQ fromlxmlImportetreeImportUrllibImportSYSImportHttplibImportdatetimeImportJSON fromSeleniumImportWebdriver fromUrlparseImportUrljoin Httplib. HTTPCONNECTION._HTTP_VSN= 10Httplib. Httpconnection._http_vsn_str='http/1.0'#Set Utf-8 modeReload (SYS) sys.setdefaultencoding ("Utf-8" )#get functions for a common time format#'%y-%m-%d ' 2017-11-18#'%y%m%d ' 20171118#%y%m%d%h ' 2017111817#empty or other 2017-11-18 17:26:35defGetTime (*format): now="' Try: Format=Format[0]except : Pass ifFormat = ='%y-%m-%d': Now= Time.strftime ('%y-%m-%d', Time.localtime (Time.time ()))elifFormat = ='%y%m%d': Now= Time.strftime ('%y%m%d', Time.localtime (Time.time ()))elifFormat = ='%y%m%d%h': Now= Time.strftime ('%y%m%d%h', Time.localtime (Time.time ()))Else: Now= Time.strftime ('%y-%m-%d%h:%m:%s', Time.localtime (Time.time ()))returnNow
5. Connect MySQL Execute SQL statement
Import TimeImportSYSImportOSImportShutilImportMySQLdbImportUrllib2 fromPyqueryImportPyquery as PQ fromlxmlImportetreeImportUrllibImportSYSImportHttplibImportdatetimeImportJSON fromSeleniumImportWebdriver fromUrlparseImportUrljoin Httplib. HTTPCONNECTION._HTTP_VSN= 10Httplib. Httpconnection._http_vsn_str='http/1.0'#Set Utf-8 modeReload (SYS) sys.setdefaultencoding ("Utf-8" )#define a MySQL database connectiondefConn_mysql (host='192.168.11.43', user='Root', passwd='Root', db='Edw'): Conn="' Try: Conn=MySQLdb.connect (Host=Host, Port= 3306, the user=User, passwd=passwd, DB=db,)#print "Connect MySQL success" except : #Pass Print "failed to connect to MySQL" returnConn#Execute SQL statement return resultdefExcute_sql (conn,sql):#conn = conn_mysql (host= ' 192.168.122.194 ', user= ' root ', passwd= ' 123456 ', db= ' label ')Cur =conn.cursor () Cur.execute ('Set character_set_client = UTF8') Cur.execute ('Set character_set_server = UTF8') Cur.execute ('Set character_set_connection = UTF8') Cur.execute ('Set character_set_results = UTF8') Cur.execute ('Set collation_connection = Utf8_general_ci') Cur.execute ('Set collation_server = Utf8_general_ci') Result=cur.fetchmany (Cur.execute (SQL)) Cur.close () Conn.commit () conn.close ()returnResult
Python Common function functions