Python Pycurl Network Programming example and automatic processing cookie detailed

Python Pycurl Network Programming example

In the use of urllib often die, debug before, is not set timing out so timeout will die.
Pycurl is a Curl Python library, although some curl features are not implemented, but they are still strong.

Curl is a very powerful tool,
Google uses it internally to debug the GDATA API. Using CURL to interact with Google data Services

You can go to http://pycurl.sourceforge.net/to download the latest Pycurl.

Simple examples of Pycurl

Import pycurl
import stringio
url = "http://www.google.com/"
CRL = Pycurl. Curl ()
crl.setopt (Pycurl. verbose,1)
crl.setopt (Pycurl). Followlocation, 1)
crl.setopt (Pycurl. Maxredirs, 5)
CRL.FP = Stringio.stringio ()
crl.setopt (Pycurl. URL, url)
crl.setopt (CRL. Writefunction, Crl.fp.write)
crl.perform ()
print crl.fp.getvalue ()

Pycurl automatically process cookies

import pycurl import stringio   url =  "http:// www.google.com/"Crl = pycurl. Curl () crl.setopt (Pycurl. verbose,1) crl.setopt (Pycurl). followlocation, 1) crl.setopt (Pycurl). maxredirs, 5) Crl.fp = stringio.stringio () crl.setopt (Pycurl. Url, url) crl.setopt (CRL. Writefunction, crl.fp.write)   # option -b/--cookie  cookie string or  file to read cookies from # note: must be a string, not
 a file object. Crl.setopt (Pycurl. cookiefile,  "Cookie_file_name")   # option -c/--cookie-jar  write cookies  to this file after operation # note: must be a string, 
Not a file object. Crl.setopt (Pycurl. cookiejar,  "Cookie_file_name")   Crl.perform () Print crl.fp.getvalue () 

Pycurl Implement POST method

import pycurl import stringio import urllib   URL  =  "http://www.google.com/" post_data_dic = {"name": "Value"} crl = pycurl. Curl () crl.setopt (Pycurl. verbose,1) crl.setopt (Pycurl). followlocation, 1) crl.setopt (Pycurl). maxredirs, 5) #crl. setopt (Pycurl. autoreferer,1)   crl.setopt (Pycurl. connecttimeout, 60) crl.setopt (Pycurl). timeout, 300) #crl. setopt (Pycurl. Proxy,proxy) crl.setopt (Pycurl). httpproxytunnel,1) #crl. setopt (Pycurl. nosignal, 1) Crl.fp = stringio.stringio () crl.setopt (Pycurl. useragent,  "Dhgu hoho")   # option -d/--data    http post  data crl.setopt (CRL. Postfields,  urllib.urlencode (post_data_dic))   crl.setopt (Pycurl. Url, url) crl.setopt (CRL. Writefunction, crl.fp.write) Crl.perform ()   Print crl.fp.getvalue () 

Urllib Timeout setting

Import Socket
Socket.setdefaulttimeout (5.0)

handling Cookies in Python: Automatically process cookies, save as cookie files, load cookies from file

The cookie-related libraries in Python

There are self-contained modules in Python: cookielib

can handle a lot of cookies-related things.

includes

    automatically processes Cookie:cookie data is saved in memory
    Automatically processing cookies: Support cookie data is stored in a file
        there are two formats:
             LWP
             Mozilla
        Further, support again:
             Save cookies to file
             Load cookies from the file

For more detailed formatting and explanations, see the code below.

 
python automatically processes cookies, saves cookies to files, reads cookies from files

Python code:

#!/usr/bin/python # -*- coding: utf-8 -*-"" "Version:    2013-01-15
Author:     crifan contact:    admin  (at)  crifan.com "" "
  import os;
import cookielib;
import urllib2;   Def pythonautohandlecookie ():      "" "         demo how to auto handle cookie in python              cookies in memory              cookies in file:                  save cookie to file                      lwp      format                     mozilla  Format                 load  cookie from file      ""       print  "1. 
demo how to auto handle cookie  (in memory) ";     cookiejarinmemory = cookielib.
Cookiejar ();     opener = urllib2.build_opener (URLLIB2.
Httpcookieprocessor (cookiejarinmemory));
    urllib2.install_opener (opener);     print  "after init, cookiejarinmemory=",cookiejarinmemory;  #after  init, cookiejarinmemory=     #!!!  following urllib2 will auto handle cookies     demoUrl 
=  "http://www.google.com/";     respoNse = urllib2.urlopen (Demourl);      #here,  we already got response cookies    
 print  "after urllib2.urlopen, cookiejarinmemory=", cookiejarinmemory;      #after  urllib2.urlopen, cookiejarinmemory= <cookielib. cookiejar[, , ]>                  print  "2.&NBSP;DEMO&NBSP;HOW&NBSP;TO&NBSP;AUTO&NBSP;HANDLE&NBSP;COOKIE&NBSP;IN&NBSP;FILE,&NBSP;LWP
 format ";
    cookieFilenameLWP =  "LocalCookiesLWP.txt";     cookiejarfilelwp = cookielib.
Lwpcookiejar (COOKIEFILENAMELWP);      #will  create  (and save to)  new cookie file  
   cookiejarfilelwp.save ();     opener = urllib2.build_opener (URLLIB2. Httpcookieprocessor (COOKIEJARFILELWP));
    urllib2.install_opener (opener);     #!!!  following urllib2 will auto handle cookies     demoUrl 
=  "http://www.google.com/";
    response = urllib2.urlopen (Demourl);      #update  cookies, save cookies into file    
 cookiejarfilelwp.save ();      #for  demo, print cookies in file     print
  "Lwp cookies:";
    print open (COOKIEFILENAMELWP). Read (Os.path.getsize (COOKIEFILENAMELWP));     #  #LWP-cookies-2.0     # set-cookie3: pref= "id= 34c1415b570a93ae:ff=0:nw=1:tm=1358236121:lm=1358236121:s=gevvojw4x37ht5n-";  path="/";  domain=". Google.com ";  path_spec; domain_dot; expires=" 2015-01-15 07:48:41z ";  version=0     # set-cookie3: nid= "67=ji_uewum5gdrq_vcwap2z_ YGU7MDLM5CLMA4CNLF7RQUTDMZRRK1EJRDDGCNPOFBHT81LAV9SPXZQQINF0MPS6LDRVCRQBBL5NOTMY8SWOZA6HWC3ITIO4-O3FO1UDKV ";  path= "/";  domain= ". google.com.hk";  path_spec; domain_dot; expires= "2013-07-17  07:48:41z " httponly=none; version=0     # set-cookie3: pref=" ID= 8F7E4EFCA89BDB1B:U=F85A4AFA4DB021AA:FF=2:LD=ZH-CN:NW=1:TM=1358236121:LM=1358236121:S=2WR59HDWUTDNUJTF ";  Path= "/";  domain= ". google.com.hk";  path_spec; domain_dot; expires= "2015-01-15  07:48:41z ";  version=0       print " 3. demo how to auto
 handle cookie in file, mozilla format ";
    cookieFilenameMozilla =  "LocalCookiesMozilla.txt";     cookiejarfilemozilla = cookielib.
Mozillacookiejar (Cookiefilenamemozilla);    &nBSP; #will  create  (and save to)  new cookie file     
Cookiejarfilemozilla.save ();     opener = urllib2.build_opener (URLLIB2.
Httpcookieprocessor (Cookiejarfilemozilla));
    urllib2.install_opener (opener);     #!!!  following urllib2 will auto handle cookies     demoUrl 
=  "http://www.google.com/";
    response = urllib2.urlopen (Demourl);      #update  cookies, save cookies into file    
 cookiejarfilemozilla.save ();      #for  demo, print cookies in file     print
  "Mozilla cookies:";
    print open (Cookiefilenamemozilla). Read (Os.path.getsize (Cookiefilenamemozilla));     # # netscape http&nbsp
Cookie file     # # http://www.netscape.com/newsref/std/cookie_spec.html     # # this is a generated file!  do not
 edit.       # .google.com   TRUE    /    false   1421308121  pref    id=0e05040dd979207c:ff=0:nw=1: TM=1358236121:LM=1358236121:S=JCFID2XGXMIHPUPL     # .google.com.hk     true    /   false   1374047321  nid 67= Klmi_z5zpwdjuyrwsuhie_kyi77_zijal0kwrougthagme86lky7h-mna2wami_ GKLIWYCD8T82QPINXZLD4GLDBMWT0OVLCXHRJ0WQDC57DTNASTS4LHVR7YJVJ2TFN     # .google.com.hk     TRUE    /   FALSE   1421308121   pref    id=028f8b736db06a9a:u=6ba6d080847c8de6:ff=2:ld=zh-cn:nw=1:tm=1358236121:lm=1358236121:s=_1bcc5v3g0zojvz8    
       print  "4. read cookies from file";     parseAndSavedCookieFile =  "ParsedAndSavedCookies.txt"      parsedcookiejarfile = cookielib.
Mozillacookiejar (Parseandsavedcookiefile);      #parsedCookieJarFile  = cookielib.
Mozillacookiejar (Cookiefilenamemozilla);     print parsedcookiejarfile; #    parsedcookiejarfile.load (Cookiefilenamemozilla);

Print parsedcookiejarfile; #<_mozillacookiejar.mozillacookiejar[ , ]>.


If __name__== "__main__":
Pythonautohandlecookie ();



Summarize

The automatic processing of cookies in Python is much better than C #.

At least one existing library is available for use. Although usage, need a little more understanding, just know how to use.