Java Crawler--Login-free, simulate login via httpclient and get the information after login

Taking Xin ' an talent network as an example
1, the use of dependence
HttpClient HTTP Request
Jsoup Parsing HTML

<dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.8.1</version>
        </dependency>
        <dependency>
            <groupId> commons-httpclient</groupid>
            <artifactId>commons-httpclient</artifactId>
            <version >3.0.1</version>
        </dependency>

2, the specific implementation of the following

Package com.ssjk.website.service.JsoupHtml;
Import Org.apache.commons.httpclient.Cookie;
Import org.apache.commons.httpclient.HttpClient;
Import Org.apache.commons.httpclient.NameValuePair;
Import Org.apache.commons.httpclient.cookie.CookiePolicy;
Import Org.apache.commons.httpclient.cookie.CookieSpec;
Import Org.apache.commons.httpclient.methods.PostMethod;
Import Org.jsoup.Jsoup;

Import org.jsoup.nodes.Document;

Import java.io.IOException;
    /** * Access to Xinan Talent Network information/public class Xinanrencaiwang {private static final String SITE = "login.goodjobs.cn";

    private static final int PORT = 80;
    private static final String loginaction = "/index.php/action/userlogin";
    private static final String Forwardurl = "http://user.goodjobs.cn/dispatcher.php/module/Personal/";
    private static final String Tourl = "d:\\jsoup_test\\";
    private static final String hostcss = "D:\\jsoup_test\\style.txt";
  private static final String Img = "Http://user.goodjobs.cn/images";  private static final String _js = "Http://user.goodjobs.cn/scripts/fValidate/fValidate.one.js";
     /** * @param logon_site * @param logon_port * @param login_action * @param params
            * @throws Exception */private static string[] loginhtml (string logon_site, int logon_port, string
        Login_action, String ... params) throws Exception {string[] result=null;
        HttpClient client = new HttpClient ();
Client.gethostconfiguration (). Sethost (Logon_site, Logon_port);
        Analog login page Postmethod post = new Postmethod (login_action);
        Namevaluepair userName = new Namevaluepair ("MemberName", params[0));
        Namevaluepair password = new Namevaluepair ("password", params[1));
        Post.setrequestbody (new namevaluepair[]{username, password});
        Client.executemethod (POST);
        System.out.println ("Execution Status:" +client.getstate ());
Post.releaseconnection (); View cookie Information Cookiespec cookiesSpec = Cookiepolicy.getdefaultspec ();
        cookie[] cookies = Cookiespec.match (Logon_site, Logon_port, "/", False, Client.getstate (). GetCookies ()); if (cookies!= null) if (cookies.length = = 0) {System.out.println ("Cookies are not Exi
            STS "); else {for (int i = 0; i < cookies.length i++) {System.out.println ("-----------
                    -----------------------------------------");
                    System.out.println (Cookies[i].tostring ());
                    result = Cookies[i].tostring (). Split ("=");
                System.out.println ("----------------------------------------------------");
    } return result; /** * * @param cookies * @return/public static Document gethtmldocument (string[) cookies
         {try {Document doc = Jsoup.connect (forwardurl). Cookie (cookies[0],cookies[1]). get ();   return doc; The catch (IOException e) {System.out.println ("page Gets an exception.
            ");
        E.printstacktrace ();
    return null;
        public static void Main (string[] args) {string[] params = {"job17093123661", "2534133662QQ"};
        String[] strings = null;
            try {strings = loginhtml (SITE, PORT, loginaction, params);
                if (strings.length>0) {Document HTMLDocument = gethtmldocument (strings);
            System.out.println (HTMLDocument);
        } catch (Exception e) {e.printstacktrace (); }

    }
}