Use JAVA to capture web page data and java to capture web pages
1. Use HttpClient to capture webpage data
Public String getHtml (String htmlurl) throws IOException {StringBuffer sb = new StringBuffer (); String acceptEncoding = "";/* 1. generate the HttpClinet object and set the parameter */HttpClient httpClient = new HttpClient (); GetMethod method = new GetMethod (htmlurl); int statusCode; try {statusCode = httpClient.exe cuteMethod (method ); // determine the access status code if (statusCode! = HttpStatus. SC _ OK) {return null;} else {if (method. getResponseHeader ("Content-Encoding ")! = Null) acceptEncoding = method. getResponseHeader ("Content-Encoding "). getValue (); if (acceptEncoding. toLowerCase (). indexOf ("gzip")>-1) {// create the gzip decompression workflow InputStream is; is = method. getResponseBodyAsStream (); GZIPInputStream gzin = new GZIPInputStream (is); InputStreamReader isr = new InputStreamReader (gzin, Charset. forName (CHARSET); // sets the encoding format of the read stream, custom encoding java. io. bufferedReader br = new java. io. bufferedReade R (isr); String tempbf; while (tempbf = br. readLine ())! = Null) {if (StringUtils. isNotBlank (tempbf) {sb. append (tempbf) ;}} isr. close (); gzin. close (); System. out. println (sb);} else {InputStreamReader isr; isr = new InputStreamReader (method. getResponseBodyAsStream (), CHARSET); java. io. bufferedReader br = new java. io. bufferedReader (isr); String tempbf; while (tempbf = br. readLine ())! = Null) {if (StringUtils. isNotBlank (tempbf) {sb. append (tempbf) ;}} isr. close () ;}} catch (HttpException e) {e. printStackTrace ();} catch (IOException e) {e. printStackTrace ();} method. abort (); method. releaseConnection (); return sb. toString ();}
Ii. Use HttpPost to capture webpage data
private static CloseableHttpClient httpClient;private static BasicHttpContext httpContext;private static BasicCookieStore cookieStore;private static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();private static RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();private static RequestConfig localConfig = RequestConfig.copy(globalConfig).setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();public String getHtml(String url){HttpClientBuilder builder = HttpClients.custom();cookieStore = new BasicCookieStore();builder.setConnectionManager(cm);builder.setDefaultCookieStore(cookieStore);builder.setDefaultRequestConfig(globalConfig);httpClient = builder.build();httpContext = new BasicHttpContext();httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);HttpPost httpPost = new HttpPost(url);httpPost.setConfig(localConfig);httpPost.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");httpPost.setHeader("Accept-Encoding","gzip, deflate");httpPost.setHeader("Accept-Language","zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");httpPost.setHeader("Connection","keep-alive");httpPost.setHeader("Cookie","ASP.NET_SessionId=11vrr4ucwsgeqtmpyfx4hmvx; _5t_trace_sid=89c4ffb8633d267e4ae322a157b52471; _5t_trace_tms=1; CheckCode=X0P64");httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0"); List <NameValuePair> nvps = new ArrayList <NameValuePair>(); nvps.add(new BasicNameValuePair("pid", "99-C3-57-35-6D-70-3D-F2")); nvps.add(new BasicNameValuePair("CurrentlyPageIndex", "2")); httpPost.setEntity(new UrlEncodedFormEntity(nvps, Consts.UTF_8));try {CloseableHttpResponse response = httpClient.execute(httpPost,httpContext);int status = response.getStatusLine().getStatusCode();if (status >= 200 && status < 300) {HttpEntity httpEntity = response.getEntity();if(httpEntity!=null){String cont = trimLineToString(httpEntity, "UTF-8");EntityUtils.consume(httpEntity);return cont;}}} catch (ClientProtocolException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}return null;}public synchronized static String trimLineToString(HttpEntity entiry,String charset) {StringBuffer sb = new StringBuffer();BufferedReader reader = null;try {InputStream instream = entiry.getContent();reader = new BufferedReader(new InputStreamReader(instream, charset));String str = null;while ((str = reader.readLine()) != null) {if(StringUtils.isNotBlank(str)) {sb.append(str.trim());}}instream.close();} catch (IllegalStateException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();} finally {if (reader != null) {try {reader.close();} catch (IOException e) {e.printStackTrace();}}}return sb.toString();}