Crawl Web content

The previous blog has introduced how to get the encoding of the Web page, after the encoding to get the corresponding stream, we will get the content of the Web page in a string type of variable can be

Package Spider;

Import Java.io.BufferedReader;
Import Java.io.InputStream;
Import Java.io.InputStreamReader;
Import java.net.HttpURLConnection;
Import java.net.InetSocketAddress;
Import Java.net.Proxy;
Import Java.net.URL;

Import Org.junit.Test;

public class Capture {
@Test
Public String K () throws exception{
URL url=new url ("http://www.baidu.com");
Get the code of Baidu homepage
String charset=i (URL);
StringBuffer Codebuffer=null;
BufferedReader In=null;
try{
Set up Proxy
Proxy proxy=new Proxy (Proxy.Type.HTTP, New inetsocketaddress ("Proxy3.bj.petrochina", 8080));
Open connection
HttpURLConnection urlcon= (httpurlconnection) url.openconnection (proxy);
Urlcon.setrequestproperty ("User-agent", "mozilla/4.0" (compatible; MSIE 5.0; Windows XP; Digext) ");
InputStream Is=urlcon.getinputstream ();
In=new BufferedReader (New InputStreamReader (Is,charset));
Codebuffer=new StringBuffer ();
String Tmpcode=null;
Read the value in buffer and save it in code
while ((Tmpcode=in.readline ())!=null) {
Codebuffer.append (Tmpcode). Append ("\ n");
}
In.close ();
}catch (Exception e) {
E.printstacktrace ();
}
String temp = codebuffer.tostring ();
SYSTEM.OUT.PRINTLN (temp);
return temp;
}
Public String I (URL url) throws exception{
Set up Proxy
Proxy proxy=new Proxy (Proxy.Type.HTTP, New inetsocketaddress ("Proxy3.bj.petrochina", 8080));

URL url=new url ("http://www.baidu.com");
Open connection
HttpURLConnection urlcon= (httpurlconnection) url.openconnection (proxy);
Urlcon.setrequestproperty ("User-agent", "mozilla/4.0" (compatible; MSIE 5.0; Windows XP; Digext) ");
String Charset=null;
String Contenttype=urlcon.getheaderfield ("Content-type");
For (String param:contentType.replace ("", ""). Split (";")) {
if (Param.startswith ("charset=")) {
Charset=param.split ("=", 2) [1];
Break
}
}
return charset;
}
}

Crawl Web content