抓取網頁內容

標籤：style   blog   http   io   ar   color   os   sp   java   

上一篇部落格已經介紹了如何得到網頁的編碼，得到編碼之後根據編碼得到相應的流，我們將網頁的內容擷取存在一個string類型的變數中即可

package Spider;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;

import org.junit.Test;

public class Capture {
@Test
public String k() throws Exception{
URL url=new URL("http://www.baidu.com");
//得到百度首頁的編碼
String charset=i(url);
StringBuffer codeBuffer=null;
BufferedReader in=null;
try{
//設定代理
Proxy proxy=new Proxy(Proxy.Type.HTTP, new InetSocketAddress("proxy3.bj.petrochina", 8080));
//開啟串連
HttpURLConnection urlcon=(HttpURLConnection) url.openConnection(proxy);
urlcon.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt)");
InputStream is=urlcon.getInputStream();
in=new BufferedReader(new InputStreamReader(is,charset));
codeBuffer=new StringBuffer();
String tmpCode=null;
//把buffer內的值讀出來，儲存到code中
while((tmpCode=in.readLine())!=null){
codeBuffer.append(tmpCode).append("\n");
}
in.close();
}catch(Exception e){
e.printStackTrace();
}
String temp = codeBuffer.toString();
System.out.println(temp);
return temp;
}
public String i(URL url) throws Exception{
//設定代理
Proxy proxy=new Proxy(Proxy.Type.HTTP, new InetSocketAddress("proxy3.bj.petrochina", 8080));

//URL url=new URL("http://www.baidu.com");
//開啟串連
HttpURLConnection urlcon=(HttpURLConnection) url.openConnection(proxy);
urlcon.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt)");
String charset=null;
String contentType=urlcon.getHeaderField("Content-Type");
for(String param:contentType.replace(" ","").split(";")){
if(param.startsWith("charset=")){
charset=param.split("=", 2)[1];
break;
}
}
return charset;
}
}

 

抓取網頁內容