Java 網路爬蟲的基本實現__Java

這是一個web搜尋的基本程式，從命令列輸入搜尋條件（起始的URL、處理url的最大數、要搜尋的字串),
它就會逐個對Internet上的URL進行即時搜尋,尋找並輸出匹配搜尋條件的頁面。 這個程式的原型來自《java編程藝術》，
為了更好的分析，站長去掉了其中的GUI部分，並稍作修改以適用jdk1.5。以這個程式為基礎，可以寫出在互連網上搜尋
諸如映像、郵件、網頁下載之類的“爬蟲”。
先請看程式啟動並執行過程：


D:/java>javac  SearchCrawler.java（編譯）

D:/java>java   SearchCrawler http://127.0.0.1:8080/zz3zcwbwebhome/index.jsp 20 java

Start searching...
result:
searchString=java
http://127.0.0.1:8080/zz3zcwbwebhome/index.jsp
http://127.0.0.1:8080/zz3zcwbwebhome/reply.jsp
http://127.0.0.1:8080/zz3zcwbwebhome/learn.jsp
http://127.0.0.1:8080/zz3zcwbwebhome/download.jsp
http://127.0.0.1:8080/zz3zcwbwebhome/article.jsp
http://127.0.0.1:8080/zz3zcwbwebhome/myexample/jlGUIOverview.htm
http://127.0.0.1:8080/zz3zcwbwebhome/myexample/Proxooldoc/index.html
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=301
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=297
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=291
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=286
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=285
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=284
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=276
http://127.0.0.1:8080/zz3zcwbwebhome/view.jsp?id=272   

又如：
D:/java>java    SearchCrawler http://www.sina.com   20 java
Start searching...
result:
searchString=java
http://sina.com
http://redirect.sina.com/WWW/sinaCN/www.sina.com.cn class=a2
http://redirect.sina.com/WWW/sinaCN/www.sina.com.cn class=a8
http://redirect.sina.com/WWW/sinaHK/www.sina.com.hk class=a2
http://redirect.sina.com/WWW/sinaTW/www.sina.com.tw class=a8
http://redirect.sina.com/WWW/sinaUS/home.sina.com class=a8
http://redirect.sina.com/WWW/smsCN/sms.sina.com.cn/ class=a2
http://redirect.sina.com/WWW/smsCN/sms.sina.com.cn/ class=a3
http://redirect.sina.com/WWW/sinaNet/www.sina.net/ class=a3


D:/java>
下面是這個程式的源碼

import java.util.*; import java.net.*; import java.io.*; import java.util.regex.*; // 搜尋Web爬行者 public class SearchCrawler implements Runnable{ /* disallowListCache緩衝robot不允許搜尋的URL。 Robot協議在Web網站的根目錄下設定一個robots.txt檔案, *規定網站上的哪些頁面是限制搜尋的。 搜尋程式應該在搜尋過程中跳過這些地區,下面是robots.txt的一個例子: # robots.txt for http://somehost.com/ User-agent: * Disallow: /cgi-bin/ Disallow: /registration # /Disallow robots on registration page Disallow: /login */ private HashMap< String,ArrayList< String>> disallowListCache = new HashMap< String,ArrayList< String>>(); ArrayList< String> errorList= new ArrayList< String>();//錯誤資訊 ArrayList< String> result=new ArrayList< String>(); //搜尋到的結果 String startUrl;//開始搜尋的起點 int maxUrl;//最大處理的url數 String searchString;//要搜尋的字串(英文) boolean caseSensitive=false;//是否區分大小寫 boolean limitHost=false;//是否在限制的主機內搜尋 public SearchCrawler(String startUrl,int maxUrl,String searchString){ this.startUrl=startUrl; this.maxUrl=maxUrl; this.searchString=searchString; } public ArrayList< String> getResult(){ return result; } public void run(){//啟動搜尋線程 crawl(startUrl,maxUrl, searchString,limitHost,caseSensitive); } //檢測URL格式 private URL verifyUrl(String url) { // 只處理HTTP URLs. if (!url.toLowerCase().startsWith("http://")) return null; URL verifiedUrl = null; try { verifiedUrl = new URL(url); } catch (Exception e) { return null; } return verifiedUrl; } // 檢測robot是否允許訪問給出的URL. private boolean isRobotAllowed(URL urlToCheck) { String host = urlToCheck.getHost().toLowerCase();//擷取給出RUL的主機 //System.out.println("主機="+host); // 擷取主機不允許搜尋的URL緩衝 ArrayList< String> disallowList =disallowListCache.get(host); // 如果還沒有緩衝,下載並緩衝。 if (disallowList == null) { disallowList = new ArrayList< String>(); try { URL robotsFileUrl =new URL("http://" + host + "/robots.txt"); BufferedReader reader =new BufferedReader(new InputStreamReader(robotsFileUrl.openStream())); // 讀robot檔案，建立不允許訪問的路徑列表。 String line; while ((line = reader.readLine()) != null) { if (line.indexOf("Disallow:") == 0) {//是否包含"Disallow:" String disallowPath =line.substring("Disallow:".length());//擷取不允許訪問路徑 // 檢查是否有注釋。 int commentIndex = disallowPath.indexOf("#"); if (commentIndex != - 1) { disallowPath =disallowPath.substring(0, commentIndex);//去掉注釋 } disallowPath = disallowPath.trim(); disallowList.add(disallowPath); } } // 緩衝此主機不允許訪問的路徑。 disallowListCache.put(host, disallowList); } catch (Exception e) { return true; //web網站根目錄下沒有robots.txt檔案,返回真 } } String file = urlToCheck.getFile(); //System.out.println("檔案getFile()="+file); for (int i = 0; i < disallowList.size(); i++) { String disallow = disallowList.get(i); if (file.startsWith(disallow)) { return false; } } return true; } private String downloadPage(URL pageUrl) { try { // Open connection to URL for reading. BufferedReader reader = new BufferedReader(new InputStreamReader(pageUrl.openStream())); // Read page into buffer. String line; StringBuffer pageBuffer = new StringBuffer(); while ((line = reader.readLine()) != null) { pageBuffer.append(line); } return pageBuffer.toString(); } catch (Exception e) { } return null; } // 從URL中去掉"www" private String removeWwwFromUrl(String url) { int index = url.indexOf("://www."); if (index != -1) { return url.substring(0, index + 3) + url.substring(index + 7); } return (url); } // 解析頁面並找出連結 private ArrayList< String> retrieveLinks(URL pageUrl, String pageContents, HashSet crawledList, boolean limitHost) { // 用Regex編譯連結的匹配模式。 Pattern p =Pattern.compile("<a//s+href//s*=//s*/"?(.*?)[/"|>]",Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(pageContents); ArrayList< String> linkList = new ArrayList< String>(); while (m.find()) { String link = m.group(1).trim(); if (link.length() < 1) { continue; } // 跳過鏈到本頁面內連結。 if (link.charAt(0) == '#') { continue; } if (link.indexOf("mailto:") != -1) { continue; } if (link.toLowerCase().indexOf("javascript") != -1) { continue; } if (link.indexOf("://") == -1){ if (link.charAt(0) == '/') {//處理絕對地 link = "http://" + pageUrl.getHost()+":"+pageUrl.getPort()+ link; } else { String file = pageUrl.getFile(); if (file.indexOf('/') == -1) {//處理相對位址 link = "http://" + pageUrl.getHost()+":"+pageUrl.getPort() + "/" + link; } else { String path =file.substring(0, file.lastIndexOf('/') + 1); link = "http://" + pageUrl.getHost() +":"+pageUrl.getPort()+ path + link; } } } int index = link.indexOf('#'); if (index != -1) { link = link.substring(0, index); } link = removeWwwFromUrl(link); URL verifiedLink = verifyUrl(link); if (verifiedLink == null) { continue; } /* 如果限定主機，排除那些不合條件的URL*/ if (limitHost && !pageUrl.getHost().toLowerCase().equals( verifiedLink.getHost().toLowerCase())) { continue; } // 跳過那些已經處理的連結. if (crawledList.contains(link)) { continue; } linkList.add(link); } return (linkList); } // 搜尋下載Web頁面的內容，判斷在該頁面內有沒有指定的搜尋字串 private boolean searchStringMatches(String pageContents, String searchString, boolean caseSensitive){ String searchContents = pageContents; if (!caseSensitive) {//如果不區分大小寫 searchContents = pageContents.toLowerCase(); } Pattern p = Pattern.compile("[//s]+"); String[] terms = p.split(searchString); for (int i = 0; i < terms.length; i++) { if (caseSensitive) { if (searchContents.indexOf(terms[i]) == -1) { return false; } } else { if (searchContents.indexOf(terms[i].toLowerCase()) == -1) { return false; } } } return true; } //執行實際的搜尋操作 public ArrayList< String> crawl(String startUrl, int maxUrls, String searchString,boolean limithost,boolean caseSensitive ) { System.out.println("searchString="+searchString); HashSet< String> crawledList = new HashSet< String>(); LinkedHashSet< String> toCrawlList = new LinkedHashSet< String>(); if (maxUrls < 1) { errorList.add("Invalid Max URLs value."); System.out.println("Invalid Max URLs value."); } if (searchString.length() < 1) { errorList.add("Missing Search String."); System.out.println("Missing search String"); } if (errorList.size() > 0) { System.out.println("err!!!"); return errorList; } // 從開始URL中移出www startUrl = removeWwwFromUrl(startUrl); toCrawlList.add(startUrl); while (toCrawlList.size() > 0) { if (maxUrls != -1) { if (crawledList.size() == maxUrls) { break; } } // Get URL at bottom of the list. String url = toCrawlList.iterator().next(); // Remove URL from the to crawl list. toCrawlList.remove(url); // Convert string url to URL object. URL verifiedUrl = verifyUrl(url); // Skip URL if robots are not allowed to access it. if (!isRobotAllowed(verifiedUrl)) { continue; } // 增加已處理的URL到crawledList crawledList.add(url); String pageContents = downloadPage(verifiedUrl); if (pageContents != null && pageContents.length() > 0){ // 從頁面中擷取有效連結 ArrayList< String> links =retrieveLinks(verifiedUrl, pageContents, crawledList,limitHost); toCrawlList.addAll(links); if (searchStringMatches(pageContents, searchString,caseSensitive)) { result.add(url); System.out.println(url); } } } return result; } // 主函數 public static void main(String[] args) { if(args.length!=3){ System.out.println("Usage:java SearchCrawler startUrl maxUrl searchString"); return; } int max=Integer.parseInt(args[1]); SearchCrawler crawler = new SearchCrawler(args[0],max,args[2]); Thread search=new Thread(crawler); System.out.println("Start searching..."); System.out.println("result:"); search.start(); } }