C # web crawler code sharing C # Simple crawl tool _c# tutorial

Company editor sister needs to crawl Web content, ask me to help make a simple crawl tool

This is crawling Web content, like this for everyone is not rare, but here are some small changes, the code to offer, you refer to

private string gethttpwebrequest (string url) {HttpWebResponse result; String strhtml = String. 
      Empty; 
        try {uri uri = new uri (URL); 
        WebRequest webreq = WebRequest.Create (URI); 
 
        WebResponse webres = Webreq.getresponse (); 
        HttpWebRequest Myreq = (HttpWebRequest) webreq; Myreq.useragent = "user-agent:mozilla/4.0" (compatible; MSIE 6.0; 
        Windows NT 5.2;. NET CLR 1.0.3705 "; 
        myreq.accept = "*/*"; 
        Myreq.keepalive = true; 
        MYREQ.HEADERS.ADD ("Accept-language", "zh-cn,en-us;q=0.5"); 
        result = (HttpWebResponse) myreq.getresponse (); Stream Recevicestream = result. 
        GetResponseStream (); 
        StreamReader Readerofstream = new StreamReader (Recevicestream, System.Text.Encoding.GetEncoding ("Utf-8")); 
        strHTML = Readerofstream.readtoend (); 
        Readerofstream.close (); 
        Recevicestream.close (); Result. 
      Close (); The catch {uri uri = New Uri (URL); 
        WebRequest webreq = WebRequest.Create (URI); 
        HttpWebRequest Myreq = (HttpWebRequest) webreq; Myreq.useragent = "user-agent:mozilla/4.0" (compatible; MSIE 6.0; 
        Windows NT 5.2;. NET CLR 1.0.3705 "; 
        myreq.accept = "*/*"; 
        Myreq.keepalive = true; 
        MYREQ.HEADERS.ADD ("Accept-language", "zh-cn,en-us;q=0.5"); 
        result = (HttpWebResponse) myreq.getresponse (); 
        try {result = (HttpWebResponse) myreq.getresponse (); The catch (WebException ex) {result = (HttpWebResponse) ex. 
        Response; Stream Recevicestream = result. 
        GetResponseStream (); 
        StreamReader Readerofstream = new StreamReader (Recevicestream, System.Text.Encoding.GetEncoding ("gb2312")); 
        strHTML = Readerofstream.readtoend (); 
        Readerofstream.close (); 
        Recevicestream.close (); Result. 
      Close (); 
    return strhtml;
 }

This is based on the URL crawl page far, there are some small changes, many pages have different coding format, and even some sites do a reverse crawl to prevent, this method can be modified to crawl

The following are links to all URLs of crawl pages

 <summary>///extract URL in html code///</summary>///<param name= "Htmlcode" ></param> <returns></returns> private static list<string> gethyperlinks (string htmlcode, String URL 
      {ArrayList Al = new ArrayList (); 
      BOOL Isgenxin = false; StringBuilder weburlsb = new StringBuilder ();//sql StringBuilder linksb = new StringBuilder ();//Show Data list&lt ;string> weburllistzx = new list<string> ();//Add list<string> weburllist = new list<string> (); 
      /Old String productioncontent = Htmlcode; Regex reg = new regex (@) http (s)?:/ /([\w-]+\.) 
      +[\w-]+/? "); String wangzhanyuming = Reg. Match (URL, 0). 
      Value; MatchCollection mc = regex.matches (Productioncontent.replace ("href=\"/"," href=\ "" + wangzhanyuming). Replace ("href= '/", "href= '" + wangzhanyuming). Replace ("href=/", "href=" + wangzhanyuming). Replace ("href=\"./"," href=\ "" + wangzhanyuming), @ "&LT;[AA][^>]* href=[^>]*> ", regexoptions.singleline); 
      int Index = 1; foreach (Match m in MC) {MatchCollection MC1 = regex.matches (M.value, @ "[a-za-z]+://[^\s]*", RegexOptions . 
        Singleline); if (MC1. Count > 0) {foreach (Match m1 in MC1) {string linkurlstr = string. 
            Empty; LINKURLSTR = M1. Value.replace ("\", ""). Replace ("'", ""). Replace (">", ""). 
            Replace (";", ""); 
            Weburlsb.append ("$-$"); 
            Weburlsb.append (LINKURLSTR); 
            Weburlsb.append ("$_$"); if (! Weburllist.contains (LINKURLSTR) &&! 
              Weburllistzx.contains (LINKURLSTR)) {isgenxin = true; 
              Weburllistzx.add (LINKURLSTR); 
            Linksb.appendformat ("{0}<br/>", linkurlstr); 
            }} else {if (M.value.indexof ("javascript") = = 1) { String amstr = StrinG.empty; String Wangzhanxiangduilujin = String. 
            Empty; Wangzhanxiangduilujin = URL. Substring (0, URL. 
            LastIndexOf ("/") + 1); Amstr = M.value.replace ("href=\" "," href=\ "" + Wangzhanxiangduilujin). 
            Replace ("href= '", "href= '" + Wangzhanxiangduilujin); 
            MatchCollection mc11 = regex.matches (Amstr, @ "[a-za-z]+://[^\s]*", regexoptions.singleline); foreach (Match m1 in Mc11) {string linkurlstr = string. 
              Empty; LINKURLSTR = M1. Value.replace ("\", ""). Replace ("'", ""). Replace (">", ""). 
              Replace (";", ""); 
              Weburlsb.append ("$-$"); 
              Weburlsb.append (LINKURLSTR); 
              Weburlsb.append ("$_$"); if (! Weburllist.contains (LINKURLSTR) &&! 
                Weburllistzx.contains (LINKURLSTR)) {isgenxin = true; 
                Weburllistzx.add (LINKURLSTR); 
        Linksb.appendformat ("{0}<br/>", linkurlstr);      }}} index++; 
    return WEBURLLISTZX;
 }

This piece of technology is actually a simple use of the regular to match! Next, get the title, and the method to store the XML file

<summary>/////write URL to xml file///</summary>///<param name= "strURL" ></param> <param name= "Alhyperlinks" ></param> private static void Writetoxml (string strurl, list<string& Gt 
      Alhyperlinks) {XmlTextWriter writer = new XmlTextWriter (@ "D:\HyperLinks.xml", Encoding.UTF8); Writer. 
      formatting = formatting.indented; Writer. 
      WriteStartDocument (FALSE); Writer. 
      Writedoctype ("HyperLinks", NULL, "URLS.DTD", null); Writer. 
      WriteComment ("Extract from" + strURL + "HYPERLINK"); Writer. 
      WriteStartElement ("HyperLinks"); Writer. 
      WriteStartElement ("HyperLinks", null); Writer. 
      WriteAttributeString ("DateTime", DateTime.Now.ToString ()); 
        foreach (String str in alhyperlinks) {string title = GetDomain (str); 
        string BODY = str; Writer. 
      WriteElementString (title, null, body); } writer. 
      WriteEndElement (); Writer. 
      WriteEndElement (); Writer. 
      Flush (); Writer. 
    Close ();  
    ///<summary>///Gets the domain name suffix of the URL///</summary>///<param name= "strURL" ></param>  
      <returns></returns> private static string GetDomain (String strurl) {string retVal; 
      String Strregex = @ "(\.com/|\.net/|\.cn/|\.org/|\.gov/)"; 
      Regex r = new Regex (Strregex, regexoptions.ignorecase); 
      Match m = R.match (strURL); 
      RetVal = M.tostring (); Strregex = @ "\.| 
      /$"; RetVal = Regex.Replace (RetVal, Strregex, ""). 
      ToString (); 
      if (RetVal = = "") RetVal = "other"; 
    return retVal; ///<summary>///get title///</summary>///<param name= "html" ></param>///& lt;returns></returns> private static string GetTitle (string html) {string titlefilter = @ "<t 
      Itle>[\s\s]*?</title> "; 
      String h1filter = @ " 

This is all the methods used, or there are many need to improve the place! If you find any deficiencies, please point out, thank you!

The above is the entire content of this article, I hope to help you learn, but also hope that we support the cloud habitat community.