C # crawls and exports all hyperlink methods in a Web page

 Public classApp {//gets the HTML code for the specified Web page         Public Static stringGetpagesource (stringURL) {URI Uri=NewUri (URL); HttpWebRequest Hwreq=(HttpWebRequest) webrequest.create (URI); HttpWebResponse Hwres=(HttpWebResponse) hwreq.getresponse (); Hwreq.method="Get"; Hwreq.keepalive=false; StreamReader Reader=NewStreamReader (Hwres.getresponsestream (), System.Text.Encoding.GetEncoding ("GB2312")); returnReader.        ReadToEnd (); }        //extracting URLs from your HTML code         Public StaticArrayList Gethyperlinks (stringHtmlcode) {ArrayList Al=NewArrayList (); stringStrregex =@"http://([\w-]+\.) +[\w-]+ (/[\w-./?%&=]*)?"; Regex R=NewRegex (Strregex, regexoptions.ignorecase); MatchCollection m=r.matches (Htmlcode);  for(inti =0; I <= M.count-1; i++)            {                BOOLRep =false; stringStrnew =M[i].                ToString (); //Filter for duplicate URLs                foreach(stringStrinchal) {                    if(Strnew = =str) {Rep=true;  Break; }                }                if(!Rep) al.            ADD (strnew); } al.            Sort (); returnal; }        //write the URL to an XML file         Public Static voidWritetoxml (stringstrURL, ArrayList alhyperlinks) {XmlTextWriter writer=NewXmlTextWriter ("Hyperlinks.xml", Encoding.UTF8); Writer. Formatting=formatting.indented; Writer. WriteStartDocument (false); Writer. Writedoctype ("HyperLinks",NULL,"URLS.DTD",NULL); Writer. WriteComment ("Extracted from"+ strURL +"of Hyperlinks"); Writer. WriteStartElement ("HyperLinks"); Writer. WriteStartElement ("HyperLinks",NULL); Writer. WriteAttributeString ("DateTime", DateTime.Now.ToString ()); foreach(stringStrinchalhyperlinks) {                stringtitle =GetDomain (str); stringBODY =str; Writer. WriteElementString (title,NULL, body); } writer.            WriteEndElement (); Writer.            WriteEndElement (); Writer.            Flush (); Writer.        Close (); }        //get the domain name suffix of the URL        Static stringGetDomain (stringstrURL) {            stringRetVal; stringStrregex =@"(\.com/|\.net/|\.cn/|\.org/|\.gov/)"; Regex R=NewRegex (Strregex, regexoptions.ignorecase); Match m=R.match (strURL); RetVal=m.tostring (); Strregex=@"\.| /$"; RetVal= Regex.Replace (RetVal, Strregex,"").            ToString (); if(RetVal = ="") RetVal=" Other"; returnRetVal; }    }

Private voidBtnkaishi_click (Objectsender, EventArgs e) {            stringStrcode;            ArrayList allinks; if(Txtapi. Text = ="") {MessageBox.Show ("Please enter the URL"); return; }            stringstrURL =Txtapi. Text.tostring ().            Trim (); if(Strurl.substring (0,7) !=@"/ http") {strURL=@"/ http"+strURL; } MessageBox.Show ("getting page code, please later ..."); Strcode=app.            Getpagesource (strURL); MessageBox.Show ("extracting hyperlink, please wait ..."); Allinks=app.            Gethyperlinks (Strcode); MessageBox.Show ("writing file, please wait ..."); App.        Writetoxml (strURL, allinks); }

C # crawls and exports all hyperlink methods in a Web page