Asp.net web page capture content, asp.net web page capture

Asp.net web page capture content, asp.net web page capture

Web page capture code

Using System; using System. collections. generic; using System. linq; using System. web; // using System. net; using System. IO; using System. text. regularExpressions; using System. text; namespace WSYL. web. common {public static class GetSteamShipInfo {public static string GetWebSite (string steamshipname, int itype) {if (steamshipname = null | steamshipname. trim () = "") return null; // step1: get html from url string urlToCrawl = @ ""; // generate http request HttpWebRequest req = (HttpWebRequest) WebRequest. create (urlToCrawl); // use GET method to get url's html req. method = "GET"; // use request to get response HttpWebResponse resp = (HttpWebResponse) req. getResponse (); // August 12, year 15 18:14:45 need to increase the judgment of Web Page parsing timeout to prevent web page false dead // string htmlCharset = "UTF-8"; string htmlCharset = "UTF-8 "; // use songtaste's html's charset GB2312 to decode html // otherwise will return messy code Encoding htmlEncoding = Encoding. getEncoding (htmlCharset); StreamReader sr = new StreamReader (resp. getResponseStream (), htmlEncoding); // read out the returned html string respHtml = sr. readToEnd (); // The third method to obtain the content. // Match TitleMatch = Regex. match (rtbExtractedHtml. text. toString (), "<td width = \" 30% \ "> ([^ <] *) </td>", RegexOptions. ignoreCase | RegexOptions. multiline );
// Start and end content of the Code to be obtained
Match TitleMatch2 = Regex. match (respHtml. toString (), "<td align = \" left \ "bgcolor = \" # EEEEEE \ "> ([^ <] *) </td>", RegexOptions. ignoreCase | RegexOptions. multiline); // txbExtractedInfo. text = TitleMatch2.Groups [1]. value + "/" + TitleMatch2.Groups [2]. value; if (TitleMatch2.Groups [1]. value. length = 0 | TitleMatch2.Groups [1]. value = "") return respHtml = ""; if (itype = 0) {respHtml = TitleMatch2.Groups [1]. val Ue. toString () ;}if (itype = 1) {respHtml = StripHtml (TitleMatch2.NextMatch (). value. toString ();} if (itype = 2) {respHtml = TitleMatch2.Groups [1]. value + "/" + StripHtml (TitleMatch2.NextMatch (). value. toString () ;}return respHtml ;}/// <summary >/// some exceptions may cause the removal of html tags and spaces, so we recommend that you convert them twice in a row. In this way, Html tags are converted to spaces. Too many consecutive spaces will affect subsequent string operations /// </summary> /// <param name = "strHtml"> tag content </param> /// <returns> </returns> private static string StripHtml (string strHtml) {Regex objRegExp = new Regex ("<(. | \ n) +?> "); String strOutput = objRegExp. replace (strHtml, ""); strOutput = strOutput. replace ("<", "& lt;"); strOutput = strOutput. replace (">", "& gt;"); // convert all spaces into a space Regex r = new Regex (@ "\ s +"); strOutput = r. replace (strOutput, ""); return strOutput. trim ();}}}