Asp.net 使用正則和網路編程抓取網頁資料(實用)

標籤：asp.net   httpwebrequest   源碼   採集   

Asp.net 使用正則和網路編程抓取網頁資料(實用)

Asp.net 使用正則和網路編程抓取網頁資料(實用)

/// <summary>        /// 抓取網頁相應內容        /// </summary>        /// <param name="strUrl">採集地址</param>        /// <param name="Begin">開始字元</param>        /// <param name="End">結束字元</param>        /// <returns></returns>        private static String GetContent(String strUrl, String Begin, String End)        {            String result = String.Empty;            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);            HttpWebResponse response = (HttpWebResponse)request.GetResponse();            using (StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.Default))            {                result = reader.ReadToEnd();            }            //抓取內容            Match table = Regex.Match(result, "(?<=" + Begin + ")[\\s\\S]*?(?=" + End + ")", RegexOptions.IgnoreCase);            result = NoHTML(table.Value);            return result;        }        ///<summary>        ///去除HTML標記        ///</summary>        ///<param   name="NoHTML">包括HTML的源碼   </param>        ///<returns>已經去除後的文字</returns>        private static string NoHTML(string Htmlstring)        {            //刪除指令碼            Htmlstring = Regex.Replace(Htmlstring, @"<script[^>]*?>.*?</script>", "",                RegexOptions.IgnoreCase);            //刪除HTML            Htmlstring = Regex.Replace(Htmlstring, @"<(.[^>]*)>", "",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"([\r\n])[\s]+", "",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"-->", "", RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"<!--.*", "", RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(quot|#34);", "\"",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(amp|#38);", "&",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(lt|#60);", "<",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(gt|#62);", ">",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(nbsp|#160);", "   ",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(iexcl|#161);", "\xa1",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(cent|#162);", "\xa2",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(pound|#163);", "\xa3",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&(copy|#169);", "\xa9",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, @"&#(\d+);", "",                RegexOptions.IgnoreCase);            Htmlstring = Regex.Replace(Htmlstring, ">", "");            Htmlstring = Regex.Replace(Htmlstring, "<", "");            Htmlstring = Regex.Replace(Htmlstring, "\r\n", "");            Htmlstring = Htmlstring.Substring(Htmlstring.IndexOf("\n") + 1);            if (Htmlstring.LastIndexOf("‘") >= 0)                Htmlstring = Htmlstring.Substring(Htmlstring.LastIndexOf("‘") + 1);            if (Htmlstring.IndexOf("class=‘tdbk‘") >= 0)                Htmlstring = Htmlstring.Substring(Htmlstring.IndexOf("class=‘tdbk‘") + "class=‘tdbk‘".Length);            return Htmlstring;        }