1.網站地圖構建:
/// <summary> /// 實現XML格式 網站地圖輸出 /// By:rhythmk.cnblogs.com /// </summary> public class SitemapResult : ActionResult { public SitemapResult(Sitemap sitemap) { this.Sitemap = sitemap; } public Sitemap Sitemap { get; private set; } public override void ExecuteResult(ControllerContext context) { context.HttpContext.Response.ContentType = "text/xml"; XmlSerializer serializer = new XmlSerializer(typeof(Sitemap)); serializer.Serialize(context.HttpContext.Response.Output, this.Sitemap); } } /// <summary> /// 網站地圖實體 /// By:rhythmk.cnblogs.com /// </summary> [XmlRoot(ElementName = "urlset", Namespace = "http://www.sitemaps.org/schemas/sitemap/0.9")] [Serializable] public class Sitemap : List<SitemapUrl> { [XmlInclude(typeof(SitemapUrl))] public void Serialize(TextWriter writer) { XmlSerializer serializer = new XmlSerializer(typeof(Sitemap)); XmlTextWriter xmlTextWriter = new XmlTextWriter(writer); serializer.Serialize(xmlTextWriter, this); } } [XmlRoot(ElementName = "url")] [XmlType(TypeName = "url")] [Serializable] public class SitemapUrl { private DateTime lastModified; [XmlElement(ElementName = "loc")] public string Location { get; set; } [XmlElement(ElementName = "lastmod")] public string LastModified { get { if (DateTime.MinValue.Equals(this.lastModified)) { this.lastModified = DateTime.Now; } return this.lastModified.ToString("yyyy-MM-dd"); } set { this.lastModified = DateTime.Parse(value); } } [XmlElement(ElementName = "changefreq")] public ChangeFrequency ChangeFrequency { get; set; } [XmlElement(ElementName = "priority")] public double Priority { get; set; } } public enum ChangeFrequency { always, hourly, daily, weekly, monthly, yearly, never }
使用:
/// <summary> /// 網站地圖 /// </summary> /// <param name="pageIndex"> 頁面索引</param> /// <param name="subjectID"> 科目 </param> /// <returns></returns> public ActionResult Index(int subjectID = 0, int pageIndex = 1) { int total = 0; int pageSize=2000; var list = GetExamList(subjectID, pageIndex, pageSize,ref total); Sitemap site = new Sitemap(); var time= DateTime.Now.ToLongDateString() ; foreach (var p in list) { site.Add(new SitemapUrl() { ChangeFrequency = ChangeFrequency.weekly, LastModified = time, Location = string.Format("http://rhythmk/home/view/{0}?t={1}", p.ExaminationID , p.ExaminationTitle), Priority = 0.7 }); } return new SitemapResult(site); }
2.判斷是否為搜尋引擎:
/// <summary> /// 判斷是否為搜尋引擎訪問 /// </summary> /// <param name="useragent"></param> /// <returns></returns> public static bool IsSearchEngine(string useragent) { bool engine = false; if (!string.IsNullOrEmpty(useragent)) { var SpiderKey = System.Configuration.ConfigurationManager.AppSettings["SpiderKey"].ToString(); // spiderkey=@"Googlebot|Feedfetcher-Google|Baiduspider|Yahoo\s*\!\s*Slurp|YodaoBot|Sosoimagespider|Sosospider|Sogou\s*Web\s*Sprider" System.Text.RegularExpressions.Regex rx = new System.Text.RegularExpressions.Regex(SpiderKey, RegexOptions.IgnoreCase | RegexOptions.Compiled); engine = rx.IsMatch(useragent); } return engine; } public ActionResult View() { bool engine = IsSearchEngine( this.HttpContext.Request.ServerVariables["Http_User_Agent"]); // 判斷是否為搜尋引擎 然後使用不同的試圖呈現資料 if (engine) { return View("SearchView", entity); } else { return View(entity); } }
3. 檢驗是否成功:
修改瀏覽器 Http_User_Agent 值,以達到類比搜尋引擎爬蟲瀏覽效果。
以Firefox為例:
3.1地址欄鍵入:about:config 斷行符號
3.2 設定:general.useragent.override–>"Baiduspider" 。可以達到類比百度爬蟲效果。
火符預設:Mozilla/5.0 (Windows; U; Windows NT 6.0; zh-CN) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.29 Safari/525.19
設定完成後可以通過瀏覽http://www.docin.com/p-259119935.html 此網址 於其他未設定的瀏覽器比較 瀏覽頁面效果。