asp.net, ASP implementation of search engine Web site Collection Check procedure _ Practical skills

Use asp.net or ASP to check a URL address, an article is search engine, such as Baidu, Google, Sogou included.

Implementation principle: Direct search your article URL address (without agreement, but the agreement also line, the code will automatically remove protocol content), if the index will return the search results, or will be prompted to find information.

asp.net check Baidu, Google, Sogou search engine is included in the article URL source code:

Using System;
Using System.Net;
Using System.Text;
Using System.IO;
Using System.Web; public class Searchengineindex {public static string[] URL = {//Search engine check Address "http://www.baidu.com/s?ie=utf-8&w D= ",//Baidu index URL check address" https://www.google.com.hk/search?q= ",//Google index URL check address" http://www.sogou.com/web?ie=utf8&qu ery= "//Sogou index URL Check address}, Nofindkeyword = {" Sorry, not found with "," can't find and your query "," not included? " "};//keyword///<summary>///gets the response encoding///</summary>///<param name=" contenttype "when the search engine does not index URL addresses >& lt;/param>///<returns></returns> private static Encoding GetEncoding (string contenttype) {if ( !string. IsNullOrEmpty (contenttype)) {ContentType = contenttype.
      ToLower (); if (contenttype. IndexOf ("gb2312")!=-1 | | ContentType.
      IndexOf ("GBK")!=-1) return encoding.getencoding (936); if (contenttype.
    IndexOf ("Big5")!=-1) return encoding.getencoding (950);
  return Encoding.UTF8; }///<summary>///useHttpWebRequest objects, automatic recognition of character set///</summary>///<param name= "url" ></param>///<param name= "AddUser" Agent > whether to add UserAgent, to collect other sites to prevent interception </param>///<returns></returns> public static string gethtml (
    String URL, bool adduseragent) {HttpWebRequest request = (HttpWebRequest) httpwebrequest.create (URL); if (adduseragent) request. useragent = "googlebot| feedfetcher-google|
    Baiduspider ";
    string html = NULL; try {httpwebresponse response = (HttpWebResponse) request.
      GetResponse (); StreamReader SRD = new StreamReader (response. GetResponseStream (), getencoding (response.
      ContentType)); html = SRD.
      ReadToEnd (); Srd.
      Close (); Response.
    Close ();
  Catch {} return HTML;
  ///<summary>///Check if a URL is indexed by a search engine///</summary>///<param name= "url" >url address </param> <param name= "Engin" >0: Baidu 1: Google 2: Sogou, other search engines such as Bing and 360 directly check the results of Web site display is not directly to get the URL, some access, do not do inspection </param>///<returns></returns> public static bool CheckIndex (string url, int engin) {if (string).
    IsNullOrEmpty (URL)) return false;
    if (Engin < 0 | | engin > 2) engin = 0; url = urls[engin] + httputility.urlencode (URL. ToLower (). Replace ("http://", "").
    Replace ("https://", ""));
    bool R = TRUE;
    String html = gethtml (URL, true); if (html = NULL | | HTML.
    IndexOf (Nofindkeyword[engin])!=-1) R = false;
  return R; }//Call Method Sample Searchengineindex.checkindex ("Www.jb51.net/article/20101014/2902.aspx", 0); check Baidu index Searchenginei Ndex. CheckIndex ("www.jb51.net/article/20101014/2902.aspx", 1);/check Google index Searchengineindex.checkindex ("www.jb51.net/
 Article/20101014/2902.aspx ", 2);/Check Sogou index

ASP check Baidu, Google, Sogou search engine is included in the article URL source code:

<% Class Searchenginindex Dim Urls,nofindkeyword Private Sub Class_Initialize ' Baidu, Google, sogou URL address index query address Urls=array ("H Ttp://www.baidu.com/s?ie=utf-8&wd= "," https://www.google.com.hk/search?q= "," http://www.sogou.com/web?ie= utf8&query= ") ' Search engine indexing URL address when keyword Nofindkeyword=array (" Sorry, not found with "," can't find and your query "," not included?) ") End Sub Private function GetEncoding (contenttype) contenttype=lcase (ContentType) If InStr (ContentType," gb2312 ") &L T;>0 and InStr (ContentType, "GBK") <>0 then getencoding= "gb2312" ElseIf InStr (ContentType, "Big5") <>0 the n getencoding= "Big5" Else getencoding= "utf-8" End If the End Function Private function bintostring (bin,encoding) ' Will 2 stream data is converted to the corresponding string content based on the encoding Dim obj Set Obj=server.createobject ("ADODB.stream") obj. Type=1:obj. Mode=3:obj. Open obj. Write bin obj. Position=0:obj. Type=2:obj. Charset=encoding Bintostring=obj. ReadText obj. Close:set obj=nothing End Function Public function gethtml (URL) Dim xhr set Xhr=server. CreateObject ("Microsoft.XMLHTTP ") Xhr.open" get ", Url,false xhr.send encoding=getencoding (Xhr.getresponseheader (" Content-type ")) Response. Charset=encoding gethtml=bintostring (xhr.responsebody,encoding) Set xhr=nothing End Function Public Function CheckIn Dex (Url,engin) If Len (URL) =0 then Exit function if Engin<0 or engin>2 then Engin=1 url=urls (Engin) &server. UrlEncode (URL) Dim html html=gethtml (URL) checkindex=instr (Html,nofindkeyword (engin)) =0 End Function End Class set Sei=new searchenginindex Response. Write sei. CheckIndex ("Www.jb51.net/article/20101014/2902.aspx", 0) ' Baidu Index response. Write sei. CheckIndex ("www.jb51.net/article/20101014/2902.aspx", 1) ' Google index response. Write sei.
 CheckIndex ("Www.jb51.net/article/20101014/2902.aspx", 2) ' Sogou index set sei=nothing%>