Linux: several methods for obtaining webpage source code: Linux Crawler

The first is to use a tool in Linux to obtain the webpage source code. I use wget or curl. Curl is more flexible, and many parameters of C ++ code can be set.
 

1. // Obtain the webpage through wget
2. String gethtmlbywget (string URL)
3. {
4. // Get the file name of the webpage to be downloaded
5. String filename = URL. substr (INT) URL. find_last_of ("/") + 1 );
6. If (filename! = "")
7. {
8. String strcom = "wget-Q"; // wget command.-Q indicates that the download information is not displayed.
9. Strcom. append (URL );
10. System (strcom. c_str (); // run wget
    
12. Ifstream fin (filename. c_str ());
13. If (! Fin)
14. {
15. Return "";
16. }
17. String strhtml = "";
18. Char chtemp [1024] = "";
19. // Read the webpage file to the memory
20. While (Fin. Getline (chtemp, 1024 ))
21. {
22. Strhtml. append (string (chtemp ));
23. Strcpy (chtemp ,"");
24. }
25. Fin. Close ();
26. Strcom = "RM-F"; // command for deleting a file.-F indicates that the file is deleted directly without any prompt.
27. Strcom. append (filename );
28. System (strcom. c_str (); // Delete the downloaded file.
29. Return strhtml; // return the webpage source code
30. }
31. Else
32. {
33. Return "";
34. }
35. }

// Get the webpage string gethtmlbywget (string URL) through wget {// get the name of the webpage to be downloaded string filename = URL. substr (INT) URL. find_last_of ("/") + 1); If (filename! = "") {String strcom = "wget-Q"; // wget command.-Q indicates that the download information strcom is not displayed. append (URL); System (strcom. c_str (); // run wget ifstream fin (filename. c_str (); If (! Fin) {return "";} string strhtml = ""; char chtemp [1024] = ""; // read the webpage file to the memory while (Fin. getline (chtemp, 1024) {strhtml. append (string (chtemp); strcpy (chtemp, "");} fin. close (); strcom = "RM-F"; // the command for deleting files.-F indicates that the file is deleted directly without prompting strcom. append (filename); System (strcom. c_str (); // Delete the downloaded file return strhtml; // return the webpage source code} else {return "";}}

The second is to use the socket to obtain the source code C ++.
 

1. // Get the webpage source code through get
2. String gethtmlbyget (string URL)
3. {
4. String strhtmlcontent = "";
5. Int sockfd;
6. Struct sockaddr_in ADDR;
7. Struct hostent * purl;
8. Char text [recvbuf];
10. // Analysis Link
11. Urlinfo = parseurl (URL );
12. String saccept = "accept: */* \ r \ naccept-language: ZH-CN \ r \ naccept-encoding: gzip, deflate ";
13. // Different host useragent
14. String suseragent = "Mozilla/5.0 (X11; U; Linux i686; en-US) applewebkit/534.10 (khtml, like gecko) Chrome/8.0.552.20.safari/534.10 ";
15. // Convert a port to a string
16. Char T [6];
17. String strport;
18. Sprintf (T, "% d", urlinfo. Port );
19. Strport = T;
20. // Construct the sending string
21. String strrequest = "";
22. Strrequest. append ("get ");
23. Strrequest. append (urlinfo. File );
24. Strrequest. append ("? ");
25. Strrequest. append (urlinfo. Body );
26. Strrequest. append ("HTTP/1.1 \ r \ n ");
27. Strrequest. append (saccept );
28. Strrequest. append ("\ r \ nuser-AGENT :");
29. Strrequest. append (suseragent );
30. Strrequest. append ("\ r \ nhost :");
31. Strrequest. append (urlinfo. HOST );
32. Strrequest. append (":");
33. Strrequest. append (strport );
34. Strrequest. append ("\ r \ nconnection: keep-alive \ r \ n ");
36. Char * Host = const_cast <char *> (urlinfo. Host. c_str ());
37. Sockfd = socket (af_inet, sock_stream, ipproto_tcp); // send in TCP Mode
38. Purl = gethostbyname (host );
39. ADDR. sin_family = af_inet;
40. ADDR. sin_addr.s_addr = * (unsigned long *) purl-> h_addr );
41. ADDR. sin_port = htons (80 );
43. // Connection
44. Connect (sockfd, (struct sockaddr *) & ADDR, sizeof (ADDR ));
45. // Send
46. Send (sockfd, const_cast <char *> (strrequest. c_str (), strrequest. Length (), 0 );
47. // Accept
48. While (Recv (sockfd, text, recvbuf, 0)> 0)
49. {
50. Strhtmlcontent. append (text );
51. Bzero (text, recvbuf );
52. }
53. // Close the socket
54. Close (sockfd );
55. // Return the acceptance result
56. Return strhtmlcontent;
57. }

// Get the webpage source code string gethtmlbyget (string URL) {string strhtmlcontent = ""; int sockfd; struct sockaddr_in ADDR; struct hostent * purl; char text [recvbuf]; // analysis link urlinfo = parseurl (URL); string saccept = "accept: */* \ r \ naccept-language: ZH-CN \ r \ naccept-encoding: gzip, deflate "; // different host useragent different string suseragent =" Mozilla/5.0 (X11; U; Linux i686; en-US) applewebkit/534.10 (khtml, like Geck O) Chrome/8.0.552.20.safari/534.10 "; // converts a port to a string char T [6]; string strport; sprintf (T," % d ", urlinfo. port); strport = T; // construct the sending string strrequest = ""; strrequest. append ("get"); strrequest. append (urlinfo. file); strrequest. append ("? "); Strrequest. append (urlinfo. body); strrequest. append ("HTTP/1.1 \ r \ n"); strrequest. append (saccept); strrequest. append ("\ r \ nuser-AGENT:"); strrequest. append (suseragent); strrequest. append ("\ r \ nhost:"); strrequest. append (urlinfo. host); strrequest. append (":"); strrequest. append (strport); strrequest. append ("\ r \ nconnection: keep-alive \ r \ n"); char * Host = const_cast <char *> (urlinfo. host. c_str (); sockfd = socket (af_inet, sock_stream, ipproto_tcp); // send purl = gethostbyname (host) in TCP mode; ADDR. sin_family = af_inet; ADDR. sin_addr.s_addr = * (unsigned long *) purl-> h_addr); ADDR. sin_port = htons (80); // connect to connect (sockfd, (struct sockaddr *) & ADDR, sizeof (ADDR); // send (sockfd, const_cast <char *> (strrequest. c_str (), strrequest. length (), 0); // accept while (Recv (sockfd, text, recvbuf, 0)> 0) {strhtmlcontent. append (text); bzero (text, recvbuf);} // close socket close (sockfd); // return the acceptance result return strhtmlcontent ;}

Use libcurl Java code
 

1. # Include <stdio. h>
2. # Include <string. h>
3. # Include <curl/curl. h>
5. # Define max_buckets 65536
7. Char wr_buf [max_buf + 1];
8. Int wr_index;
10. /*
11. * Write data callback function (called within the context
12. * Curl_easy_perform.
13. */
14. Size_t write_data (void * buffer, size_t size, size_t nmemb, void * userp)
15. {
16. Int segsize = size * nmemb;
18. /* Check to see if this data exceeds the size of our buffer. If so,
19. * Set the user-defined context value and return 0 to indicate
20. * Problem to curl.
21. */
22. If (wr_index + segsize> max_buf ){
23. * (Int *) userp = 1;
24. Return 0;
25. }
27. /* Copy the data from the curl buffer into our buffer */
28. Memcpy (void *) & wr_buf [wr_index], buffer, (size_t) segsize );
30. /* Update the write Index */
31. Wr_index + = segsize;
33. /* Null terminate the buffer */
34. Wr_buf [wr_index] = 0;
36. /* Return the number of bytes encoded ed, indicating to curl that all is okay */
37. Return segsize;
38. }
41. /*
42. * Simple curl application to read the index.html file from a web site.
43. */
44. Int main (void)
45. {
46. Curl * curl;
47. Curlcode ret;
48. Int wr_error;
50. Wr_error = 0;
51. Wr_index = 0;
53. /* First step, init curl */
54. Curl = curl_easy_init ();
55. If (! Curl ){
56. Printf ("couldn't init curl \ n ");
57. Return 0;
58. }
60. /* Tell curl the URL of the file we're re going to retrieve */
61. Curl_easy_setopt (curl, curlopt_url, "www.exampledomain.com ");
63. /* Tell curl that we'll receive data to the function write_data, and
64. * Also provide it with a context pointer for our error return.
65. */
66. Curl_easy_setopt (curl, curlopt_writedata, (void *) & wr_error );
67. Curl_easy_setopt (curl, curlopt_writefunction, write_data );
69. /* Allow curl to perform the action */
70. Ret = curl_easy_perform (curl );
72. Printf ("ret = % d (write_error = % d) \ n", RET, wr_error );
74. /* Emit the page if curl indicates that no errors occurred */
75. If (ret = 0) printf ("% s \ n", wr_buf );
77. Curl_easy_cleanup (curl );
79. Return 0;
80. }
81. Http://yang7229693.iteye.com/blog/855208