c++ 實現https網頁上的圖片爬取

標籤：object   chart   gecko   else   using   pop   技術分享   tin   iterator   

一.主要的原理

　　我們通過發送一個http請求，獲得目標網頁的html原始碼，然後通過Regex擷取到圖片的URL，把該網頁的所有的圖片都儲存到一個檔案夾，這就是整個軟體的流程。

二.具體的實踐

　　現在很多的網站都是https協議但是有一部分還是http協議，其實https就是http協議的安全版本，相當於http+ssl，SSL是介於HTTP應用程式層和TCP傳輸層，和HTTP相比HTTPS發送資料需要SSL加密，然後發送。所以說我們想通過https協議發送資料給伺服器，需要經曆一下這幾個步驟：

首先我們先和伺服器進行socket串連，然後將SSL和建立的socket通訊端進行綁定，之後我們發送資料都是通過SSL發送即可，下面介紹一下具體的流程：

首先我們先進行socket串連：

//建立TCP串連bool Connect(){//初始化通訊端WSADATA wsadata;if (0 != WSAStartup(MAKEWORD(2, 2), &wsadata)) return false;//建立通訊端g_sock = socket(AF_INET, SOCK_STREAM, 0);if (g_sock == INVALID_SOCKET) return false;//將網域名稱轉換為IP地址hostent *p = gethostbyname(g_Host);if (p == NULL) return false;sockaddr_in sa;memcpy(&sa.sin_addr, p->h_addr, 4);sa.sin_family = AF_INET;sa.sin_port = htons(443);if (SOCKET_ERROR == connect(g_sock, (sockaddr*)&sa, sizeof(sockaddr))) return false;return true;}

　　

HTTPS=HTTP + SSL，因此利用OpenSSL發送請求給HTTPS網站和第二章的SOCKET發送HTTP是非常相似的，只不過要在原生的通訊端上套上SSL層，基本流程如下：

a. WSAStartup對Winsock服務進行初始化

b. 建立socket通訊端

c. connect串連服務端

d. 建立SSL上下文

e. 建立SSL

f. 將SSL與前面建立的socket通訊端綁定

g. SSL_write()發送資料

h. SSL_read()接收資料

bool SSL_Connect(){// Register the error strings for libcrypto & libsslERR_load_BIO_strings();// SSl庫的初始化，載入SSL的所有演算法，載入所有的SSL錯誤資訊SSL_library_init();OpenSSL_add_all_algorithms();SSL_load_error_strings();// New context saying we are a client, and using SSL 2 or 3sslContext = SSL_CTX_new(SSLv23_client_method());if (sslContext == NULL){ERR_print_errors_fp(stderr);return false;}// Create an SSL struct for the connectionsslHandle = SSL_new(sslContext);if (sslHandle == NULL){ERR_print_errors_fp(stderr);return false;}// Connect the SSL struct to our connectionif (!SSL_set_fd(sslHandle, g_sock)){ERR_print_errors_fp(stderr);return false;}// Initiate SSL handshakeif (SSL_connect(sslHandle) != 1){ERR_print_errors_fp(stderr);return false;}return true;}

 

三.遇到的問題

1.首先VS控制台的編碼方式是GBK的方式，但是有的網頁就是UTF-8所以我們要進行這方面的轉換

string UtfToGbk(const char* utf8){int len = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);wchar_t* wstr = new wchar_t[len + 1];memset(wstr, 0, len + 1);MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wstr, len);len = WideCharToMultiByte(CP_ACP, 0, wstr, -1, NULL, 0, NULL, NULL);char* str = new char[len + 1];memset(str, 0, len + 1);WideCharToMultiByte(CP_ACP, 0, wstr, -1, str, len, NULL, NULL);if (wstr) delete[] wstr;return str;}

2.OPENSSL的安裝

　　網上有很多自己下載源碼然後自己編譯的，但是在這個項目中我們可以直接使用別人編譯好的庫不用自己進行編譯，直接在VS的項目目錄中添加即可

3.還有就是圖片的下載，是這種LPCWSTR資料類型的，我們通過string轉換成LPCWSTR（不知道為什麼URLDownloadToFile裡面的兩個都要這樣進行轉換不能用其他的函數）

string savepath = "E:\\c++_file\\網路爬蟲1\\網路爬蟲\\網路爬蟲\\img\\"+to_string(i)+".jpg";size_t len1 = savepath.length();wchar_t* imgsavepath = new wchar_t[len1];int nmlen1 = MultiByteToWideChar(CP_ACP, 0, savepath.c_str(), len1 + 1, imgsavepath, len1);

 這是完整的下載的代碼：

　　　　　　　　   //URL產生per = mat[1].str();size_t len = per.length();//擷取字串長度int nmlen = MultiByteToWideChar(CP_ACP, 0, per.c_str(), len + 1, NULL, 0);//如果函數運行成功，並且cchWideChar為零 //傳回值是接收到待轉換字串的緩衝區所需求的寬字元數大小。wchar_t* buffer = new wchar_t[nmlen];MultiByteToWideChar(CP_ACP, 0, per.c_str(), len + 1, buffer, nmlen);//儲存路徑string savepath = "E:\\c++_file\\網路爬蟲1\\網路爬蟲\\網路爬蟲\\img\\"+to_string(i)+".jpg";size_t len1 = savepath.length();wchar_t* imgsavepath = new wchar_t[len1];int nmlen1 = MultiByteToWideChar(CP_ACP, 0, savepath.c_str(), len1 + 1, imgsavepath, len1);cout << mat.str() << endl;cout << savepath << endl;//下載檔案HRESULT hr = URLDownloadToFile(NULL, buffer, imgsavepath, 0, NULL);if (hr == S_OK){cout << "-------ok" << endl;}　　

 

 四.完整代碼

#include "spider.h"int main(){cout << "*********************************************************" << endl;cout << "***********************爬取圖片系統***********************" << endl;cout << "*********************************************************" << endl;//建立圖片的儲存的目錄CreateDirectory(L"./img", NULL);//開始抓取string starturl = "https://www.shiyanlou.com/#sign-modal";StartCatch(starturl);//while (1);return 0;}void StartCatch(string startUrl){queue<string> q;q.push(startUrl);while (!q.empty()){string cururl = q.front();q.pop();//解析URLif (false == Analyse(cururl)){cout << "解析URL失敗，錯誤碼：" << GetLastError() << endl;continue;}//串連伺服器if (false == Connect()){cout << "串連伺服器失敗，錯誤碼：" << GetLastError() << endl;continue;}//建立ssl串連if (false == SSL_Connect()){cout << "建立SSL串連失敗，錯誤碼：" << GetLastError() << endl;continue;}//擷取網頁string html;if (false == Gethtml(html)){cout << "擷取網頁資料失敗，錯誤碼：" << GetLastError() << endl;continue;}if (false == RegexIamage(html)){cout << "擷取網頁資料失敗，錯誤碼：" << GetLastError() << endl;continue;}//cout << html << endl;}//釋放SSL_shutdown(sslHandle);SSL_free(sslHandle);SSL_CTX_free(sslContext);closesocket(g_sock);WSACleanup();}//解析urlbool Analyse(string url){char *pUrl = new char[url.length() + 1];strcpy(pUrl, url.c_str());char *pos = strstr(pUrl, "https://");//找到http://開頭的字串if (pos == NULL) return false;else pos += 8;//將http://開頭省略sscanf(pos, "%[^/]%s", g_Host, g_Object);delete[] pUrl;return true;}//建立TCP串連bool Connect(){//初始化通訊端WSADATA wsadata;if (0 != WSAStartup(MAKEWORD(2, 2), &wsadata)) return false;//建立通訊端g_sock = socket(AF_INET, SOCK_STREAM, 0);if (g_sock == INVALID_SOCKET) return false;//將網域名稱轉換為IP地址hostent *p = gethostbyname(g_Host);if (p == NULL) return false;sockaddr_in sa;memcpy(&sa.sin_addr, p->h_addr, 4);sa.sin_family = AF_INET;sa.sin_port = htons(443);if (SOCKET_ERROR == connect(g_sock, (sockaddr*)&sa, sizeof(sockaddr))) return false;return true;}bool SSL_Connect(){// Register the error strings for libcrypto & libsslERR_load_BIO_strings();// SSl庫的初始化，載入SSL的所有演算法，載入所有的SSL錯誤資訊SSL_library_init();OpenSSL_add_all_algorithms();SSL_load_error_strings();// New context saying we are a client, and using SSL 2 or 3sslContext = SSL_CTX_new(SSLv23_client_method());if (sslContext == NULL){ERR_print_errors_fp(stderr);return false;}// Create an SSL struct for the connectionsslHandle = SSL_new(sslContext);if (sslHandle == NULL){ERR_print_errors_fp(stderr);return false;}// Connect the SSL struct to our connectionif (!SSL_set_fd(sslHandle, g_sock)){ERR_print_errors_fp(stderr);return false;}// Initiate SSL handshakeif (SSL_connect(sslHandle) != 1){ERR_print_errors_fp(stderr);return false;}return true;}bool Gethtml(string & html){char temp1[100];sprintf(temp1, "%d", 166);string c_get;   c_get = c_get+ "GET " + g_Object + " HTTP/1.1\r\n"+ "Host: " + g_Host + "\r\n"+ "Content-Type: text/html; charset=UTF-8\r\n"//+ "Content-Length:" + temp1 + "\r\n"//+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299\r\n"+ "Connection:Close\r\n\r\n";//+ temp;SSL_write(sslHandle, c_get.c_str(), c_get.length());char buff[101];int nreal = 0;while ((nreal = SSL_read(sslHandle, buff, 100)) > 0){buff[nreal] = ‘\0‘;html += UtfToGbk(buff);//printf("%s\n", buff);memset(buff, 0, sizeof(buff));}//printf("%s\n", html);return true;}string UtfToGbk(const char* utf8){int len = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);wchar_t* wstr = new wchar_t[len + 1];memset(wstr, 0, len + 1);MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wstr, len);len = WideCharToMultiByte(CP_ACP, 0, wstr, -1, NULL, 0, NULL, NULL);char* str = new char[len + 1];memset(str, 0, len + 1);WideCharToMultiByte(CP_ACP, 0, wstr, -1, str, len, NULL, NULL);if (wstr) delete[] wstr;return str;}bool RegexIamage(string & html){smatch mat;regex rgx("src=\"(.*(png|svg|jpg))\"");string::const_iterator start = html.begin();string::const_iterator end = html.end();string per;int i = 1;while (regex_search(start, end, mat, rgx)){//URL產生per = mat[1].str();size_t len = per.length();//擷取字串長度int nmlen = MultiByteToWideChar(CP_ACP, 0, per.c_str(), len + 1, NULL, 0);//如果函數運行成功，並且cchWideChar為零 //傳回值是接收到待轉換字串的緩衝區所需求的寬字元數大小。wchar_t* buffer = new wchar_t[nmlen];MultiByteToWideChar(CP_ACP, 0, per.c_str(), len + 1, buffer, nmlen);//儲存路徑string savepath = "E:\\c++_file\\網路爬蟲1\\網路爬蟲\\網路爬蟲\\img\\"+to_string(i)+".jpg";size_t len1 = savepath.length();wchar_t* imgsavepath = new wchar_t[len1];int nmlen1 = MultiByteToWideChar(CP_ACP, 0, savepath.c_str(), len1 + 1, imgsavepath, len1);cout << mat.str() << endl;cout << savepath << endl;//下載檔案HRESULT hr = URLDownloadToFile(NULL, buffer, imgsavepath, 0, NULL);if (hr == S_OK){cout << "-------ok" << endl;}start = mat[0].second;i++;}return true;}LPCWSTR stringToLPCWSTR(string orig){size_t origsize = orig.length() + 1;const size_t newsize = 100;size_t convertedChars = 0;wchar_t *wcstring = (wchar_t *)malloc(sizeof(wchar_t) *(orig.length() - 1));mbstowcs_s(&convertedChars, wcstring, origsize, orig.c_str(), _TRUNCATE);return wcstring;}

　　下面是標頭檔：

#pragma once#include <iostream>#include <Windows.h>#include <string>#include <queue>#include <regex>#include <urlmon.h>#include <openssl/rand.h>#include <openssl/ssl.h>#include <openssl/err.h>#pragma comment(lib, "urlmon.lib")#pragma comment( lib, "libeay32.lib" )#pragma comment( lib, "ssleay32.lib" )#pragma comment(lib, "WS2_32")  // 連結到WS2_32.lib using namespace std;char g_Host[MAX_PATH];char g_Object[MAX_PATH];SOCKET g_sock;SSL *sslHandle;SSL_CTX *sslContext;BIO * bio;//開始抓取void StartCatch(string startUrl);//解析URLbool Analyse(string url);//串連伺服器bool Connect();//建立SSl串連bool SSL_Connect();//得到htmlbool Gethtml(string& html);//UTF轉GBKstd::string UtfToGbk(const char* utf8);//Regexbool RegexIamage(string & html);LPCWSTR stringToLPCWSTR(std::string orig);

　　

 

c++ 實現https網頁上的圖片爬取