Share next page keyword crawl components.arrow.com site code _php instance

Copy CodeThe code is as follows:
/**
* HOST:components.arrow.com
*/
Set_time_limit (0);
Base function
function Curl_get ($url, $data = Array (), $header = Array (), $timeout = all, $port = A, $reffer = ", $proxy =")
{
$ch = Curl_init ();
if (!empty ($data)) {
$data = Is_array ($data)? Http_build_query ($data): $data;
$url. = (Strpos ($url, '? ')? ' & ': "?") . $data;
}
curl_setopt ($ch, Curlopt_url, $url);
curl_setopt ($ch, Curlopt_returntransfer, true);
curl_setopt ($ch, Curlopt_connecttimeout, $timeout);
curl_setopt ($ch, curlopt_post, 0);
curl_setopt ($ch, Curlopt_port, $port);
curl_setopt ($ch, Curlopt_httpheader, $header);
curl_setopt ($ch, curlopt_followlocation, 1); Whether to crawl the page after the jump
$reffer && curl_setopt ($ch, Curlopt_referer, $reffer);
if ($proxy) {
curl_setopt ($ch, Curlopt_proxy, $proxy);
curl_setopt ($ch, Curlopt_proxyport, 1723);
curl_setopt ($ch, Curlopt_proxyuserpwd, "andhm001:andhm123");
}

$result = Array ();
$result [' result '] = curl_exec ($ch);
if (0! = Curl_errno ($ch)) {
$result [' error '] = "error:\n". Curl_error ($ch);

}
Curl_close ($ch);
return $result;
}

Copy CodeThe code is as follows:
function Curl_post ($url, $data = Array (), $header = Array (), $timeout = $port = 80)
{
$ch = Curl_init ();
curl_setopt ($ch, Curlopt_url, $url);
curl_setopt ($ch, Curlopt_returntransfer, true);
curl_setopt ($ch, Curlopt_connecttimeout, $timeout);
curl_setopt ($ch, Curlopt_port, $port);
!empty ($header) && curl_setopt ($ch, Curlopt_httpheader, $header);
curl_setopt ($ch, Curlopt_post, 1);
curl_setopt ($ch, Curlopt_postfields, $data);

$result = Array ();
$result [' result '] = curl_exec ($ch);
if (0! = Curl_errno ($ch)) {
$result [' error '] = "error:\n". Curl_error ($ch);

}
Curl_close ($ch);

return $result;
}

/**
* Get the HTML source of the list page
* @param string $keywords search keywords
* @param int $start start record number
* @return Boolean|array
*/
function getlisthtml ($keywords, $start = 0)
{
if ($start < 0)
{
return false;
}

$postData = Array (
' Search_token ' = $keywords,
' Start ' = $start,
' Limit ' = 100,
);

$result = Curl_post (' http://components.arrow.com/part/search/'. $keywords, Http_build_query ($postData));
if (Isset ($result [' Error ']))
{
return false;
Exit ($result [' Error ']);
}
$result = $result [' result '];

return $result;
}

/**
* Get list page connection href
* @param string $html HTML source
* @return Array
*/
function Getlisthref ($html)
{
$pattern = '/ ]+) ">/isu";
if (Preg_match_all ($pattern, $html, $matches))
{
return $matches [1];
} else {
No matches
return Array ();
}
}

/**
* Get Next page number start
* @param string $html HTML source
* @return Number
*/
function Getlistnextpage ($html)
{
$pattern = '/ buildpagination\ (\ ' \d+\ ', \ ' \d+\ ', \ ' (\d+) \ ', \d+\); <\/script>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
return Intval ($matches [1]);
} else {
return-1;
}
}

/**
* Get list also all the detailed list
* @param string $keywords search keywords
* @return Boolean|array
*/
function Getlisthrefall ($keywords)
{
if (empty ($keywords))
{
return false;
}

$html = getlisthtml ($keywords);
$hrefList = Getlisthref ($html);
if (empty ($hrefList))
{
No results
return Array ();
}
$nextPage = Getlistnextpage ($html);
$loop = 0;
while ($nextPage > 0)
{
$html = getlisthtml ($keywords, $nextPage);
$tmpHrefList = Getlisthref ($html);
$hrefList = Array_merge ($hrefList, $tmpHrefList);
$nextPage = Getlistnextpage ($html);
$loop + +;
}
return $hrefList;
}

/**
* Get Details Page information
* @param string $url URL address
* @return Array ()
*/
function Getdetail ($url)
{
if (empty ($url))
{
return false;
}
$host = ' http://components.arrow.com ';

$url = $host. $url;
$result = Curl_get ($url);
if (Isset ($result [' Error ']))
{
return Array ();
Exit ($result [' Error ']);
}
$html = $result [' result '];

$result = Array (
' Sup_part ' = ', '//supplier type
' sup_id ' = ', '//supplier ID
' Mfg_part ' = ', '//Manufacturer model
' Mfg_name ' = ', '//Manufacturer name
' Cat_name ' + ', '//category name
' Para ' = ', '//Properties
' desc ' and ' = ',//description
' Pdf_url ' and ' = ',//PDF address
' Sup_stock ' and ' = ',//Stock
' Min_purch ' = ' and ',//min. Order Quantity
' Price ' = ', '//Prices
' Img_url ' + ',//Picture address
' Createtime ' = ',//creation time
' Datacode ' and ' = ',//Lot number
' Package ' + = ',//encapsulation
' Page_url ' and ' = ',//page address
);

Mfg_part
$pattern = '/

[\s\n]* partno:\s*<\/strong> (. +) <\/li>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' mfg_part '] = Trim ($matches [1]);
} else {file_put_contents (' page.txt ', $html);d ie (' xxx ');
return Array ();
}

Mfg_name
$pattern = '/

[\s\n]*Manufacturer: <\/strong> (. +) <\/li>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' mfg_name '] = Trim ($matches [1]);
}

Cat_name
$pattern = '/displaycategory\ (\ ' (. [ ^\ ']+) \ ' \);/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' cat_name '] = Trim ($matches [1]);
$result [' cat_name '] = str_replace (' | ', ' > ', $result [' cat_name ']);
}

Para
$tablepattern = '/ ]*> (. +) <\/table>/isu ';
if (Preg_match ($tablepattern, $html, $matches))
{
$pattern = '/[\s\n]* (. +) <\/strong><\/td>(. +) <\/td>[\s\n]*<\/tr>/isu ';
if (Preg_match_all ($pattern, $matches [1], $matches))
{
foreach ($matches [1] as $k = $v)
{
$v = Trim ($v);
if (' package Type ' = = $v)
{
$result [' package '] = Trim ($matches [2][$k]);
Continue
}
$result [' Para '] [$v] = Trim ($matches [2][$k]);
}
}
}

Desc
$pattern = '/.+

(. +) <\/h4>[\s\n]*<\/div>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' desc '] = Trim ($matches [1]);
}

Pdf_url
$pattern = '/ [\s\n]* datasheet:<\/strong> if (Preg_match ($pattern, $html, $matches))
{
$result [' pdf_url '] = $host. Trim ($matches [1]);
}

Sup_stock
$pattern = '/ ([\d,]+) <\/td>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' sup_stock '] = Trim ($matches [1]);
$result [' sup_stock '] = Str_replace (', ', ' ", $result [' Sup_stock ']);
}

Min_purch
$pattern = '/ [\s\n]* Multiple:\s*<\/strong> (. +) <\/span>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' min_purch '] = Trim ($matches [1]);
}

Price
$pattern = '/(. [ ^<]+) <\/div>/isu ';
if (Preg_match ($pattern, $html, $matches))
{
$result [' price '][1] = Trim ($matches [1]);
}
$pattern = '/[\s\n]* ]+title= "(. [ ^ "]+") ">/isu";
if (Preg_match ($pattern, $html, $matches))
{
$priceurl = Str_replace (' & ', ' & ', $matches [1]);
$json = Curl_get ($priceurl);
$json = $json [' result '];
if (! empty ($json))
{
$jsonresult = Json_decode ($json, true);
foreach ($jsonresult [' Parts '][0][' webprice '] [' resale '] as $k = $v)
{
$result [' Price '] [$v [' minqty ']] = $v [' Price '];
}
}
}

Img_url
$pattern = '/[\s\n]* if (Preg_match ($pattern, $html, $matches))
{
$result [' img_url '] = Trim ($matches [1]);
}

Page_url
$result [' page_url '] = $url;

return $result;
}

/**
* Final Call function
* @param string $keywords search keywords
* @return Array
*/
function GetData ($keywords)
{
$hrefList = Getlisthrefall ($keywords);
$result = Array ();

foreach ($hrefList as $k = $v)
{
$result [] = Getdetail ($v);
}

return $result;
}

Test Script
$keywords = Trim ($_get[' keywords ');
$result = GetData ($keywords);

Print_r ($result);

