php代码
<?php
/**
* 采集远程网址数据
*/
class get_remote_data
{
/**
* 从远程网址采集数据
* @param string $uri 远程地址
* @param string $rand_ip 随机ip
* @param string $user_agent 模拟的user_agent
* @param string $referer_uri 模拟的来路的地址
* @param array $post_data 需要post的数据，默认为空数组
* @param string $method 选择数据提交的数据，默认get
* @return mixed
*/
public function get_data_from_remote($uri, $rand_ip, $user_agent, $referer_uri='', $post_data=array(), $method="get")
{
$ch = curl_init();
curl_setopt($ch, curlopt_url, $uri);
curl_setopt($ch, curlopt_timeout, 10);
curl_setopt($ch, curlopt_httpheader, array('x-forwarded-for:'.$rand_ip, 'client-ip:'.$rand_ip));
//追踪返回302状态码，继续抓取
// curl_setopt($ch, curlopt_header, true);
curl_setopt($ch, curlopt_returntransfer, true);
//curl_setopt($ch, curlopt_followlocation, true);
if($method == 'post') {
curl_setopt($ch, curlopt_post, true);
curl_setopt($ch, curlopt_postfields, $post_data);
}
curl_setopt($ch, curlopt_nobody, false);
if($method == 'get') {
curl_setopt($ch, curlopt_referer, $referer_uri);//模拟来路
}
curl_setopt($ch, curlopt_useragent, $user_agent);
$data = curl_exec($ch); # 获取数据
$http_code = curl_getinfo($ch,curlinfo_http_code); # 获取http状态码
curl_close($ch);
return array(
'http_status' => $http_code,
'data' => $data
);
}
/**
* 生成模拟来路地址
* @return string
*/
private function get_refer_uri()
{
$refer[0] = 'http://www.baidu.com';
$refer[1] = 'https://www.google.com';
$i = time() % count($refer);
return $refer[$i];
}
/**
* 生成agent
* @return string
*/
private function get_user_agent()
{
// windows + chrome
$str[0] = "mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.36
(khtml, like gecko) chrome/29.0.1547.66 safari/537.36";
// windows + ie
$str[1] = "mozilla/5.0 (windows nt 6.3; wow64; trident/7.0; maspjs; rv:11.0) like gecko";
$i = time() % count($str);
return $str[$i];
}
/**
* 生成随机ip
* @return string
*/
private function get_rand_ip()
{
$arr_1 = array("218","218","66","66","218","218","60","60","202","204",
"66","66","66","59","61","60","222","221","66","59","60",
"60","66","218","218","62","63","64","66","66","122","211");
$randarr= mt_rand(0,count($arr_1));
$ip1id = $arr_1[$randarr];
$ip2id= round(rand(600000, 2550000) / 10000);
$ip3id= round(rand(600000, 2550000) / 10000);
$ip4id= round(rand(600000, 2550000) / 10000);
return $ip1id . "." . $ip2id . "." . $ip3id . "." . $ip4id;
}
}