1.在scrapy工程下新建“middlewares.py”
# importing base64 library because we'll need it only in case if the proxy we are going to use requires authentication
import base64
# start your middleware class
class proxymiddleware(object):
# overwrite process request
def process_request(self, request, spider):
# set the location of the proxy
request.meta['proxy'] = "http://your_proxy_ip:port"
# use the following lines if your proxy requires authentication
proxy_user_pass = "username:password"
# setup basic authentication for the proxy
encoded_user_pass = base64.encodestring(proxy_user_pass)
request.headers['proxy-authorization'] = 'basic ' + encoded_user_pass
2.在项目配置文件里(./project_name/settings.py)添加
downloader_middlewares = {
'scrapy.contrib.downloadermiddleware.httpproxy.httpproxymiddleware': 110,
'project_name.middlewares.proxymiddleware': 100,
}
只要两步，现在请求就是通过代理的了。测试一下^_^
from scrapy.spider import basespider
from scrapy.contrib.spiders import crawlspider, rule
from scrapy.http import request
class testspider(crawlspider):
name = "test"
domain_name = "whatismyip.com"
# the following url is subject to change, you can get the last updated one from here :
# http://www.whatismyip.com/faq/automation.asp
start_urls = ["http://xujian.info"]
def parse(self, response):
open('test.html', 'wb').write(response.body)
3.使用随机user-agent
默认情况下scrapy采集时只能使用一种user-agent，这样容易被网站屏蔽，下面的代码可以从预先定义的user- agent的列表中随机选择一个来采集不同的页面
在settings.py中添加以下代码
downloader_middlewares = {
'scrapy.contrib.downloadermiddleware.useragent.useragentmiddleware' : none,
'crawler.comm.rotate_useragent.rotateuseragentmiddleware' :400
}
注意: crawler; 是你项目的名字 ，通过它是一个目录的名称 下面是蜘蛛的代码
#!/usr/bin/python
#-*-coding:utf-8-*-
import random
from scrapy.contrib.downloadermiddleware.useragent import useragentmiddleware
class rotateuseragentmiddleware(useragentmiddleware):
def __init__(self, user_agent=''):
self.user_agent = user_agent
def process_request(self, request, spider):
#这句话用于随机选择user-agent
ua = random.choice(self.user_agent_list)
if ua:
request.headers.setdefault('user-agent', ua)
#the default user_agent_list composes chrome,i e,firefox,mozilla,opera,netscape
#for more user agent strings,you can find it in http://www.useragentstring.com/pages/useragentstring.php
user_agent_list = [\
"mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.1 (khtml, like gecko) chrome/22.0.1207.1 safari/537.1"\
"mozilla/5.0 (x11; cros i686 2268.111.0) applewebkit/536.11 (khtml, like gecko) chrome/20.0.1132.57 safari/536.11",\
"mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.6 (khtml, like gecko) chrome/20.0.1092.0 safari/536.6",\
"mozilla/5.0 (windows nt 6.2) applewebkit/536.6 (khtml, like gecko) chrome/20.0.1090.0 safari/536.6",\
"mozilla/5.0 (windows nt 6.2; wow64) applewebkit/537.1 (khtml, like gecko) chrome/19.77.34.5 safari/537.1",\
"mozilla/5.0 (x11; linux x86_64) applewebkit/536.5 (khtml, like gecko) chrome/19.0.1084.9 safari/536.5",\
"mozilla/5.0 (windows nt 6.0) applewebkit/536.5 (khtml, like gecko) chrome/19.0.1084.36 safari/536.5",\
"mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1063.0 safari/536.3",\
"mozilla/5.0 (windows nt 5.1) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1063.0 safari/536.3",\
"mozilla/5.0 (macintosh; intel mac os x 10_8_0) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1063.0 safari/536.3",\
"mozilla/5.0 (windows nt 6.2) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1062.0 safari/536.3",\
"mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1062.0 safari/536.3",\
"mozilla/5.0 (windows nt 6.2) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.1 safari/536.3",\
"mozilla/5.0 (windows nt 6.1; wow64) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.1 safari/536.3",\
"mozilla/5.0 (windows nt 6.1) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.1 safari/536.3",\
"mozilla/5.0 (windows nt 6.2) applewebkit/536.3 (khtml, like gecko) chrome/19.0.1061.0 safari/536.3",\
"mozilla/5.0 (x11; linux x86_64) applewebkit/535.24 (khtml, like gecko) chrome/19.0.1055.1 safari/535.24",\
"mozilla/5.0 (windows nt 6.2; wow64) applewebkit/535.24 (khtml, like gecko) chrome/19.0.1055.1 safari/535.24"
]