写一个scrapy中间件--ip代理池

middleware文件# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.htmlimport random
from scrapy import signalsclass TutorialDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects. @classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware. # Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None def process_response(self, request, response, spider):
# Called with the response returned from the downloader. # Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception. # Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)# 创建一个中间件 ip代理池
from collections import defaultdict
from scrapy.exceptions import NotConfigured
class RandomProxyMiddleware(object): def __init__(self, settings):
# 第三步 初始化配置和变量
# 在settings中写一个 PROXIES 列表配置
# 从settings中把代理读进来（把环境变量读进来）
self.proxies = settings.getlist("PROXIES")
self.stats = defaultdict(int) # 默认值是0 统计次数
self.max_failed = 3 # 请求最多不超过3次 @classmethod
def from_cralwer(cls, crawler):
# 第一步 创建中间件对象
# 首先获取配置 HTTPPROXY_ENABLED 看看是否启用代理，
if not crawler.settings.getbool("HTTPPROXY_ENABLED"): # 如果没有启用代理
raise NotConfigured
# auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") # 读取配置，这里暂时不用
# 第二步
return cls(crawler.settings) # cls（）实际调用的是 init()函数，如果init接受参数，cls就需要参数 def process_request(self, request, spider):
# 第四步 为每个request对象随机分配一个ip代理
# 让这个请求使用代理 初始url不使用代理ip
if self.proxies and not request.meta.get("proxy") and request.url not in spider.start_urls:
request.meta["proxy"] = random.choice(self.proxies)
def process_response(self, request, response, spider):
# 第五步： 请求成功
cur_proxy = request.meta.get('proxy')
# 判断是否被对方禁封
if response.status > 400:
# 给相应的ip失败次数 +1
self.stats[cur_proxy] += 1
print("当前ip{}，第{}次出现错误状态码".format(cur_proxy, self.stats[cur_proxy]))
# 当某个ip的失败次数累计到一定数量
if self.stats[cur_proxy] >= self.max_failed: # 当前ip失败超过3次
print("当前状态码是{}，代理{}可能被封了".format(response.status, cur_proxy))
# 可以认为该ip被对方封了，从代理池中删除这个ip
self.remove_proxy(cur_proxy)
del request.meta['proxy']
# 将这个请求重新给调度器，重新下载
return request # 状态码正常的时候，正常返回
return response def process_exception(self, request, exception, spider):
# 第五步：请求失败
cur_proxy = request.meta.get('proxy') # 取出当前代理
from twisted.internet.error import ConnectionRefusedError, TimeoutError
# 如果本次请求使用了代理，并且网络请求报错，认为这个ip出了问题
if cur_proxy and isinstance(exception, (ConnectionRefusedError, TimeoutError)):
print("当前的{}和当前的{}".format(exception, cur_proxy))
self.remove_proxy(cur_proxy)
del request.meta['proxy']
# 重新下载这个请求
return request def remove_proxy(self, proxy):
if proxy in self.proxies:
self.proxies.remove(proxy)
print("从代理列表中删除{}".format(proxy))

settings 文件# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
'tutorial.middlewares.RandomProxyMiddleware': 749, # 修改下载优先级数字
}