Python3 Scrapy + Selenium + 阿布云爬取拉钩网学习笔记

{
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
}

# Location: LagouCrawler.spider.lagoucrawler.LagouCrawlerSpder def start_requests(self):
base_url = 'https://www.lagou.com'
index_flag = {'index_flag': 'fetch index page', 'brower': None, 'wait': None, 'pagenumber': None}
yield scrapy.Request(url=base_url, callback=self.parse_index, meta=index_flag, dont_filter=True)

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
def process_request(self, request, spider):
"""
middleware的核心函数，每个request都会经过该函数。此函数过滤出初始request和详情页request，
对于初始request进行验证登陆、cookies等一系列操作，然后将最后获取到的索引页response返回，对
于详情页的request则，不做任何处理。
:param request:
:param spider:
:return:
"""
# 过滤出初始的登陆、切换索引页的request
if 'index_flag' in request.meta.keys():
# 判断是否为登陆状态，若未登陆则判断是否有cookies文件存在
if not self.is_logined(request, spider):
path = os.getcwd() + '/cookies/lagou.txt'
# 若cookies文件存在，则加载cookie文件，否则进行登陆操作
if os.path.exists(path):
self.load_cookies(path)
else:
# 登陆lagou网
self.login_lagou(spider)
# 登陆成功后的索引页的响应体，若不登录，请求响应提详情页面的url时，会重定向到登陆页面
response = self.fetch_index_page(request, spider)
return response

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
def is_logined(self, request, spider):
"""
初始请求时，总会弹出切换城市的窗口，所以先关掉它，然后通过判断右上角是否显示
用户名判断是否为登陆状态,并初始化整个程序的brower实例
:param request: 初始请求request，其meta包含index_page属性
:param spider:
:return: 已经登陆返回True， 否则返回False
"""
self.brower.get(request.url)
try:
# 关掉城市选择窗口
box_close = self.wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="cboxClose"]')))
box_close.click()
# 获取右上角的登录状态
login_status = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="lg_tbar"]/div/ul/li[1]/a')))
# 若右上角显示为登陆，则说明用户还没有登陆
if login_status.text == '登录':
return False
else:
return True
except TimeoutException as e:
# 二次请求，不会出现地址框，需要重新设计
spider.logger.info('Locate Username Element Failed：%s' % e.msg)
return False

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
def load_cookies(self, path):
"""
加载本地cookies文件，实现免登录访问
:param path: 本地cookies文件路径
:return:
"""
with open(path, 'r') as f:
cookies = json.loads(f.read())
for cookie in cookies:
cookies_dict = {'name': cookie['name'], 'value': cookie['value']}
self.brower.add_cookie(cookies_dict)

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
def login_lagou(self, spider):
"""
用selenium模拟登陆流程，并将登陆成功后的cookies保存为本地文件。
:param spider:
:return:
"""
try:
# 设置等待时间，否则会出现登陆元素查找不到的异常
time.sleep(2)
# 点击进入登录页面
login_status = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="loginToolBar"]//a[@class="button bar_login passport_login_pop"]')))
login_status.click()
# 输入用户名
username = self.wait.until(EC.visibility_of_element_located((By.XPATH, '//*[@data-propertyname="username"]/input')))
username.send_keys(self.username)
# 输入用户密码
password = self.wait.until(EC.visibility_of_element_located((By.XPATH, '//*[@data-propertyname="password"]/input')))
password.send_keys(self.password)
# 点击登陆按钮
submit_button = self.wait.until(EC.visibility_of_element_located((By.XPATH, '//*[@data-propertyname="submit"]/input')))
submit_button.click()
# time.sleep(1)
# 获取登录成功后的cookies
cookies = self.brower.get_cookies()
# 保存登陆后的cookies
self.save_cookies(cookies)
except TimeoutException as e:
spider.logger.info('Locate Login Element Failed: %s' % e.msg)

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
@staticmethod
def save_cookies(cookies):
"""
登陆成功后，将cookie保存为本地文件，供下次程序运行或者以后使用
:param cookies:
:return:
"""
path = os.getcwd() + '/cookies/'
if not os.path.exists(path):
os.mkdir(path)
with open(path + 'lagou.txt', 'w') as f:
f.write(json.dumps(cookies))

# Location： LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware
def fetch_index_page(self, request, spider):
"""
该函数使用selenium完成城市切换，搜索关键字输入并点击搜索按钮操作。如果点击搜索按钮后，
页面没有成功跳转，则会因为31行的代码，抛出NoSuchElementException，而在load_cookies()
函数报一个NoneType没有get_cookies()的错误。原因是response是空的。
:param request:
:param spider:
:return:
"""
try:
# 判断是否需要切换城市
city_location = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="lg_tnav"]/div/div/div/strong')))
if city_location.text != self.city:
time.sleep(1)
city_change = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="changeCity_btn"]')))
city_change.click()
# 根据搜索城市定位到相应元素并点击切换
# time.sleep(1)
city_choice = self.wait.until(EC.presence_of_element_located((By.LINK_TEXT, self.city)))
city_choice.click()
time.sleep(1)
# 定位关键字输入框并输入关键字
keywords_input = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="search_input"]')))
keywords_input.send_keys(self.job_keywords)
# time.sleep(1)
# 定位搜索按钮并点击,有时候点击后页面不会发生跳转，原因是被重定向了。
keywords_submit = self.wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="search_button"]')))
keywords_submit.click()
# 跳转到列表页等待待抓取的内容元素加载完成,如果被重定向,则跳转不到该页面，会报NoSuchElementException
self.wait.until(EC.visibility_of_all_elements_located((By.XPATH, '//*[@id="s_position_list"]')))
pagenumber = self.wait.until(EC.presence_of_element_located((
By.XPATH, '//*[@id="s_position_list"]/div[@class="item_con_pager"]/div/span[@class="pager_next "]/preceding-sibling::span[1]'
)))
# 获取一共有多少页，供通过response传递到parse_detail函数，进行后续的翻页解析使用
request.meta['pagenumber'] = pagenumber.text
# 将brower和wait通过response传递到parse_detail函数，进行后续的翻页解析使用
request.meta['brower'] = self.brower
request.meta['wait'] = self.wait
body = self.brower.page_source
# 返回初始搜索页面，在parse_detail函数中进行相关信息的解析
response = HtmlResponse(
url=self.brower.current_url,
body=body,
encoding='utf-8',
request=request
)
return response
except TimeoutException:
spider.logger.info('Locate Index Element Failed And Use Proxy Request Again')
# except NoSuchElementException:
# 如果捕捉到该异常，说明页面被重定向了，没有正常跳转,重新请求输入关键字页面
return request

# Location: LagouCrawler.spider.lagoucrawler.LagouCrawlerSpder
def parse_index(self, response):
"""
解析第一页列表页，拿到各个招聘详情页url，并发起请求；然后进行翻页做操，拿到每页
列表页各个详情页的url，并发起请求。注意：以杭州Python爬虫职位为例详情页请求发起
大概55个后（抓取的时候，一共有4页，每页15个招聘，供60个招聘详情），最后5个总是
被重定向到最初始输入搜索关键字的页面，即使设置了DOWNLOAD_DELAY也是没用。应该是
被服务器识别出了是机器人了，初步思路是在middlewares的process_response()函数中，
通过判断response的status_code,对重定向的request加上代理后，再次发起request，
但是这个思路没能实现，需要更深层次的理解框架，所以使用了阿布云代理的动态代理，
让每个request都通过代理服务器发出请求。
:param response: 经middleware筛选并处理后的第一页详情页response
:return:
"""
self.pagenumber = response.meta['pagenumber']
# 初始化spider中的brower和wait
self.brower = response.meta['brower']
self.wait = response.meta['wait']
# 解析索引页各项招聘详情页url
for url in self.parse_url(response):
yield scrapy.Request(url=url, callback=self.parse_detail, dont_filter=True)
# 翻页并解析
for pagenumber in range(2, int(self.pagenumber) + 1):
response = self.next_page()
for url in self.parse_url(response):
yield scrapy.Request(url=url, callback=self.parse_detail, dont_filter=True)

# Location: LagouCrawler.spider.lagoucrawler.LagouCrawlerSpder
@staticmethod
def parse_url(response):
"""
解析出每页列表页各项招聘信息的url
:param response: 列表页response
:return: 该列表页各项招聘详情页的url列表
"""
url_selector = response.xpath('//*[@id="s_position_list"]/ul/li')
url_list = []
for selector in url_selector:
url = selector.xpath('.//div[@class="p_top"]/a/@href').extract_first()
url_list.append(url)
return url_list

# Location: LagouCrawler.spider.lagoucrawler.LagouCrawlerSpder
def next_page(self):
"""
用selenium模拟翻页动作。用xpath获取next_page_button控件时，花了很久时间，原因是
span标签的class="pager_next "后引号前面有一个空格！！！
:return:
"""
try:
# 用xpath找这个下一页按钮居然花了半天的时间居然是这个程序员大哥在span标签的class="pager_next "加了个空格，空格！！！
next_page_button = self.wait.until(EC.presence_of_element_located((
By.XPATH, '//*[@id="s_position_list"]/div[@class="item_con_pager"]/div/span[@class="pager_next "]'
)))
next_page_button.click()
self.wait.until(EC.visibility_of_all_elements_located((By.XPATH, '//*[@id="s_position_list"]')))
# 控制翻页速度
time.sleep(2)
body = self.brower.page_source
response = HtmlResponse(url=self.brower.current_url, body=body, encoding='utf-8')
return response
except TimeoutException:
pass

# Location: LagouCrawler.spider.lagoucrawler.LagouCrawlerSpder
@staticmethod
def parse_detail(response):
"""
解析每一页各个招聘信息的详情
:param response: 每个列表页的HtmlResponse实例
:return: 各个公司招聘详情生成器
"""
item_loader = CompanyItemLoader(item=CompanyItem(), response=response)
item_loader.add_xpath('company_name', '//*[@id="job_company"]/dt/a/div/h2/text()')
item_loader.add_xpath('company_location', 'string(//*[@id="job_detail"]/dd[@class="job-address clearfix"]/div[@class="work_addr"])')
item_loader.add_xpath('company_website', '//*[@id="job_company"]/dd/ul/li[5]/a/@href')
item_loader.add_xpath('company_figure', '//*[@id="job_company"]/dd/ul//i[@class="icon-glyph-figure"]/parent::*/text()')
item_loader.add_xpath('company_square', '//*[@id="job_company"]/dd/ul//i[@class="icon-glyph-fourSquare"]/parent::*/text()')
item_loader.add_xpath('company_trend', '//*[@id="job_company"]/dd/ul//i[@class="icon-glyph-trend"]/parent::*/text()')
item_loader.add_xpath('invest_organization', '//*[@id="job_company"]/dd/ul//p[@class="financeOrg"]/text()')
item_loader.add_xpath('job_position', '//*[@class="position-content-l"]/div[@class="job-name"]/span/text()')
item_loader.add_xpath('job_salary', '//*[@class="position-content-l"]/dd[@class="job_request"]/p/span[@class="salary"]/text()')
item_loader.add_xpath('work_experience', '//*[@class="position-content-l"]/dd[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree', '//*[@class="position-content-l"]/dd[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_category', '//*[@class="position-content-l"]/dd[@class="job_request"]/p/span[5]/text()')
item_loader.add_xpath('job_lightspot', '//*[@id="job_detail"]/dd[@class="job-advantage"]/p/text()')
item_loader.add_xpath('job_description', 'string(//*[@id="job_detail"]/dd[@class="job_bt"]/div)')
item_loader.add_xpath('job_publisher', '//*[@id="job_detail"]//div[@class="publisher_name"]/a/span/text()')
item_loader.add_xpath('resume_processing', 'string(//*[@id="job_detail"]//div[@class="publisher_data"]/div[2]/span[@class="tip"])')
item_loader.add_xpath('active_time', 'string(//*[@id="job_detail"]//div[@class="publisher_data"]/div[3]/span[@class="tip"])')
item_loader.add_xpath('publish_date', '//*[@class="position-content-l"]/dd[@class="job_request"]/p[@class="publish_time"]/text()')
item = item_loader.load_item()
yield item

# Location: LagouCrawler.items
# -*- coding: utf-8 -*- # Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html import datetime
from scrapy import Item, Field
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose, Join def formate_date(value):
"""
根据提取到的发布时间，若是当天发布则是H:M格式，则获取当天日期的年月日然后返回，否则是Y:M:D格式，
则直接返回该年月日
:param value: 提取到的时间字符串
:return: 格式化后的年月日
"""
if ':' in value:
now = datetime.datetime.now()
publish_date = now.strftime('%Y-%m-%d')
publish_date += '(今天)'
return publish_date
else:
return value class CompanyItemLoader(ItemLoader):
default_output_processor = TakeFirst() class CompanyItem(Item):
# define the fields for your item here like:
# name = scrapy.Field() # 公司名称
company_name = Field(
input_processor=MapCompose(lambda x: x.replace(' ', ''), lambda x: x.strip())
)
# 公司地址
company_location = Field(
input_processor=MapCompose(lambda x: x.replace(' ', ''), lambda x: x.replace('\n', ''), lambda x: x[:-4])
)
# 公司官网
company_website = Field()
# 公司规模
company_figure = Field(
input_processor=MapCompose(lambda x: x.replace(' ', ''), lambda x: x.replace('\n', '')),
output_processor=Join('')
)
# 公司领域
company_square = Field(
input_processor=MapCompose(lambda x: x.replace(' ', ''), lambda x: x.replace('\n', '')),
output_processor=Join('')
)
# 公司阶段
company_trend = Field(
input_processor=MapCompose(lambda x: x.replace(' ', ''), lambda x: x.replace('\n', '')),
output_processor=Join('')
)
# 投资机构
invest_organization = Field() # 岗位名称
job_position = Field()
# 岗位薪资
job_salary = Field(
input_processor=MapCompose(lambda x: x.strip())
)
# 经验需求
work_experience = Field(
input_processor=MapCompose(lambda x: x.replace(' /', ''))
)
# 学历需求
degree = Field(
input_processor=MapCompose(lambda x: x.replace(' /', ''))
)
# 工作性质
job_category = Field() # 职位亮点
job_lightspot = Field()
# 职位描述
job_description = Field(
input_processor=MapCompose(lambda x: x.replace('\xa0\xa0\xa0\xa0', '').replace('\xa0', ''), lambda x: x.replace('\n', '').replace(' ', ''))
) # 职位发布者
job_publisher = Field()
# 发布时间
publish_date = Field(
input_processor=MapCompose(lambda x: x.replace('\xa0 ', '').strip(), lambda x: x[:-6], formate_date)
)
# # 聊天意愿
# chat_will = Field()
# 简历处理
resume_processing = Field(
input_processor=MapCompose(lambda x: x.replace('\xa0', '').strip())
)
# 活跃时段
active_time = Field(
input_processor=MapCompose(str.strip)
)

# Location: LagouCrawler.piplines # -*- coding: utf-8 -*- # Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html from pymongo import MongoClient class LagoucrawlerPipeline(object): def __init__(self, host=None, db=None, collection=None):
self.mongo_uri = host
self.mongo_db = db
self.mongo_collection = collection
self.client = None
self.db = None
self.collection = None @classmethod
def from_crawler(cls, crawler):
"""
通过该函数，获取在settings.py文件中定义的Mongodb地址、数据库名称和表名
:param crawler:
:return:
"""
return cls(
host=crawler.settings.get('MONGO_URI'),
db=crawler.settings.get('MONGO_DB'),
collection=crawler.settings.get('MONGO_COLLECTION')
) def open_spider(self, spider):
"""
在spider打开时，完成mongodb数据库的初始化工作。
:param spider:
:return:
"""
self.client = MongoClient(host=self.mongo_uri)
self.db = self.client[self.mongo_db]
self.collection = self.db[self.mongo_collection] def process_item(self, item, spider):
"""
pipeline的核心函数，在该函数中执行对item的一系列操作，例如存储等。
:param item: parse_detail函数解析出来的item
:param spider: 抓取该item的spider
:return: 返回处理后的item，供其它pipeline再进行处理(如果有的话)
"""
# 以公司名称做为查询条件
condition = {'company_name': item.get('company_name')}
# upsert参数设置为True后，若数据库中没有该条记录，会执行插入操作；
# 同时，使用update_one()函数，也可以完成去重操作。
result = self.collection.update_one(condition, {'$set': item}, upsert=True)
spider.logger.debug('The Matched Item is: {} And The Modified Item is: {}'.format(result.matched_count, result.modified_count))
return item def close_spider(self, spider):
"""
在spider关闭时，关闭mongodb数据连接。
:param spider:
:return:
"""
self.client.close()

# Location: LagouCrawler.settings # Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'LagouCrawler.pipelines.LagoucrawlerPipeline': 300,
} # Mongodb地址
MONGO_URI = 'localhost'
# Mongodb库名
MONGO_DB = 'job'
# Mongodb表名
MONGO_COLLECTION = 'works'

# Location: LagouCrawler.settings # Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Location: LagouCrawler.settings # Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Host': 'www.lagou.com',
'Referer': 'https://www.lagou.com/',
'Connection': 'keep-alive'
}

# Location: LagouCrawler.middlewares class RandomUserAgentMiddleware(object):
"""
给每一个request添加随机的User-Agent
""" def __init__(self, ua_type=None):
super(RandomUserAgentMiddleware, self).__init__()
self.ua = UserAgent()
self.ua_type = ua_type @classmethod
def from_crawler(cls, crawler):
"""
获取setting.py中配置的RANDOM_UA_TYPE,如果没有配置，则使用默认值random
:param crawler:
:return:
"""
return cls(
ua_type=crawler.settings.get('RANDOM_UA_TYPE', 'random')
) def process_request(self, request, spider):
"""
UserAgentMiddleware的核心方法，getattr(A, B)相当于A.B，也就是获取A
对象的B属性，在这就相当于ua.random
:param request:
:param spider:
:return:
"""
request.headers.setdefault('User-Agent', getattr(self.ua, self.ua_type))
# 每个请求禁止重定向
request.meta['dont_redirect'] = True
request.meta['handle_httpstatus_list'] = [301, 302]
spider.logger.debug('The <{}> User Agent Is: {}'.format(request.url, getattr(self.ua, self.ua_type)))

# Location: LagouCrawler.settings DOWNLOADER_MIDDLEWARES = {
# 启用阿布云代理服务器中间件
'LagouCrawler.middlewares.AbuYunProxyMiddleware': 1,
# 在DownloaderMiddleware之前启用自定义的RandomUserAgentMiddleware
'LagouCrawler.middlewares.RandomUserAgentMiddleware': 542,
# 禁用框架默认启动的UserAgentMiddleware
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
'LagouCrawler.middlewares.LagoucrawlerDownloaderMiddleware': 543,
}

# Location: LagouCrawler.middlewares class AbuYunProxyMiddleware(object):
"""
接入阿布云代理服务器，该服务器动态IP1秒最多请求5次。需要在setting中设置下载延迟
""" def __init__(self, settings):
self.proxy_server = settings.get('PROXY_SERVER')
self.proxy_user = settings.get('PROXY_USER')
self.proxy_pass = settings.get('PROXY_PASS')
self.proxy_authorization = 'Basic ' + base64.urlsafe_b64encode(
bytes((self.proxy_user + ':' + self.proxy_pass), 'ascii')).decode('utf8') @classmethod
def from_crawler(cls, crawler):
return cls(
settings=crawler.settings
) def process_request(self, request, spider):
request.meta['proxy'] = self.proxy_server
request.headers['Proxy-Authorization'] = self.proxy_authorization
spider.logger.debug('The {} Use AbuProxy'.format(request.url))

# Location: LagouCrawler.settings # 阿布云代理服务器地址
PROXY_SERVER = "http://http-dyn.abuyun.com:9020"
# 阿布云代理隧道验证信息,注册阿布云购买服务后获取
PROXY_USER = 'H9470L5HEARXXXXX'
PROXY_PASS = '02E02D1D773XXXXX'
# 启用限速设置
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 0.2 # 初始下载延迟

from scrapy.cmdline import execute def main():
spider_name = 'lagoucrawler'
cmd_string = 'scrapy crawl {spider_name}'.format(spider_name=spider_name)
execute(cmd_string.split()) if __name__ == '__main__':
main()