爬取前尘无忧python职位信息并保存到mongo数据库

import re,os
import requests
from requests.exceptions import RequestException MAX_PAGE = 10 #最大页数
KEYWORD = 'python'
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 're_job51_python.txt' # 获取网页源码
def getHtml(page):
try:
url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{0},2,{1}.html?'.format(KEYWORD,page)
response = requests.get(url,headers=headers)
response.encoding = response.apparent_encoding
return response.text
except RequestException:
print('请求出错')
return None # 解析网页源码，得到目标信息
def getTarget(html):
reg = re.compile(
r'class="t1 ">.*? <a target="_blank" '
'title="(.*?)".*? <span class="t2"><a target="_blank" '
'title="(.*?)".*?<span '
'class="t3">(.*?)</span>.*?<span '
'class="t4">(.*?)</span>.*? <span '
'class="t5">(.*?)</span>',
re.S) # 匹配换行符
target = re.findall(reg,html)
return target # 保存到文本中
def save_to_txt(item):
with open(file_name,'a',newline='') as f: # newline参数防止两行之间有空行
for i in range(len(item)):
# 最后一个元素换行，非最后则以＇,'隔开
if i == len(item)-1:
f.write(item[i])
f.write('\n')
else:
f.write(item[i]+',') def main():
# 每次执行前检查文件是否存在，存在则删除
if os.path.exists(file_name):
os.remove(file_name) # 分页爬取
for page in range(MAX_PAGE+1):
html = getHtml(page)
content = getTarget(html)
for item in content:
save_to_txt(item) if __name__ == '__main__':
main()

import os
import requests
from requests.exceptions import RequestException
from lxml import etree
import pymongo
from spiders.前程无忧.mongo_config import * # mongo数据库设置
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB] MAX_PAGE = 5
KEYWORD = 'python'
headers = {
'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '\
'Chrome/63.0.3239.132 Safari/537.36'
}
file_name = 'xpath_job51_python.txt' # 获取网页
def get_html(page):
try:
url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{},2,{}.html?'.format(KEYWORD,page)
response = requests.get(url,headers=headers)
response.encoding = response.apparent_encoding
return response.text
except RequestException:
return None # 解析网页
def parse_html(html):
# 构造xpath解析对象，可自动修整HTML文本
html = etree.HTML(html)
# 获取文本 /text()
# 获取属性／@href
# 获取第i个标签／tar_name[i] 从1开始
# normalize-space－－＞去空格换行符
# position_name = html.xpath('normalize-space(//div[@class="el"]/p/span/a/text())') # 职位名称，
position_names = []
for name in html.xpath('//div[@class="el"]/p/span/a/text()'):
position_name = name.strip()
position_names.append(position_name) # 职位地址
position_urls = html.xpath('//div[@class="el"]/p/span/a/@href') # 公司名称
company_names = html.xpath('//div[@class="el"]/span[1]/a/text()') # 公司地址
company_urls = html.xpath('//div[@class="el"]/span[1]/a/@href') # 位置
locations = html.xpath('//div[@class="el"]/span[@class="t3"]/text()') # 薪资
salarys = html.xpath('//div[@class="el"]/span[@class="t4"]/text()') # 发布时间
release_dates = html.xpath('//div[@class="el"]/span[4]/text()') result = zip(position_names,position_urls,company_names,company_urls,locations,salarys,release_dates)
return result def save_to_txt(element):
with open(file_name,'a',newline='') as f:
for i in range(len(element)):
# data = ','.join(element[i])
if i == len(element)-1:
f.write(element[i])
f.write('\n')
else:
f.write(element[i]+',') def save_to_mongo(element):
keys = ['position_name','position_url','company_name',
'company_url','location','salary','release_date']
result = dict(zip(keys,list(element)))
if db[MONGO_TABLE_XPATH].insert(result):
print('数据成功存储到mongo数据库中')
return True
return False # 遍历字典元素
# for k,v in result.items():
# print(k,':',v)
for key in result:
print(key,':',result[key]) def main():
if os.path.exists(file_name):
os.remove(file_name)
for page in range(1,MAX_PAGE+1):
html = get_html(page)
elements = parse_html(html)
if elements:
for element in elements:
save_to_txt(element)
save_to_mongo(element) if __name__ == '__main__':
main()