爬虫之scrapy简单案例之猫眼

class TopSpider(scrapy.Spider):
name = 'top'
allowed_domains = ['maoyan.com']
start_urls = ['https://maoyan.com/board/4'] def parse(self, response):
dds = response.xpath('//dl/dd')
for dd in dds:
dic = MaoyanItem()
# dic = {}
dic['name'] = dd.xpath('.//p[@class="name"]//text()').extract_first()
dic['star'] = dd.xpath('.//p[@class="star"]/text()').extract_first().replace('\n', '').replace(' ', '')
dic['releasetime'] = dd.xpath('.//p[@class="releasetime"]/text()').extract_first()
score1 = dd.xpath('.//p[@class="score"]/i[1]/text()').extract_first()
score2 = dd.xpath('.//p[@class="score"]/i[2]/text()').extract_first()
dic['score'] = score1 + score2
# 详情页
xqy_url = 'https://maoyan.com' + dd.xpath('.//p[@class="name"]/a/@href').extract_first()
yield scrapy.Request(xqy_url, callback=self.xqy_parse, meta={'dic': dic})
# 翻页
next_url = response.xpath('//a[text()="下一页"]/@href').extract_first()
if next_url:
url = 'https://maoyan.com/board/4' + next_url
yield scrapy.Request(url, callback=self.parse) def xqy_parse(self,response):
dic = response.meta['dic']
dic['type'] = response.xpath('//ul/li[@class="ellipsis"][1]/text()').extract_first()
dic['area_time'] = response.xpath('//ul/li[@class="ellipsis"][2]/text()').extract_first().replace('\n', '').replace(' ', '')
yield dic

class DoubanItem(scrapy.Item):
title = scrapy.Field()
inf = scrapy.Field()
score = scrapy.Field()
peo = scrapy.Field()
brief = scrapy.Field()

class DoubanPipeline(object):
def open_spider(self, spider):
self.file = open('douban.txt', 'a', encoding='utf-8') def process_item(self, item, spider):
self.file.write(str(item)+'\n') def close_spider(self, spider):
self.file.close()

from pymongo import MongoClient class DoubanPipeline(object):
def open_spider(self,spider):
# self.file = open('douban.txt','a',encoding='utf8')
self.client = MongoClient()
self.collection = self.client['库名']['集合名']
self.count = 0 def process_item(self, item, spider):
# self.file.write(str(item)+'\n')
item['_id'] = self.count
self.count += 1
self.collection.insert_one(item)
return item def close_spider(self, spider):
# self.file.close()
self.client.close()