爬虫：获取动态加载数据（selenium）（某站）

提示：扫一扫查出行
【扫一扫了解最新限行尾号】
复制提示

如果网站数据是动态加载，需要不停往下拉进度条才能显示数据，用selenium模拟浏览器下拉进度条可以实现动态数据的抓取。

本文希望找到某乎某话题下讨论较多的问题，以此再寻找每一问题涉及的话题关键词（侵删）。

下面代码采用driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")模拟浏览器下拉进度条200次，获取了女性话题下近900多条回答，去重（同一话题下有重复问题）后得到600多个问题

from selenium.webdriver import Chrome

from selenium.webdriver import ChromeOptions

import time

import random

import re

from pymongo import MongoClient

client = MongoClient('localhost')

db = client['test_db']

def get_links(url, word):

    option = ChromeOptions()

    option.add_experimental_option('excludeSwitches', ['enable-automation'])

    option.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"')

    driver = Chrome(options=option)

    time.sleep(10)

    driver.get('https://www.zhihu.com')

    time.sleep(10)

    driver.delete_all_cookies()  # 清除刚才的cookie

    time.sleep(2)

    cookie = {}  # 替换为自己的cookie

    driver.add_cookie(cookie)

    driver.get(url)

    time.sleep(random.uniform(10, 11))

    for i in range(0, 200):

        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")

        time.sleep(random.uniform(3, 4))

    links = driver.find_elements_by_css_selector('h2.ContentItem-title > div > a')

    print(len(links))


    # 去重

    regex = re.compile(r'https://www.zhihu.com/question/\d+')  # 匹配问题而不是答案链接

    links_set = set()

    for link in links:

        try:

            links_set.add(regex.search(link.get_attribute("href")).group())

        except AttributeError:

            pass

    print(len(links_set))

    with open(r'知乎镜像链接' + '/' + word + '-'.join(str(i) for i in list(time.localtime())[:5]) + '.txt', 'a') as f:

        for item in links_set:

            f.write(item + '\n')

            db[word + '_' + 'links'].insert_one({"link": item})

if __name__ == '__main__':

    input_word = input('输入话题：')

    input_link = input('输入话题对应链接url：')

    get_links(input_link, input_word)