用selenium 自动爬取某一本小说章节及其内容，并存入数据库中

from selenium import webdriver
import pymysql
from selenium.webdriver.support.ui import WebDriverWait # 等待
from selenium.webdriver.support import expected_conditions as ec # 等待条件
from selenium.webdriver.common.by import By
import html
import _thread
from selenium.webdriver.chrome.options import Options def ceil(x, y):
if x % y == 0: # 相除后为整数
return int(x / y)
else: # 相除有小数
return int(x / y) + 1 # 创建一个浏览器
chrome_options = Options()
chrome_options.add_argument('--headless')
dr = webdriver.Chrome(chrome_options=chrome_options) # 设置访问的网站
dr.get('https://doupocangqiong1.com/1/list_piaotian/') # 获取所有的a标签
a = dr.find_elements_by_css_selector('.dirlist > li > a') # 连接数据库
db = pymysql.connect("localhost", "root", "root", "selenium", charset='utf8')
# 获取游标
cursor = db.cursor() for i in a:
name = i.text
href = i.get_attribute('href')
sql = "INSERT INTO novel (name,href,content) VALUES ('%s','%s','%s')"%(name,href,'')
cursor.execute(sql) # 使用execute方法执行SQL语句
db.commit()
dr.close() # 关闭浏览器 def line(lineName, start, count):
dr = webdriver.Chrome(chrome_options=chrome_options) # 创建一个浏览器
# 连接数据库
db = pymysql.connect("localhost", "root", "root", "selenium", charset='utf8')
# 获取游标
cursor = db.cursor() sql = "SELECT id,href FROM novel LIMIT %s, %s"%(start, count)
cursor.execute(sql) # 使用execute方法执行SQL语句
data = cursor.fetchall() # 使用 fetchall() 方法获取所有数据
for i in data:
dr.get(i[1])
# 放置等待
WebDriverWait(dr, 5, 0.1).until_not(ec.text_to_be_present_in_element((By.CSS_SELECTOR, '#chaptercontent'),
U'正在转码，请稍后......')) # 等待dr浏览器10秒钟，每0.1秒钟问一次
content = html.escape(dr.find_element_by_css_selector('#chaptercontent').text)
# escape()将特殊字符转为特殊的编码格式，unescape()将编码格式转回特殊字符
sql = "UPDATE novel SET content = '%s' WHERE id = %s" % (content, i[0])
cursor.execute(sql) # 使用execute方法执行SQL语句
db.commit()
print(lineName, '完成了', i[0], '的采集')
dr.close() # 关闭窗口
dr.quit() # 关闭浏览器
cursor.close()
db.close()
print(lineName, '完成了采集') def productLine(func, total, lineCount):
every = ceil(total[0][0], lineCount)
print('every', every)
for i in range(lineCount):
print('-------------', i)
print(_thread.start_new_thread(func, ('line-' + str(i) + '', i * every, every))) try:
sql = 'SELECT COUNT(*) FROM novel'
cursor.execute(sql) # 使用execute方法执行SQL语句
total = cursor.fetchall() # 使用 fetchall() 方法获取所有数据
print(total) productLine(line, total, 5) except:
print ("Error: unable to start thread") while 1:
pass