正文
python爬虫前提技术
提示：扫一扫查出行【扫一扫了解最新限行尾号】
复制提示
1、BeautifulSoup 解析html如何使用转自：http://blog.csdn.net/u013372487/article/details/51734047#!/usr/bin/python
# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import re#待分析字符串
html_doc = """
<html>
<head>
<title>The Dormouse's story</title>
</head>
<body>
<p class="title aq">
<b>
The Dormouse's story
</b>
</p><p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.
</p><p class="story">...</p>
"""# html字符串创建BeautifulSoup对象
soup = BeautifulSoup(html_doc, 'html.parser', from_encoding='utf-8')#输出第一个 title 标签
print soup.title#输出第一个 title 标签的标签名称
print soup.title.name#输出第一个 title 标签的包含内容
print soup.title.string#输出第一个 title 标签的父标签的标签名称
print soup.title.parent.name#输出第一个 p 标签
print soup.p#输出第一个 p 标签的 class 属性内容
print soup.p['class']#输出第一个 a 标签的 href 属性内容
print soup.a['href']
'''
soup的属性可以被添加,删除或修改. 再说一次, soup的属性操作方法与字典一样
'''
#修改第一个 a 标签的href属性为 http://www.baidu.com/
soup.a['href'] = 'http://www.baidu.com/'#给第一个 a 标签添加 name 属性
soup.a['name'] = u'百度'#删除第一个 a 标签的 class 属性为
del soup.a['class']##输出第一个 p 标签的所有子节点
print soup.p.contents#输出第一个 a 标签
print soup.a#输出所有的 a 标签，以列表形式显示
print soup.find_all('a')#输出第一个 id 属性等于 link3 的 a 标签
print soup.find(id="link3")#获取所有文字内容
print(soup.get_text())#输出第一个 a 标签的所有属性信息
print soup.a.attrsfor link in soup.find_all('a'):
#获取 link 的 href 属性内容
print(link.get('href'))#对soup.p的子节点进行循环输出
for child in soup.p.children:
print(child)#正则匹配，名字中带有b的标签
for tag in soup.find_all(re.compile("b")):
print(tag.name)2、cookie等使用方法以及函数爬虫参照： https://cuiqingcai.com/968.html3、header,代理,超时,认证,异常处理参照： http://blog.csdn.net/m_buddy/article/details/551937624、错误异常处理1.URLError# -*- coding: UTF-8 -*-
import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPErrorif __name__ == "__main__":
#一个不存在的连接
url = "http://www.douyu.com/Jack_Cui.html"
request = urllib.request.Request(url)
try:
response = urllib.request.urlopen(request)
# html = responese.read()
except urllib.error.HTTPError as e:
print(e.code)运行结果：C:\Python34\python.exe G:/xiaoshuo2.py
403Process finished with exit code 0# -*- coding: UTF-8 -*-
import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPErrorif __name__ == "__main__":
#一个不存在的连接
url = "http://www.douyu.com/Jack_Cui.html"
request = urllib.request.Request(url)
try:
response = urllib.request.urlopen(request)
html = response.read().decode('utf-8')
print(html)
except urllib.error.HTTPError as e:
print(e.code)运行结果：C:\Python34\python.exe G:/xiaoshuo2.py
403Process finished with exit code 0import urllib
import urllib
from urllib import request
import re
import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPErrorurl = "http://www.douyu.com/Jack_Cui.html"rep=urllib.request.Request(url)
try:
data=urllib.request.urlopen(rep)
except urllib.error.URLError as e:
if hasattr(e,'code'):
print("HTTPError")
print(e.code)
if hasattr(e,'reason' ):
print("URLError")
print(e.reason)输出结果：C:\Python34\python.exe G:/xiaoshuo2.py
HTTPError
403
URLError
ForbiddenProcess finished with exit code 05、python打印防止换行和换行https://www.cnblogs.com/kfx2007/p/5970784.html实例：# coding=utf-8
import relanguage = '''''
<table class="infobox bordered vcard" style="width: 21em; font-size: 89%; text-align: left;" cellpadding="3">
<caption style="text-align: center; font-size: larger;" class="fn"><b>jenkins</b></caption>
<tr>
<th>性別：</th>
<td>男</td>d
</tr>
<tr>
<th>異名：</th>
<td><span class="nickname">(字) 翔宇</span></td>
</tr>
<tr>
<th>爱好：</th>
<td><span class="org"><a href="../articles/%E4%B8%AD9A.html" title="篮球">篮球</a></span></td>
</tr>
<tr>
<th>籍貫：</th>
<td><a href="../articles/%E6%B5%9981.html" title="广西省">广西省</a><a href="../articles/%E7%BB%8D82.html" title="桂林市">桂林市</a></td>
</tr>
</table>
'''#获取table中tr值
res_tr = r'<tr>(.*?)</tr>'
m_tr = re.findall(res_tr,language,re.S|re.M)
for line in m_tr:
#获取表格第一列th 属性
res_th = r'<th>(.*?)</th>'
m_th = re.findall(res_th,line,re.S|re.M)
for mm in m_th:
if "href" in mm: #如果获取加粗的th中含超链接则处理
restr = r'<a href=.*?>(.*?)</a>'
h = re.findall(restr,mm,re.S|re.M)
print (h[0],end=' ') #逗号连接属性值 防止换行
else:
print (mm,end=' ') #unicode防止乱 #获取表格第二列td 属性值
res_td = r'<td>(.*?)</td>' #r'<td .*?>(.*?)</td>'
m_td = re.findall(res_td,line,re.S|re.M)
for nn in m_td:
if "href" in nn: #处理超链接<a href=../rel=..></a>
res_value = r'<a .*?>(.*?)</a>'
m_value = re.findall(res_value,nn,re.S|re.M)
for value in m_value:
print (value,end=' ')
elif "span" in nn: #处理标签<span>
res_value = r'<span .*?>(.*?)</span>'
m_value = re.findall(res_value,nn,re.S|re.M) #<td><span class="nickname">(字) 翔宇</span></td>
for value in m_value:
print (value,end=' ')
else:
print (nn,end=' ')
print (' ') #换行C:\Python34\python.exe G:/xiaoshuo2.py
性別： 男
異名： (字) 翔宇
爱好： 篮球
籍貫： 广西省 桂林市 6、python打印如何呢不换行https://www.cnblogs.com/hwd9654/p/5707920.html# -*- coding:utf-8 -*-
import urllib
import re
#import requests
import urllib.parse
import urllib.request
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPErrorclass Tool:
removeImg = re.compile('<img.*?>| {7}|')
removeAddr = re.compile('<a.*?>|</a>')
replaceLine = re.compile('<tr>|<div>|</div>|</p>')
replaceTD= re.compile('<td>')
replacePara = re.compile('<p.*?>')
replaceBR = re.compile('<br><br>|<br>')
removeExtraTag = re.compile('<.*?>')
def replace(self,x):
x = re.sub(self.removeImg,"",x)
x = re.sub(self.removeAddr,"",x)
x = re.sub(self.replaceLine,"\n",x)
x = re.sub(self.replaceTD,"\t",x)
x = re.sub(self.replacePara,"\n",x)
x = re.sub(self.replaceBR,"\n",x)
x = re.sub(self.removeExtraTag,"",x)
return x.strip()class BDTB:
def __init__(self,baseUrl,seeLZ):
self.baseURL = baseUrl
self.seeLZ = '?see_lz='+str(seeLZ)
self.tool = Tool()
def getPage(self,pageNum):
try:
url = self.baseURL+ self.seeLZ + '&pn=' + str(pageNum)
request = urllib.request.Request(url)
response = urllib.request.urlopen(request).read().decode("utf8")
#print (response) return response
except urllib.error.URLError as e:
if hasattr(e,"reason"):
print ("连接百度贴吧失败,错误原因",e.reason)
return None
def getTitle(self):
page = self.getPage(1)
pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>',re.S)
result = re.search(pattern,page)
if result:
#print (result.group(1))
return result.group(1).strip()
else:
return None def getPageNum(self):
page = self.getPage(1)
pattern = re.compile('<li class="l_reply_num.*?</span>.*?<span.*?>(.*?)</span>',re.S)
result = re.search(pattern,page)
#print (result.group(1))
if result:
return result.group(1).strip()
else:
return None
def getContent(self,page):
pattern = re.compile('<div id="post_content_.*?>(.*?)</div>',re.S)
items = re.findall(pattern,page)
floor = 1
for item in items:
print (floor,"楼-------------------------------------------------------------------------------------\n",end='')
#print ("楼---------------------------------------------------------------------------------------------------------------\n")
print (self.tool.replace(item))
floor += 1baseURLh = 'http://tieba.baidu.com/p/3138733512'
bdtb = BDTB(baseURLh,1)
bdtb.getContent(bdtb.getPage(1))打印结果：C:\Python34\python.exe C:/Users/Administrator/ceshi.py
1 楼-------------------------------------------------------------------------------------
很多媒体都在每赛季之前给球员排个名，我也有这个癖好…………，我会尽量理性的分析球队地位，个人能力等因素，评出我心目中的下赛季50大现役球员，这个50大是指预估他本赛季在篮球场上对球队的影响力……不是过去的荣誉什么的，所以难免有一定的主观性……如果把你喜欢的球星排低了，欢迎理性讨论！状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
2 今年的新秀我就不考虑了，没上赛季参照
2 楼-------------------------------------------------------------------------------------
50 惊喜新人王 迈卡威
上赛季数据
篮板 6.2 助攻 6.3 抢断 1.9 盖帽 0.6 失误 3.5 犯规 3 得分 16.7新赛季第50位，我给上赛季的新人王迈卡威。 上赛季迈卡威在彻底重建的76人中迅速掌握了球队，一开始就三双搞定了热火赢得了万千眼球。后来也屡屡有经验的表现，新秀赛季就拿过三双的球员不多，迈卡威现在可以说在76人站稳了脚跟。7、python爬虫xpath的语法http://www.cnblogs.com/lonenysky/p/4649455.html//*[@id="AD_4586850"]/div[1]/strong/i//*[@id="shop_list"]/div[1]/strong/i
//*[@id="shop_list"]8、requests用法http://cuiqingcai.com/2556.html#-*—coding:utf8-*-
from lxml import etree
import requests
import re
#编码转换
import sys
#headers构造一个字典，里面保存了user-agent
#headers= { 'User-Agent' : 'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' }
html = requests.get('http://cuiqingcai.com')
print(html.text)9、sub使用http://blog.csdn.net/lovemianmian/article/details/88676131、去除imag标签import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('<img.*?>')
s=re.sub(removeImg,"",text).strip()
print (s)C:\Python34\python.exe G:/xiaoshuo2.py
he is cool, clever, and so on...1、1 单独去除7位长空格import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('| {7}|')
s=re.sub(removeImg,"",text).strip()
print (s)打印C:\Python34\python.exe G:/xiaoshuo2.py
<imgJGood is a handsome boy,> he is cool, clever, and so on...2、去除imag标签 + 去除7位长空格import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'
removeImg = re.compile('<img.*?>| {7}|')
s=re.sub(removeImg,"",text).strip()
print (s)打印：C:\Python34\python.exe G:/xiaoshuo2.py
he is cool, clever, and so on...Process finished with exit code 03、去除imag标签 + 保留7位长空格import re
text ='<imgJGood is a handsome boy,> he is cool, clever, and so on...'removeImg = re.compile('<img.*?>{7}')
s=re.sub(removeImg,"",text).strip()
print (s)打印：C:\Python34\python.exe G:/xiaoshuo2.py
<imgJGood is a handsome boy,> he is cool, clever, and so on...Process finished with exit code 04、把两个标签中间的内容去掉import re
text='<a href="http://jump2.bdimg.com/safecheck/index?url=x+Z5)">迈卡威</a>刷出了不错的数据'
removeImg = re.compile('<a.*?>|</a>')
s=re.sub(removeImg,"",text).strip()
print (s)打印：C:\Python34\python.exe G:/xiaoshuo2.py
迈卡威刷出了不错的数据5，把<br>换行符换成/n 换行符import re
text ='height="510"><br><br><br><br>状元维金斯镇楼<br>P.S 1 我每天都至少更新一个，不TJ。<br> 2 今年的新秀我就不考虑了，没上赛季参照'
removeImg = re.compile('<br><br>|<br>')
s=re.sub(removeImg,"\n",text).strip()
print (s)C:\Python34\python.exe G:/xiaoshuo2.py
height="">状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
2 今年的新秀我就不考虑了，没上赛季参照5.1，把<br>换行符换成/n 换行符import re
text ='height="510"><br><br><br><br>状元维金斯镇楼<br>P.S 1 我每天都至少更新一个，不TJ。<br> 2 今年的新秀我就不考虑了，没上赛季参照'
removeImg = re.compile('<br>')
s=re.sub(removeImg,"\n",text).strip()
print (s)C:\Python34\python.exe G:/xiaoshuo2.py
height="">状元维金斯镇楼
P.S 1 我每天都至少更新一个，不TJ。
2 今年的新秀我就不考虑了，没上赛季参照10、正则表达式<div class="list-item">
<div class="personal-info">
<div class="pic-word">
<div class="pic s60">
<a href="//mm.taobao.com/687471686.htm" target="_blank" class="lady-avatar"> <img src="//gtd.alicdn.com/sns_logo/i2/TB1XZ1PQVXXXXaJXpXXSutbFXXX.jpg_60x60.jpg" alt="" width="" height=""/>
</a>
</div>
<p class="top">
<a class="lady-name" href="//mm.taobao.com/self/model_card.htm?user_id=687471686" target="_blank">jenkins</a>
<em><strong>27</strong>岁</em>
<span>广州市</span>pattern = re.compile('<div class="list-item">.*? href="(.*?)".*? src="(.*?)".*? target="_blank">(.*?)</a>.*?<strong>(.*?)</strong>.*?<span>(.*?)</span>',re.S)
或者pattern = re.compile('<div class="list-item">.*?<a href="(.*?)".*?<img src="(.*?)".*?<a class="lady-name".*?>(.*?)</a>.*?<strong>(.*?)</strong>.*?<span>(.*?)</span>',re.S)https://image.baidu.com/search/index?tn=baiduimage&ct=201326592&lm=-1&cl=2&ie=gbk&word=%C3%C0%C5%AE%CD%BC%C6%AC&fr=ala&ala=1&alatpl=cover&pos=0&hs=2&xthttps=111111
