正文
Python 爬虫 (二)
提示：扫一扫查出行【扫一扫了解最新限行尾号】
复制提示
cookiejar模块:
-
管理储存cookie，将传出的http请求添加cookie
-
cookie存储在内存中，CookieJar示例回收后cookie将自动消失
实例:用cookjar访问人人网主页
import json
from urllib import request,parse
from http import cookiejar
#实例化一个cookiejar对象
cookiejar_object = cookiejar.CookieJar()
#生成cookie管理器
cookie_handler = request.HTTPCookieProcessor(cookiejar_object)
#有了opener,就可以代替urlopen来进行请求网页
opener = request.build_opener(cookie_handler)
#获取网页的登陆接口
url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2018721532875'
#将网页的form信息获取
form = {
'email':'xxx',
'icode':'',
'origURL':'http://www.renren.com/home',
'domain':'renren.com',
'key_id':'',
'captcha_type':'web_login',
'password':'xxx',
'rkey':'79d8184f25d678248262a91caf3e7ea8',
'f':'http%3A%2F%2Fzhibo.renren.com%2Ftop',
}
#将数据转换成二进制
form_b = parse.urlencode(form).encode('utf-8')
#将url和表单信息,还有获取到的cookie去访问网页
response = opener.open(url,form_b)
html_b = response.read()#字节类型
# print(html_b)
res_dict = json.loads(html_b.decode('utf-8'))
#获取登陆后的个人主页url
home_url = res_dict['homeUrl']
# print(home_url)
# 访问个人主页
response = opener.open(home_url)
html_bytes = response.read()
print(html_bytes.decode('utf-8'))
python Proxy代理
from urllib import request
proxy = {
'http': 'http://219.141.153.41:80'
}
url = 'http://www.baidu.com/s?wd=ip'
# request.HTTPCookieProcessor(cookie)
handler = request.ProxyHandler(proxy)
# 生成 opener.open = urlopen
opener = request.build_opener(handler)
# 同过opener访问百度
response = opener.open(url, timeout=5)
# 存储页面
with open('baidu.html', 'wb') as f:
f.write(response.read())
有道词典翻译接口
import time
import random
import json
from Day1.tuozhan_all import post
def md5_my(need_str):
import hashlib
# 创建md5对象
md5_o = hashlib.md5()
# 需要有bytes, 作为参数
# 由str, 转换成 bytes encode-------str.encode('utf-8')
# 由bytes转换成 str, decode---------bytes.decode('utf-8')
sign_bytes = need_str.encode('utf-8')
# 更新md5 object的值
md5_o.update(sign_bytes)
sign_str = md5_o.hexdigest()
return sign_str
# url
def translate(kw):
url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'OUTFOX_SEARCH_USER_ID=-493176930@10.168.8.63; OUTFOX_SEARCH_USER_ID_NCOO=38624120.26076847; SESSION_FROM_COOKIE=unknown; JSESSIONID=aaabYcV4ZOU-JbQUha2uw; ___rl__test__cookies=1534210912076',
'Host': 'fanyi.youdao.com',
'Origin': 'http://fanyi.youdao.com',
'Referer': 'http://fanyi.youdao.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
}
# form 的生成1. i 需要确定, 2, salt, 3, sign
key= kw
# salt : ((new Date).getTime() + parseInt(10 * Math.random(), 10))
salt = int(time.time()*1000 + random.randint(0,10))
print('salt:',salt)
salt_str = str(salt)
# sign : o = u.md5(S + n + r + D);
# S = "fanyideskweb"
# D = "ebSeFb%=XZ%T[KZ)c(sy!"
# n = key
# r = salt_str
S = "fanyideskweb"
D = "ebSeFb%=XZ%T[KZ)c(sy!"
sign_str = S + key + salt_str + D
# md5 加密的方法
sign_md5_str = md5_my(sign_str)
form = {
'i': key,
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'salt': salt_str,
'sign': sign_md5_str,
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_REALTIME',
'typoResult': 'false',
}
html_bytes = post(url, form, headers=headers)
# 将 json 类型的 str, 转化成, 字典
res_dict = json.loads(html_bytes.decode('utf-8'))
#print(html_bytes.decode('utf-8'))
translate_res = res_dict['translateResult'][0][0]['tgt']
return translate_res
if __name__ == '__main__':
ret = translate('中国')
print('翻译的结果:' + ret)
import json
from urllib import request,parse
from http import cookiejar #实例化一个cookiejar对象
cookiejar_object = cookiejar.CookieJar()
#生成cookie管理器
cookie_handler = request.HTTPCookieProcessor(cookiejar_object)
#有了opener,就可以代替urlopen来进行请求网页
opener = request.build_opener(cookie_handler) #获取网页的登陆接口
url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2018721532875' #将网页的form信息获取
form = {
'email':'xxx',
'icode':'',
'origURL':'http://www.renren.com/home',
'domain':'renren.com',
'key_id':'',
'captcha_type':'web_login',
'password':'xxx',
'rkey':'79d8184f25d678248262a91caf3e7ea8',
'f':'http%3A%2F%2Fzhibo.renren.com%2Ftop',
} #将数据转换成二进制
form_b = parse.urlencode(form).encode('utf-8') #将url和表单信息,还有获取到的cookie去访问网页
response = opener.open(url,form_b)
html_b = response.read()#字节类型
# print(html_b) res_dict = json.loads(html_b.decode('utf-8'))
#获取登陆后的个人主页url
home_url = res_dict['homeUrl'] # print(home_url)
# 访问个人主页
response = opener.open(home_url)
html_bytes = response.read()
print(html_bytes.decode('utf-8'))
python Proxy代理
from urllib import request
proxy = {
'http': 'http://219.141.153.41:80'
}
url = 'http://www.baidu.com/s?wd=ip'
# request.HTTPCookieProcessor(cookie)
handler = request.ProxyHandler(proxy)
# 生成 opener.open = urlopen
opener = request.build_opener(handler)
# 同过opener访问百度
response = opener.open(url, timeout=5)
# 存储页面
with open('baidu.html', 'wb') as f:
f.write(response.read())
有道词典翻译接口
import time
import random
import json
from Day1.tuozhan_all import post
def md5_my(need_str):
import hashlib
# 创建md5对象
md5_o = hashlib.md5()
# 需要有bytes, 作为参数
# 由str, 转换成 bytes encode-------str.encode('utf-8')
# 由bytes转换成 str, decode---------bytes.decode('utf-8')
sign_bytes = need_str.encode('utf-8')
# 更新md5 object的值
md5_o.update(sign_bytes)
sign_str = md5_o.hexdigest()
return sign_str
# url
def translate(kw):
url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'OUTFOX_SEARCH_USER_ID=-493176930@10.168.8.63; OUTFOX_SEARCH_USER_ID_NCOO=38624120.26076847; SESSION_FROM_COOKIE=unknown; JSESSIONID=aaabYcV4ZOU-JbQUha2uw; ___rl__test__cookies=1534210912076',
'Host': 'fanyi.youdao.com',
'Origin': 'http://fanyi.youdao.com',
'Referer': 'http://fanyi.youdao.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
}
# form 的生成1. i 需要确定, 2, salt, 3, sign
key= kw
# salt : ((new Date).getTime() + parseInt(10 * Math.random(), 10))
salt = int(time.time()*1000 + random.randint(0,10))
print('salt:',salt)
salt_str = str(salt)
# sign : o = u.md5(S + n + r + D);
# S = "fanyideskweb"
# D = "ebSeFb%=XZ%T[KZ)c(sy!"
# n = key
# r = salt_str
S = "fanyideskweb"
D = "ebSeFb%=XZ%T[KZ)c(sy!"
sign_str = S + key + salt_str + D
# md5 加密的方法
sign_md5_str = md5_my(sign_str)
form = {
'i': key,
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'salt': salt_str,
'sign': sign_md5_str,
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_REALTIME',
'typoResult': 'false',
}
html_bytes = post(url, form, headers=headers)
# 将 json 类型的 str, 转化成, 字典
res_dict = json.loads(html_bytes.decode('utf-8'))
#print(html_bytes.decode('utf-8'))
translate_res = res_dict['translateResult'][0][0]['tgt']
return translate_res
if __name__ == '__main__':
ret = translate('中国')
print('翻译的结果:' + ret)
from urllib import request
proxy = {
'http': 'http://219.141.153.41:80'
}
url = 'http://www.baidu.com/s?wd=ip'
# request.HTTPCookieProcessor(cookie)
handler = request.ProxyHandler(proxy)
# 生成 opener.open = urlopen
opener = request.build_opener(handler)
# 同过opener访问百度
response = opener.open(url, timeout=5)
# 存储页面
with open('baidu.html', 'wb') as f:
f.write(response.read())
import time
import random
import json
from Day1.tuozhan_all import post def md5_my(need_str):
import hashlib
# 创建md5对象
md5_o = hashlib.md5()
# 需要有bytes, 作为参数
# 由str, 转换成 bytes encode-------str.encode('utf-8')
# 由bytes转换成 str, decode---------bytes.decode('utf-8')
sign_bytes = need_str.encode('utf-8')
# 更新md5 object的值
md5_o.update(sign_bytes)
sign_str = md5_o.hexdigest()
return sign_str # url def translate(kw):
url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule' headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'OUTFOX_SEARCH_USER_ID=-493176930@10.168.8.63; OUTFOX_SEARCH_USER_ID_NCOO=38624120.26076847; SESSION_FROM_COOKIE=unknown; JSESSIONID=aaabYcV4ZOU-JbQUha2uw; ___rl__test__cookies=1534210912076',
'Host': 'fanyi.youdao.com',
'Origin': 'http://fanyi.youdao.com',
'Referer': 'http://fanyi.youdao.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
} # form 的生成1. i 需要确定, 2, salt, 3, sign
key= kw # salt : ((new Date).getTime() + parseInt(10 * Math.random(), 10))
salt = int(time.time()*1000 + random.randint(0,10))
print('salt:',salt)
salt_str = str(salt) # sign : o = u.md5(S + n + r + D);
# S = "fanyideskweb"
# D = "ebSeFb%=XZ%T[KZ)c(sy!"
# n = key
# r = salt_str
S = "fanyideskweb"
D = "ebSeFb%=XZ%T[KZ)c(sy!"
sign_str = S + key + salt_str + D
# md5 加密的方法
sign_md5_str = md5_my(sign_str) form = {
'i': key,
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'salt': salt_str,
'sign': sign_md5_str,
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_REALTIME',
'typoResult': 'false',
} html_bytes = post(url, form, headers=headers) # 将 json 类型的 str, 转化成, 字典
res_dict = json.loads(html_bytes.decode('utf-8'))
#print(html_bytes.decode('utf-8')) translate_res = res_dict['translateResult'][0][0]['tgt']
return translate_res if __name__ == '__main__':
ret = translate('中国') print('翻译的结果:' + ret)
