使用python爬去国家民政最新的省份代码的程序，requests,beautifulsoup,lxml

# -*- coding: utf-8 -*-
"""
Created on Wed Jul 10 14:40:41 2019 @author: Administrator
""" import pandas as pd
import requests
from bs4 import BeautifulSoup
import time url1 = 'http://www.mca.gov.cn/article/sj/xzqh//1980/'
headers = {'content-type': 'application/json',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'} # 1. 获取所有链接========================================================================
def f1(url1):
'2018-1980年中华人民共和国行政区划代码 的所有链接'
#requests发出请求，设置url，header参数
response = requests.get(url1, headers=headers, timeout=200, verify=False)
soup = BeautifulSoup(response.text,'lxml') #将网页源码返回为BeautifulSoup类型
_tmp1 = soup.select('td.arlisttd')
end_1 = []
for i in _tmp1:
_a = i.select('a')[0].get('href')
_b = i.select('a')[0].get('title')[:4]
end_1.append(['http://www.mca.gov.cn'+_a,_b])
return end_1 end_2=[]
for i in ['','?2','?3']:
end_2 = end_2+f1(url1+i) def f2(url1='http://www.mca.gov.cn/article/sj/xzqh/2019/'):
'2019年中华人民共和国行政区划代码'
response = requests.get(url1, headers=headers, timeout=200, verify=False)
soup = BeautifulSoup(response.text,'lxml')
_tmp1 = soup.select('td.arlisttd')
end_1 = []
for i in _tmp1:
_a = i.select('a')[0].get('href')
_b = i.select('a')[0].get('title')[:7]
end_1.append(['http://www.mca.gov.cn'+_a,_b])
return end_1 end_2 = end_2+f2() # 2. 获取数据========================================================================
def f3(url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201903/20190300014989.shtml'):
#url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854922.shtml'
#url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854918.shtml'
#
response = requests.get(url1, headers=headers, timeout=200, verify=False)
soup = BeautifulSoup(response.text,'lxml')
_txt = soup.select('script')[4].get_text().strip().replace('window.location.href="','').strip('";')
if _txt[-4:]=='html':
print('script!')
url2 = _txt
else:
_tmp1 = soup.select('div.artext > div > p > a')
if len(_tmp1)==0:
_tmp1 = soup.select('div#zoom > a')
url2 = _tmp1[0].get('href')
print(url2)
#return url2
#url2='http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220856.html'
time.sleep(0.5)
response = requests.get(url2, headers=headers, timeout=200, verify=False)
#将网页源码返回为BeautifulSoup类型
soup = BeautifulSoup(response.text,'lxml')
_tmp1 = soup.select('table > tr[height="19"]')
end_1 = []
if len(_tmp1)>5:
for i in _tmp1:
_a = i.select('td')[1].get_text().strip()
if len(_a)>15: #部分数据页面，最后一行是备注。
continue
else:
_b = i.select('td')[2].get_text().strip()
end_1.append([_a,_b])
else:
_tmp1 = soup.select('table > tr[height="20"]')
for i in _tmp1:
_a = i.select('td')[0].get_text().strip()
if len(_a)>15 or _a=='行政区划代码': #部分数据页面，最后一行是备注。
continue
else:
_b = i.select('td')[1].get_text().strip()
end_1.append([_a,_b]) return end_1 #循环对每个链接 获取数据
end_3=[];#end_4=[]
for j in range(len(end_2)):
item = end_2[j]
if '' in item[1] or '' in item[1]:
print(j,item[0],item[1])
tmp2 = f3(item[0])
print('.')
end_3.extend([[item[1]]+i for i in tmp2])
#end_4.append(tmp2)
time.sleep(0.1) df_result = pd.DataFrame(end_3)
#pd.DataFrame(end_4).to_excel('所有连接.xlsx',index=False)
df_result.to_excel('地区编码.xlsx',index=False) '''
#\3 2019年5月份县以上行政区划代码_3852 > table > tbody > tr:nth-child(4)
#list_content > div.list_right > div > ul > table > tbody > tr:nth-child(1) > td.arlisttd > a
'''