1、创建表结构
本爬虫主要爬取国家统计局省市区数据，数据存储采用Mysql，首先在Mysql数据库中创建存储省市区数据的表，脚本如下：
CREATE TABLE `t_areas` (
`id` int NOT NULL AUTO_INCREMENT COMMENT '主键ID',
`pid` int DEFAULT NULL COMMENT '父ID',
`name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '名称',
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '下级数据URL',
`level` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '级别',
`code` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT '编码',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=552 DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='省市区表';
2、需要引入的包
import pymysql
import requests
import lxml.etree as etree
import os
说明：
pymysql：用于操作Mysql数据库
requests：用于处理请求
BeautifulSoup：用于解析Html Dom树
os：操作文件路径
3、创建基础代码结构
class ChineseArea(object):
"""获取省市区数据"""
def __init__(self):
"""用户初始化数据库连接及定义存储数据的属性"""
pass
def __del__(self):
"""关闭数据库连接"""
pass
def get_province_data(self, url):
"""爬行政区划代码公布页省级数据"""
pass
def parse_province(self):
"""解析省页，返回list"""
pass
def parse_sub_data(self, level, pid, url):
"""根据level解析子页数据"""
pass
def parse_villager(self, level, pid, url):
"""解析社区页数据"""
pass
def insert_to_db(self, value):
"""将数据保存到Mysql数据库中"""
pass
def parse_areas(self):
"""对外接口，用于获取所有省市区数据"""
pass
if __name__ == '__main__':
"""程序入口"""
pass
4、各个代码实现细节
4.1、初始化
def __init__(self):
"""用户初始化数据库连接及定义存储数据的属性"""
self.baseUrl = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html'
self.base = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/'
self.conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="root", db="test", charset='utf8')
self.cur = self.conn.cursor()
self.levels = {
1: '//tr[@class="provincetr"]',
2: '//tr[@class="citytr"]',
3: '//tr[@class="countytr"]',
4: '//tr[@class="towntr"]',
5: '//tr[@class="villagetr"]'
}
4.2、对象删除
def __del__(self):
"""关闭数据库连接"""
if self.cur:
self.cur.close()
if self.conn:
self.conn.close()
4.3、获取省级数据
def get_province_data(self, url):
"""爬取行政区划代码公布页省级数据"""
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
i = 0
while i < 3:
try:
html = requests.get(url, headers=headers, timeout=20)
html.encoding = 'gbk' # 这里添加一行
text = html.text
return text
except requests.exceptions.RequestException:
i += 1
print('超时' + url)
4.4、解析省级数据
def parse_province(self):
"""解析省级数据，返回省列表数据"""
html = self.get_province_data(self.baseUrl)
tree = etree.HTML(html, parser=etree.HTMLParser(encoding='gbk'))
nodes = tree.xpath('//tr[@class="provincetr"]')
values = []
for node in nodes:
items = node.xpath('./td')
for item in items:
value = {}
next_url = item.xpath('./a/@href')
province = item.xpath('./a/text()')
if province:
print(province)
value['url'] = self.base + "".join(next_url)
value['name'] = "".join(province)
value['code'] = "".join(next_url)[:2] + "0000000000"
value['pid'] = 0
value['level'] = 1
print(repr(value['name']))
last_id = self.insert_to_db(value)
value['id'] = last_id
values.append(value)
print(value)
return values
4.5、根据级别解析子页数据
def parse_sub_data(self, level, pid, url):
"""根据级别解析子页数据"""
if url.strip() == '':
return None
# url_prefix+url
html = self.get_province_data(url)
tree = etree.HTML(html, parser=etree.HTMLParser(encoding='gbk'))
if level == 3:
nodes = tree.xpath(self.levels.get(level))
if len(nodes) == 0:
nodes = tree.xpath(self.levels.get(4))
print('有镇的市：' + url)
else:
nodes = tree.xpath(self.levels.get(level))
path = os.path.basename(url)
base_url = url.replace(path, '')
values = []
# 多个城市
for node in nodes:
value = {}
next_url = node.xpath('./td[1]/a/@href')
if len(next_url) == 0:
next_url = ''
code = node.xpath('./td[1]/a/text()')
if len(code) == 0:
code = node.xpath('./td[1]/text()')
name = node.xpath('./td[2]/a/text()')
if len(name) == 0:
name = node.xpath('./td[2]/text()')
value['code'] = "".join(code)
temp_url = "".join(next_url)
if len(temp_url) != 0:
value['url'] = base_url + "".join(next_url)
else:
value['url'] = ''
value['name'] = "".join(name)
print(repr(value['name']))
print(value['url'])
value['pid'] = pid
value['level'] = level
last_id = self.insert_to_db(value)
value['id'] = last_id
values.append(value)
print(value)
return values
4.6、解析社区页数据
def parse_villager(self, level, pid, url):
"""解析社区页数据"""
html = self.get_province_data(url)
tree = etree.HTML(html, parser=etree.HTMLParser(encoding='gbk'))
nodes = tree.xpath(self.levels.get(level))
values = []
# 多个城市
for node in nodes:
value = {}
nexturl = node.xpath('./td[1]/a/@href')
code = node.xpath('./td[1]/text()')
vcode = node.xpath('./td[2]/text()')
name = node.xpath('./td[3]/text()')
value['code'] = "".join(code)
value['url'] = "".join(nexturl)
value['name'] = "".join(name)
print(repr(value['name']))
value['pid'] = pid
value['level'] = level
values.append(value)
last_id = self.insert_to_db(value)
value['id'] = last_id
values.append(value)
print(value)
return values
4.7、保存数据到数据库
def insert_to_db(self, value):
"""将数据保存到Mysql数据库中"""
last_id = 0
try:
sql = 'INSERT INTO t_areas(pid,name,url,level,code) values(%s,%s,%s,%s,%s)'
param = (value.get("pid"), value.get("name"), value.get("url"), value.get("level"), value.get("code"))
self.cur.execute(sql, param)
last_id = self.cur.lastrowid
self.conn.commit()
except Exception as e:
print(e)
self.conn.rollback()
return last_id
4.8、对外获取数据接口
def parse_areas(self):
"""对外接口，用于获取所有省市区数据"""
values = self.parse_province()
for value in values:
cities = self.parse_sub_data(2, value['id'], value['url'])
if cities:
for city in cities:
counties = self.parse_sub_data(3, city['id'], city['url'])
# 这个下面是获取 乡镇和居委会数据 如果不需要删除就可以了
# if counties:
# for county in counties:
# towns = self.parse_sub_data(4, county['id'], county['url'])
# if towns:
# for town in towns:
# self.parse_villager(5, town['id'], town['url'])
4.9、程序入口
if __name__ == '__main__':
"""程序入口"""
chinese_area = ChineseArea()
chinese_area.parse_areas()
5、代码运行结果
6、爬取完成后存储结果
【生长吧！Python】有奖征文火热进行中：https://bbs.huaweicloud.com/blogs/278897