爬取行政区划代码

恰克与飞鸟

爬取国家统计局统计用区划代码和城乡划分代码 2023 版

python 实现

一、打开国家统计局官网

https://www.stats.gov.cn/sj/tjbz/qhdm/

二、分析每一级URL找到规律

省级:https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/index.html
地市级：https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/61.html 61为陕西编码
区县级：https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/61/6101.html

找到规律 当前路径+href 路径即可跳入下一级

打码

1. import json
2. import time
4. import requests
5. from bs4 import BeautifulSoup
7. main_url = "https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023"
10. class area_code:
11.     name = ""
12.     code = ""
13.     url = ""
14.     child = []
15.     urban_rural_type = 0
16.     lng = 0
17.     lat = 0
19.     def __init__(self, name, code, url, child, urban_rural_type=0):
20.         self.name = name
21.         self.code = code
22.         self.url = url
23.         self.child = child
24.         self.urban_rural_type = urban_rural_type
25.         self.lng = 0
26.         self.lat = 0
29. # 爬取全国统计用区划代码和城乡划分代码
30. # pip install beautifulsoup4
31. def get_code(suffix_url="index.html"):
32.     _province_url = "{}/{}".format(main_url, suffix_url)
33.     response = requests.get(_province_url)
34.     response.encoding = "utf-8"
35.     _html = response.text
36.     _soup = BeautifulSoup(_html, "html.parser")
37.     _province_code = {}
38.     for a in _soup.find_all("a"):
39.         if a.get("href") and a.get("href").endswith(".html"):
40.             _province_code[a.text] = a.get("href")
41.     return _province_code
44. def get_child_code(_url, _parent_url=None, _retry=3):
45.     """
46.     输出 [{name:"呼和浩特市", code:"150100000000", url:"15/1501.html"},{name:"包头市", code:"150200000000", url:"15/1502.html"}]
47.     :param _parent_url: 父级url
48.     :param _retry: 重试次数
49.     :param _url: 当前url
50.     :return:
51.     """
52.     _city_code = []
53.     if _parent_url is not None and len(_parent_url) > 0:
54.         # 截取最后一个"/"之前的字符串
55.         _parent_path = _parent_url.rsplit("/", 1)[0]
56.         _req_url = "{}/{}".format(_parent_path, _url)
57.     else:
58.         _req_url = "{}/{}".format(main_url, _url)
59.     try:
60.         response = requests.get(_req_url)
61.     except Exception as e:
62.         if _retry > 0:
63.             time.sleep(1)
64.             print("请求出错：{},第{}次重试".format(e, 4 - _retry))
65.             return get_child_code(_url, _parent_url, _retry - 1)
66.         else:
67.             raise e
68.     response.encoding = "utf-8"
69.     _html = response.text
70.     _soup = BeautifulSoup(_html, "html.parser")
72.     # class_="citytr" or class_="towntr" or class_="countytr" or class_="villagetr"
73.     for tr in _soup.find_all("tr", class_=["citytr", "towntr", "countytr"]):
74.         _tds = tr.find_all("td")
75.         print("开始处理 - {}".format(_tds[1].text))
76.         _child_url = ""
77.         if _tds[0].find("a") is not None and _tds[0].find("a").get("href") is not None:
78.             _child_url = _tds[0].find("a").get("href")
79.             if _child_url.endswith(".html"):
80.                 _child = get_child_code(_child_url, _req_url)
81.                 _city_code.append(area_code(_tds[1].text, _tds[0].text, _child_url, _child))
82.         else:
83.             _city_code.append(area_code(_tds[1].text, _tds[0].text, _child_url, []))
84.     for tr in _soup.find_all("tr", class_=["villagetr"]):
85.         _tds = tr.find_all("td")
86.         code = _tds[0].text
87.         urban_rural_type = _tds[1].text
88.         name = _tds[2].text
89.         _city_code.append(area_code(name, code, "", [], urban_rural_type))
90.     return _city_code
93. def get_province_list():
94.     """
95.     # 获取省份、直辖市、自治区代码
96.     :return:
97.     """
98.     province_map = get_code()
99.     _province_list = []
100.     for _name, _url in province_map.items():
101.         _province_list.append(area_code(_name, _url.split(".")[0], _url, []))
102.     return _province_list
105. if __name__ == '__main__':
106.     province_list = get_province_list()
107.     # 获取市级代码
108.     for province in province_list:
109.         print("开始处理 - {}".format(province.name))
110.         city_code = get_child_code(province.url)
111.         province.child = city_code
112.     # 输出到文件json
113.     with open("area_code.json", "w", encoding="utf-8") as f:
114.         f.write(json.dumps(province_list, default=lambda obj: obj.__dict__, ensure_ascii=False))

复制代码

缺陷

• json格式太大了，建议直接入库或者生成cvs
  
  
  
• 不支持退出续爬，后续完善....
  

来源:https://www.cnblogs.com/bigroc/p/18037559
免责声明：由于采集信息均来自互联网，如果侵犯了您的权益，请联系我们【E-Mail:cb@itdo.tech】 我们会及时删除侵权内容，谢谢合作！