网页编程网

没遇到反爬，暂不用控制爬速度。

# 下面这行一定要加不会会报错
# coding=utf-8
import json
import time

import pymysql
import requests

# 获取全国城市编码
def getCity():
    for pn in range(1, 60):
        # print('----------第' + str(pn) + '页----------' )
        parameter = {
            'query_type': 'TQUERY',
            'pagesize': '20',
            'pagenum': pn,
            'qii': 'true',
            'cluster_state': '5',
            'need_utd': 'true',
            'utd_sceneid': '1000',
            'div': 'PC1000',
            'addr_poi_merge': 'true',
            'is_classify': 'true',
            'zoom': '5',
            'city': '100000',
            'geoobj': '70.064603|26.107167|133.345871|41.927674',
            'keywords': '农贸市场'
        }

        url = 'https://www.amap.com/service/poiInfo?'
        try:
            res = requests.get(url, params=parameter)
            # time.sleep(1)
            res_json = json.loads(res.text)
            for word in (res_json["data"])["suggestion"]['regions']:
                if word["adcode"] != "":
                    print(word["adcode"] + '\t' + word["name"])
                    amap(word["adcode"], '宾馆')
        except:
            pass

# 查询所在城市关键词信息
def amap(adcode, keyword):
    for pn in range(1, 60):
        # print('----------第' + str(pn) + '页----------' )
        parameter = {
            'query_type': 'TQUERY',
            'pagesize': '20',
            'pagenum': pn,
            'qii': 'true',
            'cluster_state': '5',
            'need_utd': 'true',
            'utd_sceneid': '1000',
            'div': 'PC1000',
            'addr_poi_merge': 'true',
            'is_classify': 'true',
            'zoom': '11',
            'city': adcode,
            'geoobj': '37.325347|17.613722|163.887862|48.959349',
            'keywords': keyword
        }

        url = 'https://www.amap.com/service/poiInfo?'
        try:
            res = requests.get(url, params=parameter)
            res_json = json.loads(res.text)
            # time.sleep(2)
            for word in (res_json["data"])["poi_list"]:
                # if word["tel"] != "":
                print(word["cityname"] + '\t' + word["name"] + "\t" + word["tel"] + "\t" + word["address"])
                addMarketMysql(word)
                # file.write(word["cityname"] + '\t' + word["name"] + "\t" + word["tel"] + "\t" + word["address"] + '\n')
        except:
            pass

# 写入mysql
def addMarketMysql(word):
    # 连接 mysql
    db = pymysql.connect("localhost", "root", "root", "data")
    cursor = db.cursor()
    # 数据库的相关操作
    sql = "insert into market_info(city, marketName, phone, address) values (%s,%s,%s,%s)"
    # 执行数据库语句
    cursor.execute(sql, (word["cityname"], word["name"], word["tel"], word["address"]))
    # 提交数据库
    db.commit()
    print('save market done')
    db.close()


if __name__ == '__main__':
    # file = open('dance.txt', 'w', encoding='utf-8')
    keyword2 = '农贸市场'
    # file.write('----------高德----------' + '\n')
    getCity()
    # file.close()