Skip to content

Instantly share code, notes, and snippets.

@alitrack

alitrack/amap.city.py

Forked from witwall/amap.city.py
Created August 23, 2016 03:57
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alitrack/6b93032165c8d9687b84adc26a33ca70 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alitrack/6b93032165c8d9687b84adc26a33ca70 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
amap.city.py
# -*- coding: utf-8 -*-
#encoding=utf-8
import urllib2
import sys, json
from StringIO import StringIO
import gzip
import MySQLdb
import datetime
reload(sys)
sys.setdefaultencoding('utf-8')
provinceList = ['北京市', '天津市', '河北省', '山西省', '内蒙古自治区', '辽宁省', '吉林省','黑龙江省', '上海市', '江苏省', '浙江省', '安徽省', '福建省', '江西省', '山东省','河南省', '湖北省', '湖南省', '广东省', '广西壮族自治区', '海南省', '重庆市','四川省', '贵州省', '云南省', '西藏自治区', '陕西省', '甘肃省', '青海省', '宁夏回族自治区', '新疆维吾尔自治区', '台灣', '香港特别行>政区', '澳门特别行政区']
send_headers = {
'Accept':'*/*',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'zh-CN,zh;q=0.8',
'Connection':'keep-alive',
'Host':'restapi.amap.com',
'Referer':'http://lbs.amap.com/fn/iframe/?id=3556',
}
def getCity(province):
print "Province->" + province
url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=province&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + province;
data = getData(url)
item = data['districts'][0];
if item['citycode'] == []:
item['citycode'] = ''
save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], ''])
for item in data['districts'][0]['districts']:
save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], ''])
while 1:
try:
getDistrict(item)
break
except Exception, e:
print 'retry:' + item['name'] + "->" + str(e)
def getCity1(province):
#print "Province->" + province
url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=province&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + province;
data = getData(url)
if data['districts'] == []:
return
item = data['districts'][0];
if item['citycode'] == []:
item['citycode'] = ''
save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], ''])
for item in data['districts'][0]['districts']:
if item['citycode'] == []:
item['citycode'] = ''
save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], ''])
jsonp = 9999
def getJsonP():
global jsonp
jsonp = jsonp + 1
if (jsonp > 99999):
jsonp = 10000
return str(jsonp)
def getDataWithEx(url):
req = urllib2.Request(url,headers=send_headers)
r = urllib2.urlopen(req,timeout=30)
if r.info().get('Content-Encoding') == 'gzip':
buf = StringIO(r.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
else:
data = r.read()
return json.loads(data[13:-1])
def getData(url):
while 1:
try:
response = getDataWithEx(url)
break
except Exception, e:
print 'retry:' + url + " with error " + str(e)
return response
def save1(value):
print "insert into amap(citycode, adcode, name, center, level, areacode) values(%s);" % ', '.join(value)
def save(value):
try:
conn=MySQLdb.connect(host='localhost',user='root',port=3306,charset="utf8")
conn.select_db('base')
cur=conn.cursor()
cur.execute("insert into amap(citycode, adcode, name, center, level, areacode) values(%s,%s,%s,%s,%s,%s)",value)
conn.commit()
cur.close()
conn.close()
except MySQLdb.Error,e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
print value
def getDistrict(city):
print "->City->" + city['name']
url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=city&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + city['name']
data = getData(url)
for possible in data['districts']:
if possible['adcode'] == city['adcode']:
for item in possible['districts']:
save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], ''])
while 1:
try:
getBusiness(item)
break
except Exception, e:
print 'retry:' + item['name'] + "->" + str(e)
break
def getBusiness(district):
print "->->District->" + district['name']
url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=district&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + district['name']
data = getData(url)
for possible in data['districts']:
if possible['adcode'] == district['adcode']:
values=[]
for item in possible['districts']:
values.append((item['citycode'], item['adcode'], item['name'], item['center'], item['level'], item['areacode']))
saveAll(values)
break
def saveAll(values):
try:
conn=MySQLdb.connect(host='www.xxx.com',user='xxx',passwd='xxx',port=3306,charset="utf8")
conn.select_db('test')
cur=conn.cursor()
cur.executemany("insert into amap(citycode, adcode, name, center, level, areacode) values(%s,%s,%s,%s,%s,%s)",values)
conn.commit()
cur.close()
conn.close()
except MySQLdb.Error,e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
starttime = datetime.datetime.now()
for province in provinceList:
getCity1(province)
print 'over'
endtime = datetime.datetime.now()
print (endtime - starttime).seconds
Raw
create_table.sql
DROP TABLE IF EXISTS `amap`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `amap` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`citycode` varchar(10) NOT NULL DEFAULT '',
`adcode` varchar(20) NOT NULL DEFAULT '',
`name` varchar(20) NOT NULL DEFAULT '',
`center` varchar(50) NOT NULL DEFAULT '',
`level` varchar(10) NOT NULL DEFAULT '',
`areacode` varchar(20) NOT NULL DEFAULT '',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1224 DEFAULT CHARSET=utf8;
Raw
export2csv.sql
select * from amap
into outfile '/tmp/test.csv'
fields terminated by ',' optionally enclosed by '"' escaped by '"'
lines terminated by '\r\n';
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment