-
-
Save alitrack/6b93032165c8d9687b84adc26a33ca70 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| #encoding=utf-8 | |
| import urllib2 | |
| import sys, json | |
| from StringIO import StringIO | |
| import gzip | |
| import MySQLdb | |
| import datetime | |
| reload(sys) | |
| sys.setdefaultencoding('utf-8') | |
| provinceList = ['北京市', '天津市', '河北省', '山西省', '内蒙古自治区', '辽宁省', '吉林省','黑龙江省', '上海市', '江苏省', '浙江省', '安徽省', '福建省', '江西省', '山东省','河南省', '湖北省', '湖南省', '广东省', '广西壮族自治区', '海南省', '重庆市','四川省', '贵州省', '云南省', '西藏自治区', '陕西省', '甘肃省', '青海省', '宁夏回族自治区', '新疆维吾尔自治区', '台灣', '香港特别行>政区', '澳门特别行政区']; | |
| send_headers = { | |
| 'Accept':'*/*', | |
| 'Accept-Encoding':'gzip, deflate, sdch', | |
| 'Accept-Language':'zh-CN,zh;q=0.8', | |
| 'Connection':'keep-alive', | |
| 'Host':'restapi.amap.com', | |
| 'Referer':'http://lbs.amap.com/fn/iframe/?id=3556', | |
| } | |
| def getCity(province): | |
| print "Province->" + province | |
| url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=province&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + province; | |
| data = getData(url) | |
| item = data['districts'][0]; | |
| if item['citycode'] == []: | |
| item['citycode'] = '' | |
| save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], '']) | |
| for item in data['districts'][0]['districts']: | |
| save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], '']) | |
| while 1: | |
| try: | |
| getDistrict(item) | |
| break | |
| except Exception, e: | |
| print 'retry:' + item['name'] + "->" + str(e) | |
| jsonp = 9999 | |
| def getJsonP(): | |
| global jsonp | |
| jsonp = jsonp + 1 | |
| if (jsonp > 99999): | |
| jsonp = 10000 | |
| return str(jsonp) | |
| def getDataWithEx(url): | |
| req = urllib2.Request(url,headers=send_headers) | |
| r = urllib2.urlopen(req,timeout=30) | |
| if r.info().get('Content-Encoding') == 'gzip': | |
| buf = StringIO(r.read()) | |
| f = gzip.GzipFile(fileobj=buf) | |
| data = f.read() | |
| else: | |
| data = r.read() | |
| return json.loads(data[13:-1]) | |
| def getData(url): | |
| while 1: | |
| try: | |
| response = getDataWithEx(url) | |
| break | |
| except Exception, e: | |
| print 'retry:' + url + " with error " + str(e) | |
| return response | |
| def save(value): | |
| try: | |
| conn=MySQLdb.connect(host='www.xxx.com',user='xxx',passwd='xxx',port=3306,charset="utf8") | |
| conn.select_db('test') | |
| cur=conn.cursor() | |
| cur.execute("insert into amap(citycode, adcode, name, center, level, areacode) values(%s,%s,%s,%s,%s,%s)",value) | |
| conn.commit() | |
| cur.close() | |
| conn.close() | |
| except MySQLdb.Error,e: | |
| print "Mysql Error %d: %s" % (e.args[0], e.args[1]) | |
| def getDistrict(city): | |
| print "->City->" + city['name'] | |
| url = "http://restapi.amap.com/v3/config/district?subdistrict=1&extensions=all&level=city&key=608d75903d29ad471362f8c58c550daf&s=rsv3&output=json&callback=jsonp_" + getJsonP() + "_&keywords=" + city['name'] | |
| data = getData(url) | |
| for possible in data['districts']: | |
| if possible['adcode'] == city['adcode']: | |
| for item in possible['districts']: | |
| save([item['citycode'], item['adcode'], item['name'], item['center'], item['level'], '']) | |
| while 1: | |
| try: | |
| getBusiness(item) | |
| break | |
| except Exception, e: | |
| print 'retry:' + item['name'] + "->" + str(e) | |
| break | |
| starttime = datetime.datetime.now() | |
| for province in provinceList: | |
| getCity(province) | |
| print 'over' | |
| endtime = datetime.datetime.now() | |
| print (endtime - starttime).seconds |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment