爬虫面试案例系列01

### 需求:爬取https://m.vmall.com/help/hnrstoreaddr.htm荣耀线下门店中的门店详情信息。页面显示如下:

- 首页显示

 

 

 

- 详情页显示

 

 

 ### 基于抓包工具分析如下:

 

 

 ### 查看定位到数据包的请求头信息:

- 请求的url和请求方式如下:

 

 

 

- 请求携带的请求参数如下:

 

 

   - 注意:请求参数为字典格式并非常规的键值对,所以在代码实现中需要使用dumps将字典转成json串作为请求参数

### 代码实现:爬取到首页对应的门店信息

import requests
import json
url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList'
data = {
    "portal":"2",
    "lang":"zh-CN",
    "country":"CN",
    "brand":"1",
    "province":"河北省",
    "city":"邯郸市",
    "pageNo":"2",
    "pageSize":"20"}

#必须使用dumps操作
json_data = requests.post(url,data=json.dumps(data)).json()

 - 请求到的数据为:

{
	"code": "0",
	"shopInfos": [{
		"address": "河北省邯郸市磁县建设西路与京深线交叉口美食林一楼底商",
		"brand": 1,
		"city": "邯郸市",
		"dist": "磁县",
		"distance": 0.0,
		"id": 111616,
		"isStar": 1,
		"latitude": "36.364752",
		"longitude": "114.390096",
		"name": "荣耀授权体验店(建设西路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "9:00-18:00",
		"shopCode": "RYRA03100602",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市邱县育新街100号移动公司斜对过",
		"brand": 1,
		"city": "邯郸市",
		"dist": "邱县",
		"distance": 0.0,
		"id": 111897,
		"isStar": 1,
		"latitude": "36.815125",
		"longitude": "115.173926",
		"name": "荣耀授权体验店(育新街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100954",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市磁县友谊北大街199号",
		"brand": 1,
		"city": "邯郸市",
		"dist": "磁县",
		"distance": 0.0,
		"id": 108067,
		"isStar": 1,
		"latitude": "36.387417",
		"longitude": "114.379099",
		"name": "荣耀授权体验店(友谊北大街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03104185",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市肥乡县建安路中段",
		"brand": 1,
		"city": "邯郸市",
		"dist": "肥乡县",
		"distance": 0.0,
		"id": 111932,
		"isStar": 1,
		"latitude": "36.559004",
		"longitude": "114.813224",
		"name": "荣耀授权体验店(建安路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100613",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市临漳县邺都南大街移动公司旁",
		"brand": 1,
		"city": "邯郸市",
		"dist": "临漳县",
		"distance": 0.0,
		"id": 111678,
		"isStar": 1,
		"latitude": "36.33081",
		"longitude": "114.613037",
		"name": "荣耀授权体验店(邺都大街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100610",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市成安县西关口西行200路南",
		"brand": 1,
		"city": "邯郸市",
		"dist": "成安县",
		"distance": 0.0,
		"id": 107586,
		"isStar": 1,
		"latitude": "36.447977",
		"longitude": "114.692388",
		"name": "荣耀授权体验店(政府街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03103736",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市馆陶县新华街国土资源局斜对过",
		"brand": 1,
		"city": "邯郸市",
		"dist": "馆陶县",
		"distance": 0.0,
		"id": 111751,
		"isStar": 1,
		"latitude": "36.548521",
		"longitude": "115.294026",
		"name": "荣耀授权体验店(新华街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100615",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市武安市中兴路1666号",
		"brand": 1,
		"city": "邯郸市",
		"dist": "武安市",
		"distance": 0.0,
		"id": 106905,
		"isStar": 1,
		"latitude": "36.692459",
		"longitude": "114.191067",
		"name": "荣耀授权体验店(中兴路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100255",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市永年区洺兴路电子一条街口",
		"brand": 1,
		"city": "邯郸市",
		"dist": "永年区",
		"distance": 0.0,
		"id": 111771,
		"isStar": 1,
		"latitude": "36.782304",
		"longitude": "114.494515",
		"name": "荣耀授权体验店(洺兴路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100611",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市鸡泽县鸡泽镇会盟南大街89",
		"brand": 1,
		"city": "邯郸市",
		"dist": "鸡泽县",
		"distance": 0.0,
		"id": 111786,
		"isStar": 1,
		"latitude": "36.919418",
		"longitude": "114.882834",
		"name": "荣耀授权体验店(会盟南大街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:00-18:30",
		"shopCode": "RYRA03100614",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市曲周县振兴路移动手机大卖场",
		"brand": 1,
		"city": "邯郸市",
		"dist": "曲周县",
		"distance": 0.0,
		"id": 107216,
		"isStar": 1,
		"latitude": "36.780474",
		"longitude": "114.964419",
		"name": "荣耀授权体验店(振兴路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "9:00-18:30",
		"shopCode": "RYRA03101174",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市武安市新华大街145号",
		"brand": 1,
		"city": "邯郸市",
		"dist": "武安县",
		"distance": 0.0,
		"id": 107729,
		"isStar": 1,
		"latitude": "36.695474",
		"longitude": "114.184811",
		"name": "荣耀授权体验店(新华大街店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03103735",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市大名县大名府路东段路南(妇幼保健院对过)",
		"brand": 1,
		"city": "邯郸市",
		"dist": "大名县",
		"distance": 0.0,
		"id": 111832,
		"isStar": 1,
		"latitude": "36.288208",
		"longitude": "115.171058",
		"name": "荣耀授权体验店(大名府路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:00-18:30",
		"shopCode": "RYRA03100612",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市广平县人民路166号",
		"brand": 1,
		"city": "邯郸市",
		"dist": "广平县",
		"distance": 0.0,
		"id": 108001,
		"isStar": 1,
		"latitude": "36.488035",
		"longitude": "114.957763",
		"name": "荣耀授权体验店(人民路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03103980",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市邯山区陵园路114号金茂大厦一楼",
		"brand": 1,
		"city": "邯郸市",
		"dist": "邯山区",
		"distance": 0.0,
		"id": 126945,
		"isStar": 1,
		"latitude": "36.59796329",
		"longitude": "114.4948675",
		"name": "荣耀授权体验店(陵园路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "8:30-18:30",
		"shopCode": "RYRA03100100",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "河北省邯郸市涉县振兴路与龙山大街交叉口东南角",
		"brand": 1,
		"city": "邯郸市",
		"dist": "涉县",
		"distance": 0.0,
		"id": 111607,
		"isStar": 1,
		"latitude": "36.567255",
		"longitude": "113.680533",
		"name": "荣耀授权体验店(振兴路店)",
		"pictures": [],
		"province": "河北省",
		"score": 5,
		"serviceTime": "9:00-18:00",
		"shopCode": "RYRA03100599",
		"starShopPic": "https://res.vmallres.com/pimages//offlineshop/FOMuBZyeqTWYYJgMLPP2.jpg",
		"type": 1
	}, {
		"address": "中长街老检查院对面",
		"brand": 1,
		"city": "邯郸市",
		"dist": "鸡泽县",
		"distance": 0.0,
		"id": 137612,
		"isStar": 0,
		"latitude": "36.921823",
		"longitude": "114.882514",
		"name": "邯郸鸡泽航天中长街旗舰店",
		"pictures": [],
		"province": "河北省",
		"score": 4,
		"serviceTime": "10:00-18:30",
		"shopCode": "RY0000000001",
		"starShopPic": "",
		"type": 0
	}, {
		"address": "磁州路130号",
		"brand": 1,
		"city": "邯郸市",
		"dist": "磁县",
		"distance": 0.0,
		"id": 137613,
		"isStar": 0,
		"latitude": "36.379835",
		"longitude": "114.395066",
		"name": "邯郸磁县盛尧磁州路电信店",
		"pictures": [],
		"province": "河北省",
		"score": 4,
		"serviceTime": "10:00-18:30",
		"shopCode": "RY0000000001",
		"starShopPic": "",
		"type": 0
	}, {
		"address": "滏阳北大街新世纪商业广场1楼",
		"brand": 1,
		"city": "邯郸市",
		"dist": "峰峰矿区",
		"distance": 0.0,
		"id": 137614,
		"isStar": 0,
		"latitude": "36.428341",
		"longitude": "114.217691",
		"name": "邯郸峰峰太行新世纪店",
		"pictures": [],
		"province": "河北省",
		"score": 4,
		"serviceTime": "10:00-18:30",
		"shopCode": "RY0000000001",
		"starShopPic": "",
		"type": 0
	}, {
		"address": "建设街移动公司路南西侧泰山通讯",
		"brand": 1,
		"city": "邯郸市",
		"dist": "肥乡区",
		"distance": 0.0,
		"id": 137625,
		"isStar": 0,
		"latitude": "36.559058",
		"longitude": "114.811038",
		"name": "邯郸肥乡泰山建设街店",
		"pictures": [],
		"province": "河北省",
		"score": 4,
		"serviceTime": "10:00-18:30",
		"shopCode": "RY0000000001",
		"starShopPic": "",
		"type": 0
	}],
	"success": true,
	"totalRows": 60
}

- 请求到的数据分析:

  - 数据为门店相关数据,其中每一个门店有其对应的一个id值,我们需要将id值解析出来,在后面请求详情页会使用到

import requests
import json
url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList'
data = {
    "portal":"2",
    "lang":"zh-CN",
    "country":"CN",
    "brand":"1",
    "province":"河北省",
    "city":"邯郸市",
    "pageNo":"2",
    "pageSize":"20"}
#从中解析出id
json_data = requests.post(url,data=json.dumps(data)).json()
for dic in json_data['shopInfos']:
    _id = dic['id'] #解析出门店的id值

### 请求每一个门店详情页的数据

 

 

- 请求的url和请求方式:

 

 - 请求参数:

 

 发现只有shopId为动态变化的请求参数其他都是固定不变的,然后该shopId就是上一步我们解析出来的门店id,则基于门店id作为请求详情页的请求参数

import requests
import json
url = 'https://openapi.vmall.com/mcp/offlineshop/getShopList'
data = {
    "portal":"2",
    "lang":"zh-CN",
    "country":"CN",
    "brand":"1",
    "province":"河北省",
    "city":"邯郸市",
    "pageNo":"2",
    "pageSize":"20"}
#从中解析出id
json_data = requests.post(url,data=json.dumps(data)).json()
for dic in json_data['shopInfos']:
    _id = dic['id']
    #拼接详情页的url
    detail_url = 'https://openapi.vmall.com/mcp/offlineshop/getShopById?portal=2&version=10&country=CN&shopId={}&lang=zh-CN'.format(_id)
    finally_data = requests.get(url=detail_url).json()
    print(finally_data)#每一页详情页url的数据
    

  

 

posted @ 2020-02-17 10:32  波晓张  阅读(787)  评论(0编辑  收藏