爬取58同城二手电脑一页的商品数据(35个商品),不要网站推广的商品,只爬取普通个人的商品;每个商品爬取的数据有:'标题'、价格'、'分类'、'区域'、'浏览量'(浏览量没能正确获取,用selenium能获取浏览量,在此没有具体描述)
58同城平板电脑页:https://bj.58.com/pbdn/0/
详情页:
https://bj.58.com/pingbandiannao/41525432516648x.shtml?link_abtest=&psid=195682258207684760008335092&entinfo=41525432516648_p&slot=-1&iuType=p_1&PGTID=0d305a36-0000-1ad2-2601-c5a3525b5373&ClickID=1
元素中有子孙级元素的多段文字,用stripped_strings处理,即:list(soup.select('div.nav')[0].stripped_strings)
import requests
from bs4 import BeautifulSoup
import time
# 给定商品概览页,爬取数据,返回多个电脑详情的url,列表形式
def get_computer_urls(url):
computer_urls = []
raw_page = requests.get(url).text
print('正在打开网页:',url)
time.sleep(3)
soup = BeautifulSoup(raw_page,'lxml')
eles = soup.select('tr[infotag*=commoninfo]>td>a') # 普通个人的商品,不是网站推广的商品
print('正在定位元素,选择器是:','tr[infotag*=commoninfo]>td>a')
for e in eles:
url = e.get('href')
computer_urls.append(url)
return computer_urls
# 给定电脑详情页,返回电脑信息,字典格式
def get_computer_info(url):
raw_page = requests.get(url).text
print('正在打开网页:', url)
time.sleep(3)
soup = BeautifulSoup(raw_page,'lxml')
title = soup.select('h1.detail-title__name')[0].get_text().strip()
category = list(soup.select('div.nav')[0].stripped_strings)
category = ''.join(category)
price = soup.select('span.infocard__container__item__main__text--price')[0].get_text().strip()
address = list(soup.select('div.infocard__container__item__main')[1].stripped_strings)
address = ''.join(address)
look_num = soup.select('span#totalcount')[0].get_text().strip()
#print({'标题':title,'价格':price,'分类':category,'区域':address,'浏览量':look_num})
return {'标题':title,'价格':price,'分类':category,'区域':address,'浏览量':look_num}
start_url = 'https://bj.58.com/pbdn/0/'
computer_urls = get_computer_urls(start_url)
print(computer_urls)
print('len(computer_urls):',len(computer_urls))
time.sleep(3)
i = 0
for url in computer_urls:
i +=1
dict = get_computer_info(url)
print(i, dict)