html_parser.py

coding=UTF-8

HTML解释器

import re

from bs4 import BeautifulSoup

class htmlParser():
def parse(self, url, html_cont):
if url is None or html_cont is None:
return
soup = BeautifulSoup(html_cont, 'html.parser',from_encoding='utf-8')
new_urls = self._get_new_urls(url, soup)
new_data = self._get_new_datas(url, soup)
return new_urls, new_data

def _get_new_urls(self, url, soup):
urls = set()
# 解释器
links = soup.find_all('a', href=re.compile(r'/item/\S+'))
for link in links:
new_url = link['href']
new_full_url = 'http://baike.baidu.com' + new_url
urls.add(new_full_url)
return urls

def _get_new_datas(self, url, soup):
rst_data = {}

url

rst_data['url'] = url

Python

title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find('h1') # class为关键字，需要后面加下划线
rst_data['title'] = title_node.get_text()

summary_node = soup.find('div', class_="lemma-summary")
rst_data['summary'] = summary_node.get_text()

return rst_data