数据分析案例之39药品网
前期准备
获取39药品网所有药品ID
1.下载39药品网所有药品页面
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/10/15
@Author: Zhang Yafei
"""
import re
import logging
import requests
import os
import time
from retrying import retry
from urllib.request import urljoin
from urllib.parse import urlsplit
# from scrapy import Selector
from lxml import etree
# from fake_useragent import UserAgent
from multiprocessing import Pool
from ids import Diabetes_ids
# ua = UserAgent()
# headers = {'User-Agent':ua.random}
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
proxies = {'http':'http://61.135.217.7:80','https':'http://171.113.156.168:8010'}
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
# DOWNLOAD_DIR = os.path.join(BASE_DIR,'药品')
DOWNLOAD_DIR = os.path.join(BASE_DIR,'糖尿病')
file_path = os.path.join(BASE_DIR,'drug_ruls.txt')
RUN_LOG_FILE = os.path.join(BASE_DIR,'log','run.log')
ERROR_LOG_FILE = os.path.join(BASE_DIR,'log','error_log')
if not os.path.exists(DOWNLOAD_DIR):
os.makedirs(DOWNLOAD_DIR)
class Logger(object):
"""
logger对象：打印日志
"""
def __init__(self):
self.run_log_file = RUN_LOG_FILE
self.error_log_file = ERROR_LOG_FILE
self.run_log = None
self.error_log = None
self.initialize_run_log()
self.initialize_error_log()
@staticmethod
def check_path_exist(log_abs_file):
log_path = os.path.split(log_abs_file)[0]
if not os.path.exists(log_path):
os.mkdir(log_path)
def initialize_run_log(self):
self.check_path_exist(self.run_log_file)
fh = logging.FileHandler(self.run_log_file, 'a', encoding='utf-8')
sh = logging.StreamHandler()
# fmt = logging.Formatter(fmt="%(asctime)s - %(levelname)s : %(message)s")
# fh.setFormatter(fmt)
# sh.setFormatter(fmt)
logger1 = logging.Logger('run_log', level=logging.INFO)
logger1.addHandler(fh)
logger1.addHandler(sh)
self.run_logger = logger1
def initialize_error_log(self):
self.check_path_exist(self.error_log_file)
fh = logging.FileHandler(self.error_log_file, 'a', encoding='utf-8')
sh = logging.StreamHandler()
# fmt = logging.Formatter(fmt="%(asctime)s - %(levelname)s : %(message)s")
# fh.setFormatter(fmt)
# sh.setFormatter(fmt)
logger1 = logging.Logger('error_log', level=logging.ERROR)
logger1.addHandler(fh)
logger1.addHandler(sh)
self.error_logger = logger1
def log(self, message, mode=True):
"""
写入日志
:param message: 日志信息
:param mode: True表示运行信息，False表示错误信息
:return:
"""
if mode:
self.run_logger.info(message)
else:
self.error_logger.error(message)
logger = Logger()
class Drug(object):
"""
self.base_url 药物抓取基础url=药品概述url
self.manual_url 药品详细说明书url
self.comment_url 药品用药经验url
self.ask_url 药品咨询url
self.logger 打印日志
"""
def __init__(self,base_url):
self.base_url = base_url
self.drug_id = self.base_url.split('/')[-2]
self.manual_url = urljoin(base_url,'manual')
self.comment_url = urljoin(base_url,'comment')
self.ask_url = urljoin(base_url,'ask')
self.make_drug_dir()
method_list = [self.summary,self.manual,self.comment,self.ask]
map(lambda x:x,[x() for x in method_list])
def make_drug_dir(self):
"""
创建每一种药品所有网页文件文件夹
:return:
"""
# self.check_download_dir()
response = requests.get(self.base_url,headers=headers)
response.encoding = response.apparent_encoding
html = etree.HTML(response.text)
# selector = Selector(response)
# drug_name = selector.css('.t1 h1 a::text').extract_first()
try:
drug_name = html.xpath('//div[@class="t1"]/h1/a/text()')[0]
except IndexError:
drug_name = html.xpath('//div[@class="t1"]/h1/text()')[0]
self.drug_name = self.validateTitle(drug_name)
self.drug_dir_path = os.path.join(DOWNLOAD_DIR,'{}[{}]'.format(self.drug_name,self.drug_id))
if not os.path.exists(self.drug_dir_path):
os.mkdir(self.drug_dir_path)
def validateTitle(self,title):
rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/ \ : * ? " < > |'
new_title = re.sub(rstr, "_", title) # 替换为下划线
return new_title
@retry(stop_max_attempt_number=3)
def retry_download(self, url):
"""
通过装饰器封装重试下载模块，最多重试三次
:param url_str: 下载网页的最终地址
:param data: Post传输数据
:param method: 下载方法GET或POST
:param proxies: 代理服务器
:return: 下载结果
"""
result = requests.get(url, headers=headers, proxies=proxies,timeout=3)
assert result.status_code == 200 # 使用断言判断下载状态，成功则返回结果，失败抛出异常
return result
def download(self, url):
"""
真正的下载类,代理模式
:param url_str:下载的链接
:param data:post需要传输的数据
:param method:请求方法
:param proxies:代理
:return:下载的结果
"""
try:
result = self.retry_download(url)
except Exception as e: # 异常处理尽量使用具体的异常
print(e)
# logger.log(url,False)
result = None
return result
def summary(self):
"""
抓取药品概述页
:return:
"""
summary_path = os.path.join(self.drug_dir_path,'{}[{}]-药品概述.html'.format(self.drug_name,self.drug_id))
if os.path.exists(summary_path):
print('{}药品概述已经下载过了'.format(self.drug_name))
else:
response = requests.get(self.base_url,headers=headers)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-药品概述下载失败-{}'.format(self.drug_name,self.drug_id,self.base_url),False)
logger.log('{}'.format(self.base_url),False)
return
response = response.content.decode('gb2312','ignore')
with open(summary_path,'w',encoding='gb2312') as file:
file.write(response)
logger.log('{}[{}]-药品概述下载完成'.format(self.drug_name,self.drug_id))
def manual(self):
"""
抓取药品详细说明书
:return:
"""
manual_path = os.path.join(self.drug_dir_path,'{}[{}]-详细说明书.html'.format(self.drug_name,self.drug_id))
if os.path.exists(manual_path):
print('{}详细说明书已经下载过了'.format(self.drug_name))
else:
response = requests.get(self.manual_url,headers=headers)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-详细说明书下载失败-{}'.format(self.drug_name,self.drug_id,self.manual_url),False)
logger.log('{}'.format(self.manual_url),False)
return
response = response.content.decode('gb2312','ignore')
with open(manual_path,'w',encoding='gb2312') as file:
file.write(response)
logger.log('{}[{}]-详细说明书下载完成'.format(self.drug_name,self.drug_id))
def comment(self):
"""
药品用药经验页
:return:
"""
response = requests.get(self.comment_url,headers=headers)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-用药经验下载失败'.format(self.drug_name,self.drug_id,self.comment_url),False)
logger.log('{}'.format(self.comment_url),False)
return
response = response.content.decode('gb2312','ignore')
html = etree.HTML(response)
try:
comment_nums = int(html.xpath('//div[@class="dps"]/cite/font/text()')[0])
except IndexError as e:
logger.log('{}[{}]-用药经验页评论数为零'.format(self.drug_name,self.drug_id))
comment_nums = 0
# selector = Selector(response)
# comment_nums = int(selector.css('.dps cite font::text').extract_first())
num,remainder = divmod(comment_nums,20)
for x in range(1,num+2):
url = urljoin(self.base_url,'comment/k0_p{}'.format(x))
self.comment_page(url)
def comment_page(self,url):
"""
抓取用药经验详情页
:param url:
:return:
"""
comment_path = os.path.join(self.drug_dir_path,'{}[{}]-用药经验{}.html'.format(self.drug_name,self.drug_id,url[-1]))
if os.path.exists(comment_path):
print('{}[{}]-用药经验{}已经下载过了'.format(self.drug_name,self.drug_id,url[-1]))
else:
response = requests.get(url,headers=headers)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-用药经验{}下载失败-{}'.format(self.drug_name,self.drug_id,url[-1],url),False)
logger.log('{}'.format(url),False)
return
response = response.content.decode('gb2312','ignore')
with open(comment_path,'w',encoding='gb2312') as file:
file.write(response)
logger.log('{}[{}]-用药经验{}下载完成'.format(self.drug_name,self.drug_id,url[-1]))
def ask(self):
"""
药品用药咨询页
:return:
"""
response = requests.get(self.ask_url)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-用药咨询下载失败-{}'.format(self.drug_name,self.drug_id,self.ask_url),False)
logger.log('{}'.format(self.ask_url),False)
return
response = response.content.decode('gb2312','ignore')
html = etree.HTML(response)
try:
ask_nums = html.xpath('//span[@class="pages"]/span[@class="pgleft"]/b/text()')[0]
ask_nums = int(re.match('.*?(\d+).*',ask_nums).group(1))
except Exception as e:
ask_nums = 0
logger.log('{}[{}]-用药咨询页无人提问'.format(self.drug_name,self.drug_id))
# selector = Selector(response)
# ask_nums = int(selector.css('.pages .pgleft b::text').re('\d+')[0])
num,remainder = divmod(ask_nums,5)
for x in range(1,num+2):
url = urljoin(self.base_url,'ask/p{}'.format(x))
self.ask_page(url)
def ask_page(self,url):
"""
抓取用药咨询详情页
:param url:
:return:
"""
ask_path = os.path.join(self.drug_dir_path,'{}[{}]-用药咨询{}.html'.format(self.drug_name,self.drug_id,url[-1]))
if os.path.exists(ask_path):
print('{}[{}]-用药咨询{}已经下载过了'.format(self.drug_name,self.drug_id,url[-1]))
else:
response = requests.get(url,headers=headers)
if response.status_code != 200:
response = self.download(self.base_url)
if not response:
# self.logger.log('{}[{}]-用药咨询{}下载失败-{}'.format(self.drug_name,self.drug_id,url[-1],url),False)
logger.log('{}'.format(url),False)
return
response = response.content.decode('gb2312','ignore')
with open(ask_path,'w',encoding='gb2312') as file:
file.write(response)
logger.log('{}[{}]-用药咨询{}下载完成'.format(self.drug_name,self.drug_id,url[-1]))
def transform_urls(filename):
drug_id = re.findall(r'.*?\[(\d+)\]', filename)[-1]
drug_url = 'http://ypk.39.net/{}/'.format(drug_id)
return drug_url
def check_downloaded(func):
def inner(drug_urls):
file_list = os.listdir(DOWNLOAD_DIR)
file_list = map(transform_urls,[filename for filename in file_list])
# print(len(list(file_list)))
files = set(drug_urls)-set(file_list)
# print(len(drug_urls))
# print(len(files))
func(list(files))
return inner
def get_drug_urls():
"""读取所有要抓取药品的url地址"""
with open(file_path,'r',encoding='utf-8') as f:
drug_urls = f.readlines()
drug_urls = list(map(lambda x: x.strip(), list(drug_urls)))
return drug_urls
def get_diabetes_urls():
return list(set(list(map(lambda x:'http://ypk.39.net/{}/'.format(x),Diabetes_ids))))
def main(drug_base_url):
"""创建Drug类实例，进行每一种药品的抓取"""
Drug(drug_base_url)
def validateTitle(title):
rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/ \ : * ? " < > |'
new_title = re.sub(rstr, "_", title) # 替换为下划线
return new_title
def spider(url):
url_path = urlsplit(url)
drug_id = url_path.path.strip('/')
try:
response = requests.get(url=url,headers=headers,timeout=3)
# response.encoding = response.apparent_encoding
response = response.content.decode('gb2312','ignore')
html = etree.HTML(response)
drug_name = html.xpath('//div[@class="t1"]/h1/text()')[0]
drug_name = validateTitle(drug_name)
except Exception as e:
print(e)
logger.log(url,False)
return
drug_dir_path = os.path.join(DOWNLOAD_DIR, '{}[{}]'.format(drug_name, drug_id))
if not os.path.exists(drug_dir_path):
os.mkdir(drug_dir_path)
drug_html_detail = os.path.join(drug_dir_path,'{}[{}].html'.format(drug_name,drug_id))
if not os.path.exists(drug_html_detail):
with open(drug_html_detail,'w',encoding='gb2312') as file:
file.write(response)
print(drug_name,'下载成功')
@check_downloaded
def run(drug_urls):
"""创建进程池"""
print(drug_urls)
print(len(drug_urls))
pool = Pool(5)
pool.map(main,drug_urls) #drug_urls[7010:12000]
# pool.map(spider,drug_urls)
pool.close()
pool.join()
if __name__ == '__main__':
drug_urls = get_drug_urls()
run(drug_urls)
# urls = get_diabetes_urls()
# run(urls)
具体实现：进程池，requests+lxml，打印日志，类，重新下载排除已下载页面
2.解析所有药品页面提取有价值信息
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/10/13
@Author: Zhang Yafei
"""
import csv
import json
import os
import re
from scrapy.selector import Selector
import logging
import pandas
# import numpy as np
BASE_DIRS = os.path.dirname(os.path.abspath(__file__))
drug_path = os.path.join(BASE_DIRS,'药品')
dirs_list = os.listdir(drug_path)
analysis_file_path = os.path.join(BASE_DIRS,'drug_info.tsv')
RUN_LOG_FILE = os.path.join(BASE_DIRS,'analysis_log','run.log')
ERROR_LOG_FILE = os.path.join(BASE_DIRS,'analysis_log','error_log')
def get_unresoved_drug_list():
"""得到未解析药品列表"""
data = pandas.read_csv('drug_info.tsv',sep='\t',encoding='utf-8')
try:
resoved_drug_list = data.apply(lambda row:'{}[{}]'.format(row['药品名称'],row['药品ID']),axis=1).tolist()
except AttributeError as e:
resoved_drug_list = []
for index,row in data.iterrows():
drug_name = '{}[{}]'.format(row['药品名称'],row['药品ID'])
resoved_drug_list.append(drug_name)
unresoved_drug_list = list(set(dirs_list) - set(resoved_drug_list))
return unresoved_drug_list
#1.index方式
# resoved_drug_list = []
# for row in data.index:
# drug_name = '{}[{}]'.format(data.iloc[row]['药品名称'],data.iloc[row]['药品ID'])
# resoved_drug_list.append(drug_name)
#2.iterrows方式
# for index,row in data.iterrows():
# drug_name = '{}[{}]'.format(row['药品名称'],row['药品ID'])
# resoved_drus_list.append(drug_name)
# print(dirs_list.__len__(),resoved_drug_list.__len__(),unresoved_drug_list.__len__())
def write_resoved_drag_list(drag):
"""将解析完成的药品写入文件"""
if not os.path.exists('resolved_drag_list.py'):
resoved_drag_list = set()
resoved_drag_list.add(drag)
else:
with open('resolved_drag_list.py', 'r', encoding='utf-8') as f:
resoved_drag_list = set(json.load(f))
resoved_drag_list.add(drag)
with open('resolved_drag_list.py','w',encoding='utf-8') as f:
json.dump(list(resoved_drag_list),f)
def write_error_drag_list(drag):
"""将错误drug写入文件"""
if not os.path.exists('error_drag_list.py'):
error_drag_list = set()
error_drag_list.add(drag)
else:
with open('error_drag_list.py', 'r', encoding='utf-8') as f:
error_drag_list = set(json.load(f))
error_drag_list.add(drag)
with open('error_drag_list.py','w',encoding='utf-8') as f:
json.dump(list(error_drag_list),f)
class Logger(object):
def __init__(self):
self.run_log_file = RUN_LOG_FILE
self.error_log_file = ERROR_LOG_FILE
self.run_log = None
self.error_log = None
self.initialize_run_log()
self.initialize_error_log()
@staticmethod
def check_path_exist(log_abs_file):
log_path = os.path.split(log_abs_file)[0]
if not os.path.exists(log_path):
os.mkdir(log_path)
def initialize_run_log(self):
self.check_path_exist(self.run_log_file)
fh = logging.FileHandler(self.run_log_file, 'a', encoding='utf-8')
sh = logging.StreamHandler()
fmt = logging.Formatter(fmt="%(asctime)s - %(levelname)s : %(message)s")
# fh.setFormatter(fmt)
sh.setFormatter(fmt)
logger1 = logging.Logger('run_log', level=logging.INFO)
logger1.addHandler(fh)
logger1.addHandler(sh)
self.run_logger = logger1
def initialize_error_log(self):
self.check_path_exist(self.error_log_file)
fh = logging.FileHandler(self.error_log_file, 'a', encoding='utf-8')
sh = logging.StreamHandler()
fmt = logging.Formatter(fmt="%(asctime)s - %(levelname)s : %(message)s")
# fh.setFormatter(fmt)
sh.setFormatter(fmt)
logger1 = logging.Logger('error_log', level=logging.ERROR)
logger1.addHandler(fh)
logger1.addHandler(sh)
self.error_logger = logger1
def log(self, message, mode=True):
"""
写入日志
:param message: 日志信息
:param mode: True表示运行信息，False表示错误信息
:return:
"""
if mode:
self.run_logger.info(message)
else:
self.error_logger.error(message)
class DrugInfo(object):
"""
提取的药品信息：
self.drug_name #药品名称
self.category #药品类型
self.cite #国家标准
self.company #生产厂家
self.address #厂家地址
self.license_number #批准文号
self.approval_date #批准日期
self.form_drug #剂型
self.spec #规格
self.store #储存方法
self.period_valid #有效期限
self.attention_rank #关注度排名
self.indication #适应症
self.component #成分
self.function #功能主治
self.usage_dosage #用法用量
self.contraindication #禁忌症
self.special_population #特殊人群用药
self.indications #适应症概况
self.is_or_not_medical_insurance #是否属于医保
self.is_or_not_infections #是否有传染性
self.related_symptoms #相关症状
self.related_examination #相关检查
self.adverse_reaction #不良反应
self.attention_matters #注意事项
self.interaction #药物相互作用
self.pharmacological_action #药理作用
self.revision_date #说明书修订日期
self.drug_use_consult #用药咨询
self.drug_use_experience #用药经验
"""
def __init__(self,drug):
drug_dir = os.path.join(drug_path, drug)
self.drug_name = re.findall('(.*?)\[\d+\]',drug)[0]
self.drug_id = re.findall('.*?\[(\d+)\].*',drug)[0]
self.drug_dir = drug_dir
self.drug_use_experience = ''
self.drug_use_consult = ''
self.file_list = os.listdir(self.drug_dir)
self.logger = Logger()
self.result = True
self.dispatch()
if self.drug_use_consult.__len__()==0:self.drug_use_consult = '无'
if self.drug_use_experience.__len__()==0:self.drug_use_experience = '无'
def dispatch(self):
for file in self.file_list:
if file.endswith('药品概述.html'):
self.drug_summary(self.file_path(file))
elif file.endswith('详细说明书.html'):
self.drug_instruction(self.file_path(file))
elif re.match('.*?用药咨询.*',file):
self.drug_consultation(self.file_path(file))
elif re.match('.*?用药经验.*',file):
self.drug_experience(self.file_path(file))
else:
self.result = False
break
def file_path(self,file):
return os.path.join(self.drug_dir,file)
def read_file(self,file):
with open(file,'r') as f:
html = f.read()
return html
def drug_summary(self,file):
"""药品概况"""
html = self.read_file(file)
selector = Selector(text=html)
self.category = selector.xpath('//div[@class="t1"]/cite[1]/span/text()').extract_first() #药品类型
if not self.category:
self.category = '未知'
self.cite = selector.xpath('//div[@class="t1"]/cite[2]/span/text()').extract_first() #国家标准
if not self.cite:
self.cite = '未知'
try:
self.company = selector.css('.t3 .company a::text').extract()[0] #生产厂家
except IndexError as e:
self.company = '未知'
try:
self.address = selector.css('.t3 .address::text').extract()[0] #厂家地址
except IndexError as e:
self.address = '未知'
try:
self.license_number = selector.xpath('//ul[@class="xxs"]/li[1]/text()').extract_first().strip() #批准文号
except AttributeError:
self.license_number = '未知'
try:
self.approval_date = selector.xpath('//ul[@class="xxs"]/li[2]/text()').extract_first().strip() #批准日期
except AttributeError:
self.approval_date = '未知'
try:
self.form_drug = selector.xpath('//ul[@class="showlis"]/li[1]/text()').extract_first().strip() #剂型
except AttributeError:
self.form_drug = '未知'
try:
self.spec = selector.xpath('//ul[@class="showlis"]/li[2]/text()').extract_first().strip() #规格
except AttributeError:
self.spec = '未知'
try:
self.store = selector.xpath('//ul[@class="showlis"]/li[3]/text()').extract_first().strip().strip('。') #储存方法
except AttributeError:
self.store = '未知'
try:
self.period_valid = selector.xpath('//ul[@class="showlis"]/li[4]/text()').extract_first().strip('。').replace('\n','') #有效期限
except AttributeError:
self.period_valid = '未知'
self.attention_rank = selector.css('.guanzhu cite font::text').extract_first() #关注度排名
if not self.attention_rank:
self.attention_rank = '未知'
self.indication = ','.join(selector.css('.whatsthis li::text').extract()) #适应症
if self.indication == '':
self.indication = '未知'
usage_dosage = selector.css('.ps p:nth-child(3)::text').extract_first() #用法用量
if usage_dosage:
self.usage_dosage = re.sub('<.*?>','',usage_dosage).strip().replace('\n','') #禁忌症
else:
self.usage_dosage = '未知'
indications = selector.css('#diseaseintro::text').extract_first() #适应症概况
if indications:
self.indications = re.sub('<.*?>','',indications).strip().replace('\n','') #禁忌症
else:
self.indications = '未知'
try:
self.is_or_not_medical_insurance = selector.css('.syz_cons p:nth-child(2)::text').extract_first().split('：')[1] #是否属于医保
except AttributeError as e:
self.is_or_not_medical_insurance = '未知'
try:
self.is_or_not_infections = selector.css('.syz_cons p:nth-child(3)::text').extract_first().split('：')[1].strip() #是否有传染性
except AttributeError as e:
self.is_or_not_infections = '未知'
self.related_symptoms = ','.join(selector.css('.syz_cons p:nth-child(4) a::text').extract()[:-1]) #相关症状
if len(self.related_symptoms) == 0:
self.related_symptoms = '未知'
self.related_examination = ','.join(selector.css('.syz_cons p:nth-child(5) a::text').extract()[:-1]) #相关检查
if len(self.related_examination) == 0:
self.related_examination = '未知'
def drug_instruction(self,file):
"""详细说明书"""
html = self.read_file(file)
selector = Selector(text=html)
#注：不同药品之间网页结构有差别，提取的时候应注意
component = selector.xpath('//dt[text()="【成份】"]/following::*[1]').extract_first()
if not component:
self.component = '未知'
else:
self.component = re.sub('<.*?>','',component).strip() #成分
contraindication= selector.xpath('//dt[text()="【禁忌】"]/following::*[1]').extract_first()
if contraindication:
self.contraindication = re.sub('<.*?>','',contraindication).strip().replace('\n','') #禁忌症
else:
self.contraindication = '未知'
function = selector.xpath('//dt[text()="【功能主治】"]/following::*[1]').extract_first()
if function:
self.function = re.sub('<.*?>','',function).strip() #功能主治
else:
self.function = '未知'
try:
self.adverse_reaction = selector.xpath('//dt[text()="【不良反应】"]/following::*[1]/p/text()').extract_first().strip('。') #不良反应
except AttributeError as e:
try:
self.adverse_reaction = selector.xpath('//dt[text()="【不良反应】"]/following::*[1]/text()').extract_first().strip('。') #不良反应
self.adverse_reaction = re.sub('<.*?>','',self.adverse_reaction).strip().replace('\n','') #注意事项
except AttributeError:
self.adverse_reaction = '未知'
attention_matters = selector.xpath('//dt[text()="【注意事项】"]/following::*[1]').extract_first()
if attention_matters:
self.attention_matters = re.sub('<.*?>','',attention_matters).strip().replace('\n','') #注意事项
else:
self.attention_matters = '未知'
self.logger.log('{}[{}]-注意事项为空'.format(self.drug_name,self.drug_id),False)
try:
self.interaction = selector.xpath('//dt[text()="【药物相互作用】"]/following::*[1]/p/text()').extract_first() #药物相互作用
self.interaction = re.sub('<.*?>','',self.interaction).strip().replace('\n','') #注意事项
except TypeError:
self.interaction = '未知'
try:
self.pharmacological_action = selector.xpath('//dt[text()="【药理作用】"]/following::*[1]/p/text()').extract_first() #药理作用
self.pharmacological_action = re.sub('<.*?>','',self.pharmacological_action).strip().replace('\n','')
except TypeError:
self.pharmacological_action = '未知'
try:
self.revision_date = selector.xpath('//dt[text()="【说明书修订日期】"]/following::*[1]/text()').extract_first().strip() #说明书修订日期
except AttributeError:
self.revision_date = '未知'
try:
self.special_population = selector.xpath('//dt[text()="【特殊人群用药】"]/following::*[1]/text()').extract_first() #特殊人群用药
self.special_population = re.sub('<.*?>','',self.special_population).strip().replace('\n','') #特殊人群用药
except TypeError:
self.special_population = '未知'
def drug_consultation(self,file):
"""用药咨询"""
html = self.read_file(file)
selector = Selector(text=html)
drug_use_consult = selector.css('.dpzx_con .zx p::text').extract()
drug_use_consult = ''.join(drug_use_consult)
drug_use_consult = re.sub('<.*?>','',drug_use_consult).strip().replace('\n','') #用药咨询
self.drug_use_consult += drug_use_consult
def drug_experience(self,file):
"""用药经验"""
html = self.read_file(file)
selector = Selector(text=html)
drug_use_experience = selector.css('.pls_box .pls_mid p::text').extract()
drug_use_experience = ''.join(drug_use_experience)
drug_use_experience = re.sub('<.*?>','',drug_use_experience).strip().replace('\n','') #用药经验
self.drug_use_experience += drug_use_experience.strip()
@staticmethod
def write_to_fileheader():
with open('drug_info.tsv','w',newline='',encoding='utf-8') as MyFile:
writer = csv.writer(MyFile,dialect='excel-tab')
drug_header = ['药品名称','药品ID','药品类型','国家标准','生产厂家','厂家地址','批准文号','批准日期','剂型','规格','储存方法','有效期限','关注度排名','适应症','成分','功能主治','用发用量','禁忌症','特殊人群用药','适应症概况','是否用于医保','是否具有传染性','相关症状','相关检查','不良反应','注意事项','药物相互作用','药理作用','说明书修订日期','用药经验','用药咨询']
writer.writerow(drug_header)
def write_to_file(self):
with open('drug_info.tsv', 'a', newline='', encoding='utf-8') as MyFile:
writer = csv.writer(MyFile, dialect='excel-tab')
druginfo_list = [self.drug_name,self.drug_id,self.category,self.cite,self.company,self.address,self.license_number,self.approval_date,
self.form_drug,self.spec,self.store,self.period_valid,self.attention_rank,
self.indication,self.component,self.function,self.usage_dosage,self.contraindication,
self.special_population,self.indications,self.is_or_not_medical_insurance,self.is_or_not_infections,
self.related_symptoms,self.related_examination,self.adverse_reaction,self.attention_matters,
self.interaction,self.pharmacological_action,self.revision_date,self.drug_use_experience,
self.drug_use_consult,
]
writer.writerow(druginfo_list)
self.logger.log('{}[{}]信息写入文件完毕'.format(self.drug_name,self.drug_id))
def main(drug):
"""主函数"""
druginfo = DrugInfo(drug)
# print(druginfo.drug_name,druginfo.drug_id)
# print(druginfo.drug_use_experience)
# print(druginfo.drug_use_consult)
if druginfo.result:
druginfo.write_to_file()
write_resoved_drag_list(drug)
else:
druginfo.logger.log('{}[{}]'.format(druginfo.drug_name,druginfo.drug_id),False)
write_error_drag_list(drug)
def new_data(row):
"""增加别名列"""
# print(row['药品名称'],row['别名'])
drug_name = row['药品名称']
try:
row['别名'] = drug_name.rsplit('(',1)[1].strip(')')
row['药品名称'] = drug_name.rsplit('(',1)[0]
except IndexError as e:
row['别名'] = np.NAN
return row
def update_drug_name():
"""更新药品文件信息"""
data = pandas.read_csv('drug_info.tsv',sep='\t',encoding='utf-8')
col_name = data.columns.tolist()
# print(dir(col_name))
col_name.insert(col_name.index('药品名称')+1,'别名')
# col_name.insert(1,'别名')
data = data.reindex(columns=col_name)
new_drug = data.apply(new_data,axis=1)
new_drug.to_csv('new_drug_info.tsv',index=False,sep='\t',encoding='utf-8')
print('文件保存成功')
# print(new_drug[['药品名称','别名']])
# for row in data[10:13].iterrows():
# drug_name = row['药品名称'].values
# drug_alias = drug_name.rsplit('(',1)[1].strip(')')
# print(drug_name)
# print(drug_alias)
# print(data.tail(10).index)
# print(data.iloc[:2,1:8])
# print(data.iloc[1]['注意事项'].replace('\n',''))
# print(data.iloc[2]['注意事项'].replace('\n',''))
# print(data.__len__())
# resoved_drus_list = data.apply(lambda row:'{}[{}]'.format(row['药品名称'],row['药品ID']),axis=1).tolist()
# print(resoved_drus_list.__len__())
# unresoved_drug_list = set(dirs_list) - set(resoved_drus_list)
# print(unresoved_drug_list.__len__())
if __name__ == '__main__':
if not os.path.exists(analysis_file_path):
DrugInfo.write_to_fileheader()
drug_list = get_unresoved_drug_list()
print(drug_list.__len__())
list(map(main,drug_list))
# with open('error_drag_list.py','r',encoding='utf-8') as f:
# data = json.load(f)
# print(data)
# update_drug_name()
# newdata = pandas.read_csv('new_drug_info.tsv',sep='\t',encoding='utf-8')
# print(newdata.head())
3.下载糖尿病相关药品页面
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/11/10
@Author: Zhang Yafei
"""
import json
import requests
from scrapy.selector import Selector
from lxml import etree
from multiprocessing import Pool
ids_list = []
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
def spider(url):
response = requests.get(url,headers=headers)
# selector = Selector(response=response)
html = etree.HTML(response.text)
# ids = selector.css('.search_ul li a:nth-child(1)::attr(href)').extract()
ids = html.xpath('//ul[@class="search_ul search_ul_yb"]/li/a/@href')
ids = list(map(lambda x:x.strip('/'),ids))
ids_list.extend(ids)
if __name__ == '__main__':
urls = ['http://ypk.39.net/tangniaobing/p{}'.format(i) for i in range(1,135)]
pool = Pool(4)
pool.map(spider,urls)
list(map(spider,urls))
with open('ids.py','w',encoding='utf-8') as f:
json.dump(ids_list,f)
4.更新药品信息，拆分药名列分为药品名称列和别名列
def new_data(row):
"""增加别名列"""
drug_name = row['药品名称']
try:
row['别名'] = drug_name.rsplit('(',1)[1].strip(')')
row['药品名称'] = drug_name.rsplit('(',1)[0]
except IndexError as e:
row['别名'] = np.NAN
return row
def update_drug_name():
"""更新药品文件信息"""
data = pandas.read_csv('drug_info.tsv',sep='\t',encoding='utf-8')
col_name = data.columns.tolist()
# print(dir(col_name))
col_name.insert(col_name.index('药品名称')+1,'别名')
# col_name.insert(1,'别名')
data = data.reindex(columns=col_name)
new_drug = data.apply(new_data,axis=1)
new_drug.to_csv('new_drug_info.tsv',index=False,sep='\t',encoding='utf-8')
print('文件保存成功')
if __name__ == '__main__':
update_drug_name()
5.抓取所有药品评论数，并构建药品评论数字典
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/11/10
@Author: Zhang Yafei
"""
import pandas
import os
import re
# import jieba
from multiprocessing.pool import Pool
from scrapy import Selector
import json
import numpy
import time
import csv
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
drug_path = os.path.join(BASE_DIR, '药品')
dirs_list = os.listdir(drug_path)
result = {}
k_list = []
v_list = []
class_list = []
# comment_data = pandas.read_csv('comment_num_grade.csv',encoding='utf-8')
# data = list(comment_data.药品名称.values)
# comment_data['类别'] = ''
count = 0
class DrugInfo(object):
"""构造药品评论数字典"""
def __init__(self, drug):
self.drug = drug
drug_dir = os.path.join(drug_path, drug)
self.drug_name = re.findall('(.*?)\[\d+\]', drug)[0]
self.drug_id = re.findall('.*?\[(\d+)\].*', drug)[0]
self.drug_dir = drug_dir
self.file_list = os.listdir(self.drug_dir)
self.dispatch()
def dispatch(self):
for file in self.file_list:
# if file.endswith('药品概述.html'):
# self.drug_summary(self.file_path(file))
# if re.match('.*?用药咨询.*',file):
# self.drug_consultation(self.file_path(file))
if re.match('.*?用药经验.*', file):
self.drug_experience(self.file_path(file))
def file_path(self, file):
return os.path.join(self.drug_dir, file)
def read_file(self, file):
with open(file, 'r') as f:
html = f.read()
return html
def drug_summary(self, file):
"""药品概况"""
html = self.read_file(file)
selector = Selector(text=html)
category = selector.xpath('//div[@class="subs"]/p/a[last()]/text()').extract_first()
print(category)
# class_list.append(category)
index = comment_data.loc[comment_data.药品名称 == self.drug, '类别'].index.values[0]
comment_data.loc[index, '类别'] = category
def drug_experience(self, file):
# print(file)
"""用药经验"""
html = self.read_file(file)
selector = Selector(text=html)
drug_use_experience_num = selector.css('.dps cite font::text').extract_first()
if not drug_use_experience_num:
self.drug_use_experience_num = 0
else:
self.drug_use_experience_num = int(drug_use_experience_num)
result[self.drug] = self.drug_use_experience_num
print(self.drug,self.drug_use_experience_num)
def write_to_file(self):
try:
with open('comment_num_grade.csv', 'a', newline='', encoding='utf_8_sig') as MyFile:
writer = csv.writer(MyFile)
druginfo_list = [self.drug,self.drug_use_experience_num]
writer.writerow(druginfo_list)
print('{}写入文件完毕'.format(self.drug))
except AttributeError:
return
def write_num():
with open('comment.py', 'w', encoding='utf-8') as f:
json.dump(result, f)
# for k,v in result.items():
# k_list.append(k)
# v_list.append(v)
data = {'药品名称': list(result.keys()), '评论数': list(result.values())}
df = pandas.DataFrame(data)
comment_data = df.sort_values(by='评论数', ascending=False)
comment_data.to_csv('comment_num_grade.csv', sep=',', encoding='utf_8_sig', mode='w', index=False)
return comment_data
def read_num():
with open('comment.py', 'r', encoding='utf-8') as f:
num = json.load(f)
for k, v in num.items():
k_list.append(k)
v_list.append(v)
data = {'药品名称': k_list, '评论数': v_list}
df = pandas.DataFrame(data)
comment_data = df.sort_values(by='评论数', ascending=False)
comment_data.to_csv('comment_num_grade.csv', sep=',', encoding='utf_8_sig', mode='w', index=False)
return comment_data
def main(drug):
"""主函数"""
DrugInfo(drug)
# try:
# result[d.drug] = d.drug_use_experience_num
# except:
# result[d.drug] = 0
# write_to_file(d)
if __name__ == '__main__':
start = time.time()
# pool = Pool(4)
# pool.map(main,dirs_list)
# pool.close()
# pool.join()
list(map(main,dirs_list))
write_num()
# comment_data.to_csv('new_comment_num_grade.csv',encoding='utf_8_sig',mode='w',index=False)
print('总花费:{}秒'.format(time.time() - start))
# comment_data = read_num()
# print(comment_data)
# print(len(num))
6.提取评论数量最多的前10个药品评论信息
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/11/10
@Author: Zhang Yafei
"""
import csv
import numpy
import pandas
import os
import re
import jieba
from scrapy import Selector
import re
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
drug_path = os.path.join(BASE_DIR,'药品')
dirs_list = os.listdir(drug_path)
comment_info_filename = 'first50_comment.csv'
class DrugInfo(object):
"""
提取的用户评论信息：
"""
def __init__(self,drug):
drug_dir = os.path.join(drug_path, drug)
self.drug_name = re.findall('(.*?)\[\d+\]',drug)[0]
self.drug_id = re.findall('.*?\[(\d+)\].*',drug)[0]
self.drug_dir = drug_dir
self.drug_use_experience = ''
self.file_list = os.listdir(self.drug_dir)
self.result = True
self.dispatch()
def dispatch(self):
for file in self.file_list:
# if re.match('.*?用药咨询.*',file):
# self.drug_consultation(self.file_path(file))
if re.match('.*?用药经验.*',file):
self.drug_experience(self.file_path(file))
def file_path(self,file):
return os.path.join(self.drug_dir,file)
def read_file(self,file):
with open(file,'r') as f:
html = f.read()
return html
def drug_experience(self,file):
print(file)
"""用药经验"""
html = self.read_file(file)
selector = Selector(text=html)
drug_use_experience = selector.css('.pls_box')
try:
page = selector.css('.dpzx .pages .pgleft span::text').extract()[0]
except IndexError:
page = 1
drug_url = 'http://ypk.39.net/{}/comment/k0_p{}'.format(self.drug_id,page)
if not drug_use_experience:
self.write_to_file(numpy.NAN,numpy.NAN,numpy.NAN,drug_url)
return
for drug in drug_use_experience:
self.drug_use_experience = drug.css('.pls_mid p::text').extract()[0].replace(' ','').strip('\n')
commter_info = drug.css('.pls_top cite::text').extract()[0].replace('\n','').strip('来自').strip(' ').replace(' ','/').rstrip('/')
cut_info = '/'.join(list(jieba.cut(self.drug_use_experience)))
cut_info = cut_info.strip('/ /')
time = drug.css('.pls_top i::text').extract()[0].strip().strip('点评时间：')
if not time:
time = numpy.NAN
self.write_to_file(commter_info,cut_info,time,drug_url)
def write_to_file(self,commter_info,cut_info,time,drug_url):
with open(comment_info_filename, 'a', newline='', encoding='utf_8_sig') as MyFile:
writer = csv.writer(MyFile)
druginfo_list = [self.drug_name,self.drug_id,commter_info,self.drug_use_experience,cut_info,time,drug_url]
writer.writerow(druginfo_list)
print('{}写入文件完毕'.format(drug_url))
def write_to_fileheader():
with open(comment_info_filename,'w',newline='',encoding='utf_8_sig') as MyFile:
writer = csv.writer(MyFile)
drug_header = ['药品名称','药品ID','评论者信息','评论','分词','评论时间','url']
writer.writerow(drug_header)
def main(drug):
"""主函数"""
DrugInfo(drug)
print('解析完成')
def read_comment_num_first50():
"""
读取前评论数前10多药品
:return: 评论数前10多药品名称的列表
"""
data = pandas.read_csv('concat_first50_comment.csv',encoding='utf-8')
drugs = data.药品名称.values.tolist()
drugs_id = list(map(lambda x:re.findall('\d+',x)[-1],drugs))
df = pandas.DataFrame({'drug_name':dirs_list})
drugs = list(map(lambda x:df[df.drug_name.str.contains(x)].drug_name.values,drugs_id))
drugs = list(filter(lambda x:x.__len__(),drugs))
return [x[0] for x in drugs]
if __name__ == '__main__':
if not os.path.exists(os.path.join(BASE_DIR,comment_info_filename)):
write_to_fileheader()
drugs = read_comment_num_first50()
print(drugs.__len__())
list(map(main,drugs))
print(drugs.__len__())
7.分析药品评论数量所占比例
# -*- coding: utf-8 -*-
"""
Created on Mon Nov 12 19:28:09 2018
@author: Zhang Yafei
"""
import json
import os
import pandas
#from wordcloud import WordCloud as wc
#from pyecharts import WordCloud
import matplotlib as mpl
from matplotlib import pyplot as plt
#import wordcloud
import numpy as np
from PIL import Image
data = pandas.read_csv('new_comment_num_grade.csv',encoding='utf-8')
drug_type_num = data.类别.value_counts()
drug_type_names = data.类别.value_counts().index.values
drug_type_dict = {}
def parse(drug_type_name):
drug_type_frequence = data[data['类别']==drug_type_name].评论数.sum()
drug_type_dict[drug_type_name] = int(drug_type_frequence)
def plot_wordcloud(drug_dict=None):
if drug_dict:
label = drug_dict.keys()
attr = drug_dict.values()
else:
label = drug_type_dict.keys()
attr = drug_type_dict.values()
wordcloud = WordCloud(width=800, height=620)
wordcloud.add('', label, attr, word_size_range=[20, 100])
wordcloud.render('drug_comment_wordcloud.html')
def plot_wc(drug_dict=None):
mask = np.array(Image.open('mask1.jpg'))
word_plot = wc(
font_path='font/simsun.ttc', # 设置字体格式
mask=mask, # 设置背景图
max_words=200, # 最多显示词数
max_font_size=100 # 字体最大值
)
if drug_dict:
word_plot = word_plot.generate_from_frequencies(drug_dict) # 从字典生成词云
else:
word_plot = word_plot.generate_from_frequencies(drug_type_dict) # 从字典生成词云
image_colors = wordcloud.ImageColorGenerator(mask) # 从背景图建立颜色方案
word_plot.recolor(color_func=image_colors) # 将词云颜色设置为背景图方案
plt.imshow(word_plot) # 显示词云
plt.axis('off') # 关闭坐标轴
plt.show() # 显示图像
word_plot.to_file('comment_num.jpg')
def plot_series_pie():
mpl.rcParams['font.sans-serif'] = ['SimHei']
pie_data = pandas.read_csv('drug_type_num_sum.csv',encoding='utf-8')
numbers = np.array(pie_data[pie_data.评论总数>0].评论总数)
drug_type = pie_data[pie_data.评论总数>0].类别名称
series = pandas.Series(numbers,index=drug_type,name='药物类型评论数饼状图')
series.plot.pie(figsize=(8,8),autopct='%.2f')
def plot_mpl_pie():
font = {
'family': 'SimHei'
}
mpl.rc('font', **font)
pie_data = pandas.read_csv('drug_type_num_sum.csv', encoding='utf-8')
numbers = np.array(pie_data[pie_data.评论总数 > 0].评论总数)
drug_type = pie_data.类别名称
plt.pie(numbers, labels=drug_type, autopct='%.2f%%',
shadow=True, labeldistance=1.1, startangle=90, pctdistance=0.6)
plt.title('药物类型评论数饼状图')
plt.savefig('药物类别与评论数量饼状图(mpl).png')
plt.show()
def type_drug_num_pie():
font = {
'family': 'SimHei'
}
mpl.rc('font', **font)
pie_data = pandas.read_csv('drug_type_num_sum.csv', encoding='utf-8')
numbers = np.array(pie_data.药品数量)
drug_type = pie_data.类别名称
plt.pie(numbers, labels=drug_type, autopct='%.2f%%',
shadow=True, labeldistance=1.1, startangle=90, pctdistance=0.6)
plt.title('药物类型药品数量数饼状图')
plt.savefig('药物类别与药品数量饼状图(mpl).png')
plt.show()
def wirte_to_file():
with open('comment_num_dict.py','w',encoding='utf-8') as f:
json.dump(drug_type_dict,f)
def read_from_file():
with open('comment_num_dict.py','r',encoding='utf-8') as f:
drug_type_dict = json.load(f)
return drug_type_dict
def write_type_num_to_file():
drug_type_dict = read_from_file()
type_name = list(drug_type_dict.keys())
type_num = list(drug_type_dict.values())
drug_type_nums = data.类别.value_counts().values
df_data = {'类别名称':type_name,'药品数量':drug_type_nums,'评论总数':type_num,}
df = pandas.DataFrame(df_data)
df.to_csv('drug_type_num_sum.csv',mode='w',encoding='utf_8_sig',index=False)
def write_new_file():
new_data = pandas.read_csv('drug_type_num_sum.csv', encoding='utf-8')
new_data['药品数量所占比例'] = round(new_data.药品数量/new_data.药品数量.sum(),4)
new_data['评论数量所占比例'] = round(new_data.评论总数/new_data.评论总数.sum(),4)
new_data.to_csv('drug_type_num_sum.csv',mode='w',encoding='utf_8_sig',index=False)
def main():
if os.path.exists('comment_num_dict.py'):
drug_dict = read_from_file()
# plot_wordcloud(drug_dict)
plot_wc(drug_dict)
else:
list(map(parse,drug_type_names))
wirte_to_file()
# plot_wordcloud()
plot_wc()
if __name__ == '__main__':
# 1.计算每人评论数量所占比例，并生成词云
# main()
# write_type_num_to_file()
# 2.画饼状图
# plot_series_pie()
# plot_mpl_pie()
# type_drug_num_pie()
# write_new_file()
8.前50药品数据合并
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 3 20:50:12 2018
@author: Zhang Yafei
"""
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
data1 = pd.read_csv('comment_num_grade_zhangyafei.csv',encoding='utf-8')
data2 = pd.read_csv('comment_num_grade_wangyuxin.csv',encoding='utf-8')
data3 = pd.read_csv('comment_num_grade_liangwenqi.csv',encoding='utf-8')
data4 = pd.read_csv('comment_num_grade_zhangxinrui.csv',encoding='utf-8')
data5 = pd.read_table('macaizhen.txt',encoding='utf-8',header=None,names=['药品名称','评论数'])
data6 = pd.read_csv('comment_num_grade_wangshuai.csv',encoding='utf-8')
data7 = pd.read_csv('comment_num_grade_wangqi.csv',encoding='utf-8')
data8 = pd.read_csv('tangao.txt',encoding='utf-8',delimiter='\t',header=None,names=['药品名称','评论数'])
data1['who'] = '张亚飞'
data2['who'] = '王于心'
data3['who'] = '梁雯琪'
data4['who'] = '张昕瑞'
data5['who'] = '马彩珍'
data6['who'] = '王帅'
data7['who'] = '王琪'
data8['who'] = '唐奥'
data_concat = pd.concat([data1,data2,data3,data4,data5,data6,data7,data8],ignore_index=True,sort=True).sort_values('评论数',ascending=False).reset_index().drop('index',axis=1)[:50]
print(data_concat)
data_concat.who.value_counts()
data_concat.评论数.sum()
groupby_data = data_concat.groupby(by='who')['评论数'].agg(np.sum)
data9 = pd.read_csv('first50_comment_zhangyafei.csv',encoding='utf-8')
data10 = pd.read_csv('first50_comment_zhangxinrui.csv',encoding='utf-8')
data11 = pd.read_csv('first50_comment_wangqi.csv',encoding='utf-8')
data12 = pd.read_csv('first50_comment_tangao.csv',encoding='utf-8')
data13 = pd.read_csv('first50_comment_wangshuai.csv',encoding='utf-8')
data14 = pd.read_csv('first50_comment_wangyuxin.csv',encoding='utf-8')
data15 = pd.read_csv('first50_comment_liangwenqi.csv',encoding='utf-8')
data16 = pd.read_csv('first50_comment_macaizhen.csv',encoding='utf-8')
data_concat2 = pd.concat([data9,data10,data11,data12,data13,data14,data15,data16],ignore_index=True)
def plot_hist():
"""画出评论数量分布直方图"""
font = {'family' : 'SimHei'}
matplotlib.rc('font', **font)
plt.figure(figsize=(15,8),dpi=80)
# x = data_concat.评论数.values
x = data_concat2.药品ID.value_counts().values
# num_bins 分组数
num_bins = int((max(x)-min(x))//10)
plt.hist(x,num_bins,facecolor='blue')
plt.xticks(range(int(min(x)),int(max(x))+10,10))
plt.grid(alpha=0.5)
plt.title('评论总数前50名药品数量分布状况')
plt.xlabel('评论数量')
plt.ylabel('分布情况')
plt.savefig('评论总数前50名药品数量分布状况1.png')
plt.show()
def plot_bar():
"""画出每个人的评论数量对比条形图"""
font = {'family':'SimHei'}
matplotlib.rc('font', **font)
plt.figure(figsize=(11,6),dpi=80)
plt.bar(groupby_data.index,groupby_data.values)
plt.xlabel('姓名')
plt.ylabel('评论数')
plt.title('评论数量前50名个人所占评论总数对比')
plt.savefig('评论数量前50名个人所占评论总数对比.png')
plt.show()
#plot_bar()
##
#plot_hist()
# df = pd.DataFrame(np.arange(24).reshape(6,4),columns=['A','B','C','D'])
## df[2] = 1
# df
# df[:1]
def label_recognition(df):
"""标注识别"""
# label1 = df[df.apply(lambda x:x.分词 == x.分词2,axis=1)]
label1 = df[df.分词 == df.分词2]
label2 = df[(df.分词 == df.分词2) & (df.分词 == df.分词3)]
return label1, label2
if __name__ == '__main__':
# data_concat.to_csv('concat_first50_comment.csv',encoding='utf_8_sig',index=False)
# data_concat2.to_csv('first50_comment.csv',encoding='utf_8_sig',index=False)
label1 = pd.read_excel(io='first50_comment_zhangxinrui2.xlsx',encoding='utf-8')
label,label2 = label_recognition(label1)
writer = pd.ExcelWriter('three_people_same_label.xlsx')
label2.to_excel(writer,'diabetes')
writer.save()
new_label = label.drop('分词2',axis=1)
new_label.to_csv('label.csv',encoding='utf_8_sig',index=False)
9.适应症和不良反应数据字典的构建
# -*- coding: utf-8 -*-
"""
@Datetime: 2018/1/10
@Author: Zhang Yafei
"""
import numpy
import re
from scrapy.selector import Selector
import pandas
from twisted.web.client import getPage, defer
from twisted.internet import reactor
pandas.set_option('display.max_columns', None)
data_list = []
n = 0
re_data = pandas.DataFrame(columns=['通用名称', '商品名称', '适应症', '不良反应', 'url'])
def parse(content, url):
"""
详细说明书中提取适应症和不良反应
:param content:
:param url:
:return: 通用名称 商品名称 适应症 不良反应 url
"""
global n
n += 1
print(n, url)
# text = content.decode('GB2312')
text = content.decode('gbk')
selector = Selector(text=text)
# 注：不同药品之间网页结构有差别，提取的时候应注意
drug_name = selector.xpath('//dt[text()="【药品名称】"]/following::*[1]').extract_first()
if not drug_name:
drug_name = selector.xpath('//dt[text()="【产品名称】"]/following::*[1]').extract_first()
generic_name = re.findall('通用名称：(.*)<br>', drug_name)[0]
trade_name = re.findall('商品名称：(.*)<br>', drug_name)[0]
# trade_name = numpy.NAN
function = selector.xpath('//dt[text()="【功能主治】"]/following::*[1]').extract_first()
if function:
function = re.sub('<.*?>', '', function).strip() # 功能主治
else:
function = numpy.NAN
indiction = selector.xpath('//dt[text()="【适应症】"]/following::*[1]')
if indiction:
indiction = indiction.xpath('string(.)').extract_first().strip().replace('\n', '')
else:
indiction = numpy.NAN
indictions = indiction if indiction is not numpy.NAN else function
try:
adverse_reaction = selector.xpath('//dt[text()="【不良反应】"]/following::*[1]/p/text()').extract_first().strip(
'。') # 不良反应
except AttributeError:
try:
adverse_reaction = selector.xpath('//dt[text()="【不良反应】"]/following::*[1]/text()').extract_first().strip(
'。') # 不良反应
adverse_reaction = re.sub('<.*?>', '', adverse_reaction).strip().replace('\n', '')
except AttributeError:
adverse_reaction = numpy.NAN
data = {'通用名称': generic_name, '商品名称': trade_name,
'适应症': indictions, '不良反应': adverse_reaction,
'url': url,
}
data_list.append(data)
def stop_loop(arg):
reactor.stop()
def main(url_list):
"""
主函数：利用twisted实现基于事件循环的异步非阻塞IO
:param url_list:
:return:
"""
# 制定任务计划：分配请求任务和添加回调函数
defered_list = []
for url in url_list:
defered = getPage(bytes(url, encoding='utf-8'))
defered.addCallback(callback=parse, url=url)
defered_list.append(defered)
# 将任务计划告诉领导和下属，并通知任务结束之后停止
dlist = defer.DeferredList(defered_list)
dlist.addBoth(stop_loop)
# 开始执行任务
reactor.run()
if __name__ == '__main__':
# 1.读取数据url下载响应信息
# data = pandas.read_excel('three_people_same_label.xlsx')
# url_list = ['http://ypk.39.net/{}/manual'.format(i) for i in data.药品ID.unique().tolist()]
# data = pandas.read_excel('drug_dict.xlsx')
# has_url = set(data.url.tolist())
# urls = list(set(url_list) - has_url)
# main(urls)
#
# # 2. 将下载信息写入文件
# df = pandas.DataFrame(data=data_list)
# df = df.loc[:, ['通用名称','商品名称','适应症','不良反应','url']]
# result = pandas.concat([data, df])
# writer = pandas.ExcelWriter('drug_dict.xlsx')
# result.to_excel(writer, 'drug_dict', index=False)
# writer.save()
# 3.合并39药品数据和不良反应数据库数据
# df1 = pandas.read_excel('adverse_reaction_database.xlsx')
# df2 = pandas.read_excel('drug_dict.xlsx')
# df2['适应症2'] = numpy.NAN
# df2['不良反应2'] = numpy.NAN
# print(df1.药品通用名称)
# print(df2.通用名称)
# index = df2.通用名称.apply(lambda x: x in df1.药品通用名称.values)
# df3 = df2.loc[index, :]
# df4 = pandas.DataFrame(columns=['药品通用名称', '适应症', '不良反应'])
# # df3.通用名称.apply(judge)
# for k in df3.通用名称.values:
# data = df1[df1.药品通用名称 == k]
# df4 = df4.append(data, ignore_index=True)
# writer = pandas.ExcelWriter('drug_dict2.xlsx')
# df4.to_excel(writer, 'drug_dict', index=False)
# writer.save()
# 4.读取drug_dict2.xlsx,合并相关数据
df4 = pandas.read_excel('drug_dict2.xlsx')
drug_list = []
for name in df4.药品通用名称.unique():
result = df4[df4.药品通用名称 == name]
indiction = '/'.join(str(s) for s in result.适应症.values if s is not numpy.NAN).strip()
adverse = '/'.join(str(s) for s in result.不良反应.values if s is not numpy.NAN).strip()
dict = {
'药品通用名称': name,
'适应症': indiction,
'不良反应': adverse,
}
drug_list.append(dict)
df5 = pandas.DataFrame(data=drug_list)
df5 = df5.loc[:, ['药品通用名称','适应症','不良反应']]
writer = pandas.ExcelWriter('database_dict.xlsx')
df5.to_excel(writer, sheet_name='database_dict', index=False)
writer.save()
浙公网安备 33010602011771号