爬虫练习-爬取小说

# 程序启动文件    start.py
#
!/usr/bin/python # -*- coding: utf-8 -*- import os, sys BASEPATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) print(BASEPATH) sys.path.append(BASEPATH) from core import SpiderMan if __name__ == '__main__': s=SpiderMan.SpiderMan() s.async()
# 爬虫调度器
#
!/usr/bin/python # -*- coding: utf-8 -*- from gevent import spawn,monkey,joinall;monkey.patch_all() from concurrent.futures import ThreadPoolExecutor from core.UrlManager import UrlManager from core.Htmldown import Htmldown from core.Htmlparser import Htmlparser # from core.DataOutput import DataOutput class SpiderMan: def __init__(self): self.manager=UrlManager() #url管理 self.downloader=Htmldown() #HTML下载 self.parser=Htmlparser() #HTML解析 # self.output=DataOutput() def index_work(self): ''' 爬取凌霄主页 ''' url = 'http://www.lingxiaozhishang.com' self.manager.oldurls.add(url) #列表中添加每次传进来的url html_dict = self.downloader.down_page(url) #下载器下载 if html_dict is None: # raise print("爬取主页出错了") print("爬取主页出错了") return None new_urls = self.parser.parser_index(html_dict,url) # 解析二层链接 self.manager.add_urls(new_urls) # 所有的a标签存放的列表 print("爬取 主页 + 所有文章url 完成") def async(self): ''' 开启协程 ''' self.index_work() pool = ThreadPoolExecutor(10) # 开启十个线程池 while True: url = self.manager.get_url() # 从url管理器中获取url if url is None: break pool.submit(self.downloader.down_page,url).add_done_callback(self.parser.parser_page) # 提交下载任务,解析 pool.shutdown(wait=True) #最后得关闭线程池 print("完了-----------------------")
# URL管理器
#
!/usr/bin/python # -*- coding: utf-8 -*- class UrlManager: def __init__(self): self.newurls=set() self.oldurls=set() def add_url(self,newurl): ''' 添加小说章节的url :return: ''' if newurl not in self.oldurls: self.newurls.add(newurl) def add_urls(self,newurls): ''' 添加多个小说章节的url :param newurls: :return: ''' if len(newurls)==0:return for url in newurls: self.add_url(url) def get_url(self): ''' 取出一个小说章节的url :return: ''' try: url = self.newurls.pop() if url is not None: self.oldurls.add(url) return url except KeyError: pass def has_oldurls(self): ''' 返回已爬小说章节的数量 :return: ''' return len(self.oldurls)
# HTML下载器
#
!/usr/bin/python # -*- coding: utf-8 -*- import requests class Htmldown: def down_page(self,url): ''' 下载网页内容 ''' headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0'} r=requests.get(url,headers=headers) r.encoding='utf8' if r.status_code==200: return r.text
# HTML解析器      解析完直接存储到文件了,应该持久化到MongoDB中
#
!/usr/bin/python # -*- coding: utf-8 -*- from bs4 import BeautifulSoup class Htmlparser: def parser_index(self,html_conf,url): soup = BeautifulSoup(html_conf, 'html.parser') list_a = soup.find(class_="chapterlist").find_all('a') new_urls=[] for a in list_a: #url=http://www.lingxiaozhishang.com #/book/439.html new_url ="%s%s"%(url,a.attrs["href"]) new_urls.append(new_url) return new_urls def parser_page(self,html_conf): ''' 解析小说章节页面 :param html_conf: :return: ''' html_conf =html_conf.result() soup=BeautifulSoup(html_conf,'html.parser') title = soup.find('h1').get_text() text = soup.find(id="BookText").get_text() filepath = r"C:\Users\Administrator\Desktop\Article\db\%s.txt"%title with open(filepath,"w") as f: f.write(text) print("%s 下载完成"%title)

 

posted @ 2018-01-24 18:33  前方、有光  阅读(1105)  评论(0编辑  收藏  举报