python学习之小说爬虫 

 1 # coding:utf8
 2 from multiprocessing.dummy import Pool as ThreadPool
 3 import multiprocessing
 4 import requests, os, codecs, time
 5 from lxml import etree
 6 
 7 url = 'https://www.biquge5200.cc/79_79883/'  # 要下载的小说章节列表页面url
 8 
 9 
10 def getsource(url):
11     try:
12         s = requests.get(url)
13     except:
14         print('访问异常，跳过~！')
15     else:
16         s.encoding = 'gbk'
17         return s.text
18 
19 
20 def getlist(url):
21     global txtname, txtzz
22     #解析地址
23     html = getsource(url)
24     ehtml = etree.HTML(html)
25     u = ehtml.xpath('//*[@id="list"]/dl/dd/a/@href')
26     t = ehtml.xpath('//*[@id="list"]/dl/dd/a/text()')
27     txtname = ehtml.xpath('//*[@id="info"]/h1/text()')[0].replace('\\', '').replace('/', '').replace(':', '').replace(
28         '*', '').replace('?', '').replace('"', '').replace('<', '').replace('>', '').replace('|', '')
29     txtzz = ehtml.xpath('//*[@id="info"]/p[1]/text()')[0].replace('\xa0', '')
30     num = 0
31     #循环urllist
32     for i in range(9, len(u)):
33         urllist.append(u[i] + '|' + t[i] + '|' + str(num))
34         print(urllist)
35         print(u[i] + '|' + t[i] + '|' + str(num))
36         num += 1
37 
38 
39 def downtxt(url):
40     global downcount
41     u = url.split('|')[0]
42     t = url.split('|')[1]
43     num = url.split('|')[2]
44     content = ''
45     while len(content) == 0:
46         html = getsource(u)
47         ehtml = etree.HTML(html)
48         content = ehtml.xpath('string(//*[@id="content"])').replace('    ', '\r\n').replace('　　', '\r\n').replace(
49             '\xa0', '').replace('\ufffd', '').replace('\u266a', '').replace('readx;', '')
50     if os.path.exists(savepath + num + '.txt'):
51         print(num + '.txt 已经存在!')
52     else:
53         with codecs.open(savepath + num + '.txt', 'a')as f:
54             f.write('\r\n' + t + '\r\n' + content)
55         print(t + ' 下载完成!')
56         downcount += 1
57 
58 
59 time_start = time.time();
60 downcount = 0
61 urllist = []
62 getlist(url)
63 savepath = os.getcwd() + '\\' + txtname + '\\'
64 if os.path.exists(savepath) == False:
65     os.makedirs(savepath)
66 pool = ThreadPool(multiprocessing.cpu_count())
67 results = pool.map(downtxt, urllist)
68 pool.close()
69 pool.join()
70 print('开始合并txt...')
71 with codecs.open(savepath + txtname + '.txt', 'a')as f:
72     f.write(txtname)
73     f.write('\r\n')
74     f.write(txtzz)
75     f.write('\r\n')
76     for i in range(0, len(urllist)):
77         with open(savepath + str(i) + '.txt', "r") as fr:
78             txt = fr.read()
79             f.write(txt)
80             f.write('===========================')
81             fr.close()
82             os.remove(savepath + str(i) + '.txt')
83 print('小说合并完成~！')
84 
85 print('')
86 print('*' * 15 + ' 任务完成，结果如下：' + '*' * 15)
87 print('')
88 print('<' + txtname + '> 下载完成' + '，获取并下载章节页面：' + str(downcount) + ' 个')
89 print('')
90 print('耗时：' + str(time.time() - time_start) + ' s')
91 print('')
92 print('*' * 51)

 

posted @ 2019-06-05 09:58  疾风不弃  阅读(446)  评论(0)    收藏  举报