1 class xiaoshuoSpider(scrapy.Spider):
2 name = "freenovel"
3 headers={
4 'Upgrade - Insecure - Requests': '1',
5 }
6 def start_requests(self):
7 #完本、免费小说
8 start_url=["url起始网址"]
9 for url in start_url:
10 yield scrapy.Request(url=url, headers=self.headers,callback=self.first_parse)
11
12 def first_parse(self, response):
13 sel=Selector(response)
14 category=sel.css('div[class="select-list"] div ul[type="category"] li a::text').extract()
15 category_url=sel.css('div[class="select-list"] div ul[type="category"] li a::attr(href)').extract()
16 items=[]
17 for i in range(1,len(category_url)):
18 item=XiaoshuoItem()
19 item['category']=category[i]
20 item['category_url']="https:"+category_url[i]
21 items.append(item)
22 for item in items:
23 yield scrapy.Request(url=item['category_url'],meta={"category":item['category']},callback=self.second_parse,headers=self.headers)
24
25 def second_parse(self,response):
26 sel=Selector(response)
27 novel_url=sel.css('div[class="book-mid-info"] h4 a::attr(href)').extract()
28 item=XiaoshuoItem()
29 item['category']=response.meta['category']
30 yield scrapy.Request(url="https:" + novel_url[1] + "#Catalog",callback=self.article_parse,
31 headers=self.headers)
32 for i in range(len(novel_url)):
33 novel_url[i]="https:" + novel_url[i] + "#Catalog"
34 yield scrapy.Request(url=novel_url[i], meta={"category":item['category']},callback=self.article_parse, headers=self.headers)
35
36 def article_parse(self, response):
37 sel=Selector(response)
38 article_name=sel.xpath('//h1/em/text()').extract_first()
39 article_url=sel.css(
40 'div[id="j-catalogWrap"] div[class="volume-wrap"] div[class="volume"] ul li a::attr(href)').extract_first()
41 article_url="https:" + article_url
42 item=XiaoshuoItem()
43 item['article_name']=article_name
44 item['category']=response.meta['category']
45 yield scrapy.Request(url=article_url, meta={'article_name': item['article_name'],"category":item['category']}, callback=self.detail_parse,
46 headers=self.headers)
47
48 def detail_parse(self, response):
49 sel=Selector(response)
50 content=""
51 item=XiaoshuoItem()
52 content_list=sel.css(
53 'div[id="j_chapterBox"] div[class="text-wrap"] div[class="main-text-wrap"] div[class="read-content j_readContent"] p::text').extract()
54 content_name=sel.css('h3[class="j_chapterName"]::text').extract_first()
55 next_page=sel.css('a[id="j_chapterNext"]::attr(href)').extract_first()
56 for content_one in content_list:
57 content+=content_one
58 item['content']=content
59 item['content_name']=content_name
60 item['article_name']=response.meta['article_name']
61 item['category']=response.meta['category']
62 yield item
63 if next_page is not None:
64 next_page="https:" + next_page
65 yield scrapy.Request(url=next_page, meta={'article_name': item['article_name'],"category":item['category']}, callback=self.detail_parse,
66 headers=self.headers)
