python学习 例：爬虫解析文件（队列queque和线程的结合使用）

 

import requests
from queue import Queue
from threading import Thread
from lxml import etree
import json

CRAW_EXIT = True
PARSE_EXIT = True

class ThreadCrawl(Thread):
    def __init__(self,threadName,pageQueque,dataQueque):
        # 调用父类初始化方法
        super(ThreadCrawl,self).__init__()
        self.threadName=threadName   # 线程名
        self.pageQueque=pageQueque   # 页码队列
        self.dataQueque=dataQueque   # 数据队列

    def run(self):
        print('启动'+self.threadName)
        while CRAW_EXIT:
            try:
                # 取出一个数字，先进先出
                # 可选参数block，默认值为True
                # 1. 如果对列为空，block为True的话，不会结束，会进入阻塞状态，直到队列有新的数据
                # 2. 如果队列为空，block为False的话，就弹出一个Queue.empty()异常
                page=self.pageQueque.get(block=False)
                # print(self.pageQueque.qsize())
                url = 'http://www.waduanzi.com/page/'+str(page)
                # url = 'http://www.waduanzi.com'
                header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36"}
                content=requests.get(url,headers=header).text
                self.dataQueque.put(content)
                # print(content)
                # print(len(content))
            except:
                pass


class ThreadParse(Thread):
    def __init__(self,parseName,dataQueque,lock):
        super(ThreadParse,self).__init__()
        self.parseName=parseName
        self.dataQueque=dataQueque
        self.lock=lock    # 线程锁

    def run(self):
        print('启动'+self.parseName)
        while PARSE_EXIT:
            try:
                html=self.dataQueque.get(False)
                self.parse(html)
            except:
                pass

    def parse(self,html):
        # 解析为HTML DOM
        text=etree.HTML(html)
        node = "//div[contains(@class,'panel panel20')]"
        node_list = text.xpath(node)

        for node in node_list:
            # xpath返回的列表，这个列表就这一个参数，用索引方式取出来，用户名
            img = node.xpath("./div[1]/img/@src")[0]
            title = node.xpath("./div[2]/h2/a/@title")[0]
            # 取出标签下的内容,段子内容
            # content = node.xpath("./div[2]/div")[0].text     //遇到有&nbsp;和<br>后，文本内容没有取全
            content = node.xpath("normalize-space(./div[2]/div)")  # normalize-space（）遇到有&nbsp;和<br>后，文本内容正常取
            zan = node.xpath("./div[3]/ul/li[1]/a")[0].text
            buzan = node.xpath("./div[3]/ul/li[2]/a")[0].text

            items = {
                "img": img,
                "title": title,
                "content": content.replace("\xa0", ""),
                "zan": zan,
                "buzan": buzan,
            }

            self.lock.acquire()   #锁定
            with open("waduanzi.json", "a") as f:
                f.write(json.dumps(items,ensure_ascii=False)+",\n")
                self.lock.release()   #释放


def main():
    # 页码的队列，表示10个页面
    pageQueque=Queue(10)
    for i in range(1,11):
        pageQueque.put(i)

    # 采集结果(每页的HTML源码)的数据队列，参数为空表示不限制
    dataQueque=Queue(0)

    import threading
    lock=threading.Lock()   #创建锁

    # 三个采集线程的名字
    crawList=['采集线程1','采集线程2','采集线程3']
    # 存储三个采集线程的列表集合
    threadcrawl=[]
    for threadName in crawList:
        thread=ThreadCrawl(threadName,pageQueque,dataQueque)
        thread.start()
        threadcrawl.append(thread)

    # 三个解析线程的名字
    parseList=['解析县城1','解析线程2','解析线程3']
    # 存储三个解析线程
    threadparse=[]
    for parseName in parseList:
        thread=ThreadParse(parseName,dataQueque,lock)
        thread.start()
        threadparse.append(thread)

    # 等待pageQueue队列为空，也就是等待之前的操作执行完毕
    while not pageQueque.empty():
        pass

    # 如果pageQueue为空，采集线程退出循环
    global CRAW_EXIT
    CRAW_EXIT=False
    print("pageQueue为空")

    for thread in threadcrawl:
        thread.join()   #守护线程
        print('end')


    while not dataQueque.empty():
        pass

    global PARSE_EXIT
    PARSE_EXIT=False
    print("dataQueue为空")

    for thread1 in threadparse:
        thread1.join()
        print('end1')

    print("谢谢使用！")

if __name__ == '__main__':
    main()