微型scrapy

from twisted.web.client import getPage  #模块功能：socket对象，自动完成移除
from twisted.internet import reactor    #模块功能：事件循环（所有的socket对象都移除）
from twisted.internet import defer      #模块功能：defer.Deferred，特殊的socket对象，不发请求，需手动移除
from queue import Queue

class Request(object):
    """
    用于封装用户请求相关信息
    """
    def __init__(self,url,callback):
        self.url = url
        self.callback = callback

class HttpResponse(object):
    #将下载结果和request封装成一个类，以后方法解析好，类.xxx就能取到所有内容
    def __init__(self,content,request):
        self.content = content
        self.request = request

class Scheduler(object):
    """
    任务调度器
    """
    def __init__(self):
        self.q = Queue()

    def open(self):
        pass

    def next_request(self):
        try:
            req = self.q.get(block=False)
        except Exception as e:
            req = None
        return req

    def enqueue_request(self,req):
        self.q.put(req)

    def size(self):
        return self.q.qsize()

class ExecutionEngine(object):
    """
    引擎：所有的调度
    """
    def __init__(self):
        self._close = None
        self.scheduler = None
        self.max = 5
        self.crawlling = []

    def get_response_callback(self,content,request):
        """回调函数，传给parse和接受parse的返回值"""
        self.crawlling.remove(request)  # 移除
        response = HttpResponse(content, request)
        result = request.callback(response)  # 即调用了parse
        import types
        if isinstance(result, types.GeneratorType):  # 解析返回生成器的时候，再放到队列中
            for req in result:
                self.scheduler.enqueue_request(req)

    def _next_request(self):
        """取"""
        if self.scheduler.size() == 0 and len(self.crawlling) == 0:
            self._close.callback(None)
            return
        while len(self.crawlling)<self.max:
            req = self.scheduler.next_request()
            if not req: # 还可以try，没有就except即可
                return
            self.crawlling.append(req)
            d = getPage(req.url.encode('utf-8'))
            d.addCallback(self.get_response_callback,req)
            d.addCallback(lambda _: reactor.callLater(0, self._next_request))

    @defer.inlineCallbacks
    def open_spider(self,start_requests):
        """将初始请求加入调度器，然后开始取任务"""
        self.scheduler = Scheduler()
        while True:
            try:
                req = next(start_requests)
                self.scheduler.enqueue_request(req)
            except StopIteration as e:
                break
        yield self.scheduler.open()      #加这个装饰器必须有yield，否则twisted报错，none表示没有影响，这个方法没有返回，用途是调用方法
        reactor.callLater(0,self._next_request)

    @defer.inlineCallbacks
    def start(self):
        """创建Deferred对象"""
        self._close = defer.Deferred()
        yield self._close

class Crawler(object):
    """
    用户封装调度器以及引擎，将初始爬虫交给引擎调度
    """
    def _create_engine(self):
        """创建引擎对象"""
        return ExecutionEngine()

    def _creat_spider(self,spider_cls_path):
        """
        根据spider路径创建spider对象
        :param spider_cls_path:
        :return:
        """
        module_path,cls_name = spider_cls_path.rsplit('.',maxsplit=1)   #模块路径，类名
        import importlib            #反射
        m = importlib.import_module(module_path)
        cls = getattr(m,cls_name)
        return cls()

    @defer.inlineCallbacks
    def crawl(self,spider_cls_path):
        """创建引擎，创建spider，将初始url传给引擎open_spider加入队列，然后yield defered对象elf._close"""
        engine = self._create_engine()
        spider = self._creat_spider(spider_cls_path)
        start_requests = iter(spider.start_requests())
        yield engine.open_spider(start_requests)    # 相当于将yield写到open_spider里
        yield engine.start()    #yield self._close

class CrawlerProcess(object):
    """
    调用Crawler创建爬虫，开启事件循环
    """
    def __init__(self):
        self._active = set()

    def crawl(self,spider_cls_path):
        # 创建爬虫Crawler对象，这样把创建爬虫和将爬虫添加到循环的工作分开
        crawler = Crawler()
        d = crawler.crawl(spider_cls_path)
        self._active.add(d)

    def start(self):
        """
        启动一次reactor，就算多个spider
        :return:
        """
        dd = defer.DeferredList(self._active)
        dd.addBoth(lambda _:reactor.stop())
        reactor.run()

class Command(object):

    def run(self):
        crawl_process = CrawlerProcess()
        spider_cls_path_list = ['spider.chouti.ChoutiSpider',]
        for spider_cls_path in spider_cls_path_list:
            crawl_process.crawl(spider_cls_path)
        crawl_process.start()

if __name__ == '__main__':
    cmd = Command()
    cmd.run()

　

from ..engine import Request
class ChoutiSpider(object):
    name = 'chouti'

    def start_requests(self):
        start_url = ['https://www.baidu.com','https://www.bing.com',]
        for url in start_url:
            yield Request(url,self.parse)

    def parse(self,response):
        print(response)
        # yield Request('https://www.baidu.com', callback=self.parse)
        #1.crawling移除
        #2.获取parser yield值
        #3.再次取队列中获取