scrapy爬虫main.py可以直接run整个项目

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

process = CrawlerProcess(get_project_settings())

# spider_name 为项目爬虫名
# process.crawl('spider_name')

for spider_name in process.spider_loader.list():
    process.crawl(spider_name)


process.start()