scrapy运行多爬虫的方法3

# 通过CrawlerRunner和链接(chaining) deferred来线性运行
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging

class MySpider1(scrapy.Spider):
    # Your first spider definition
    name = "Standard"
    # 只爬取包含这里面域名的网站
    allowed_domains = []
    start_urls = ["http://www.ip138.com"]
    ...

class MySpider2(scrapy.Spider):
    # Your second spider definition
    ...

configure_logging()
runner = CrawlerRunner()

@defer.inlineCallbacks
def crawl():
    yield runner.crawl(MySpider1)
    yield runner.crawl(MySpider2)
    reactor.stop()

crawl()
reactor.run()