Scrapy, a fast high-level web crawling & scraping framework for Python.
Use a non-zero exit code when a pipeline's open_spider method throws an exception (#4207)
gunblues034e2c31c7d5
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.pyindex 8093fd402..7b417e2eb 100644--- a/scrapy/commands/crawl.py+++ b/scrapy/commands/crawl.py@@ -54,8 +54,13 @@ class Command(ScrapyCommand):raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")spname = args[0]- self.crawler_process.crawl(spname, **opts.spargs)- self.crawler_process.start()+ crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)- if self.crawler_process.bootstrap_failed:+ if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):self.exitcode = 1+ else:+ self.crawler_process.start()++ if self.crawler_process.bootstrap_failed or \+ (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):+ self.exitcode = 1diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pynew file mode 100644index 000000000..d341888d3--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -0,0 +1,20 @@+import os+import sys+import unittest+from subprocess import Popen, PIPE+++class CmdlineCrawlPipelineTest(unittest.TestCase):++ def _execute(self, spname):+ args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)+ cwd = os.path.dirname(os.path.abspath(__file__))+ proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)+ proc.communicate()+ return proc.returncode++ def test_open_spider_normally_in_pipeline(self):+ self.assertEqual(self._execute('normal'), 0)++ def test_exception_at_open_spider_in_pipeline(self):+ self.assertEqual(self._execute('exception'), 1)diff --git a/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfgnew file mode 100644index 000000000..2f238dba3--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg@@ -0,0 +1,2 @@+[settings]+default = test_spider.settingsdiff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.pynew file mode 100644index 000000000..ce916f699--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py@@ -0,0 +1,16 @@+class TestSpiderPipeline(object):++ def open_spider(self, spider):+ pass++ def process_item(self, item, spider):+ return item+++class TestSpiderExceptionPipeline(object):++ def open_spider(self, spider):+ raise Exception('exception')++ def process_item(self, item, spider):+ return itemdiff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.pynew file mode 100644index 000000000..ae782c0d8--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py@@ -0,0 +1,2 @@+BOT_NAME = 'test_spider'+SPIDER_MODULES = ['test_spider.spiders']diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.pynew file mode 100644index 000000000..300f45ebf--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py@@ -0,0 +1,14 @@+import scrapy+++class ExceptionSpider(scrapy.Spider):+ name = 'exception'++ custom_settings = {+ 'ITEM_PIPELINES': {+ 'test_spider.pipelines.TestSpiderExceptionPipeline': 300+ }+ }++ def parse(self, response):+ passdiff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.pynew file mode 100644index 000000000..87a40fdcb--- /dev/null+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py@@ -0,0 +1,14 @@+import scrapy+++class NormalSpider(scrapy.Spider):+ name = 'normal'++ custom_settings = {+ 'ITEM_PIPELINES': {+ 'test_spider.pipelines.TestSpiderPipeline': 300+ }+ }++ def parse(self, response):+ pass