Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use a non-zero exit code when a pipeline's open_spider method throws an exception (#4207)

gunblues034e2c31c7d5
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 8093fd402..7b417e2eb 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -54,8 +54,13 @@ class Command(ScrapyCommand):
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
spname = args[0]
- self.crawler_process.crawl(spname, **opts.spargs)
- self.crawler_process.start()
+ crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
- if self.crawler_process.bootstrap_failed:
+ if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
self.exitcode = 1
+ else:
+ self.crawler_process.start()
+
+ if self.crawler_process.bootstrap_failed or \
+ (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+ self.exitcode = 1
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
new file mode 100644
index 000000000..d341888d3
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -0,0 +1,20 @@
+import os
+import sys
+import unittest
+from subprocess import Popen, PIPE
+
+
+class CmdlineCrawlPipelineTest(unittest.TestCase):
+
+ def _execute(self, spname):
+ args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
+ cwd = os.path.dirname(os.path.abspath(__file__))
+ proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
+ proc.communicate()
+ return proc.returncode
+
+ def test_open_spider_normally_in_pipeline(self):
+ self.assertEqual(self._execute('normal'), 0)
+
+ def test_exception_at_open_spider_in_pipeline(self):
+ self.assertEqual(self._execute('exception'), 1)
diff --git a/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg
new file mode 100644
index 000000000..2f238dba3
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg
@@ -0,0 +1,2 @@
+[settings]
+default = test_spider.settings
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
new file mode 100644
index 000000000..ce916f699
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
@@ -0,0 +1,16 @@
+class TestSpiderPipeline(object):
+
+ def open_spider(self, spider):
+ pass
+
+ def process_item(self, item, spider):
+ return item
+
+
+class TestSpiderExceptionPipeline(object):
+
+ def open_spider(self, spider):
+ raise Exception('exception')
+
+ def process_item(self, item, spider):
+ return item
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
new file mode 100644
index 000000000..ae782c0d8
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py
@@ -0,0 +1,2 @@
+BOT_NAME = 'test_spider'
+SPIDER_MODULES = ['test_spider.spiders']
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
new file mode 100644
index 000000000..300f45ebf
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py
@@ -0,0 +1,14 @@
+import scrapy
+
+
+class ExceptionSpider(scrapy.Spider):
+ name = 'exception'
+
+ custom_settings = {
+ 'ITEM_PIPELINES': {
+ 'test_spider.pipelines.TestSpiderExceptionPipeline': 300
+ }
+ }
+
+ def parse(self, response):
+ pass
diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
new file mode 100644
index 000000000..87a40fdcb
--- /dev/null
+++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py
@@ -0,0 +1,14 @@
+import scrapy
+
+
+class NormalSpider(scrapy.Spider):
+ name = 'normal'
+
+ custom_settings = {
+ 'ITEM_PIPELINES': {
+ 'test_spider.pipelines.TestSpiderPipeline': 300
+ }
+ }
+
+ def parse(self, response):
+ pass