Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix open_spider exception handling to set exit code 1 (#7255)

pierreeurope3fe89a211be0
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 33c2bb44c..95cd04b51 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -439,12 +439,16 @@ class CrawlerRunner(CrawlerRunnerBase):
self.crawlers.add(crawler)
d = crawler.crawl(*args, **kwargs)
self._active.add(d)
+ failed = False
try:
yield d
+ except Exception:
+ failed = True
+ raise
finally:
self.crawlers.discard(crawler)
self._active.discard(d)
- self.bootstrap_failed |= not getattr(crawler, "spider", None)
+ self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed
def stop(self) -> Deferred[Any]:
"""
@@ -538,7 +542,15 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
# or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
loop = asyncio.get_event_loop()
self.crawlers.add(crawler)
- task = loop.create_task(crawler.crawl_async(*args, **kwargs))
+
+ async def _crawl_and_track() -> None:
+ try:
+ await crawler.crawl_async(*args, **kwargs)
+ except Exception:
+ self.bootstrap_failed = True
+ raise # re-raise so asyncio still logs it to stderr naturally
+
+ task = loop.create_task(_crawl_and_track())
self._active.add(task)
def _done(_: asyncio.Task[None]) -> None:
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
index 2ad8fb8a4..f17543575 100644
--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -2,8 +2,6 @@ import sys
from pathlib import Path
from subprocess import PIPE, Popen
-from tests import TWISTED_KEEPS_TRACEBACKS
-
class TestCmdlineCrawlPipeline:
def _execute(self, spname):
@@ -18,10 +16,6 @@ class TestCmdlineCrawlPipeline:
assert returncode == 0
def test_exception_at_open_spider_in_pipeline(self):
- returncode, stderr = self._execute("exception")
- # An unhandled exception in a pipeline should not stop the crawl
- assert returncode == 0
- if TWISTED_KEEPS_TRACEBACKS:
- assert b'RuntimeError("exception")' in stderr
- else:
- assert b"RuntimeError: exception" in stderr
+ returncode, _ = self._execute("exception")
+ # An exception in pipeline's open_spider should result in a non-zero exit code
+ assert returncode == 1