Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix open_spider exception handling to set exit code 1 (#7255)
pierreeurope3fe89a211be0
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 33c2bb44c..95cd04b51 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -439,12 +439,16 @@ class CrawlerRunner(CrawlerRunnerBase):self.crawlers.add(crawler)d = crawler.crawl(*args, **kwargs)self._active.add(d)+ failed = Falsetry:yield d+ except Exception:+ failed = True+ raisefinally:self.crawlers.discard(crawler)self._active.discard(d)- self.bootstrap_failed |= not getattr(crawler, "spider", None)+ self.bootstrap_failed |= not getattr(crawler, "spider", None) or faileddef stop(self) -> Deferred[Any]:"""@@ -538,7 +542,15 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):# or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).loop = asyncio.get_event_loop()self.crawlers.add(crawler)- task = loop.create_task(crawler.crawl_async(*args, **kwargs))++ async def _crawl_and_track() -> None:+ try:+ await crawler.crawl_async(*args, **kwargs)+ except Exception:+ self.bootstrap_failed = True+ raise # re-raise so asyncio still logs it to stderr naturally++ task = loop.create_task(_crawl_and_track())self._active.add(task)def _done(_: asyncio.Task[None]) -> None:diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pyindex 2ad8fb8a4..f17543575 100644--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -2,8 +2,6 @@ import sysfrom pathlib import Pathfrom subprocess import PIPE, Popen-from tests import TWISTED_KEEPS_TRACEBACKS-class TestCmdlineCrawlPipeline:def _execute(self, spname):@@ -18,10 +16,6 @@ class TestCmdlineCrawlPipeline:assert returncode == 0def test_exception_at_open_spider_in_pipeline(self):- returncode, stderr = self._execute("exception")- # An unhandled exception in a pipeline should not stop the crawl- assert returncode == 0- if TWISTED_KEEPS_TRACEBACKS:- assert b'RuntimeError("exception")' in stderr- else:- assert b"RuntimeError: exception" in stderr+ returncode, _ = self._execute("exception")+ # An exception in pipeline's open_spider should result in a non-zero exit code+ assert returncode == 1