Scrapy, a fast high-level web crawling & scraping framework for Python.
Return exit code 1 from check, parse and fetch on bootstrap errors (#7920)
Adriand772acdf1925
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex 2113c19d2..4d9e02a73 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -119,4 +119,6 @@ class Command(ScrapyCommand):result.printErrors()result.printSummary(start_time, stop)- self.exitcode = int(not result.wasSuccessful())+ self.exitcode = int(+ not result.wasSuccessful() or self.crawler_process.bootstrap_failed+ )diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex 0b8311efb..809f6918d 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -97,3 +97,5 @@ class Command(ScrapyCommand):self.crawler_process.crawl(spidercls)self.crawler_process.start()+ if self.crawler_process.bootstrap_failed:+ self.exitcode = 1diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex b6255ec7e..5ef83ab6f 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -269,6 +269,8 @@ class Command(BaseRunSpiderCommand):self.crawler_process.crawl(self.spidercls, **opts.spargs)self.pcrawler = next(iter(self.crawler_process.crawlers))self.crawler_process.start()+ if self.crawler_process.bootstrap_failed:+ self.exitcode = 1if not self.first_response:logger.error("No response downloaded for: %(url)s", {"url": url})diff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex 240f44584..36af92c37 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -61,6 +61,12 @@ class CheckSpider(scrapy.Spider):assert "OK" in errassert ret == 0+ def test_bootstrap_failure(self, proj_path: Path) -> None:+ self._write_contract(proj_path, "", "pass")+ self._break_bootstrap(proj_path / self.project_name)+ ret, _, _ = proc("check", cwd=proj_path)+ assert ret == 1+def test_check_returns_requests_contract(self, proj_path: Path) -> None:contracts = """@returns requests 1@@ -218,7 +224,9 @@ class CheckSpider(scrapy.Spider):)cm_cls_mock.return_value = cm_mock = Mock()spider_loader_mock = Mock()- cmd.crawler_process = Mock(spider_loader=spider_loader_mock)+ cmd.crawler_process = Mock(+ spider_loader=spider_loader_mock, bootstrap_failed=False+ )spider_name = "FakeSpider"spider_cls_mock = Mock()spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex c6a3afc91..e25540f50 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -68,3 +68,10 @@ class MySpider(scrapy.Spider):"fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path)assert "myspider-user-agent" in out, err++ def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:+ self._break_bootstrap(proj_path / self.project_name)+ returncode, _, _ = proc(+ "fetch", "--spider", "myspider", mockserver.url("/text"), cwd=proj_path+ )+ assert returncode == 1diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex e434055b7..f06816b8c 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -211,6 +211,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}""")+ def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:+ self._break_bootstrap(proj_path / self.project_name)+ returncode, _, _ = proc(+ "parse",+ "--spider",+ self.spider_name,+ "-c",+ "parse",+ mockserver.url("/html"),+ cwd=proj_path,+ )+ assert returncode == 1+def test_spider_arguments(self, proj_path: Path, mockserver: MockServer) -> None:_, _, stderr = proc("parse",diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.pyindex 55ef686fa..3a01488a7 100644--- a/tests/utils/bases/commands.py+++ b/tests/utils/bases/commands.py@@ -38,3 +38,20 @@ class TestProjectBase:"""Add text to the end of the project settings.py."""with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:f.write(text)++ @classmethod+ def _break_bootstrap(cls, proj_mod_path: Path) -> None:+ """Enable a spider middleware that raises an exception when built, so+ that crawling any spider of the project fails before it starts."""+ cls._append_settings(+ proj_mod_path,+ """++class BreakingMiddleware:+ def __init__(self):+ raise RuntimeError("Breaking middleware")+++SPIDER_MIDDLEWARES = {f"{__name__}.BreakingMiddleware": 100}+""",+ )