Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Return exit code 1 from check, parse and fetch on bootstrap errors (#7920)

Adriand772acdf1925
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 2113c19d2..4d9e02a73 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -119,4 +119,6 @@ class Command(ScrapyCommand):
result.printErrors()
result.printSummary(start_time, stop)
- self.exitcode = int(not result.wasSuccessful())
+ self.exitcode = int(
+ not result.wasSuccessful() or self.crawler_process.bootstrap_failed
+ )
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 0b8311efb..809f6918d 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -97,3 +97,5 @@ class Command(ScrapyCommand):
self.crawler_process.crawl(spidercls)
self.crawler_process.start()
+ if self.crawler_process.bootstrap_failed:
+ self.exitcode = 1
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index b6255ec7e..5ef83ab6f 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -269,6 +269,8 @@ class Command(BaseRunSpiderCommand):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
self.pcrawler = next(iter(self.crawler_process.crawlers))
self.crawler_process.start()
+ if self.crawler_process.bootstrap_failed:
+ self.exitcode = 1
if not self.first_response:
logger.error("No response downloaded for: %(url)s", {"url": url})
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 240f44584..36af92c37 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -61,6 +61,12 @@ class CheckSpider(scrapy.Spider):
assert "OK" in err
assert ret == 0
+ def test_bootstrap_failure(self, proj_path: Path) -> None:
+ self._write_contract(proj_path, "", "pass")
+ self._break_bootstrap(proj_path / self.project_name)
+ ret, _, _ = proc("check", cwd=proj_path)
+ assert ret == 1
+
def test_check_returns_requests_contract(self, proj_path: Path) -> None:
contracts = """
@returns requests 1
@@ -218,7 +224,9 @@ class CheckSpider(scrapy.Spider):
)
cm_cls_mock.return_value = cm_mock = Mock()
spider_loader_mock = Mock()
- cmd.crawler_process = Mock(spider_loader=spider_loader_mock)
+ cmd.crawler_process = Mock(
+ spider_loader=spider_loader_mock, bootstrap_failed=False
+ )
spider_name = "FakeSpider"
spider_cls_mock = Mock()
spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index c6a3afc91..e25540f50 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -68,3 +68,10 @@ class MySpider(scrapy.Spider):
"fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path
)
assert "myspider-user-agent" in out, err
+
+ def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:
+ self._break_bootstrap(proj_path / self.project_name)
+ returncode, _, _ = proc(
+ "fetch", "--spider", "myspider", mockserver.url("/text"), cwd=proj_path
+ )
+ assert returncode == 1
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index e434055b7..f06816b8c 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -211,6 +211,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
"""
)
+ def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:
+ self._break_bootstrap(proj_path / self.project_name)
+ returncode, _, _ = proc(
+ "parse",
+ "--spider",
+ self.spider_name,
+ "-c",
+ "parse",
+ mockserver.url("/html"),
+ cwd=proj_path,
+ )
+ assert returncode == 1
+
def test_spider_arguments(self, proj_path: Path, mockserver: MockServer) -> None:
_, _, stderr = proc(
"parse",
diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.py
index 55ef686fa..3a01488a7 100644
--- a/tests/utils/bases/commands.py
+++ b/tests/utils/bases/commands.py
@@ -38,3 +38,20 @@ class TestProjectBase:
"""Add text to the end of the project settings.py."""
with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
f.write(text)
+
+ @classmethod
+ def _break_bootstrap(cls, proj_mod_path: Path) -> None:
+ """Enable a spider middleware that raises an exception when built, so
+ that crawling any spider of the project fails before it starts."""
+ cls._append_settings(
+ proj_mod_path,
+ """
+
+class BreakingMiddleware:
+ def __init__(self):
+ raise RuntimeError("Breaking middleware")
+
+
+SPIDER_MIDDLEWARES = {f"{__name__}.BreakingMiddleware": 100}
+""",
+ )