Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Exit with code 1 when Spider.start() raises an exception (#8203)

Adrianafd71bf477d0
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index d85cac2e7..632e2a27a 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -243,10 +243,9 @@ class BaseRunSpiderCommand(ScrapyCommand):
crawler = self._create_crawler(spidercls)
self.crawler_process.crawl(crawler, **opts.spargs)
self.crawler_process.start()
- if (
- self.crawler_process.bootstrap_failed
- or crawler.stats.get_value("finish_reason") == "closespider_errorcount"
- ):
+ if self.crawler_process.bootstrap_failed or crawler.stats.get_value(
+ "finish_reason"
+ ) in {"closespider_errorcount", "start_error"}:
self.exitcode = 1
def _warn_if_feeds_unused(self, sender: Crawler, **kwargs: Any) -> None:
diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py
index 5261df562..6719127c2 100644
--- a/tests/test_command_crawl.py
+++ b/tests/test_command_crawl.py
@@ -134,6 +134,21 @@ class MySpider(scrapy.Spider):
assert "Spider closed (closespider_errorcount)" in err
assert returncode != 0
+ def test_start_error_exit_code(self, proj_path: Path) -> None:
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ async def start(self):
+ raise Exception('Expected exception')
+ yield
+"""
+ returncode, _, err = self.crawl(spider_code, proj_path)
+ assert "Spider closed (start_error)" in err
+ assert returncode == 1
+
def test_default_reactor(self, proj_path: Path) -> None:
spider_code = """
import scrapy
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 46be5cdcf..c2fccd54f 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -197,6 +197,21 @@ class MySpider(scrapy.Spider):
assert "Spider closed (closespider_errorcount)" in err
assert returncode != 0
+ def test_start_error_exit_code(self, tmp_path: Path) -> None:
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ async def start(self):
+ raise Exception('Expected exception')
+ yield
+"""
+ returncode, _, err = self.runspider(tmp_path, spider_code)
+ assert "Spider closed (start_error)" in err
+ assert returncode == 1
+
def test_asyncio_enabled_true(self, tmp_path: Path) -> None:
log = self.get_log(
tmp_path,