Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Report crawl errors in scrapy check (#7991)

Adriana45b75813b04
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index fdb9c7483..7fe80adff 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,10 +1,14 @@
import argparse
+import asyncio
import time
from collections import defaultdict
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Awaitable
from typing import Any, ClassVar
+from unittest import TestCase, TextTestRunner
from unittest import TextTestResult as _TextTestResult
-from unittest import TextTestRunner
+
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.commands import ScrapyCommand
@@ -44,6 +48,42 @@ class TextTestResult(_TextTestResult):
write("\n")
+def _report_crawl_errors(
+ crawl: Awaitable[None], spidername: str, result: TextTestResult
+) -> None:
+ """Make an exception that stops *crawl* before its contracts can run show
+ up as an error in *result*, instead of being silently discarded."""
+
+ class CrawlTestCase(TestCase):
+ # unittest requires a test method, but this one is only reported, never run.
+ runTest = staticmethod(lambda: None)
+
+ def __str__(self) -> str:
+ return f"[{spidername}] crawl"
+
+ def report(exception: BaseException) -> None:
+ result.addError(
+ CrawlTestCase(),
+ (type(exception), exception, exception.__traceback__), # type: ignore[arg-type]
+ )
+
+ if isinstance(crawl, Deferred):
+
+ def on_failure(failure: Failure) -> None:
+ assert failure.value is not None
+ report(failure.value)
+
+ crawl.addErrback(on_failure)
+ else:
+ assert isinstance(crawl, asyncio.Task)
+
+ def on_done(task: asyncio.Task[None]) -> None:
+ if not task.cancelled() and (exception := task.exception()) is not None:
+ report(exception)
+
+ crawl.add_done_callback(on_done)
+
+
class Command(ScrapyCommand):
requires_project = True
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
@@ -128,7 +168,8 @@ class Command(ScrapyCommand):
for method in tested_methods:
contract_reqs[spidercls.name].append(method)
elif tested_methods:
- self.crawler_process.crawl(spidercls, **opts.spargs)
+ crawl = self.crawler_process.crawl(spidercls, **opts.spargs)
+ _report_crawl_errors(crawl, spidercls.name, result)
# start checks
if opts.list:
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 00b5c463f..c28d5fb0a 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -6,6 +6,8 @@ from typing import TYPE_CHECKING
from unittest import TestCase
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
+import pytest
+
from scrapy.commands.check import Command, TextTestResult
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
@@ -93,6 +95,27 @@ class CheckSpider(scrapy.Spider):
"""
self._test_contract(proj_path, contracts, parse_def, use_reactor=False)
+ @pytest.mark.parametrize(
+ "settings",
+ [
+ "",
+ "TWISTED_REACTOR_ENABLED = False\n",
+ "FORCE_CRAWLER_PROCESS = True\n",
+ ],
+ ids=["async", "no_reactor", "crawler_process"],
+ )
+ def test_check_crawl_error(self, proj_path: Path, settings: str) -> None:
+ self._write_contract(proj_path, "@returns requests 0", "pass")
+ self._append_settings(
+ proj_path / self.project_name,
+ f"\nEXTENSIONS = {{'nonexistent.module.Extension': 300}}\n{settings}",
+ )
+ ret, _, err = proc("check", cwd=proj_path)
+ assert f"[{self.spider_name}] crawl" in err
+ assert "ModuleNotFoundError" in err
+ assert "FAILED (errors=1)" in err
+ assert ret == 1
+
def test_check_returns_items_contract(self, proj_path: Path) -> None:
contracts = """
@returns items 1