Scrapy, a fast high-level web crawling & scraping framework for Python.
Report crawl errors in scrapy check (#7991)
Adriana45b75813b04
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex fdb9c7483..7fe80adff 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -1,10 +1,14 @@import argparse+import asyncioimport timefrom collections import defaultdict-from collections.abc import AsyncIterator+from collections.abc import AsyncIterator, Awaitablefrom typing import Any, ClassVar+from unittest import TestCase, TextTestRunnerfrom unittest import TextTestResult as _TextTestResult-from unittest import TextTestRunner++from twisted.internet.defer import Deferred+from twisted.python.failure import Failurefrom scrapy import Spiderfrom scrapy.commands import ScrapyCommand@@ -44,6 +48,42 @@ class TextTestResult(_TextTestResult):write("\n")+def _report_crawl_errors(+ crawl: Awaitable[None], spidername: str, result: TextTestResult+) -> None:+ """Make an exception that stops *crawl* before its contracts can run show+ up as an error in *result*, instead of being silently discarded."""++ class CrawlTestCase(TestCase):+ # unittest requires a test method, but this one is only reported, never run.+ runTest = staticmethod(lambda: None)++ def __str__(self) -> str:+ return f"[{spidername}] crawl"++ def report(exception: BaseException) -> None:+ result.addError(+ CrawlTestCase(),+ (type(exception), exception, exception.__traceback__), # type: ignore[arg-type]+ )++ if isinstance(crawl, Deferred):++ def on_failure(failure: Failure) -> None:+ assert failure.value is not None+ report(failure.value)++ crawl.addErrback(on_failure)+ else:+ assert isinstance(crawl, asyncio.Task)++ def on_done(task: asyncio.Task[None]) -> None:+ if not task.cancelled() and (exception := task.exception()) is not None:+ report(exception)++ crawl.add_done_callback(on_done)++class Command(ScrapyCommand):requires_project = Truedefault_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}@@ -128,7 +168,8 @@ class Command(ScrapyCommand):for method in tested_methods:contract_reqs[spidercls.name].append(method)elif tested_methods:- self.crawler_process.crawl(spidercls, **opts.spargs)+ crawl = self.crawler_process.crawl(spidercls, **opts.spargs)+ _report_crawl_errors(crawl, spidercls.name, result)# start checksif opts.list:diff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex 00b5c463f..c28d5fb0a 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -6,6 +6,8 @@ from typing import TYPE_CHECKINGfrom unittest import TestCasefrom unittest.mock import MagicMock, Mock, PropertyMock, call, patch+import pytest+from scrapy.commands.check import Command, TextTestResultfrom tests.utils.bases.commands import TestProjectBasefrom tests.utils.cmdline import proc@@ -93,6 +95,27 @@ class CheckSpider(scrapy.Spider):"""self._test_contract(proj_path, contracts, parse_def, use_reactor=False)+ @pytest.mark.parametrize(+ "settings",+ [+ "",+ "TWISTED_REACTOR_ENABLED = False\n",+ "FORCE_CRAWLER_PROCESS = True\n",+ ],+ ids=["async", "no_reactor", "crawler_process"],+ )+ def test_check_crawl_error(self, proj_path: Path, settings: str) -> None:+ self._write_contract(proj_path, "@returns requests 0", "pass")+ self._append_settings(+ proj_path / self.project_name,+ f"\nEXTENSIONS = {{'nonexistent.module.Extension': 300}}\n{settings}",+ )+ ret, _, err = proc("check", cwd=proj_path)+ assert f"[{self.spider_name}] crawl" in err+ assert "ModuleNotFoundError" in err+ assert "FAILED (errors=1)" in err+ assert ret == 1+def test_check_returns_items_contract(self, proj_path: Path) -> None:contracts = """@returns items 1