Scrapy, a fast high-level web crawling & scraping framework for Python.
Type the request attribute Scrapy adds to Failure objects (#8078)
Adrian6d53f9b93405
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex a9e35cb12..6da8ff527 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -7,7 +7,7 @@ import loggingimport warningsfrom collections import dequefrom collections.abc import AsyncIterator-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, castfrom twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.python.failure import Failure@@ -49,6 +49,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.logformatter import LogFormatterfrom scrapy.signalmanager import SignalManager+ from scrapy.utils.defer import FailureWithRequestlogger = logging.getLogger(__name__)@@ -355,8 +356,7 @@ class Scraper:stacklevel=2,)else: # result is a Failure- # TODO: properly type adding this attribute to a Failure- result.request = request # type: ignore[attr-defined]+ cast("FailureWithRequest", result).request = requestparent_id = request.idif not request.errback:result.raiseException()diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 9daad7bdc..c2fc11304 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -29,6 +29,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http.request import CallbackT+ from scrapy.utils.defer import FailureWithRequest_T = TypeVar("_T")@@ -284,7 +285,8 @@ class CrawlSpider(Spider):)def _errback(self, failure: Failure) -> Iterable[Any]:- rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined]+ request = cast("FailureWithRequest", failure).request+ rule = self._rules[cast("int", request.meta["rule"])]return self._handle_failure(failure, cast("Callable[[Failure], Any]", rule.errback))diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 8b1d3e343..d3fa4d214 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -18,6 +18,7 @@ from typing import (Concatenate,Generic,ParamSpec,+ Protocol,TypeVar,cast,overload,@@ -36,6 +37,16 @@ if TYPE_CHECKING:from twisted.python.failure import Failure+ from scrapy.http.request import Request++ class FailureWithRequest(Protocol):+ """A :class:`~twisted.python.failure.Failure` that has had a+ *request* attribute attached, identifying the request whose+ processing raised it.+ """++ request: Request+_T = TypeVar("_T")_T2 = TypeVar("_T2")