Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type the request attribute Scrapy adds to Failure objects (#8078)

Adrian6d53f9b93405
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index a9e35cb12..6da8ff527 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -7,7 +7,7 @@ import logging
import warnings
from collections import deque
from collections.abc import AsyncIterator
-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
@@ -49,6 +49,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatter
from scrapy.signalmanager import SignalManager
+ from scrapy.utils.defer import FailureWithRequest
logger = logging.getLogger(__name__)
@@ -355,8 +356,7 @@ class Scraper:
stacklevel=2,
)
else: # result is a Failure
- # TODO: properly type adding this attribute to a Failure
- result.request = request # type: ignore[attr-defined]
+ cast("FailureWithRequest", result).request = request
parent_id = request.id
if not request.errback:
result.raiseException()
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 9daad7bdc..c2fc11304 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
+ from scrapy.utils.defer import FailureWithRequest
_T = TypeVar("_T")
@@ -284,7 +285,8 @@ class CrawlSpider(Spider):
)
def _errback(self, failure: Failure) -> Iterable[Any]:
- rule = self._rules[cast("int", failure.request.meta["rule"])] # type: ignore[attr-defined]
+ request = cast("FailureWithRequest", failure).request
+ rule = self._rules[cast("int", request.meta["rule"])]
return self._handle_failure(
failure, cast("Callable[[Failure], Any]", rule.errback)
)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 8b1d3e343..d3fa4d214 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -18,6 +18,7 @@ from typing import (
Concatenate,
Generic,
ParamSpec,
+ Protocol,
TypeVar,
cast,
overload,
@@ -36,6 +37,16 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
+ from scrapy.http.request import Request
+
+ class FailureWithRequest(Protocol):
+ """A :class:`~twisted.python.failure.Failure` that has had a
+ *request* attribute attached, identifying the request whose
+ processing raised it.
+ """
+
+ request: Request
+
_T = TypeVar("_T")
_T2 = TypeVar("_T2")