Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve type hints for copy() and replace() in Request and Response.

Andrey Rakhmatullina6cee787dd45
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 56a58a750..d44eb933a 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import io
import zlib
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
@@ -74,12 +74,12 @@ class HttpCompressionMiddleware:
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
- kwargs = dict(cls=respcls, body=decoded_body)
+ kwargs: Dict[str, Any] = dict(body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs["encoding"] = None
- response = response.replace(**kwargs)
+ response = response.replace(cls=respcls, **kwargs)
if not content_encoding:
del response.headers["Content-Encoding"]
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 814b1a561..7b1401ac8 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -27,6 +27,7 @@ def _build_redirect_request(
redirect_request = source_request.replace(
url=url,
**kwargs,
+ cls=None,
cookies=None,
)
if "Cookie" in redirect_request.headers:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a1c5a5e51..4effc2178 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,8 +4,11 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
+from __future__ import annotations
+
import inspect
from typing import (
+ TYPE_CHECKING,
Any,
AnyStr,
Callable,
@@ -19,7 +22,7 @@ from typing import (
Type,
TypeVar,
Union,
- cast,
+ overload,
)
from w3lib.url import safe_url_string
@@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
@@ -173,23 +181,36 @@ class Request(object_ref):
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
- def copy(self) -> "Request":
+ def copy(self) -> Self:
return self.replace()
- def replace(self, *args: Any, **kwargs: Any) -> "Request":
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+ ) -> RequestTypeVar:
+ ...
+
+ @overload
+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+ ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+ ) -> Request:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
- cls = kwargs.pop("cls", self.__class__)
- return cast(Request, cls(*args, **kwargs))
+ if cls is None:
+ cls = self.__class__
+ return cls(*args, **kwargs)
@classmethod
def from_curl(
- cls: Type[RequestTypeVar],
+ cls,
curl_command: str,
ignore_unknown_options: bool = True,
**kwargs: Any,
- ) -> RequestTypeVar:
+ ) -> Self:
"""Create a Request object from a string containing a `cURL
<https://curl.haxx.se/>`_ command. It populates the HTTP method, the
URL, the headers, the cookies and the body. It accepts the same
@@ -221,7 +242,7 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
- def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+ def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 1dd9e6c87..5c09835e4 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
+from __future__ import annotations
+
import copy
import json
import warnings
-from typing import Any, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
+
+from scrapy.http.request import Request, RequestTypeVar
-from scrapy.http.request import Request
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class JsonRequest(Request):
@@ -44,7 +50,19 @@ class JsonRequest(Request):
def dumps_kwargs(self) -> dict:
return self._dumps_kwargs
- def replace(self, *args: Any, **kwargs: Any) -> Request:
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+ ) -> RequestTypeVar:
+ ...
+
+ @overload
+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+ ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+ ) -> Request:
body_passed = kwargs.get("body", None) is not None
data = kwargs.pop("data", None)
data_passed = data is not None
@@ -54,7 +72,7 @@ class JsonRequest(Request):
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)
- return super().replace(*args, **kwargs)
+ return super().replace(*args, cls=cls, **kwargs)
def _dumps(self, data: dict) -> str:
"""Convert to JSON"""
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 6eae3e8b3..e889a6460 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -19,8 +19,10 @@ from typing import (
Mapping,
Optional,
Tuple,
+ Type,
+ TypeVar,
Union,
- cast,
+ overload,
)
from urllib.parse import urljoin
@@ -33,9 +35,15 @@ from scrapy.link import Link
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
from scrapy.selector import SelectorList
+ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
+
+
class Response(object_ref):
"""An object that represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -132,16 +140,29 @@ class Response(object_ref):
def __repr__(self) -> str:
return f"<{self.status} {self.url}>"
- def copy(self) -> Response:
+ def copy(self) -> Self:
"""Return a copy of this Response"""
return self.replace()
- def replace(self, *args: Any, **kwargs: Any) -> Response:
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
+ ) -> ResponseTypeVar:
+ ...
+
+ @overload
+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+ ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
+ ) -> Response:
"""Create a new Response with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
- cls = kwargs.pop("cls", self.__class__)
- return cast(Response, cls(*args, **kwargs))
+ if cls is None:
+ cls = self.__class__
+ return cls(*args, **kwargs)
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an