Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve type hints for copy() and replace() in Request and Response.
Andrey Rakhmatullina6cee787dd45
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 56a58a750..d44eb933a 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport ioimport zlib-from typing import TYPE_CHECKING, List, Optional, Union+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom scrapy import Request, Spiderfrom scrapy.crawler import Crawler@@ -74,12 +74,12 @@ class HttpCompressionMiddleware:respcls = responsetypes.from_args(headers=response.headers, url=response.url, body=decoded_body)- kwargs = dict(cls=respcls, body=decoded_body)+ kwargs: Dict[str, Any] = dict(body=decoded_body)if issubclass(respcls, TextResponse):# force recalculating the encoding until we make sure the# responsetypes guessing is reliablekwargs["encoding"] = None- response = response.replace(**kwargs)+ response = response.replace(cls=respcls, **kwargs)if not content_encoding:del response.headers["Content-Encoding"]diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 814b1a561..7b1401ac8 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -27,6 +27,7 @@ def _build_redirect_request(redirect_request = source_request.replace(url=url,**kwargs,+ cls=None,cookies=None,)if "Cookie" in redirect_request.headers:diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex a1c5a5e51..4effc2178 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -4,8 +4,11 @@ requests in Scrapy.See documentation in docs/topics/request-response.rst"""+from __future__ import annotations+import inspectfrom typing import (+ TYPE_CHECKING,Any,AnyStr,Callable,@@ -19,7 +22,7 @@ from typing import (Type,TypeVar,Union,- cast,+ overload,)from w3lib.url import safe_url_string@@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import escape_ajax+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")@@ -173,23 +181,36 @@ class Request(object_ref):def __repr__(self) -> str:return f"<{self.method} {self.url}>"- def copy(self) -> "Request":+ def copy(self) -> Self:return self.replace()- def replace(self, *args: Any, **kwargs: Any) -> "Request":+ @overload+ def replace(+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any+ ) -> RequestTypeVar:+ ...++ @overload+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:+ ...++ def replace(+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any+ ) -> Request:"""Create a new Request with the same attributes except for those given new values"""for x in self.attributes:kwargs.setdefault(x, getattr(self, x))- cls = kwargs.pop("cls", self.__class__)- return cast(Request, cls(*args, **kwargs))+ if cls is None:+ cls = self.__class__+ return cls(*args, **kwargs)@classmethoddef from_curl(- cls: Type[RequestTypeVar],+ cls,curl_command: str,ignore_unknown_options: bool = True,**kwargs: Any,- ) -> RequestTypeVar:+ ) -> Self:"""Create a Request object from a string containing a `cURL<https://curl.haxx.se/>`_ command. It populates the HTTP method, theURL, the headers, the cookies and the body. It accepts the same@@ -221,7 +242,7 @@ class Request(object_ref):request_kwargs.update(kwargs)return cls(**request_kwargs)- def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:+ def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:"""Return a dictionary containing the Request's data.Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.pyindex 1dd9e6c87..5c09835e4 100644--- a/scrapy/http/request/json_request.py+++ b/scrapy/http/request/json_request.py@@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient classSee documentation in docs/topics/request-response.rst"""+from __future__ import annotations+import copyimport jsonimport warnings-from typing import Any, Optional, Tuple+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload++from scrapy.http.request import Request, RequestTypeVar-from scrapy.http.request import Request+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Selfclass JsonRequest(Request):@@ -44,7 +50,19 @@ class JsonRequest(Request):def dumps_kwargs(self) -> dict:return self._dumps_kwargs- def replace(self, *args: Any, **kwargs: Any) -> Request:+ @overload+ def replace(+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any+ ) -> RequestTypeVar:+ ...++ @overload+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:+ ...++ def replace(+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any+ ) -> Request:body_passed = kwargs.get("body", None) is not Nonedata = kwargs.pop("data", None)data_passed = data is not None@@ -54,7 +72,7 @@ class JsonRequest(Request):elif not body_passed and data_passed:kwargs["body"] = self._dumps(data)- return super().replace(*args, **kwargs)+ return super().replace(*args, cls=cls, **kwargs)def _dumps(self, data: dict) -> str:"""Convert to JSON"""diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 6eae3e8b3..e889a6460 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -19,8 +19,10 @@ from typing import (Mapping,Optional,Tuple,+ Type,+ TypeVar,Union,- cast,+ overload,)from urllib.parse import urljoin@@ -33,9 +35,15 @@ from scrapy.link import Linkfrom scrapy.utils.trackref import object_refif TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self+from scrapy.selector import SelectorList+ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")++class Response(object_ref):"""An object that represents an HTTP response, which is usuallydownloaded (by the Downloader) and fed to the Spiders for processing.@@ -132,16 +140,29 @@ class Response(object_ref):def __repr__(self) -> str:return f"<{self.status} {self.url}>"- def copy(self) -> Response:+ def copy(self) -> Self:"""Return a copy of this Response"""return self.replace()- def replace(self, *args: Any, **kwargs: Any) -> Response:+ @overload+ def replace(+ self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any+ ) -> ResponseTypeVar:+ ...++ @overload+ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:+ ...++ def replace(+ self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any+ ) -> Response:"""Create a new Response with the same attributes except for those given new values"""for x in self.attributes:kwargs.setdefault(x, getattr(self, x))- cls = kwargs.pop("cls", self.__class__)- return cast(Response, cls(*args, **kwargs))+ if cls is None:+ cls = self.__class__+ return cls(*args, **kwargs)def urljoin(self, url: str) -> str:"""Join this Response's url with a possible relative url to form an