Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into typing-request-response-cls
Andrey Rakhmatullinb6587575a1d2
diff --cc scrapy/downloadermiddlewares/httpcompression.pyindex d44eb933a,aa3abe853..aebdfb3e4--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@@ -1,12 -1,13 +1,13 @@@from __future__ import annotations- import io- import zlib+ import warnings+ from itertools import chain+ from logging import getLogger-from typing import TYPE_CHECKING, List, Optional, Union+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union- from scrapy import Request, Spider+ from scrapy import Request, Spider, signalsfrom scrapy.crawler import Crawler- from scrapy.exceptions import NotConfigured+ from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.statscollectors import StatsCollector@@@ -74,7 -135,7 +135,7 @@@ class HttpCompressionMiddlewarerespcls = responsetypes.from_args(headers=response.headers, url=response.url, body=decoded_body)- kwargs: Dict[str, Any] = dict(body=decoded_body)- kwargs = {"cls": respcls, "body": decoded_body}++ kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body}if issubclass(respcls, TextResponse):# force recalculating the encoding until we make sure the# responsetypes guessing is reliablediff --cc scrapy/downloadermiddlewares/redirect.pyindex 7b1401ac8,24089afea..e5ebf2273--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@@ -27,10 -27,11 +27,12 @@@ def _build_redirect_requestredirect_request = source_request.replace(url=url,**kwargs,+ cls=None,cookies=None,)- if "Cookie" in redirect_request.headers:+ has_cookie_header = "Cookie" in redirect_request.headers+ has_authorization_header = "Authorization" in redirect_request.headers+ if has_cookie_header or has_authorization_header:source_request_netloc = urlparse_cached(source_request).netlocredirect_request_netloc = urlparse_cached(redirect_request).netlocif source_request_netloc != redirect_request_netloc:diff --cc scrapy/http/request/__init__.pyindex 4effc2178,6269ee86a..e7fdd5d54--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@@ -4,11 -4,9 +4,12 @@@ requests in ScrapySee documentation in docs/topics/request-response.rst"""++from __future__ import annotations+import inspectfrom typing import (+ TYPE_CHECKING,Any,AnyStr,Callable,@@@ -181,22 -174,10 +182,20 @@@ class Request(object_ref)def __repr__(self) -> str:return f"<{self.method} {self.url}>"- def copy(self) -> "Request":+ def copy(self) -> Self:return self.replace()- def replace(self, *args: Any, **kwargs: Any) -> "Request":+ @overload+ def replace(+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any- ) -> RequestTypeVar:- ...++ ) -> RequestTypeVar: ...++ @overload- def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:- ...++ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...++ def replace(+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any+ ) -> Request:"""Create a new Request with the same attributes except for those given new values"""for x in self.attributes:kwargs.setdefault(x, getattr(self, x))diff --cc scrapy/http/response/__init__.pyindex e889a6460,d73dfce4b..166b19b0b--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@@ -144,19 -137,7 +145,17 @@@ class Response(object_ref)"""Return a copy of this Response"""return self.replace()- def replace(self, *args: Any, **kwargs: Any) -> Response:+ @overload+ def replace(+ self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any- ) -> ResponseTypeVar:- ...++ ) -> ResponseTypeVar: ...++ @overload- def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:- ...++ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...++ def replace(+ self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any+ ) -> Response:"""Create a new Response with the same attributes except for those given new values"""for x in self.attributes:kwargs.setdefault(x, getattr(self, x))