Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into typing-request-response-cls

Andrey Rakhmatullinb6587575a1d2
diff --cc scrapy/downloadermiddlewares/httpcompression.py
index d44eb933a,aa3abe853..aebdfb3e4
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@@ -1,12 -1,13 +1,13 @@@
from __future__ import annotations
- import io
- import zlib
+ import warnings
+ from itertools import chain
+ from logging import getLogger
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
- from scrapy import Request, Spider
+ from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
- from scrapy.exceptions import NotConfigured
+ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.statscollectors import StatsCollector
@@@ -74,7 -135,7 +135,7 @@@ class HttpCompressionMiddleware
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
- kwargs: Dict[str, Any] = dict(body=decoded_body)
- kwargs = {"cls": respcls, "body": decoded_body}
++ kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
diff --cc scrapy/downloadermiddlewares/redirect.py
index 7b1401ac8,24089afea..e5ebf2273
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@@ -27,10 -27,11 +27,12 @@@ def _build_redirect_request
redirect_request = source_request.replace(
url=url,
**kwargs,
+ cls=None,
cookies=None,
)
- if "Cookie" in redirect_request.headers:
+ has_cookie_header = "Cookie" in redirect_request.headers
+ has_authorization_header = "Authorization" in redirect_request.headers
+ if has_cookie_header or has_authorization_header:
source_request_netloc = urlparse_cached(source_request).netloc
redirect_request_netloc = urlparse_cached(redirect_request).netloc
if source_request_netloc != redirect_request_netloc:
diff --cc scrapy/http/request/__init__.py
index 4effc2178,6269ee86a..e7fdd5d54
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@@ -4,11 -4,9 +4,12 @@@ requests in Scrapy
See documentation in docs/topics/request-response.rst
"""
+
+from __future__ import annotations
+
import inspect
from typing import (
+ TYPE_CHECKING,
Any,
AnyStr,
Callable,
@@@ -181,22 -174,10 +182,20 @@@ class Request(object_ref)
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
- def copy(self) -> "Request":
+ def copy(self) -> Self:
return self.replace()
- def replace(self, *args: Any, **kwargs: Any) -> "Request":
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
- ) -> RequestTypeVar:
- ...
++ ) -> RequestTypeVar: ...
+
+ @overload
- def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
- ...
++ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+ ) -> Request:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
diff --cc scrapy/http/response/__init__.py
index e889a6460,d73dfce4b..166b19b0b
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@@ -144,19 -137,7 +145,17 @@@ class Response(object_ref)
"""Return a copy of this Response"""
return self.replace()
- def replace(self, *args: Any, **kwargs: Any) -> Response:
+ @overload
+ def replace(
+ self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
- ) -> ResponseTypeVar:
- ...
++ ) -> ResponseTypeVar: ...
+
+ @overload
- def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
- ...
++ def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
+
+ def replace(
+ self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
+ ) -> Response:
"""Create a new Response with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))