Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into typing-request-response-cls

Andrey Rakhmatullinb950ed77b6d7
diff --cc scrapy/downloadermiddlewares/httpcompression.py
index c4b620743,8e170a1c7..f3647e05f
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@@ -3,7 -3,7 +3,7 @@@ from __future__ import annotation
import warnings
from itertools import chain
from logging import getLogger
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
-from typing import TYPE_CHECKING, List, Optional, Tuple, Union
++from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
diff --cc scrapy/downloadermiddlewares/redirect.py
index e5ebf2273,aa08827c4..371e2fd3b
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@@ -27,9 -27,16 +27,17 @@@ def _build_redirect_request
redirect_request = source_request.replace(
url=url,
**kwargs,
+ cls=None,
cookies=None,
)
+ if "_scheme_proxy" in redirect_request.meta:
+ source_request_scheme = urlparse_cached(source_request).scheme
+ redirect_request_scheme = urlparse_cached(redirect_request).scheme
+ if source_request_scheme != redirect_request_scheme:
+ redirect_request.meta.pop("_scheme_proxy")
+ redirect_request.meta.pop("proxy", None)
+ redirect_request.meta.pop("_auth_proxy", None)
+ redirect_request.headers.pop(b"Proxy-Authorization", None)
has_cookie_header = "Cookie" in redirect_request.headers
has_authorization_header = "Authorization" in redirect_request.headers
if has_cookie_header or has_authorization_header: