Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into typing-request-response-cls
Andrey Rakhmatullinb950ed77b6d7
diff --cc scrapy/downloadermiddlewares/httpcompression.pyindex c4b620743,8e170a1c7..f3647e05f--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@@ -3,7 -3,7 +3,7 @@@ from __future__ import annotationimport warningsfrom itertools import chainfrom logging import getLogger- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union-from typing import TYPE_CHECKING, List, Optional, Tuple, Union++from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Unionfrom scrapy import Request, Spider, signalsfrom scrapy.crawler import Crawlerdiff --cc scrapy/downloadermiddlewares/redirect.pyindex e5ebf2273,aa08827c4..371e2fd3b--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@@ -27,9 -27,16 +27,17 @@@ def _build_redirect_requestredirect_request = source_request.replace(url=url,**kwargs,+ cls=None,cookies=None,)+ if "_scheme_proxy" in redirect_request.meta:+ source_request_scheme = urlparse_cached(source_request).scheme+ redirect_request_scheme = urlparse_cached(redirect_request).scheme+ if source_request_scheme != redirect_request_scheme:+ redirect_request.meta.pop("_scheme_proxy")+ redirect_request.meta.pop("proxy", None)+ redirect_request.meta.pop("_auth_proxy", None)+ redirect_request.headers.pop(b"Proxy-Authorization", None)has_cookie_header = "Cookie" in redirect_request.headershas_authorization_header = "Authorization" in redirect_request.headersif has_cookie_header or has_authorization_header: