Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop the Authorization header on cross-domain redirect
Adrián Chaves080fecd8900b
diff --git a/docs/news.rst b/docs/news.rstindex fd8fa3ea3..b19ec2e99 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,20 @@Release notes=============+.. _release-2.11.1:++Scrapy 2.11.1 (unreleased)+--------------------------++**Security bug fix:**++- The ``Authorization`` header is now dropped on redirects to a different+ domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more+ information.++ .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv++.. _release-2.11.0:Scrapy 2.11.0 (2023-09-18)@@ -2869,6 +2883,19 @@ affect subclasses:(:issue:`3884`)+.. _release-1.8.4:++Scrapy 1.8.4 (unreleased)+-------------------------++**Security bug fix:**++- The ``Authorization`` header is now dropped on redirects to a different+ domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more+ information.++ .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv+.. _release-1.8.3:diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 65f1d2224..3176ed930 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -17,11 +17,17 @@ def _build_redirect_request(source_request, *, url, **kwargs):**kwargs,cookies=None,)- if "Cookie" in redirect_request.headers:+ has_cookie_header = "Cookie" in redirect_request.headers+ has_authorization_header = "Authorization" in redirect_request.headers+ if has_cookie_header or has_authorization_header:source_request_netloc = urlparse_cached(source_request).netlocredirect_request_netloc = urlparse_cached(redirect_request).netlocif source_request_netloc != redirect_request_netloc:- del redirect_request.headers["Cookie"]+ if has_cookie_header:+ del redirect_request.headers["Cookie"]+ # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name+ if has_authorization_header:+ del redirect_request.headers["Authorization"]return redirect_requestdiff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex dc15b672c..10b8ca9af 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -247,6 +247,37 @@ class RedirectMiddlewareTest(unittest.TestCase):perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o"self.assertEqual(perc_encoded_utf8_url, req_result.url)+ def test_cross_domain_header_dropping(self):+ safe_headers = {"A": "B"}+ original_request = Request(+ "https://example.com",+ headers={"Cookie": "a=b", "Authorization": "a", **safe_headers},+ )++ internal_response = Response(+ "https://example.com",+ headers={"Location": "https://example.com/a"},+ status=301,+ )+ internal_redirect_request = self.mw.process_response(+ original_request, internal_response, self.spider+ )+ self.assertIsInstance(internal_redirect_request, Request)+ self.assertEqual(original_request.headers, internal_redirect_request.headers)++ external_response = Response(+ "https://example.com",+ headers={"Location": "https://example.org/a"},+ status=301,+ )+ external_redirect_request = self.mw.process_response(+ original_request, external_response, self.spider+ )+ self.assertIsInstance(external_redirect_request, Request)+ self.assertEqual(+ safe_headers, external_redirect_request.headers.to_unicode_dict()+ )+class MetaRefreshMiddlewareTest(unittest.TestCase):def setUp(self):