Scrapy, a fast high-level web crawling & scraping framework for Python.
Only send http_user/http_pass to the origin they were first used on (#8329)
Adrian502caa2e2bab
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex b20fe73d3..1fec3405c 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -955,6 +955,10 @@ http_auth_domainOverrides :setting:`HTTPAUTH_DOMAIN` for this request.+If this key is not set, :reqmeta:`http_user` and :reqmeta:`http_pass` are only+sent to the origin (scheme, host and port) of the request where they were first+used, e.g. they are not sent after a cross-origin redirect.+.. reqmeta:: http_passhttp_passdiff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex 106d96402..a48fcbfef 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -15,6 +15,7 @@ from scrapy import Request, Spider, signalsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import SETTINGS_PRIORITIESfrom scrapy.utils.decorators import _warn_spider_arg+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.url import url_is_from_any_domainif TYPE_CHECKING:@@ -22,6 +23,32 @@ if TYPE_CHECKING:from scrapy.http import Response+_DEFAULT_PORTS = {+ "http": 80,+ "https": 443,+}+++def _origin(request: Request) -> str:+ parsed_url = urlparse_cached(request)+ scheme = parsed_url.scheme+ netloc = (+ parsed_url.netloc+ if parsed_url.port != _DEFAULT_PORTS[scheme]+ else parsed_url.hostname+ )+ return f"{scheme}://{netloc}"+++def _setdefault_auth_origin(request: Request) -> str:+ origin: str | None = request.meta.get("_auth_origin")+ if origin:+ return origin+ origin = _origin(request)+ request.meta["_auth_origin"] = origin+ return origin++class HttpAuthMiddleware:"""Set Basic HTTP Authorization header."""@@ -67,14 +94,22 @@ class HttpAuthMiddleware:def process_request(self, request: Request, spider: Spider | None = None) -> Request | Response | None:- if b"Authorization" in request.headers:+ if (+ b"Authorization" in request.headers+ or urlparse_cached(request).scheme not in _DEFAULT_PORTS+ ):return None# Per-request meta overridesusr = request.meta.get("http_user", "")pwd = request.meta.get("http_pass", "")if usr or pwd:domain = request.meta.get("http_auth_domain")- if not domain or url_is_from_any_domain(request, [domain]):+ allowed = (+ url_is_from_any_domain(request, [domain])+ if domain+ else _setdefault_auth_origin(request) == _origin(request)+ )+ if allowed:request.headers[b"Authorization"] = basic_auth_header(usr, pwd)return None# Middleware-level authdiff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex e4b414f45..5366c6633 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -177,3 +177,49 @@ class TestHttpAuthMiddlewareMeta:)mw.process_request(req)assert req.headers["Authorization"] == b"Digest 123"++ @pytest.mark.parametrize(+ ("url", "expected"),+ [+ ("https://example.com/b", True),+ ("https://example.com:443/b", True),+ ("http://example.com/a", False),+ ("https://example.com:8443/a", False),+ ("https://sub.example.com/a", False),+ ("https://other.example/a", False),+ ],+ )+ def test_meta_auth_origin(self, url: str, expected: bool):+ mw = make_mw()+ req = Request(+ "https://example.com/a", meta={"http_user": "u", "http_pass": "p"}+ )+ mw.process_request(req)+ assert req.headers["Authorization"] == basic_auth_header("u", "p")+ redirected = req.replace(url=url, headers={})+ mw.process_request(redirected)+ assert ("Authorization" in redirected.headers) is expected++ def test_meta_auth_domain_ignores_origin(self):+ mw = make_mw()+ meta = {"http_user": "u", "http_pass": "p", "http_auth_domain": "example.com"}+ req = Request("https://example.com/a", meta=meta)+ mw.process_request(req)+ redirected = req.replace(url="http://sub.example.com/a", headers={})+ mw.process_request(redirected)+ assert redirected.headers["Authorization"] == basic_auth_header("u", "p")+++ ("mw_kwargs", "meta"),+ [+ ({"user": "foo", "passwd": "bar", "domain": None}, {}),+ ({}, {"http_user": "u", "http_pass": "p"}),+ ],+)[email protected]("url", ["ftp://example.com/a", "s3://example.com/a"])+def test_non_http(mw_kwargs: dict[str, Any], meta: dict[str, Any], url: str):+ mw = make_mw(**mw_kwargs)+ req = Request(url, meta=meta)+ mw.process_request(req)+ assert "Authorization" not in req.headers