Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Only send http_user/http_pass to the origin they were first used on (#8329)

Adrian502caa2e2bab
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b20fe73d3..1fec3405c 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -955,6 +955,10 @@ http_auth_domain
Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
+If this key is not set, :reqmeta:`http_user` and :reqmeta:`http_pass` are only
+sent to the origin (scheme, host and port) of the request where they were first
+used, e.g. they are not sent after a cross-origin redirect.
+
.. reqmeta:: http_pass
http_pass
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 106d96402..a48fcbfef 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -15,6 +15,7 @@ from scrapy import Request, Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import SETTINGS_PRIORITIES
from scrapy.utils.decorators import _warn_spider_arg
+from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
@@ -22,6 +23,32 @@ if TYPE_CHECKING:
from scrapy.http import Response
+_DEFAULT_PORTS = {
+ "http": 80,
+ "https": 443,
+}
+
+
+def _origin(request: Request) -> str:
+ parsed_url = urlparse_cached(request)
+ scheme = parsed_url.scheme
+ netloc = (
+ parsed_url.netloc
+ if parsed_url.port != _DEFAULT_PORTS[scheme]
+ else parsed_url.hostname
+ )
+ return f"{scheme}://{netloc}"
+
+
+def _setdefault_auth_origin(request: Request) -> str:
+ origin: str | None = request.meta.get("_auth_origin")
+ if origin:
+ return origin
+ origin = _origin(request)
+ request.meta["_auth_origin"] = origin
+ return origin
+
+
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header."""
@@ -67,14 +94,22 @@ class HttpAuthMiddleware:
def process_request(
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
- if b"Authorization" in request.headers:
+ if (
+ b"Authorization" in request.headers
+ or urlparse_cached(request).scheme not in _DEFAULT_PORTS
+ ):
return None
# Per-request meta overrides
usr = request.meta.get("http_user", "")
pwd = request.meta.get("http_pass", "")
if usr or pwd:
domain = request.meta.get("http_auth_domain")
- if not domain or url_is_from_any_domain(request, [domain]):
+ allowed = (
+ url_is_from_any_domain(request, [domain])
+ if domain
+ else _setdefault_auth_origin(request) == _origin(request)
+ )
+ if allowed:
request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
return None
# Middleware-level auth
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index e4b414f45..5366c6633 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -177,3 +177,49 @@ class TestHttpAuthMiddlewareMeta:
)
mw.process_request(req)
assert req.headers["Authorization"] == b"Digest 123"
+
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ ("https://example.com/b", True),
+ ("https://example.com:443/b", True),
+ ("http://example.com/a", False),
+ ("https://example.com:8443/a", False),
+ ("https://sub.example.com/a", False),
+ ("https://other.example/a", False),
+ ],
+ )
+ def test_meta_auth_origin(self, url: str, expected: bool):
+ mw = make_mw()
+ req = Request(
+ "https://example.com/a", meta={"http_user": "u", "http_pass": "p"}
+ )
+ mw.process_request(req)
+ assert req.headers["Authorization"] == basic_auth_header("u", "p")
+ redirected = req.replace(url=url, headers={})
+ mw.process_request(redirected)
+ assert ("Authorization" in redirected.headers) is expected
+
+ def test_meta_auth_domain_ignores_origin(self):
+ mw = make_mw()
+ meta = {"http_user": "u", "http_pass": "p", "http_auth_domain": "example.com"}
+ req = Request("https://example.com/a", meta=meta)
+ mw.process_request(req)
+ redirected = req.replace(url="http://sub.example.com/a", headers={})
+ mw.process_request(redirected)
+ assert redirected.headers["Authorization"] == basic_auth_header("u", "p")
+
+
+ ("mw_kwargs", "meta"),
+ [
+ ({"user": "foo", "passwd": "bar", "domain": None}, {}),
+ ({}, {"http_user": "u", "http_pass": "p"}),
+ ],
+)
[email protected]("url", ["ftp://example.com/a", "s3://example.com/a"])
+def test_non_http(mw_kwargs: dict[str, Any], meta: dict[str, Any], url: str):
+ mw = make_mw(**mw_kwargs)
+ req = Request(url, meta=meta)
+ mw.process_request(req)
+ assert "Authorization" not in req.headers