Skip to content

The agent that grows with you

fix(url-safety): refuse URLs whose authority parses differently across clients

unsupportedpastels01fbd2663dcc
diff --git a/tests/tools/test_url_safety_authority.py b/tests/tools/test_url_safety_authority.py
new file mode 100644
index 00000000000..ed5c7400ca7
--- /dev/null
+++ b/tests/tools/test_url_safety_authority.py
@@ -0,0 +1,104 @@
+"""URLs whose authority urllib.parse and WHATWG clients (browsers, Node) read differently.
+
+Every case resolves to a public address, so DNS is never what blocks it: only the parse
+does. ``http://127.0.0.1:8080\\@1.1.1.1`` is host 1.1.1.1 to urlsplit but 127.0.0.1:8080
+to a browser.
+"""
+
+import logging
+import socket
+from unittest.mock import patch
+
+import pytest
+
+from tools.url_safety import async_is_safe_url, is_always_blocked_url, is_safe_url
+
+
[email protected](autouse=True)
+def _public_dns():
+ """Every hostname resolves to a public address."""
+ answer = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 0))]
+ with patch("socket.getaddrinfo", return_value=answer):
+ yield
+
+
+AMBIGUOUS = [
+ "http://127.0.0.1:8080\\@1.1.1.1", # reported: browser dials 127.0.0.1:8080
+ "http://127.0.0.1:8080\\\\@1.1.1.1",
+ "http://127.0.0.1\\@1.1.1.1/",
+ "http://1.1.1.1\\.127.0.0.1/",
+ "http://\\127.0.0.1/",
+ "http://%31%32%37.0.0.1/", # WHATWG percent-decodes the host
+ "http://127.0.0.1:8080\[email protected]",
+ "http://127.0.0.1:8080\[email protected]",
+ "http://127.0.0.1:8080\[email protected]",
+ "http://127.0.0.1:8080\[email protected]",
+ "http://exa\tmple.com/",
+ "https://example.com/a\nb",
+ "http://exa mple.com/",
+ "http://1.1.1.1:99999/",
+ "http://1.1.1.1:80x/",
+]
+
+USERINFO = [
+ "http://127.0.0.1:8080%[email protected]",
+ "http://user:[email protected]/",
+ "http://[email protected]/",
+ "https://@example.com/",
+]
+
+LEGITIMATE = [
+ "https://example.com/image.png",
+ "https://example.com/a\\b?q=c\\d", # backslash after the authority
+ "https://example.com/%5C@x",
+ "https://example.com/p@q?x=@y#frag@z", # '@' after the authority
+ "https://example.com/r?next=http://127.0.0.1\\@x",
+ "https://example.com/path%20with%20spaces?q=a%40b",
+ "https://example.com:8443/x",
+ "http://[2606:4700:4700::1111]:8080/",
+ "HTTPS://EXAMPLE.COM/",
+ " https://example.com/ ", # surrounding whitespace is stripped by every parser
+]
+
+
+class TestIsSafeUrlRefusesAmbiguousAuthority:
+ @pytest.mark.parametrize("url", AMBIGUOUS + USERINFO)
+ def test_refused(self, url):
+ assert is_safe_url(url) is False
+
+ @pytest.mark.parametrize("url", LEGITIMATE)
+ def test_legitimate_urls_still_allowed(self, url):
+ assert is_safe_url(url) is True
+
+ @pytest.mark.asyncio
+ async def test_async_variant_refuses_reported_url(self):
+ assert await async_is_safe_url("http://127.0.0.1:8080\\@1.1.1.1") is False
+
+ def test_log_omits_userinfo_credentials(self, caplog):
+ with caplog.at_level(logging.WARNING, logger="tools.url_safety"):
+ assert is_safe_url("https://alice:[email protected]/") is False
+ assert "userinfo" in caplog.text
+ assert "hunter2" not in caplog.text
+
+
+class TestAlwaysBlockedFloorRefusesAmbiguousAuthority:
+ """The floor guards browser backends that skip ``is_safe_url``; Chromium reads the WHATWG host."""
+
+ @pytest.mark.parametrize("url", [
+ "http://169.254.169.254\\@1.1.1.1/latest/meta-data/",
+ "http:\\\\169.254.169.254\\latest",
+ "http:///169.254.169.254/latest",
+ "http://%31%36%39.254.169.254/",
+ "http://169.254.169.254\[email protected]/",
+ ])
+ def test_refused(self, url):
+ assert is_always_blocked_url(url) is True
+
+ @pytest.mark.parametrize("url", [
+ "https://user:[email protected]/", # userinfo alone parses the same everywhere
+ "https://example.com/a\\b",
+ "about:blank",
+ "data:text/plain,line1\nline2",
+ ])
+ def test_unaffected(self, url):
+ assert is_always_blocked_url(url) is False
diff --git a/tools/url_safety.py b/tools/url_safety.py
index 8455e392f4c..3fabfed6654 100644
--- a/tools/url_safety.py
+++ b/tools/url_safety.py
@@ -226,6 +226,42 @@ def _normalize_hostname(host: Optional[str]) -> str:
return (host or "").strip().lower().rstrip(".")
+# urllib.parse and WHATWG parsers (browsers, Node-based fetch backends, also urllib3) can take a
+# different host from one string: WHATWG ends the authority at ``\``, percent-decodes the host and
+# accepts ``http:\\host`` or ``http:///host``; urlsplit does none of these. So
+# ``http://127.0.0.1:8080\@1.1.1.1`` is 1.1.1.1 here and 127.0.0.1:8080 in a browser. Such
+# URLs are refused rather than guessed at. Both parsers strip C0 controls and space from the ends.
+_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(0x21))
+
+
+def _authority_ambiguity(url: str, *, allow_userinfo: bool = False) -> Optional[str]:
+ """Why another client could read a different host or port from an http(s) *url* than urlsplit
+ does, else None. ``\\`` and ``@`` after the authority (path, query, fragment) are accepted: every
+ parser has fixed the host by then, and WHATWG's ``\\`` -> ``/`` rewrite only changes the path."""
+ url = url.strip(_C0_CONTROL_OR_SPACE)
+ parts = urlsplit(url)
+ if parts.scheme.lower() not in _HTTP_SCHEMES:
+ return None
+ if any(ord(c) < 0x20 or ord(c) == 0x7F for c in url):
+ return "control character in URL" # urlsplit silently drops tab/CR/LF anywhere
+ if not parts.netloc:
+ return "authority not introduced by //"
+ if "\\" in parts.netloc:
+ return "backslash in authority"
+ _, at, host = parts.netloc.rpartition("@")
+ if at and not allow_userinfo:
+ return "userinfo in authority"
+ if "%" in host:
+ return "percent-encoding in host"
+ if any(c.isspace() for c in host):
+ return "whitespace in host"
+ try:
+ parts.port
+ except ValueError:
+ return "invalid port"
+ return None
+
+
def _parse_ip(hostname: str) -> Optional[_IPAddress]:
"""IP object for a literal-IP hostname, else None."""
try:
@@ -287,8 +323,13 @@ def is_always_blocked_url(url: str) -> bool:
hostnames/IPs, regardless of backend, routing, or ``allow_private_urls``. For callers that
deliberately bypass the full check (e.g. hybrid cloud browser routing private URLs to a local
sidecar) but must still enforce the floor. False for ordinary private/loopback URLs, DNS
- failures and parse errors (the caller's ordinary fail-closed path handles those)."""
+ failures and parse errors (the caller's ordinary fail-closed path handles those); True for an
+ http(s) URL whose host another client could read differently, which may hide a metadata target."""
try:
+ reason = _authority_ambiguity(url, allow_userinfo=True)
+ if reason is not None:
+ logger.warning("Blocked request — ambiguous URL authority (always-blocked floor): %s", reason)
+ return True
hostname = _normalize_hostname(urlparse(url).hostname)
if not hostname:
return False
@@ -338,8 +379,9 @@ def _resolved_ip_block_reason(ip: _IPAddress, allow_private: bool) -> Optional[s
def is_safe_url(url: str) -> bool:
"""True if the URL target is not a private/internal address. Resolves the hostname and checks
- every answer; fails closed on DNS errors and unexpected exceptions. ``allow_private_urls``
- skips private-IP blocking, but cloud metadata endpoints remain blocked regardless."""
+ every answer; fails closed on DNS errors, unexpected exceptions and URLs whose authority other
+ clients could parse differently (userinfo included). ``allow_private_urls`` skips private-IP
+ blocking, but cloud metadata endpoints remain blocked regardless."""
try:
parsed = urlparse(url)
hostname = _normalize_hostname(parsed.hostname)
@@ -347,6 +389,11 @@ def is_safe_url(url: str) -> bool:
if scheme not in _HTTP_SCHEMES:
logger.warning("Blocked request — unsupported URL scheme: %s", scheme or "<empty>")
return False
+ reason = _authority_ambiguity(url)
+ if reason is not None:
+ # The reason only: the URL may carry userinfo credentials.
+ logger.warning("Blocked request — ambiguous URL authority: %s", reason)
+ return False
if not hostname:
return False