The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(url-safety): refuse URLs whose authority parses differently across clients
unsupportedpastels01fbd2663dcc
diff --git a/tests/tools/test_url_safety_authority.py b/tests/tools/test_url_safety_authority.pynew file mode 100644index 00000000000..ed5c7400ca7--- /dev/null+++ b/tests/tools/test_url_safety_authority.py@@ -0,0 +1,104 @@+"""URLs whose authority urllib.parse and WHATWG clients (browsers, Node) read differently.++Every case resolves to a public address, so DNS is never what blocks it: only the parse+does. ``http://127.0.0.1:8080\\@1.1.1.1`` is host 1.1.1.1 to urlsplit but 127.0.0.1:8080+to a browser.+"""++import logging+import socket+from unittest.mock import patch++import pytest++from tools.url_safety import async_is_safe_url, is_always_blocked_url, is_safe_url++[email protected](autouse=True)+def _public_dns():+ """Every hostname resolves to a public address."""+ answer = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 0))]+ with patch("socket.getaddrinfo", return_value=answer):+ yield+++AMBIGUOUS = [+ "http://127.0.0.1:8080\\@1.1.1.1", # reported: browser dials 127.0.0.1:8080+ "http://127.0.0.1:8080\\\\@1.1.1.1",+ "http://127.0.0.1\\@1.1.1.1/",+ "http://1.1.1.1\\.127.0.0.1/",+ "http://\\127.0.0.1/",+ "http://%31%32%37.0.0.1/", # WHATWG percent-decodes the host+ "http://127.0.0.1:8080\[email protected]",+ "http://127.0.0.1:8080\[email protected]",+ "http://127.0.0.1:8080\[email protected]",+ "http://127.0.0.1:8080\[email protected]",+ "http://exa\tmple.com/",+ "https://example.com/a\nb",+ "http://exa mple.com/",+ "http://1.1.1.1:99999/",+ "http://1.1.1.1:80x/",+]++USERINFO = [+ "http://127.0.0.1:8080%[email protected]",+ "http://user:[email protected]/",+ "http://[email protected]/",+ "https://@example.com/",+]++LEGITIMATE = [+ "https://example.com/image.png",+ "https://example.com/a\\b?q=c\\d", # backslash after the authority+ "https://example.com/%5C@x",+ "https://example.com/p@q?x=@y#frag@z", # '@' after the authority+ "https://example.com/r?next=http://127.0.0.1\\@x",+ "https://example.com/path%20with%20spaces?q=a%40b",+ "https://example.com:8443/x",+ "http://[2606:4700:4700::1111]:8080/",+ "HTTPS://EXAMPLE.COM/",+ " https://example.com/ ", # surrounding whitespace is stripped by every parser+]+++class TestIsSafeUrlRefusesAmbiguousAuthority:+ @pytest.mark.parametrize("url", AMBIGUOUS + USERINFO)+ def test_refused(self, url):+ assert is_safe_url(url) is False++ @pytest.mark.parametrize("url", LEGITIMATE)+ def test_legitimate_urls_still_allowed(self, url):+ assert is_safe_url(url) is True++ @pytest.mark.asyncio+ async def test_async_variant_refuses_reported_url(self):+ assert await async_is_safe_url("http://127.0.0.1:8080\\@1.1.1.1") is False++ def test_log_omits_userinfo_credentials(self, caplog):+ with caplog.at_level(logging.WARNING, logger="tools.url_safety"):+ assert is_safe_url("https://alice:[email protected]/") is False+ assert "userinfo" in caplog.text+ assert "hunter2" not in caplog.text+++class TestAlwaysBlockedFloorRefusesAmbiguousAuthority:+ """The floor guards browser backends that skip ``is_safe_url``; Chromium reads the WHATWG host."""++ @pytest.mark.parametrize("url", [+ "http://169.254.169.254\\@1.1.1.1/latest/meta-data/",+ "http:\\\\169.254.169.254\\latest",+ "http:///169.254.169.254/latest",+ "http://%31%36%39.254.169.254/",+ "http://169.254.169.254\[email protected]/",+ ])+ def test_refused(self, url):+ assert is_always_blocked_url(url) is True++ @pytest.mark.parametrize("url", [+ "https://user:[email protected]/", # userinfo alone parses the same everywhere+ "https://example.com/a\\b",+ "about:blank",+ "data:text/plain,line1\nline2",+ ])+ def test_unaffected(self, url):+ assert is_always_blocked_url(url) is Falsediff --git a/tools/url_safety.py b/tools/url_safety.pyindex 8455e392f4c..3fabfed6654 100644--- a/tools/url_safety.py+++ b/tools/url_safety.py@@ -226,6 +226,42 @@ def _normalize_hostname(host: Optional[str]) -> str:return (host or "").strip().lower().rstrip(".")+# urllib.parse and WHATWG parsers (browsers, Node-based fetch backends, also urllib3) can take a+# different host from one string: WHATWG ends the authority at ``\``, percent-decodes the host and+# accepts ``http:\\host`` or ``http:///host``; urlsplit does none of these. So+# ``http://127.0.0.1:8080\@1.1.1.1`` is 1.1.1.1 here and 127.0.0.1:8080 in a browser. Such+# URLs are refused rather than guessed at. Both parsers strip C0 controls and space from the ends.+_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(0x21))+++def _authority_ambiguity(url: str, *, allow_userinfo: bool = False) -> Optional[str]:+ """Why another client could read a different host or port from an http(s) *url* than urlsplit+ does, else None. ``\\`` and ``@`` after the authority (path, query, fragment) are accepted: every+ parser has fixed the host by then, and WHATWG's ``\\`` -> ``/`` rewrite only changes the path."""+ url = url.strip(_C0_CONTROL_OR_SPACE)+ parts = urlsplit(url)+ if parts.scheme.lower() not in _HTTP_SCHEMES:+ return None+ if any(ord(c) < 0x20 or ord(c) == 0x7F for c in url):+ return "control character in URL" # urlsplit silently drops tab/CR/LF anywhere+ if not parts.netloc:+ return "authority not introduced by //"+ if "\\" in parts.netloc:+ return "backslash in authority"+ _, at, host = parts.netloc.rpartition("@")+ if at and not allow_userinfo:+ return "userinfo in authority"+ if "%" in host:+ return "percent-encoding in host"+ if any(c.isspace() for c in host):+ return "whitespace in host"+ try:+ parts.port+ except ValueError:+ return "invalid port"+ return None++def _parse_ip(hostname: str) -> Optional[_IPAddress]:"""IP object for a literal-IP hostname, else None."""try:@@ -287,8 +323,13 @@ def is_always_blocked_url(url: str) -> bool:hostnames/IPs, regardless of backend, routing, or ``allow_private_urls``. For callers thatdeliberately bypass the full check (e.g. hybrid cloud browser routing private URLs to a localsidecar) but must still enforce the floor. False for ordinary private/loopback URLs, DNS- failures and parse errors (the caller's ordinary fail-closed path handles those)."""+ failures and parse errors (the caller's ordinary fail-closed path handles those); True for an+ http(s) URL whose host another client could read differently, which may hide a metadata target."""try:+ reason = _authority_ambiguity(url, allow_userinfo=True)+ if reason is not None:+ logger.warning("Blocked request — ambiguous URL authority (always-blocked floor): %s", reason)+ return Truehostname = _normalize_hostname(urlparse(url).hostname)if not hostname:return False@@ -338,8 +379,9 @@ def _resolved_ip_block_reason(ip: _IPAddress, allow_private: bool) -> Optional[sdef is_safe_url(url: str) -> bool:"""True if the URL target is not a private/internal address. Resolves the hostname and checks- every answer; fails closed on DNS errors and unexpected exceptions. ``allow_private_urls``- skips private-IP blocking, but cloud metadata endpoints remain blocked regardless."""+ every answer; fails closed on DNS errors, unexpected exceptions and URLs whose authority other+ clients could parse differently (userinfo included). ``allow_private_urls`` skips private-IP+ blocking, but cloud metadata endpoints remain blocked regardless."""try:parsed = urlparse(url)hostname = _normalize_hostname(parsed.hostname)@@ -347,6 +389,11 @@ def is_safe_url(url: str) -> bool:if scheme not in _HTTP_SCHEMES:logger.warning("Blocked request — unsupported URL scheme: %s", scheme or "<empty>")return False+ reason = _authority_ambiguity(url)+ if reason is not None:+ # The reason only: the URL may carry userinfo credentials.+ logger.warning("Blocked request — ambiguous URL authority: %s", reason)+ return Falseif not hostname:return False