Skip to content

The agent that grows with you

fix(url-safety): look for controls on the string the host checks read

unsupportedpastels55bfc47de1ea
diff --git a/tests/tools/test_url_safety_authority_floor.py b/tests/tools/test_url_safety_authority_floor.py
index ae1912e057b..883c9dc3f87 100644
--- a/tests/tools/test_url_safety_authority_floor.py
+++ b/tests/tools/test_url_safety_authority_floor.py
@@ -51,6 +51,9 @@ TEMPLATED_CODE = [
'js("fetch(`http://localhost:${port}/api`)")',
]
+# Trailing C0 controls (and DEL) that urlsplit keeps in the host and WHATWG clients drop.
+TRAILING_CONTROLS = ["\x00", "\x01", "\x08", "\x1b", "\x1f", "\x7f"]
+
class TestFloorAcceptsTemplatedUrlLiterals:
@pytest.mark.parametrize("code", TEMPLATED_CODE)
def test_local_browser_runs_templated_dev_server_urls(self, local_browser, code):
@@ -84,3 +87,22 @@ class TestFloorJudgesPercentEncodedHostDecoded:
@pytest.mark.parametrize("url", ["http://%6c%6f%63%61%6c%68%6f%73%74/", "http://exa%20mple.com/"])
def test_decoded_ordinary_host_not_blocked(self, url):
assert is_always_blocked_url(url) is False
+
+
+class TestTrailingControlCannotHideMetadataHost:
+ @pytest.mark.parametrize("control", TRAILING_CONTROLS)
+ def test_floor_blocks_despite_dns_failure(self, control):
+ assert is_always_blocked_url("http://169.254.169.254" + control) is True
+
+ @pytest.mark.parametrize("control", TRAILING_CONTROLS)
+ def test_is_safe_url_refuses_with_proxy_configured(self, proxy, control):
+ assert is_safe_url("http://169.254.169.254" + control) is False
+
+ def test_browser_scan_blocks_raw_control_literal(self, local_browser):
+ verdict = browser_use_cli._code_url_verdict('new_tab("http://169.254.169.254\x01")')
+ assert verdict.blocked is not None
+
+ @pytest.mark.parametrize("url", [" https://example.com/ ", "https://example.com/\n", "\thttps://example.com/\r\n"])
+ def test_surrounding_whitespace_still_accepted(self, proxy, url):
+ assert is_always_blocked_url(url) is False
+ assert is_safe_url(url) is True
diff --git a/tools/url_safety.py b/tools/url_safety.py
index 8a7ef15d974..98f24b985da 100644
--- a/tools/url_safety.py
+++ b/tools/url_safety.py
@@ -230,8 +230,18 @@ def _normalize_hostname(host: Optional[str]) -> str:
# different host from one string: WHATWG ends the authority at ``\``, percent-decodes the host and
# accepts ``http:\\host`` or ``http:///host``; urlsplit does none of these. So
# ``http://127.0.0.1:8080\@1.1.1.1`` is 1.1.1.1 here and 127.0.0.1:8080 in a browser. Such
-# URLs are refused rather than guessed at. Both parsers strip C0 controls and space from the ends.
-_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(0x21))
+# URLs are refused rather than guessed at. WHATWG strips every C0 control from the ends, but urlsplit
+# keeps a trailing one in the host, so only ordinary whitespace is stripped before looking for controls.
+_URL_EDGE_WHITESPACE = " \t\n\r\x0b\x0c"
+
+
+def _url_hostname(url: str, *, percent_decode: bool = False) -> str:
+ """The host of *url* as every check compares it: the same edge stripping as the ambiguity check,
+ lowercased, trailing dot dropped, optionally percent-decoded first."""
+ host = urlsplit(url.strip(_URL_EDGE_WHITESPACE)).hostname or ""
+ if percent_decode:
+ host = unquote(host)
+ return _normalize_hostname(host)
def _authority_ambiguity(url: str, *, floor: bool = False) -> Optional[str]:
@@ -241,7 +251,7 @@ def _authority_ambiguity(url: str, *, floor: bool = False) -> Optional[str]:
``floor`` keeps only what can move the host itself (controls, no ``//``, ``\\`` in the authority):
userinfo and an unparseable port (templated ``{port}``/``%d`` literals) leave the host as is, and
the floor checks a percent-encoded host after decoding it."""
- url = url.strip(_C0_CONTROL_OR_SPACE)
+ url = url.strip(_URL_EDGE_WHITESPACE)
parts = urlsplit(url)
if parts.scheme.lower() not in _HTTP_SCHEMES:
return None
@@ -336,7 +346,7 @@ def is_always_blocked_url(url: str) -> bool:
if reason is not None:
logger.warning("Blocked request — ambiguous URL authority (always-blocked floor): %s", reason)
return True
- hostname = _normalize_hostname(unquote(urlparse(url).hostname or ""))
+ hostname = _url_hostname(url, percent_decode=True)
if not hostname:
return False
if hostname in _BLOCKED_HOSTNAMES:
@@ -389,9 +399,7 @@ def is_safe_url(url: str) -> bool:
clients could parse differently (userinfo included). ``allow_private_urls`` skips private-IP
blocking, but cloud metadata endpoints remain blocked regardless."""
try:
- parsed = urlparse(url)
- hostname = _normalize_hostname(parsed.hostname)
- scheme = (parsed.scheme or "").strip().lower()
+ scheme = (urlparse(url).scheme or "").strip().lower()
if scheme not in _HTTP_SCHEMES:
logger.warning("Blocked request — unsupported URL scheme: %s", scheme or "<empty>")
return False
@@ -400,6 +408,7 @@ def is_safe_url(url: str) -> bool:
# The reason only: the URL may carry userinfo credentials.
logger.warning("Blocked request — ambiguous URL authority: %s", reason)
return False
+ hostname = _url_hostname(url)
if not hostname:
return False