Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): expand IDNA and nested host templates
ldm020746549c8e8
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 0fe06f077..4b9dddaac 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -41,6 +41,7 @@ from collections.abc import Callable, Mappingfrom html import escape as html_escapefrom email.utils import formatdatefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from itertools import productfrom pathlib import Pathfrom urllib.parse import parse_qs, parse_qsl, quote, unquote, urlparse, urlsplit, urlunsplit@@ -831,6 +832,21 @@ _REQUEST_TEMPLATE_RE = re.compile()_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")+_HOST_TEMPLATE_RE = re.compile(+ rb"\{\{(?:domains\[(?P<domain>[^\]\r\n]*)\]|"+ rb"hosts\[(?P<namespace>[^\]\r\n]*)\]\[(?P<subdomain>[^\]\r\n]*)\])\}\}"+)+# wptserve configures these labels and their two-label combinations. Template+# keys retain Unicode while the resulting hostnames use ASCII IDNA labels.+_WPT_SUBDOMAIN_LABELS = ("www", "www1", "www2", "天気の良い日", "élève")+_WPT_SUBDOMAIN_PREFIXES = {+ b"": b"",+ **{+ ".".join(labels).encode("utf-8"): ".".join(labels).encode("idna") + b"."+ for depth in (1, 2)+ for labels in product(_WPT_SUBDOMAIN_LABELS, repeat=depth)+ },+}_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")_TRICKLE_DELAY_RE = re.compile(r"d([0-9]+(?:\.[0-9]+)?)")_MAX_TRICKLE_DELAY_SECONDS = 10.0@@ -1390,15 +1406,6 @@ def _substitute_wpt_template_variables(b"{{location[host]}}": request_host_bytes,b"{{location[hostname]}}": request_hostname_bytes,b"{{location[path]}}": request_path_bytes,- b"{{hosts[][]}}": b"localhost",- b"{{hosts[][www]}}": b"www.localhost",- b"{{hosts[][www1]}}": b"www1.localhost",- b"{{hosts[][www2]}}": b"www2.localhost",- b"{{hosts[alt][]}}": b"alt.localhost",- b"{{hosts[alt][www]}}": b"www.alt.localhost",- b"{{domains[www]}}": b"www.localhost",- b"{{domains[www1]}}": b"www1.localhost",- b"{{domains[www2]}}": b"www2.localhost",b"{{location[port]}}": str(port).encode("ascii"),b"{{ports[http][0]}}": str(port).encode("ascii"),b"{{ports[http][1]}}": str(alternate_port).encode("ascii"),@@ -1460,6 +1467,20 @@ def _substitute_wpt_template_variables(}for marker, value in replacements.items():body = body.replace(marker, value)++ def replace_host(match: re.Match[bytes]) -> bytes:+ subdomain = match.group("domain")+ namespace = b""+ if subdomain is None:+ subdomain = match.group("subdomain")+ namespace = match.group("namespace")+ prefix = _WPT_SUBDOMAIN_PREFIXES.get(subdomain)+ host = {b"": b"localhost", b"alt": b"alt.localhost"}.get(namespace)+ if prefix is None or host is None:+ return match.group(0)+ return prefix + host++ body = _HOST_TEMPLATE_RE.sub(replace_host, body)if template_ids is None:template_ids = {}diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_resources.py b/moli-benchmark/tests/test_wpt_cross_fixture_resources.pyindex 034612688..4b477548f 100644--- a/moli-benchmark/tests/test_wpt_cross_fixture_resources.py+++ b/moli-benchmark/tests/test_wpt_cross_fixture_resources.py@@ -372,6 +372,67 @@ class WptCrossFixtureResourcesTests(WptCrossTestCase):b" ws-url=ws://example.test:12345/socket"b" wss-url=ws://example.test:12345/socket",)+ def test_fixture_server_substitutes_idna_domain_aliases(self) -> None:+ cases = [+ ("{{domains[天気の良い日]}}", b"xn--n8j6ds53lwwkrqhv28a.localhost"),+ ("{{hosts[][天気の良い日]}}", b"xn--n8j6ds53lwwkrqhv28a.localhost"),+ ("{{domains[élève]}}", b"xn--lve-6lad.localhost"),+ ("{{hosts[alt][élève]}}", b"xn--lve-6lad.alt.localhost"),+ ("{{domains[www.élève]}}", b"www.xn--lve-6lad.localhost"),+ ("{{hosts[alt][www1.www2]}}", b"www1.www2.alt.localhost"),+ ("{{hosts[alt][www2]}}", b"www2.alt.localhost"),+ ("{{hosts[unknown][élève]}}", "{{hosts[unknown][élève]}}".encode()),+ ("{{domains[unconfigured]}}", b"{{domains[unconfigured]}}"),+ ]+ for marker, expected in cases:+ with self.subTest(marker=marker):+ self.assertEqual(+ _substitute_wpt_template_variables(+ b"\xff" + marker.encode("utf-8") + b"\xfe",+ port=12345,+ request_hostname="www1.localhost",+ primary_hostname="localhost",+ ),+ b"\xff" + expected + b"\xfe",+ )++ def test_fixture_server_serves_idna_domain_aliases_in_body_and_headers(self) -> None:+ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text("", encoding="utf-8")+ body = (+ "{{domains[élève]}} {{hosts[alt][天気の良い日]}} {{GET[value]}}"+ ).encode("utf-8")+ for name in ("aliases.sub.js", "aliases.txt"):+ (root_path / name).write_bytes(body)+ (root_path / (name + ".sub.headers")).write_text(+ "Access-Control-Allow-Origin: http://{{hosts[alt][élève]}}:{{ports[http][0]}}\n",+ encoding="utf-8",+ )+ with WptFixtureServer(root_path) as server:+ for path in ("aliases.sub.js?", "aliases.txt?pipe=sub&"):+ with self.subTest(path=path):+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)+ try:+ connection.request(+ "GET", "/" + path + "value=%7B%7Bdomains%5Bwww%5D%7D%7D",+ headers={"Host": f"www1.localhost:{server.port}"},+ )+ response = connection.getresponse()+ self.assertEqual(response.status, 200)+ self.assertEqual(+ response.read(),+ b"xn--lve-6lad.localhost "+ b"xn--n8j6ds53lwwkrqhv28a.alt.localhost {{domains[www]}}",+ )+ self.assertEqual(+ response.getheader("Access-Control-Allow-Origin"),+ f"http://xn--lve-6lad.alt.localhost:{server.port}",+ )+ finally:+ connection.close()+def test_fixture_server_pipe_sub_requests_template_substitution(self) -> None:self.assertTrue(_needs_wpt_template_substitution(