Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(wpt): expand IDNA and nested host templates

ldm020746549c8e8
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 0fe06f077..4b9dddaac 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -41,6 +41,7 @@ from collections.abc import Callable, Mapping
from html import escape as html_escape
from email.utils import formatdate
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from itertools import product
from pathlib import Path
from urllib.parse import parse_qs, parse_qsl, quote, unquote, urlparse, urlsplit, urlunsplit
@@ -831,6 +832,21 @@ _REQUEST_TEMPLATE_RE = re.compile(
)
_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")
_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")
+_HOST_TEMPLATE_RE = re.compile(
+ rb"\{\{(?:domains\[(?P<domain>[^\]\r\n]*)\]|"
+ rb"hosts\[(?P<namespace>[^\]\r\n]*)\]\[(?P<subdomain>[^\]\r\n]*)\])\}\}"
+)
+# wptserve configures these labels and their two-label combinations. Template
+# keys retain Unicode while the resulting hostnames use ASCII IDNA labels.
+_WPT_SUBDOMAIN_LABELS = ("www", "www1", "www2", "天気の良い日", "élève")
+_WPT_SUBDOMAIN_PREFIXES = {
+ b"": b"",
+ **{
+ ".".join(labels).encode("utf-8"): ".".join(labels).encode("idna") + b"."
+ for depth in (1, 2)
+ for labels in product(_WPT_SUBDOMAIN_LABELS, repeat=depth)
+ },
+}
_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
_TRICKLE_DELAY_RE = re.compile(r"d([0-9]+(?:\.[0-9]+)?)")
_MAX_TRICKLE_DELAY_SECONDS = 10.0
@@ -1390,15 +1406,6 @@ def _substitute_wpt_template_variables(
b"{{location[host]}}": request_host_bytes,
b"{{location[hostname]}}": request_hostname_bytes,
b"{{location[path]}}": request_path_bytes,
- b"{{hosts[][]}}": b"localhost",
- b"{{hosts[][www]}}": b"www.localhost",
- b"{{hosts[][www1]}}": b"www1.localhost",
- b"{{hosts[][www2]}}": b"www2.localhost",
- b"{{hosts[alt][]}}": b"alt.localhost",
- b"{{hosts[alt][www]}}": b"www.alt.localhost",
- b"{{domains[www]}}": b"www.localhost",
- b"{{domains[www1]}}": b"www1.localhost",
- b"{{domains[www2]}}": b"www2.localhost",
b"{{location[port]}}": str(port).encode("ascii"),
b"{{ports[http][0]}}": str(port).encode("ascii"),
b"{{ports[http][1]}}": str(alternate_port).encode("ascii"),
@@ -1460,6 +1467,20 @@ def _substitute_wpt_template_variables(
}
for marker, value in replacements.items():
body = body.replace(marker, value)
+
+ def replace_host(match: re.Match[bytes]) -> bytes:
+ subdomain = match.group("domain")
+ namespace = b""
+ if subdomain is None:
+ subdomain = match.group("subdomain")
+ namespace = match.group("namespace")
+ prefix = _WPT_SUBDOMAIN_PREFIXES.get(subdomain)
+ host = {b"": b"localhost", b"alt": b"alt.localhost"}.get(namespace)
+ if prefix is None or host is None:
+ return match.group(0)
+ return prefix + host
+
+ body = _HOST_TEMPLATE_RE.sub(replace_host, body)
if template_ids is None:
template_ids = {}
diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_resources.py b/moli-benchmark/tests/test_wpt_cross_fixture_resources.py
index 034612688..4b477548f 100644
--- a/moli-benchmark/tests/test_wpt_cross_fixture_resources.py
+++ b/moli-benchmark/tests/test_wpt_cross_fixture_resources.py
@@ -372,6 +372,67 @@ class WptCrossFixtureResourcesTests(WptCrossTestCase):
b" ws-url=ws://example.test:12345/socket"
b" wss-url=ws://example.test:12345/socket",
)
+ def test_fixture_server_substitutes_idna_domain_aliases(self) -> None:
+ cases = [
+ ("{{domains[天気の良い日]}}", b"xn--n8j6ds53lwwkrqhv28a.localhost"),
+ ("{{hosts[][天気の良い日]}}", b"xn--n8j6ds53lwwkrqhv28a.localhost"),
+ ("{{domains[élève]}}", b"xn--lve-6lad.localhost"),
+ ("{{hosts[alt][élève]}}", b"xn--lve-6lad.alt.localhost"),
+ ("{{domains[www.élève]}}", b"www.xn--lve-6lad.localhost"),
+ ("{{hosts[alt][www1.www2]}}", b"www1.www2.alt.localhost"),
+ ("{{hosts[alt][www2]}}", b"www2.alt.localhost"),
+ ("{{hosts[unknown][élève]}}", "{{hosts[unknown][élève]}}".encode()),
+ ("{{domains[unconfigured]}}", b"{{domains[unconfigured]}}"),
+ ]
+ for marker, expected in cases:
+ with self.subTest(marker=marker):
+ self.assertEqual(
+ _substitute_wpt_template_variables(
+ b"\xff" + marker.encode("utf-8") + b"\xfe",
+ port=12345,
+ request_hostname="www1.localhost",
+ primary_hostname="localhost",
+ ),
+ b"\xff" + expected + b"\xfe",
+ )
+
+ def test_fixture_server_serves_idna_domain_aliases_in_body_and_headers(self) -> None:
+ with tempfile.TemporaryDirectory() as root:
+ root_path = Path(root)
+ (root_path / "resources").mkdir()
+ (root_path / "resources" / "testharness.js").write_text("", encoding="utf-8")
+ body = (
+ "{{domains[élève]}} {{hosts[alt][天気の良い日]}} {{GET[value]}}"
+ ).encode("utf-8")
+ for name in ("aliases.sub.js", "aliases.txt"):
+ (root_path / name).write_bytes(body)
+ (root_path / (name + ".sub.headers")).write_text(
+ "Access-Control-Allow-Origin: http://{{hosts[alt][élève]}}:{{ports[http][0]}}\n",
+ encoding="utf-8",
+ )
+ with WptFixtureServer(root_path) as server:
+ for path in ("aliases.sub.js?", "aliases.txt?pipe=sub&"):
+ with self.subTest(path=path):
+ connection = HTTPConnection("127.0.0.1", server.port, timeout=2)
+ try:
+ connection.request(
+ "GET", "/" + path + "value=%7B%7Bdomains%5Bwww%5D%7D%7D",
+ headers={"Host": f"www1.localhost:{server.port}"},
+ )
+ response = connection.getresponse()
+ self.assertEqual(response.status, 200)
+ self.assertEqual(
+ response.read(),
+ b"xn--lve-6lad.localhost "
+ b"xn--n8j6ds53lwwkrqhv28a.alt.localhost {{domains[www]}}",
+ )
+ self.assertEqual(
+ response.getheader("Access-Control-Allow-Origin"),
+ f"http://xn--lve-6lad.alt.localhost:{server.port}",
+ )
+ finally:
+ connection.close()
+
def test_fixture_server_pipe_sub_requests_template_substitution(self) -> None:
self.assertTrue(
_needs_wpt_template_substitution(