Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
test(wpt): substitute request headers with template escaping
ldm01f635fa30d38
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/pipes.py b/moli-benchmark/moli_benchmark/wpt_cross/pipes.pynew file mode 100644index 000000000..1aef03616--- /dev/null+++ b/moli-benchmark/moli_benchmark/wpt_cross/pipes.py@@ -0,0 +1,93 @@+"""Parse wptserve pipe syntax for the fixture server's response helpers.++This mirrors tools/wptserve/wptserve/pipes.py and handlers.wrap_pipeline in+WPT. Recognizing a command does not implement its response transformation;+the static fixture server still only implements its documented pipe subset.+"""++from __future__ import annotations++from urllib.parse import parse_qsl+++class WptPipeError(ValueError):+ """A pipe that wptserve would fail instead of serving the requested file."""+++_ARGUMENT_COUNTS = {+ "header": (2, 3),+ "status": (1, 1),+ "sub": (0, 1),+ "trickle": (1, 1),+ "slice": (1, 2),+ "gzip": (0, 0),+}+_ESCAPES = {"n": "\n", "r": "\r", "t": "\t"}+++def parse_pipe_commands(query: str) -> list[tuple[str, list[str]]]:+ # wrap_pipeline selects the last nonempty pipe parameter. Query decoding+ # precedes tokenization, so a percent-encoded comma is still a separator.+ pipe_string = ""+ for name, value in parse_qsl(query):+ if name == "pipe":+ pipe_string = value++ commands: list[tuple[str, list[str]]] = []+ token: list[str] = []+ in_arguments = False+ chars = iter(pipe_string)+ for char in chars:+ if in_arguments:+ if char == "\\":+ escaped = next(chars, None)+ if escaped is None:+ raise WptPipeError("Unterminated pipe escape")+ token.append(_ESCAPES.get(escaped, escaped))+ elif char in {",", ")"}:+ commands[-1][1].append("".join(token))+ token.clear()+ in_arguments = char == ","+ else:+ # '(' and '|' are ordinary characters inside an argument.+ token.append(char)+ elif char == "(":+ commands.append(("".join(token), []))+ token.clear()+ in_arguments = True+ elif char == "|":+ if token:+ commands.append(("".join(token), []))+ token.clear()+ else:+ token.append(char)+ if in_arguments:+ # Upstream accepts an argument terminated by EOF without a final ')'.+ commands[-1][1].append("".join(token))+ elif token:+ commands.append(("".join(token), []))++ for name, args in commands:+ counts = _ARGUMENT_COUNTS.get(name)+ if counts is None:+ raise WptPipeError(f"Unknown pipe: {name!r}")+ if not counts[0] <= len(args) <= counts[1]:+ raise WptPipeError(f"Invalid argument count for {name}")+ try:+ if name == "header":+ if len(args) == 3 and args[2].lower() not in {"true", "false", "1", "0"}:+ raise ValueError("Invalid append flag")+ # ResponseHeaders uses an isomorphic (Latin-1) encoding, after+ # the query was decoded as UTF-8. Do not silently send UTF-8+ # or let the HTTP connection abort for unencodable values.+ args[0].encode("latin-1")+ args[1].encode("latin-1")+ elif name == "status":+ int(args[0])+ elif name == "slice":+ for arg in args:+ if arg.lower() != "null":+ int(arg)+ except ValueError as error:+ raise WptPipeError(f"Invalid arguments for {name}") from error+ return commandsdiff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 01ca33390..fed0d3c63 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -37,12 +37,15 @@ import sysimport threadingimport timeimport uuid+from collections.abc import Callable, Mappingfrom html import escape as html_escapefrom email.utils import formatdatefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathfrom urllib.parse import parse_qs, parse_qsl, unquote, urlparse, urlsplit, urlunsplit+from .pipes import WptPipeError, parse_pipe_commands+from .any_js import (ANY_JS_DEDICATED_WORKER_GLOBAL,ANY_JS_WINDOW_GLOBAL,@@ -795,6 +798,12 @@ _TRICKLE_PIPE_RE = re.compile(r"(?:^|[|,])trickle\(d([0-9]+(?:\.[0-9]+)?)\)(?:$|_HEADER_PIPE_RE = re.compile(r"^header\(([^,()]+),([^()]*)\)$")_STATUS_PIPE_RE = re.compile(r"^status\(([0-9]{3})\)$")_GET_TEMPLATE_RE = re.compile(rb"\{\{GET\[([^\]\r\n]+)\]\}\}")+_REQUEST_TEMPLATE_RE = re.compile(+ rb"\{\{(?:GET\[(?P<query>[^\]\r\n]+)\]|"+ rb"headers\[(?P<header>[^\]\r\n]+)\]|"+ rb"(?i:header_or_default)\(\s*(?P<optional_header>[^,()]+?)\s*,"+ rb"\s*(?P<default>[^()]*)\))\}\}",+)_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")@@ -806,13 +815,19 @@ _EMPTY_WASM_MODULE = b"\0asm\1\0\0\0"def _pipe_requests_template_substitution(query: str) -> bool:- for name, value in parse_qsl(query, keep_blank_values=True):- if name != "pipe":- continue- for command in value.split("|"):- if command.strip() == "sub":- return True- return False+ return any(name == "sub" for name, _ in parse_pipe_commands(query))+++def _template_escape_type(file_name: str, query: str) -> str:+ # Filename substitution runs before explicit pipes in wptserve. Markup+ # filenames escape HTML; scripts and other resources preserve raw values.+ if ".sub." in file_name:+ markup_extensions = {".html", ".htm", ".xht", ".xhtml", ".xml", ".svg"}+ return "html" if Path(file_name).suffix in markup_extensions else "none"+ for name, args in parse_pipe_commands(query):+ if name == "sub":+ return args[0] if args else "html"+ return "html"def _needs_wpt_template_substitution(file_name: str, body: bytes, query: str = "") -> bool:@@ -991,7 +1006,11 @@ def _valid_static_response_header(name: str, value: str) -> bool:return bool(_HTTP_TOKEN_RE.match(name)) and "\r" not in value and "\n" not in value-def _sidecar_response_headers(file_path: Path) -> list[tuple[str, str]]:+def _sidecar_response_headers(+ file_path: Path,+ *,+ substitute: Callable[[bytes], bytes] | None = None,+) -> list[tuple[str, str]]:"""Return immediate-directory and file-specific WPT sidecar headers."""sidecars: list[Path] = []@@ -1014,9 +1033,12 @@ def _sidecar_response_headers(file_path: Path) -> list[tuple[str, str]]:headers: list[tuple[str, str]] = []for sidecar in sidecars:try:- lines = sidecar.read_bytes().decode("latin-1").splitlines()+ body = sidecar.read_bytes()except OSError:continue+ if substitute is not None and sidecar.name.endswith(".sub.headers"):+ body = substitute(body)+ lines = body.decode("latin-1").splitlines()for line in lines:stripped = line.strip()if (@@ -1174,31 +1196,30 @@ def _static_response_headers(request_path: str = "/",request_hostname: str = "localhost",primary_hostname: str | None = None,+ request_headers: Mapping[str, str] | None = None,) -> list[tuple[str, str]]:- headers = _apply_header_operations(- _sidecar_response_headers(file_path),- _pipe_response_header_operations(query),- )- if port is None:- return headerstemplate_ids: dict[bytes, bytes] = {}- return [- (- name,- _substitute_wpt_template_variables(- value.encode("latin-1"),- port=port,- alternate_port=alternate_port,- remote_port=remote_port,- query=query,- request_path=request_path,- request_hostname=request_hostname,- primary_hostname=primary_hostname,- template_ids=template_ids,- ).decode("latin-1"),++ def substitute(body: bytes) -> bytes:+ assert port is not None+ return _substitute_wpt_template_variables(+ body,+ port=port,+ alternate_port=alternate_port,+ remote_port=remote_port,+ query=query,+ request_path=request_path,+ request_hostname=request_hostname,+ primary_hostname=primary_hostname,+ request_headers=request_headers,+ template_ids=template_ids,+ escape_type="none",)- for name, value in headers- ]++ return _apply_header_operations(+ _sidecar_response_headers(file_path, substitute=substitute if port is not None else None),+ _pipe_response_header_operations(query),+ )def _static_response_header_block(@@ -1239,8 +1260,12 @@ def _substitute_wpt_template_variables(request_path: str = "/",request_hostname: str = "localhost",primary_hostname: str | None = None,+ request_headers: Mapping[str, str] | None = None,template_ids: dict[bytes, bytes] | None = None,+ escape_type: str = "html",) -> bytes:+ if escape_type not in {"html", "none"}:+ raise WptPipeError("Unknown template escape type")if alternate_port is None:alternate_port = portif remote_port is None:@@ -1427,11 +1452,41 @@ def _substitute_wpt_template_variables(name.encode("utf-8", errors="replace"): value.encode("utf-8", errors="replace")for name, value in parse_qsl(query, keep_blank_values=True)}- body = _GET_TEMPLATE_RE.sub(- lambda match: get_params.get(match.group(1), b""),- body,- )- return body+ normalized_request_headers: dict[str, str] = {}+ for name, value in (request_headers or {}).items():+ name = name.lower()+ if name in normalized_request_headers:+ normalized_request_headers[name] += ", " + value+ else:+ normalized_request_headers[name] = value++ def replace_request_value(match: re.Match[bytes]) -> bytes:+ parameter = match.group("query")+ if parameter is not None:+ return get_params.get(parameter, b"")+ required_name = match.group("header")+ if required_name is not None:+ name = required_name.decode("utf-8", errors="replace").lower()+ else:+ name = match.group("optional_header").decode("utf-8", errors="replace").strip().lower()+ if name in normalized_request_headers:+ # HTTPMessage stores header octets as Latin-1 text. wptserve's+ # template engine decodes the original header bytes as UTF-8.+ try:+ value = normalized_request_headers[name].encode("latin-1").decode("utf-8")+ except UnicodeError as error:+ raise WptPipeError("Template request header is not valid UTF-8") from error+ elif required_name is not None:+ raise WptPipeError("Missing template request header")+ else:+ value = match.group("default").decode("utf-8", errors="replace").strip()+ if escape_type == "html":+ value = html_escape(value, quote=True)+ return value.encode("utf-8")++ # Substitute request data in one pass, after configuration and UUID values,+ # so a header or query value containing template syntax remains literal data.+ return _REQUEST_TEMPLATE_RE.sub(replace_request_value, body)def _host_header_hostname(host_header: str | None) -> str:@@ -1883,6 +1938,12 @@ def _make_handler(returndef _serve(self, *, emit_body: bool) -> None:+ try:+ self._serve_response(emit_body=emit_body)+ except WptPipeError:+ self.send_error(500, "Invalid WPT template or pipe")++ def _serve_response(self, *, emit_body: bool) -> None:if self._serve_empty_location_resource(emit_body=emit_body):returnif self._serve_xhr_response_resource(emit_body=emit_body):@@ -2104,8 +2165,11 @@ def _make_handler(request_path=path,request_hostname=_host_header_hostname(self.headers.get("Host")),primary_hostname=primary_hostname,+ request_headers=self.headers,+ escape_type=_template_escape_type(file_path.name, parsed.query),)static_header_context = {+ "request_headers": self.headers,"port": int(getattr(self.server,diff --git a/moli-benchmark/tests/test_wpt_template_headers_server.py b/moli-benchmark/tests/test_wpt_template_headers_server.pynew file mode 100644index 000000000..ad5676165--- /dev/null+++ b/moli-benchmark/tests/test_wpt_template_headers_server.py@@ -0,0 +1,130 @@+from __future__ import annotations++import tempfile+import unittest+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch+from urllib.parse import urlencode++from moli_benchmark.wpt_cross.server import WptFixtureServer+++class TemplateHeaderFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,+ ))+ self.server = self.stack.enter_context(WptFixtureServer(self.root))++ def request(self, path: str, headers=(), *, method: str = "GET"):+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)+ try:+ connection.putrequest(method, path)+ for name, value in headers:+ connection.putheader(name, value)+ connection.endheaders()+ response = connection.getresponse()+ return response.status, response.headers, response.read()+ finally:+ connection.close()++ def test_required_header_lookup_is_case_insensitive_and_preserves_empty_values(self) -> None:+ (self.root / "page.sub.html").write_bytes(+ b"{{headers[referer]}}|{{headers[X-Empty]}}|{{header_or_default(X-Missing,<none>)}}"+ )+ headers = [("rEfErEr", "https://example.test/page?a=1&b=2"), ("X-Empty", "")]+ expected = b"https://example.test/page?a=1&b=2||<none>"+ for method in ("GET", "HEAD"):+ with self.subTest(method=method):+ status, response_headers, body = self.request("/page.sub.html", headers, method=method)+ self.assertEqual(status, 200)+ self.assertEqual(body, b"" if method == "HEAD" else expected)+ self.assertEqual(response_headers["Content-Length"], str(len(expected)))++ def test_duplicate_header_fields_are_combined_in_request_order(self) -> None:+ (self.root / "page.sub.txt").write_bytes(+ b"{{headers[x-test]}}|{{header_or_default(X-Test,missing)}}"+ )+ status, _, body = self.request(+ "/page.sub.txt", [("X-Test", "first"), ("x-test", "second"), ("X-Test", "third")],+ )+ self.assertEqual(status, 200)+ self.assertEqual(body, b"first, second, third|first, second, third")++ def test_header_escaping_follows_filename_and_explicit_sub_mode(self) -> None:+ template = b"{{headers[X-Test]}}|{{header_or_default(X-Test,missing)}}"+ for name in ("page.sub.html", "page.sub.xml", "page.sub.js", "page.sub.txt", "page.txt"):+ (self.root / name).write_bytes(template)+ value = '<&"\''+ escaped = b"<&"'"+ for path, expected in (+ ("/page.sub.html", escaped), ("/page.sub.xml", escaped),+ ("/page.sub.js", value.encode()), ("/page.sub.txt", value.encode()),+ ("/page.txt?pipe=sub", escaped), ("/page.txt?pipe=sub(html)", escaped),+ ("/page.txt?pipe=sub(none)", value.encode()),+ ("/page.sub.html?pipe=sub(none)", escaped),+ ):+ with self.subTest(path=path):+ status, _, body = self.request(path, [("X-Test", value)])+ self.assertEqual(status, 200)+ self.assertEqual(body, expected + b"|" + expected)+ self.assertEqual(self.request("/page.txt", [("X-Test", value)])[2], template)++ def test_sub_headers_use_raw_values_and_plain_headers_and_pipe_values_stay_literal(self) -> None:+ (self.root / "page.txt").write_bytes(b"body")+ (self.root / "page.txt.sub.headers").write_text(+ "X-Echo: {{headers[X-Test]}}\nX-Default: {{header_or_default(X-Test,missing)}}\n"+ )+ value = '<&"\''+ status, headers, body = self.request("/page.txt", [("X-Test", value)])+ self.assertEqual((status, body), (200, b"body"))+ self.assertEqual(headers["X-Echo"], value)+ self.assertEqual(headers["X-Default"], value)+ (self.root / "page.txt.sub.headers").unlink()+ (self.root / "page.txt.headers").write_text("X-Echo: {{headers[X-Test]}}\n")+ self.assertEqual(self.request("/page.txt", [("X-Test", value)])[1]["X-Echo"],+ "{{headers[X-Test]}}")+ query = urlencode({"pipe": "header(X-Echo,{{headers[X-Test]}})"})+ self.assertEqual(self.request("/page.txt?" + query, [("X-Test", value)])[1]["X-Echo"],+ "{{headers[X-Test]}}")++ def test_missing_required_headers_fail_without_closing_the_connection_abruptly(self) -> None:+ (self.root / "body.sub.txt").write_bytes(b"{{headers[X-Missing]}}")+ (self.root / "sidecar.txt").write_bytes(b"body")+ (self.root / "sidecar.txt.sub.headers").write_text("X-Echo: {{headers[X-Missing]}}\n")+ for path in ("/body.sub.txt", "/sidecar.txt"):+ for method in ("GET", "HEAD"):+ with self.subTest(path=path, method=method):+ status, _, body = self.request(path, method=method)+ self.assertEqual(status, 500)+ if method == "HEAD":+ self.assertEqual(body, b"")++ def test_request_values_are_not_interpreted_as_more_templates(self) -> None:+ (self.root / "page.sub.txt").write_bytes(+ b"{{headers[X-Test]}}|{{header_or_default(X-Test,missing)}}|{{GET[q]}}"+ )+ value = "{{GET[q]}} {{headers[X-Missing]}} {{$id:uuid()}} {{host}}"+ query_value = "{{headers[X-Missing]}}"+ status, _, body = self.request(+ "/page.sub.txt?" + urlencode({"q": query_value}), [("X-Test", value)],+ )+ self.assertEqual(status, 200)+ self.assertEqual(body, (value + "|" + value + "|" + query_value).encode())++ def test_template_header_values_decode_utf8_request_bytes(self) -> None:+ (self.root / "page.sub.html").write_bytes(b"{{headers[X-Test]}}")+ status, _, body = self.request("/page.sub.html", [("X-Test", b"caf\xc3\xa9")])+ self.assertEqual((status, body), (200, b"caf\xc3\xa9"))+ self.assertEqual(self.request("/page.sub.html", [("X-Test", b"caf\xe9")])[0], 500)+++if __name__ == "__main__":+ unittest.main()