Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

test(wpt): substitute request headers with template escaping

ldm01f635fa30d38
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/pipes.py b/moli-benchmark/moli_benchmark/wpt_cross/pipes.py
new file mode 100644
index 000000000..1aef03616
--- /dev/null
+++ b/moli-benchmark/moli_benchmark/wpt_cross/pipes.py
@@ -0,0 +1,93 @@
+"""Parse wptserve pipe syntax for the fixture server's response helpers.
+
+This mirrors tools/wptserve/wptserve/pipes.py and handlers.wrap_pipeline in
+WPT. Recognizing a command does not implement its response transformation;
+the static fixture server still only implements its documented pipe subset.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import parse_qsl
+
+
+class WptPipeError(ValueError):
+ """A pipe that wptserve would fail instead of serving the requested file."""
+
+
+_ARGUMENT_COUNTS = {
+ "header": (2, 3),
+ "status": (1, 1),
+ "sub": (0, 1),
+ "trickle": (1, 1),
+ "slice": (1, 2),
+ "gzip": (0, 0),
+}
+_ESCAPES = {"n": "\n", "r": "\r", "t": "\t"}
+
+
+def parse_pipe_commands(query: str) -> list[tuple[str, list[str]]]:
+ # wrap_pipeline selects the last nonempty pipe parameter. Query decoding
+ # precedes tokenization, so a percent-encoded comma is still a separator.
+ pipe_string = ""
+ for name, value in parse_qsl(query):
+ if name == "pipe":
+ pipe_string = value
+
+ commands: list[tuple[str, list[str]]] = []
+ token: list[str] = []
+ in_arguments = False
+ chars = iter(pipe_string)
+ for char in chars:
+ if in_arguments:
+ if char == "\\":
+ escaped = next(chars, None)
+ if escaped is None:
+ raise WptPipeError("Unterminated pipe escape")
+ token.append(_ESCAPES.get(escaped, escaped))
+ elif char in {",", ")"}:
+ commands[-1][1].append("".join(token))
+ token.clear()
+ in_arguments = char == ","
+ else:
+ # '(' and '|' are ordinary characters inside an argument.
+ token.append(char)
+ elif char == "(":
+ commands.append(("".join(token), []))
+ token.clear()
+ in_arguments = True
+ elif char == "|":
+ if token:
+ commands.append(("".join(token), []))
+ token.clear()
+ else:
+ token.append(char)
+ if in_arguments:
+ # Upstream accepts an argument terminated by EOF without a final ')'.
+ commands[-1][1].append("".join(token))
+ elif token:
+ commands.append(("".join(token), []))
+
+ for name, args in commands:
+ counts = _ARGUMENT_COUNTS.get(name)
+ if counts is None:
+ raise WptPipeError(f"Unknown pipe: {name!r}")
+ if not counts[0] <= len(args) <= counts[1]:
+ raise WptPipeError(f"Invalid argument count for {name}")
+ try:
+ if name == "header":
+ if len(args) == 3 and args[2].lower() not in {"true", "false", "1", "0"}:
+ raise ValueError("Invalid append flag")
+ # ResponseHeaders uses an isomorphic (Latin-1) encoding, after
+ # the query was decoded as UTF-8. Do not silently send UTF-8
+ # or let the HTTP connection abort for unencodable values.
+ args[0].encode("latin-1")
+ args[1].encode("latin-1")
+ elif name == "status":
+ int(args[0])
+ elif name == "slice":
+ for arg in args:
+ if arg.lower() != "null":
+ int(arg)
+ except ValueError as error:
+ raise WptPipeError(f"Invalid arguments for {name}") from error
+ return commands
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 01ca33390..fed0d3c63 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -37,12 +37,15 @@ import sys
import threading
import time
import uuid
+from collections.abc import Callable, Mapping
from html import escape as html_escape
from email.utils import formatdate
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, parse_qsl, unquote, urlparse, urlsplit, urlunsplit
+from .pipes import WptPipeError, parse_pipe_commands
+
from .any_js import (
ANY_JS_DEDICATED_WORKER_GLOBAL,
ANY_JS_WINDOW_GLOBAL,
@@ -795,6 +798,12 @@ _TRICKLE_PIPE_RE = re.compile(r"(?:^|[|,])trickle\(d([0-9]+(?:\.[0-9]+)?)\)(?:$|
_HEADER_PIPE_RE = re.compile(r"^header\(([^,()]+),([^()]*)\)$")
_STATUS_PIPE_RE = re.compile(r"^status\(([0-9]{3})\)$")
_GET_TEMPLATE_RE = re.compile(rb"\{\{GET\[([^\]\r\n]+)\]\}\}")
+_REQUEST_TEMPLATE_RE = re.compile(
+ rb"\{\{(?:GET\[(?P<query>[^\]\r\n]+)\]|"
+ rb"headers\[(?P<header>[^\]\r\n]+)\]|"
+ rb"(?i:header_or_default)\(\s*(?P<optional_header>[^,()]+?)\s*,"
+ rb"\s*(?P<default>[^()]*)\))\}\}",
+)
_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")
_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")
_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
@@ -806,13 +815,19 @@ _EMPTY_WASM_MODULE = b"\0asm\1\0\0\0"
def _pipe_requests_template_substitution(query: str) -> bool:
- for name, value in parse_qsl(query, keep_blank_values=True):
- if name != "pipe":
- continue
- for command in value.split("|"):
- if command.strip() == "sub":
- return True
- return False
+ return any(name == "sub" for name, _ in parse_pipe_commands(query))
+
+
+def _template_escape_type(file_name: str, query: str) -> str:
+ # Filename substitution runs before explicit pipes in wptserve. Markup
+ # filenames escape HTML; scripts and other resources preserve raw values.
+ if ".sub." in file_name:
+ markup_extensions = {".html", ".htm", ".xht", ".xhtml", ".xml", ".svg"}
+ return "html" if Path(file_name).suffix in markup_extensions else "none"
+ for name, args in parse_pipe_commands(query):
+ if name == "sub":
+ return args[0] if args else "html"
+ return "html"
def _needs_wpt_template_substitution(file_name: str, body: bytes, query: str = "") -> bool:
@@ -991,7 +1006,11 @@ def _valid_static_response_header(name: str, value: str) -> bool:
return bool(_HTTP_TOKEN_RE.match(name)) and "\r" not in value and "\n" not in value
-def _sidecar_response_headers(file_path: Path) -> list[tuple[str, str]]:
+def _sidecar_response_headers(
+ file_path: Path,
+ *,
+ substitute: Callable[[bytes], bytes] | None = None,
+) -> list[tuple[str, str]]:
"""Return immediate-directory and file-specific WPT sidecar headers."""
sidecars: list[Path] = []
@@ -1014,9 +1033,12 @@ def _sidecar_response_headers(file_path: Path) -> list[tuple[str, str]]:
headers: list[tuple[str, str]] = []
for sidecar in sidecars:
try:
- lines = sidecar.read_bytes().decode("latin-1").splitlines()
+ body = sidecar.read_bytes()
except OSError:
continue
+ if substitute is not None and sidecar.name.endswith(".sub.headers"):
+ body = substitute(body)
+ lines = body.decode("latin-1").splitlines()
for line in lines:
stripped = line.strip()
if (
@@ -1174,31 +1196,30 @@ def _static_response_headers(
request_path: str = "/",
request_hostname: str = "localhost",
primary_hostname: str | None = None,
+ request_headers: Mapping[str, str] | None = None,
) -> list[tuple[str, str]]:
- headers = _apply_header_operations(
- _sidecar_response_headers(file_path),
- _pipe_response_header_operations(query),
- )
- if port is None:
- return headers
template_ids: dict[bytes, bytes] = {}
- return [
- (
- name,
- _substitute_wpt_template_variables(
- value.encode("latin-1"),
- port=port,
- alternate_port=alternate_port,
- remote_port=remote_port,
- query=query,
- request_path=request_path,
- request_hostname=request_hostname,
- primary_hostname=primary_hostname,
- template_ids=template_ids,
- ).decode("latin-1"),
+
+ def substitute(body: bytes) -> bytes:
+ assert port is not None
+ return _substitute_wpt_template_variables(
+ body,
+ port=port,
+ alternate_port=alternate_port,
+ remote_port=remote_port,
+ query=query,
+ request_path=request_path,
+ request_hostname=request_hostname,
+ primary_hostname=primary_hostname,
+ request_headers=request_headers,
+ template_ids=template_ids,
+ escape_type="none",
)
- for name, value in headers
- ]
+
+ return _apply_header_operations(
+ _sidecar_response_headers(file_path, substitute=substitute if port is not None else None),
+ _pipe_response_header_operations(query),
+ )
def _static_response_header_block(
@@ -1239,8 +1260,12 @@ def _substitute_wpt_template_variables(
request_path: str = "/",
request_hostname: str = "localhost",
primary_hostname: str | None = None,
+ request_headers: Mapping[str, str] | None = None,
template_ids: dict[bytes, bytes] | None = None,
+ escape_type: str = "html",
) -> bytes:
+ if escape_type not in {"html", "none"}:
+ raise WptPipeError("Unknown template escape type")
if alternate_port is None:
alternate_port = port
if remote_port is None:
@@ -1427,11 +1452,41 @@ def _substitute_wpt_template_variables(
name.encode("utf-8", errors="replace"): value.encode("utf-8", errors="replace")
for name, value in parse_qsl(query, keep_blank_values=True)
}
- body = _GET_TEMPLATE_RE.sub(
- lambda match: get_params.get(match.group(1), b""),
- body,
- )
- return body
+ normalized_request_headers: dict[str, str] = {}
+ for name, value in (request_headers or {}).items():
+ name = name.lower()
+ if name in normalized_request_headers:
+ normalized_request_headers[name] += ", " + value
+ else:
+ normalized_request_headers[name] = value
+
+ def replace_request_value(match: re.Match[bytes]) -> bytes:
+ parameter = match.group("query")
+ if parameter is not None:
+ return get_params.get(parameter, b"")
+ required_name = match.group("header")
+ if required_name is not None:
+ name = required_name.decode("utf-8", errors="replace").lower()
+ else:
+ name = match.group("optional_header").decode("utf-8", errors="replace").strip().lower()
+ if name in normalized_request_headers:
+ # HTTPMessage stores header octets as Latin-1 text. wptserve's
+ # template engine decodes the original header bytes as UTF-8.
+ try:
+ value = normalized_request_headers[name].encode("latin-1").decode("utf-8")
+ except UnicodeError as error:
+ raise WptPipeError("Template request header is not valid UTF-8") from error
+ elif required_name is not None:
+ raise WptPipeError("Missing template request header")
+ else:
+ value = match.group("default").decode("utf-8", errors="replace").strip()
+ if escape_type == "html":
+ value = html_escape(value, quote=True)
+ return value.encode("utf-8")
+
+ # Substitute request data in one pass, after configuration and UUID values,
+ # so a header or query value containing template syntax remains literal data.
+ return _REQUEST_TEMPLATE_RE.sub(replace_request_value, body)
def _host_header_hostname(host_header: str | None) -> str:
@@ -1883,6 +1938,12 @@ def _make_handler(
return
def _serve(self, *, emit_body: bool) -> None:
+ try:
+ self._serve_response(emit_body=emit_body)
+ except WptPipeError:
+ self.send_error(500, "Invalid WPT template or pipe")
+
+ def _serve_response(self, *, emit_body: bool) -> None:
if self._serve_empty_location_resource(emit_body=emit_body):
return
if self._serve_xhr_response_resource(emit_body=emit_body):
@@ -2104,8 +2165,11 @@ def _make_handler(
request_path=path,
request_hostname=_host_header_hostname(self.headers.get("Host")),
primary_hostname=primary_hostname,
+ request_headers=self.headers,
+ escape_type=_template_escape_type(file_path.name, parsed.query),
)
static_header_context = {
+ "request_headers": self.headers,
"port": int(
getattr(
self.server,
diff --git a/moli-benchmark/tests/test_wpt_template_headers_server.py b/moli-benchmark/tests/test_wpt_template_headers_server.py
new file mode 100644
index 000000000..ad5676165
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_template_headers_server.py
@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from contextlib import ExitStack
+from http.client import HTTPConnection
+from pathlib import Path
+from unittest.mock import patch
+from urllib.parse import urlencode
+
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+class TemplateHeaderFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,
+ ))
+ self.server = self.stack.enter_context(WptFixtureServer(self.root))
+
+ def request(self, path: str, headers=(), *, method: str = "GET"):
+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)
+ try:
+ connection.putrequest(method, path)
+ for name, value in headers:
+ connection.putheader(name, value)
+ connection.endheaders()
+ response = connection.getresponse()
+ return response.status, response.headers, response.read()
+ finally:
+ connection.close()
+
+ def test_required_header_lookup_is_case_insensitive_and_preserves_empty_values(self) -> None:
+ (self.root / "page.sub.html").write_bytes(
+ b"{{headers[referer]}}|{{headers[X-Empty]}}|{{header_or_default(X-Missing,<none>)}}"
+ )
+ headers = [("rEfErEr", "https://example.test/page?a=1&b=2"), ("X-Empty", "")]
+ expected = b"https://example.test/page?a=1&amp;b=2||&lt;none&gt;"
+ for method in ("GET", "HEAD"):
+ with self.subTest(method=method):
+ status, response_headers, body = self.request("/page.sub.html", headers, method=method)
+ self.assertEqual(status, 200)
+ self.assertEqual(body, b"" if method == "HEAD" else expected)
+ self.assertEqual(response_headers["Content-Length"], str(len(expected)))
+
+ def test_duplicate_header_fields_are_combined_in_request_order(self) -> None:
+ (self.root / "page.sub.txt").write_bytes(
+ b"{{headers[x-test]}}|{{header_or_default(X-Test,missing)}}"
+ )
+ status, _, body = self.request(
+ "/page.sub.txt", [("X-Test", "first"), ("x-test", "second"), ("X-Test", "third")],
+ )
+ self.assertEqual(status, 200)
+ self.assertEqual(body, b"first, second, third|first, second, third")
+
+ def test_header_escaping_follows_filename_and_explicit_sub_mode(self) -> None:
+ template = b"{{headers[X-Test]}}|{{header_or_default(X-Test,missing)}}"
+ for name in ("page.sub.html", "page.sub.xml", "page.sub.js", "page.sub.txt", "page.txt"):
+ (self.root / name).write_bytes(template)
+ value = '<&"\''
+ escaped = b"&lt;&amp;&quot;&#x27;"
+ for path, expected in (
+ ("/page.sub.html", escaped), ("/page.sub.xml", escaped),
+ ("/page.sub.js", value.encode()), ("/page.sub.txt", value.encode()),
+ ("/page.txt?pipe=sub", escaped), ("/page.txt?pipe=sub(html)", escaped),
+ ("/page.txt?pipe=sub(none)", value.encode()),
+ ("/page.sub.html?pipe=sub(none)", escaped),
+ ):
+ with self.subTest(path=path):
+ status, _, body = self.request(path, [("X-Test", value)])
+ self.assertEqual(status, 200)
+ self.assertEqual(body, expected + b"|" + expected)
+ self.assertEqual(self.request("/page.txt", [("X-Test", value)])[2], template)
+
+ def test_sub_headers_use_raw_values_and_plain_headers_and_pipe_values_stay_literal(self) -> None:
+ (self.root / "page.txt").write_bytes(b"body")
+ (self.root / "page.txt.sub.headers").write_text(
+ "X-Echo: {{headers[X-Test]}}\nX-Default: {{header_or_default(X-Test,missing)}}\n"
+ )
+ value = '<&"\''
+ status, headers, body = self.request("/page.txt", [("X-Test", value)])
+ self.assertEqual((status, body), (200, b"body"))
+ self.assertEqual(headers["X-Echo"], value)
+ self.assertEqual(headers["X-Default"], value)
+ (self.root / "page.txt.sub.headers").unlink()
+ (self.root / "page.txt.headers").write_text("X-Echo: {{headers[X-Test]}}\n")
+ self.assertEqual(self.request("/page.txt", [("X-Test", value)])[1]["X-Echo"],
+ "{{headers[X-Test]}}")
+ query = urlencode({"pipe": "header(X-Echo,{{headers[X-Test]}})"})
+ self.assertEqual(self.request("/page.txt?" + query, [("X-Test", value)])[1]["X-Echo"],
+ "{{headers[X-Test]}}")
+
+ def test_missing_required_headers_fail_without_closing_the_connection_abruptly(self) -> None:
+ (self.root / "body.sub.txt").write_bytes(b"{{headers[X-Missing]}}")
+ (self.root / "sidecar.txt").write_bytes(b"body")
+ (self.root / "sidecar.txt.sub.headers").write_text("X-Echo: {{headers[X-Missing]}}\n")
+ for path in ("/body.sub.txt", "/sidecar.txt"):
+ for method in ("GET", "HEAD"):
+ with self.subTest(path=path, method=method):
+ status, _, body = self.request(path, method=method)
+ self.assertEqual(status, 500)
+ if method == "HEAD":
+ self.assertEqual(body, b"")
+
+ def test_request_values_are_not_interpreted_as_more_templates(self) -> None:
+ (self.root / "page.sub.txt").write_bytes(
+ b"{{headers[X-Test]}}|{{header_or_default(X-Test,missing)}}|{{GET[q]}}"
+ )
+ value = "{{GET[q]}} {{headers[X-Missing]}} {{$id:uuid()}} {{host}}"
+ query_value = "{{headers[X-Missing]}}"
+ status, _, body = self.request(
+ "/page.sub.txt?" + urlencode({"q": query_value}), [("X-Test", value)],
+ )
+ self.assertEqual(status, 200)
+ self.assertEqual(body, (value + "|" + value + "|" + query_value).encode())
+
+ def test_template_header_values_decode_utf8_request_bytes(self) -> None:
+ (self.root / "page.sub.html").write_bytes(b"{{headers[X-Test]}}")
+ status, _, body = self.request("/page.sub.html", [("X-Test", b"caf\xc3\xa9")])
+ self.assertEqual((status, body), (200, b"caf\xc3\xa9"))
+ self.assertEqual(self.request("/page.sub.html", [("X-Test", b"caf\xe9")])[0], 500)
+
+
+if __name__ == "__main__":
+ unittest.main()