Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(wpt): inspect Fetch request headers across HTTP methods

ldm0cccefdc7d9cd
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
index 9e08c01b0..4445a3e76 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
@@ -1701,7 +1701,10 @@ def _service_worker_script_handler_reference_patterns(directory: str) -> tuple[r
@lru_cache(maxsize=None)
def _fetch_resource_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
- names = ("preflight.py", "clean-stash.py", "inspect-headers.py", "redirect.py")
+ names = (
+ "preflight.py", "clean-stash.py", "inspect-headers.py", "redirect.py",
+ "redirect-empty-location.py",
+ )
references = []
for name in names:
resource = "fetch/api/resources/" + name
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 07a822c6d..9bbe662d3 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -131,6 +131,7 @@ IFRAME_STASH_PATH = "/html/semantics/embedded-content/the-iframe-element/stash.p
COMMON_ECHO_PATH = "/common/echo.py"
COMMON_REDIRECT_PATH = "/common/redirect.py"
+FETCH_INSPECT_HEADERS_PATH = "/fetch/api/resources/inspect-headers.py"
FETCH_REDIRECT_RESOURCE_PATH = "/fetch/api/resources/redirect.py"
FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"
XHR_URL_RESOURCE_PATHS = {
@@ -1366,6 +1367,61 @@ def _workers_modules_export_on_load_script_response() -> tuple[bytes, list[tuple
)
+def _inspect_headers_response_headers(
+ query: str,
+ request_headers: list[tuple[str, str]],
+) -> list[tuple[str, str]]:
+ """Model fetch/api/resources/inspect-headers.py's response headers."""
+
+ params = parse_qsl(query, keep_blank_values=True)
+ checked_headers_value = next(
+ (value for name, value in params if name == "headers"),
+ None,
+ )
+ checked_headers = (
+ checked_headers_value.split("|") if checked_headers_value is not None else []
+ )
+ request_headers_by_name: dict[str, str] = {}
+ for name, value in request_headers:
+ request_headers_by_name.setdefault(name.lower(), value)
+
+ response_headers = []
+ for name in checked_headers:
+ value = request_headers_by_name.get(name.lower())
+ if value is not None:
+ response_headers.append((f"x-request-{name}", value))
+
+ if any(name == "cors" for name, _ in params):
+ response_headers.extend(
+ [
+ (
+ "Access-Control-Allow-Origin",
+ request_headers_by_name.get("origin", "*"),
+ ),
+ ("Access-Control-Allow-Credentials", "true"),
+ ("Access-Control-Allow-Methods", "GET, POST, HEAD"),
+ (
+ "Access-Control-Expose-Headers",
+ ", ".join(f"x-request-{name}" for name in checked_headers),
+ ),
+ ]
+ )
+ allow_headers = next(
+ (value for name, value in params if name == "allow_headers"),
+ None,
+ )
+ response_headers.append(
+ (
+ "Access-Control-Allow-Headers",
+ allow_headers
+ if allow_headers is not None
+ else ", ".join(name for name, _ in request_headers),
+ )
+ )
+
+ return response_headers
+
+
def _url_host_literal(hostname: str) -> str:
if hostname.startswith("[") and hostname.endswith("]"):
return hostname
@@ -2126,6 +2182,11 @@ requestExecutor("{executor_uuid}", {start_on_js});
self._serve(emit_body=False)
def do_OPTIONS(self) -> None: # noqa: N802
+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ self._serve_fetch_inspect_headers(
+ urlsplit(self.path).query, emit_body=self.command != "HEAD",
+ )
+ return
if unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
self._serve_fetch_redirect_resource(
urlsplit(self.path).query, emit_body=self.command != "HEAD",
@@ -2189,6 +2250,11 @@ requestExecutor("{executor_uuid}", {start_on_js});
self.send_error(404)
def do_POST(self) -> None: # noqa: N802
+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ self._serve_fetch_inspect_headers(
+ urlsplit(self.path).query, emit_body=self.command != "HEAD",
+ )
+ return
if unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
self._serve_fetch_redirect_resource(
urlsplit(self.path).query, emit_body=self.command != "HEAD",
@@ -2284,6 +2350,11 @@ requestExecutor("{executor_uuid}", {start_on_js});
self.end_headers()
def _serve_fetch_resource_method(self) -> None:
+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ self._serve_fetch_inspect_headers(
+ urlsplit(self.path).query, emit_body=self.command != "HEAD",
+ )
+ return
if unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
self._serve_fetch_redirect_resource(
urlsplit(self.path).query, emit_body=self.command != "HEAD",
@@ -2346,6 +2417,11 @@ requestExecutor("{executor_uuid}", {start_on_js});
do_DELETE = _serve_fetch_resource_method
def do_YO(self) -> None: # noqa: N802 (WPT custom method)
+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ self._serve_fetch_inspect_headers(
+ urlsplit(self.path).query, emit_body=self.command != "HEAD",
+ )
+ return
if unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
self._serve_fetch_redirect_resource(
urlsplit(self.path).query, emit_body=self.command != "HEAD",
@@ -2598,6 +2674,11 @@ requestExecutor("{executor_uuid}", {start_on_js});
self.send_error(500, "Invalid WPT template or pipe")
def _serve_response(self, *, emit_body: bool) -> None:
+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ self._serve_fetch_inspect_headers(
+ urlsplit(self.path).query, emit_body=self.command != "HEAD",
+ )
+ return
if unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
self._serve_fetch_redirect_resource(
urlsplit(self.path).query, emit_body=self.command != "HEAD",
@@ -3488,6 +3569,25 @@ requestExecutor("{executor_uuid}", {start_on_js});
self.send_error(500)
+ def _serve_fetch_inspect_headers(self, query: str, *, emit_body: bool) -> None:
+ try:
+ status_code = _pipe_response_status(query) or 200
+ except WptPipeError:
+ self.send_error(500, "Invalid WPT pipe")
+ return
+ headers = _inspect_headers_response_headers(query, list(self.headers.items()))
+ # The upstream handler only reads headers. Return immediately and
+ # close connections with unread uploads, as for redirect fixtures.
+ if (self.headers.get("Transfer-Encoding") is not None
+ or self.headers.get("Content-Length", "0").strip() not in {"", "0"}):
+ self.close_connection = True
+ headers.append(("Connection", "close"))
+ self._send_bytes(
+ "text/plain", b"", emit_body=emit_body, extra_headers=headers,
+ status_code=status_code,
+ )
+
+
def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:
delay_seconds = _wpt_delay_seconds(query)
if delay_seconds is None:
@@ -3533,6 +3633,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
)
def __getattr__(self, name: str):
+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:
+ return self._serve_fetch_resource_method
if name.startswith("do_") and unquote(urlsplit(self.path).path) == PRELOAD_COUNT_PATH:
return self._serve_preload_count_resource
if name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_responses.py b/moli-benchmark/tests/test_wpt_cross_fixture_responses.py
index ed207f461..709dbb3b9 100644
--- a/moli-benchmark/tests/test_wpt_cross_fixture_responses.py
+++ b/moli-benchmark/tests/test_wpt_cross_fixture_responses.py
@@ -505,3 +505,65 @@ class WptCrossFixtureResponsesTests(WptCrossTestCase):
self.assertEqual(
response.read(), b"FAIL" if method == "GET" else b""
)
+
+ def test_fixture_server_models_fetch_inspect_headers_handler(self) -> None:
+ self.assertEqual(
+ _inspect_headers_response_headers(
+ "headers=referer%7Corigin%7Cmissing&cors&allow_headers=x-test",
+ [
+ ("Referer", "http://source.test/path"),
+ ("Origin", "http://origin.test"),
+ ("X-Test", "value"),
+ ],
+ ),
+ [
+ ("x-request-referer", "http://source.test/path"),
+ ("x-request-origin", "http://origin.test"),
+ ("Access-Control-Allow-Origin", "http://origin.test"),
+ ("Access-Control-Allow-Credentials", "true"),
+ ("Access-Control-Allow-Methods", "GET, POST, HEAD"),
+ (
+ "Access-Control-Expose-Headers",
+ "x-request-referer, x-request-origin, x-request-missing",
+ ),
+ ("Access-Control-Allow-Headers", "x-test"),
+ ],
+ )
+
+ with tempfile.TemporaryDirectory() as root:
+ root_path = Path(root)
+ (root_path / "resources").mkdir()
+ (root_path / "resources" / "testharness.js").write_text(
+ "// testharness", encoding="utf-8"
+ )
+ with WptFixtureServer(root_path) as server:
+ url = (
+ f"{server.base_url}/fetch/api/resources/"
+ "inspect-headers.py?headers=referer%7Corigin&cors"
+ )
+ request = Request(
+ url,
+ headers={
+ "Referer": "http://source.test/path",
+ "Origin": "http://origin.test",
+ },
+ )
+ with urlopen(request, timeout=2) as response:
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.read(), b"")
+ self.assertEqual(
+ response.headers["x-request-referer"],
+ "http://source.test/path",
+ )
+ self.assertEqual(
+ response.headers["x-request-origin"],
+ "http://origin.test",
+ )
+ self.assertEqual(
+ response.headers["Access-Control-Allow-Origin"],
+ "http://origin.test",
+ )
+ self.assertEqual(
+ response.headers["Access-Control-Expose-Headers"],
+ "x-request-referer, x-request-origin",
+ )
diff --git a/moli-benchmark/tests/test_wpt_fetch_headers_server.py b/moli-benchmark/tests/test_wpt_fetch_headers_server.py
new file mode 100644
index 000000000..6d999179e
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_fetch_headers_server.py
@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import socket
+import tempfile
+import unittest
+from contextlib import ExitStack
+from http.client import HTTPConnection
+from pathlib import Path
+from unittest.mock import patch
+
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+RESOURCE = "/fetch/api/resources/inspect-headers.py"
+
+
+class FetchHeaderFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None
+ ))
+ self.server = self.stack.enter_context(WptFixtureServer(self.root))
+
+ def request(self, method: str, query: str, headers: dict[str, str]):
+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)
+ try:
+ connection.request(method, RESOURCE + "?" + query, headers=headers)
+ response = connection.getresponse()
+ return response.status, response.headers, response.read()
+ finally:
+ connection.close()
+
+ def test_reflects_headers_for_standard_and_extension_methods(self) -> None:
+ for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "patcH", "DELETE",
+ "OPTIONS", "YO", "chicken", "*"):
+ with self.subTest(method=method):
+ status, headers, body = self.request(
+ method, "headers=x-value|empty|missing", {
+ "X-Value": "x\x01\t\x1f\x80\xffx", "Empty": "",
+ },
+ )
+ self.assertEqual((status, body), (200, b""))
+ self.assertEqual(headers["Content-Type"], "text/plain")
+ self.assertEqual(headers["x-request-x-value"], "x\x01\t\x1f\x80\xffx")
+ self.assertEqual(headers["x-request-empty"], "")
+ self.assertIsNone(headers["x-request-missing"])
+ self.assertFalse(any(name.lower().startswith("access-control-") for name in headers))
+
+ def test_cors_and_status_pipe_work_for_each_method(self) -> None:
+ for method in ("GET", "HEAD", "POST", "PUT", "OPTIONS", "YO", "chicken"):
+ with self.subTest(method=method):
+ status, headers, body = self.request(
+ method, "headers=x-test&cors&allow_headers=X-Test&pipe=status(202)", {
+ "Origin": "https://caller.test", "X-Test": "value",
+ },
+ )
+ self.assertEqual((status, body), (202, b""))
+ self.assertEqual(headers["x-request-x-test"], "value")
+ self.assertEqual(headers["Access-Control-Allow-Origin"], "https://caller.test")
+ self.assertEqual(headers["Access-Control-Allow-Credentials"], "true")
+ self.assertEqual(headers["Access-Control-Allow-Methods"], "GET, POST, HEAD")
+ self.assertEqual(headers["Access-Control-Allow-Headers"], "X-Test")
+ self.assertEqual(headers["Access-Control-Expose-Headers"], "x-request-x-test")
+
+ def test_inspection_does_not_wait_for_unread_uploads(self) -> None:
+ for framing in (b"Content-Length: 10", b"Transfer-Encoding: chunked"):
+ with self.subTest(framing=framing), socket.create_connection(
+ ("127.0.0.1", self.server.port), timeout=2
+ ) as connection:
+ connection.sendall(
+ b"POST " + RESOURCE.encode() + b"?headers=x-test HTTP/1.1\r\n"
+ b"Host: localhost\r\nX-Test: value\r\n" + framing + b"\r\n\r\n"
+ )
+ with connection.makefile("rb") as stream:
+ response = stream.read()
+ head, body = response.split(b"\r\n\r\n", 1)
+ self.assertTrue(head.startswith(b"HTTP/1.0 200 OK\r\n"))
+ self.assertIn(b"x-request-x-test: value\r\n", head + b"\r\n")
+ self.assertIn(b"Connection: close\r\n", head + b"\r\n")
+ self.assertEqual(body, b"")
+
+ def test_can_send_and_receive_all_253_header_values(self) -> None:
+ request_headers = {
+ f"val{byte}": "x" + chr(byte) + "x"
+ for byte in range(256) if byte not in (0, 10, 13)
+ }
+ for method in ("GET", "POST"):
+ with self.subTest(method=method):
+ status, headers, body = self.request(
+ method, "headers=" + "|".join(request_headers), request_headers
+ )
+ self.assertEqual((status, body), (200, b""))
+ for name, value in request_headers.items():
+ self.assertEqual(headers["x-request-" + name], value, name)
+
+ def test_header_count_and_line_length_remain_bounded(self) -> None:
+ for request_headers in (
+ {f"val{index}": "value" for index in range(512)},
+ {"X-Long": "x" * (64 * 1024)},
+ ):
+ with self.subTest(header_count=len(request_headers)):
+ status, _, _ = self.request("GET", "", request_headers)
+ self.assertEqual(status, 431)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/moli-benchmark/tests/wpt_cross_test_support.py b/moli-benchmark/tests/wpt_cross_test_support.py
index b4e268ecb..cf9a17e78 100644
--- a/moli-benchmark/tests/wpt_cross_test_support.py
+++ b/moli-benchmark/tests/wpt_cross_test_support.py
@@ -125,6 +125,7 @@ from moli_benchmark.wpt_cross.server import (
_host_header_hostname,
_headers_include,
_normalize_harness_case_key,
+ _inspect_headers_response_headers,
_needs_wpt_template_substitution,
_legacy_wpt_resource_alias,
_pipe_response_header_operations,