Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): inspect Fetch request headers across HTTP methods
ldm0cccefdc7d9cd
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 9e08c01b0..4445a3e76 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -1701,7 +1701,10 @@ def _service_worker_script_handler_reference_patterns(directory: str) -> tuple[r@lru_cache(maxsize=None)def _fetch_resource_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:- names = ("preflight.py", "clean-stash.py", "inspect-headers.py", "redirect.py")+ names = (+ "preflight.py", "clean-stash.py", "inspect-headers.py", "redirect.py",+ "redirect-empty-location.py",+ )references = []for name in names:resource = "fetch/api/resources/" + namediff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 07a822c6d..9bbe662d3 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -131,6 +131,7 @@ IFRAME_STASH_PATH = "/html/semantics/embedded-content/the-iframe-element/stash.pCOMMON_ECHO_PATH = "/common/echo.py"COMMON_REDIRECT_PATH = "/common/redirect.py"+FETCH_INSPECT_HEADERS_PATH = "/fetch/api/resources/inspect-headers.py"FETCH_REDIRECT_RESOURCE_PATH = "/fetch/api/resources/redirect.py"FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"XHR_URL_RESOURCE_PATHS = {@@ -1366,6 +1367,61 @@ def _workers_modules_export_on_load_script_response() -> tuple[bytes, list[tuple)+def _inspect_headers_response_headers(+ query: str,+ request_headers: list[tuple[str, str]],+) -> list[tuple[str, str]]:+ """Model fetch/api/resources/inspect-headers.py's response headers."""++ params = parse_qsl(query, keep_blank_values=True)+ checked_headers_value = next(+ (value for name, value in params if name == "headers"),+ None,+ )+ checked_headers = (+ checked_headers_value.split("|") if checked_headers_value is not None else []+ )+ request_headers_by_name: dict[str, str] = {}+ for name, value in request_headers:+ request_headers_by_name.setdefault(name.lower(), value)++ response_headers = []+ for name in checked_headers:+ value = request_headers_by_name.get(name.lower())+ if value is not None:+ response_headers.append((f"x-request-{name}", value))++ if any(name == "cors" for name, _ in params):+ response_headers.extend(+ [+ (+ "Access-Control-Allow-Origin",+ request_headers_by_name.get("origin", "*"),+ ),+ ("Access-Control-Allow-Credentials", "true"),+ ("Access-Control-Allow-Methods", "GET, POST, HEAD"),+ (+ "Access-Control-Expose-Headers",+ ", ".join(f"x-request-{name}" for name in checked_headers),+ ),+ ]+ )+ allow_headers = next(+ (value for name, value in params if name == "allow_headers"),+ None,+ )+ response_headers.append(+ (+ "Access-Control-Allow-Headers",+ allow_headers+ if allow_headers is not None+ else ", ".join(name for name, _ in request_headers),+ )+ )++ return response_headers++def _url_host_literal(hostname: str) -> str:if hostname.startswith("[") and hostname.endswith("]"):return hostname@@ -2126,6 +2182,11 @@ requestExecutor("{executor_uuid}", {start_on_js});self._serve(emit_body=False)def do_OPTIONS(self) -> None: # noqa: N802+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ self._serve_fetch_inspect_headers(+ urlsplit(self.path).query, emit_body=self.command != "HEAD",+ )+ returnif unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:self._serve_fetch_redirect_resource(urlsplit(self.path).query, emit_body=self.command != "HEAD",@@ -2189,6 +2250,11 @@ requestExecutor("{executor_uuid}", {start_on_js});self.send_error(404)def do_POST(self) -> None: # noqa: N802+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ self._serve_fetch_inspect_headers(+ urlsplit(self.path).query, emit_body=self.command != "HEAD",+ )+ returnif unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:self._serve_fetch_redirect_resource(urlsplit(self.path).query, emit_body=self.command != "HEAD",@@ -2284,6 +2350,11 @@ requestExecutor("{executor_uuid}", {start_on_js});self.end_headers()def _serve_fetch_resource_method(self) -> None:+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ self._serve_fetch_inspect_headers(+ urlsplit(self.path).query, emit_body=self.command != "HEAD",+ )+ returnif unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:self._serve_fetch_redirect_resource(urlsplit(self.path).query, emit_body=self.command != "HEAD",@@ -2346,6 +2417,11 @@ requestExecutor("{executor_uuid}", {start_on_js});do_DELETE = _serve_fetch_resource_methoddef do_YO(self) -> None: # noqa: N802 (WPT custom method)+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ self._serve_fetch_inspect_headers(+ urlsplit(self.path).query, emit_body=self.command != "HEAD",+ )+ returnif unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:self._serve_fetch_redirect_resource(urlsplit(self.path).query, emit_body=self.command != "HEAD",@@ -2598,6 +2674,11 @@ requestExecutor("{executor_uuid}", {start_on_js});self.send_error(500, "Invalid WPT template or pipe")def _serve_response(self, *, emit_body: bool) -> None:+ if unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ self._serve_fetch_inspect_headers(+ urlsplit(self.path).query, emit_body=self.command != "HEAD",+ )+ returnif unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:self._serve_fetch_redirect_resource(urlsplit(self.path).query, emit_body=self.command != "HEAD",@@ -3488,6 +3569,25 @@ requestExecutor("{executor_uuid}", {start_on_js});self.send_error(500)+ def _serve_fetch_inspect_headers(self, query: str, *, emit_body: bool) -> None:+ try:+ status_code = _pipe_response_status(query) or 200+ except WptPipeError:+ self.send_error(500, "Invalid WPT pipe")+ return+ headers = _inspect_headers_response_headers(query, list(self.headers.items()))+ # The upstream handler only reads headers. Return immediately and+ # close connections with unread uploads, as for redirect fixtures.+ if (self.headers.get("Transfer-Encoding") is not None+ or self.headers.get("Content-Length", "0").strip() not in {"", "0"}):+ self.close_connection = True+ headers.append(("Connection", "close"))+ self._send_bytes(+ "text/plain", b"", emit_body=emit_body, extra_headers=headers,+ status_code=status_code,+ )++def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:delay_seconds = _wpt_delay_seconds(query)if delay_seconds is None:@@ -3533,6 +3633,8 @@ requestExecutor("{executor_uuid}", {start_on_js});)def __getattr__(self, name: str):+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_INSPECT_HEADERS_PATH:+ return self._serve_fetch_resource_methodif name.startswith("do_") and unquote(urlsplit(self.path).path) == PRELOAD_COUNT_PATH:return self._serve_preload_count_resourceif name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:diff --git a/moli-benchmark/tests/test_wpt_cross_fixture_responses.py b/moli-benchmark/tests/test_wpt_cross_fixture_responses.pyindex ed207f461..709dbb3b9 100644--- a/moli-benchmark/tests/test_wpt_cross_fixture_responses.py+++ b/moli-benchmark/tests/test_wpt_cross_fixture_responses.py@@ -505,3 +505,65 @@ class WptCrossFixtureResponsesTests(WptCrossTestCase):self.assertEqual(response.read(), b"FAIL" if method == "GET" else b"")++ def test_fixture_server_models_fetch_inspect_headers_handler(self) -> None:+ self.assertEqual(+ _inspect_headers_response_headers(+ "headers=referer%7Corigin%7Cmissing&cors&allow_headers=x-test",+ [+ ("Referer", "http://source.test/path"),+ ("Origin", "http://origin.test"),+ ("X-Test", "value"),+ ],+ ),+ [+ ("x-request-referer", "http://source.test/path"),+ ("x-request-origin", "http://origin.test"),+ ("Access-Control-Allow-Origin", "http://origin.test"),+ ("Access-Control-Allow-Credentials", "true"),+ ("Access-Control-Allow-Methods", "GET, POST, HEAD"),+ (+ "Access-Control-Expose-Headers",+ "x-request-referer, x-request-origin, x-request-missing",+ ),+ ("Access-Control-Allow-Headers", "x-test"),+ ],+ )++ with tempfile.TemporaryDirectory() as root:+ root_path = Path(root)+ (root_path / "resources").mkdir()+ (root_path / "resources" / "testharness.js").write_text(+ "// testharness", encoding="utf-8"+ )+ with WptFixtureServer(root_path) as server:+ url = (+ f"{server.base_url}/fetch/api/resources/"+ "inspect-headers.py?headers=referer%7Corigin&cors"+ )+ request = Request(+ url,+ headers={+ "Referer": "http://source.test/path",+ "Origin": "http://origin.test",+ },+ )+ with urlopen(request, timeout=2) as response:+ self.assertEqual(response.status, 200)+ self.assertEqual(response.read(), b"")+ self.assertEqual(+ response.headers["x-request-referer"],+ "http://source.test/path",+ )+ self.assertEqual(+ response.headers["x-request-origin"],+ "http://origin.test",+ )+ self.assertEqual(+ response.headers["Access-Control-Allow-Origin"],+ "http://origin.test",+ )+ self.assertEqual(+ response.headers["Access-Control-Expose-Headers"],+ "x-request-referer, x-request-origin",+ )diff --git a/moli-benchmark/tests/test_wpt_fetch_headers_server.py b/moli-benchmark/tests/test_wpt_fetch_headers_server.pynew file mode 100644index 000000000..6d999179e--- /dev/null+++ b/moli-benchmark/tests/test_wpt_fetch_headers_server.py@@ -0,0 +1,112 @@+from __future__ import annotations++import socket+import tempfile+import unittest+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch++from moli_benchmark.wpt_cross.server import WptFixtureServer+++RESOURCE = "/fetch/api/resources/inspect-headers.py"+++class FetchHeaderFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None+ ))+ self.server = self.stack.enter_context(WptFixtureServer(self.root))++ def request(self, method: str, query: str, headers: dict[str, str]):+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)+ try:+ connection.request(method, RESOURCE + "?" + query, headers=headers)+ response = connection.getresponse()+ return response.status, response.headers, response.read()+ finally:+ connection.close()++ def test_reflects_headers_for_standard_and_extension_methods(self) -> None:+ for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "patcH", "DELETE",+ "OPTIONS", "YO", "chicken", "*"):+ with self.subTest(method=method):+ status, headers, body = self.request(+ method, "headers=x-value|empty|missing", {+ "X-Value": "x\x01\t\x1f\x80\xffx", "Empty": "",+ },+ )+ self.assertEqual((status, body), (200, b""))+ self.assertEqual(headers["Content-Type"], "text/plain")+ self.assertEqual(headers["x-request-x-value"], "x\x01\t\x1f\x80\xffx")+ self.assertEqual(headers["x-request-empty"], "")+ self.assertIsNone(headers["x-request-missing"])+ self.assertFalse(any(name.lower().startswith("access-control-") for name in headers))++ def test_cors_and_status_pipe_work_for_each_method(self) -> None:+ for method in ("GET", "HEAD", "POST", "PUT", "OPTIONS", "YO", "chicken"):+ with self.subTest(method=method):+ status, headers, body = self.request(+ method, "headers=x-test&cors&allow_headers=X-Test&pipe=status(202)", {+ "Origin": "https://caller.test", "X-Test": "value",+ },+ )+ self.assertEqual((status, body), (202, b""))+ self.assertEqual(headers["x-request-x-test"], "value")+ self.assertEqual(headers["Access-Control-Allow-Origin"], "https://caller.test")+ self.assertEqual(headers["Access-Control-Allow-Credentials"], "true")+ self.assertEqual(headers["Access-Control-Allow-Methods"], "GET, POST, HEAD")+ self.assertEqual(headers["Access-Control-Allow-Headers"], "X-Test")+ self.assertEqual(headers["Access-Control-Expose-Headers"], "x-request-x-test")++ def test_inspection_does_not_wait_for_unread_uploads(self) -> None:+ for framing in (b"Content-Length: 10", b"Transfer-Encoding: chunked"):+ with self.subTest(framing=framing), socket.create_connection(+ ("127.0.0.1", self.server.port), timeout=2+ ) as connection:+ connection.sendall(+ b"POST " + RESOURCE.encode() + b"?headers=x-test HTTP/1.1\r\n"+ b"Host: localhost\r\nX-Test: value\r\n" + framing + b"\r\n\r\n"+ )+ with connection.makefile("rb") as stream:+ response = stream.read()+ head, body = response.split(b"\r\n\r\n", 1)+ self.assertTrue(head.startswith(b"HTTP/1.0 200 OK\r\n"))+ self.assertIn(b"x-request-x-test: value\r\n", head + b"\r\n")+ self.assertIn(b"Connection: close\r\n", head + b"\r\n")+ self.assertEqual(body, b"")++ def test_can_send_and_receive_all_253_header_values(self) -> None:+ request_headers = {+ f"val{byte}": "x" + chr(byte) + "x"+ for byte in range(256) if byte not in (0, 10, 13)+ }+ for method in ("GET", "POST"):+ with self.subTest(method=method):+ status, headers, body = self.request(+ method, "headers=" + "|".join(request_headers), request_headers+ )+ self.assertEqual((status, body), (200, b""))+ for name, value in request_headers.items():+ self.assertEqual(headers["x-request-" + name], value, name)++ def test_header_count_and_line_length_remain_bounded(self) -> None:+ for request_headers in (+ {f"val{index}": "value" for index in range(512)},+ {"X-Long": "x" * (64 * 1024)},+ ):+ with self.subTest(header_count=len(request_headers)):+ status, _, _ = self.request("GET", "", request_headers)+ self.assertEqual(status, 431)+++if __name__ == "__main__":+ unittest.main()diff --git a/moli-benchmark/tests/wpt_cross_test_support.py b/moli-benchmark/tests/wpt_cross_test_support.pyindex b4e268ecb..cf9a17e78 100644--- a/moli-benchmark/tests/wpt_cross_test_support.py+++ b/moli-benchmark/tests/wpt_cross_test_support.py@@ -125,6 +125,7 @@ from moli_benchmark.wpt_cross.server import (_host_header_hostname,_headers_include,_normalize_harness_case_key,+ _inspect_headers_response_headers,_needs_wpt_template_substitution,_legacy_wpt_resource_alias,_pipe_response_header_operations,