Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve service worker registration error fixtures
ldm03ef21f694eec
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 139573e13..51a0a9b00 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -837,6 +837,8 @@ def _supported_wptserve_handler_references(rel: str | None,) -> tuple[re.Pattern[str], ...]:supported: tuple[re.Pattern[str], ...] = ()+ if rel is not None:+ supported += _service_worker_registration_handler_reference_patterns(posixpath.dirname(rel) or ".")if rel is not None and rel.startswith("fetch/api/"):supported += _empty_location_handler_reference_patterns(posixpath.dirname(rel))if rel is not None:@@ -1631,3 +1633,26 @@ def explicit_case(wpt_root: Path, case_path: str) -> WptCase:timeout_multiplier=timeout_multiplier,test_type="crashtest" if _is_crashtest_path(source_case_path) else "testharness",)+++@lru_cache(maxsize=None)+def _service_worker_registration_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:+ references = []+ resources = (+ "service-workers/service-worker/resources/mime-type-worker.py",+ "service-workers/service-worker/resources/import-mime-type-worker.py",+ "service-workers/service-worker/resources/malformed-worker.py",+ "service-workers/service-worker/resources/invalid-chunked-encoding.py",+ "service-workers/service-worker/resources/invalid-chunked-encoding-with-flush.py",+ )+ for resource in resources:+ relative = posixpath.relpath(resource, directory)+ references.extend(("/" + resource, relative, "./" + relative))+ return tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for reference in references+ )+diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 4c31bac60..e7e0f4099 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -137,6 +137,27 @@ NAVIGATION_SECOND_VISIT_PATH = ("/navigation-api/navigation-methods/return-value/resources/""204-205-download-on-second-visit.py")+SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS = {+ "/service-workers/service-worker/resources/mime-type-worker.py",+ "/service-workers/service-worker/resources/import-mime-type-worker.py",+ "/service-workers/service-worker/resources/malformed-worker.py",+ "/service-workers/service-worker/resources/invalid-chunked-encoding.py",+ "/service-workers/service-worker/resources/invalid-chunked-encoding-with-flush.py",+}+SERVICE_WORKER_MALFORMED_SCRIPTS = {+ "parse-error": 'var foo = function() {;',+ "undefined-error": 'foo.bar = 42;',+ "uncaught-exception": 'throw new DOMException("AbortError");',+ "caught-exception": 'try { throw new Error; } catch(e) {}',+ "import-malformed-script": 'importScripts("malformed-worker.py?parse-error");',+ "import-no-such-script": 'importScripts("no-such-script.js");',+ "top-level-await": 'await Promise.resolve(1);',+ "instantiation-error": 'import nonexistent from "./imported-module-script.js";',+ "instantiation-error-and-top-level-await":+ 'import nonexistent from "./imported-module-script.js"; await Promise.resolve(1);',+}++BENCH_TIMEOUT_MULTIPLIER_QUERY = "__moli_bench_timeout_multiplier"BENCH_REPORT_BRIDGE_SRC_RE = re.compile(rb"(?P<prefix>\bsrc\s*=\s*)(?P<quote>['\"])"@@ -1897,6 +1918,9 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):return+ if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ self._serve_service_worker_registration_resource()+ returnif self._serve_xhr_response_resource():returnparsed = urlparse(self.path)@@ -1935,6 +1959,9 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):return+ if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ self._serve_service_worker_registration_resource()+ returnif self._serve_xhr_response_resource():returnparsed = urlparse(self.path)@@ -1994,6 +2021,9 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):return+ if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ self._serve_service_worker_registration_resource()+ returnif self._serve_xhr_response_resource():returnparsed = urlparse(self.path)@@ -2038,6 +2068,9 @@ requestExecutor("{executor_uuid}", {start_on_js});} | XHR_BODY_RESOURCE_PATHS:self._serve_xhr_response_resource()return+ if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ self._serve_service_worker_registration_resource()+ returnparsed = urlparse(self.path)if unquote(parsed.path) == NAVIGATION_SECOND_VISIT_PATH:self._serve_navigation_second_visit()@@ -2256,6 +2289,9 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_empty_location_resource(emit_body=emit_body):return+ if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ self._serve_service_worker_registration_resource()+ returnif self._serve_xhr_response_resource(emit_body=emit_body):returnparsed = urlparse(self.path)@@ -2743,6 +2779,66 @@ requestExecutor("{executor_uuid}", {start_on_js});emit_body=emit_body, cache_control=None,)+ def _serve_service_worker_invalid_chunked(self, *, delayed: bool) -> None:+ self.close_connection = True+ self.protocol_version = self.request_version+ try:+ self.send_response(200)+ self.send_header("Content-Type", "application/javascript")+ self.send_header("Transfer-Encoding", "chunked")+ # wptserve adds a length for the returned body, but not for+ # the explicit writer used by the delayed variant.+ if not delayed:+ self.send_header("Content-Length", "6")+ self.end_headers()+ self.wfile.flush()+ if delayed and stopping.wait(1):+ return+ # An explicit upstream writer also emits its bytes for HEAD.+ if delayed or self.command != "HEAD":+ self.wfile.write(b"XX\r\n\r\n")+ self.wfile.flush()+ except OSError:+ return+++ def _serve_service_worker_registration_resource(self) -> None:+ parsed = urlsplit(self.path)+ path = unquote(parsed.path)+ if path.endswith(("/invalid-chunked-encoding.py", "/invalid-chunked-encoding-with-flush.py")):+ self._serve_service_worker_invalid_chunked(delayed=path.endswith("-with-flush.py"))+ return+ if not self._consume_request_body():+ return+ params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")+ headers: list[tuple[str, str]] = []+ body = b""+ try:+ if path.endswith("/mime-type-worker.py"):+ if "mime" in params:+ headers.append(("Content-Type", params["mime"][0]))+ elif path.endswith("/import-mime-type-worker.py"):+ headers.append(("Content-Type", "application/javascript"))+ suffix = "?mime=" + params["mime"][0] if "mime" in params else ""+ body = f"importScripts('./mime-type-worker.py{suffix}');".encode("latin-1")+ else:+ script = SERVICE_WORKER_MALFORMED_SCRIPTS.get(parsed.query)+ if script is None:+ self.send_error(500)+ return+ headers.append(("Content-Type", "application/javascript"))+ body = script.encode("utf-8")+ for _, value in headers:+ if "\r" in value or "\n" in value:+ raise ValueError("invalid response header")+ value.encode("latin-1")+ except (ValueError, UnicodeError):+ self.send_error(400)+ return+ self._send_bytes(None, body, emit_body=self.command != "HEAD",+ extra_headers=headers, cache_control=None)++def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:delay_seconds = _wpt_delay_seconds(query)if delay_seconds is None:@@ -2798,6 +2894,8 @@ requestExecutor("{executor_uuid}", {start_on_js});return self._serve_navigation_second_visitif name.startswith("do_") and unquote(urlparse(self.path).path) == FETCH_EMPTY_LOCATION_PATH:return self._serve_empty_location_resource+ if name.startswith("do_") and unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:+ return self._serve_service_worker_registration_resourceif name.startswith("do_") and unquote(urlparse(self.path).path) in XHR_RESPONSE_RESOURCE_PATHS:return self._serve_xhr_response_resourceif name.startswith("do_") and unquote(urlparse(self.path).path) in FETCH_RANGE_RESOURCE_PATHS:diff --git a/moli-benchmark/tests/test_wpt_service_worker_registration_server.py b/moli-benchmark/tests/test_wpt_service_worker_registration_server.pynew file mode 100644index 000000000..c9d2b9cbf--- /dev/null+++ b/moli-benchmark/tests/test_wpt_service_worker_registration_server.py@@ -0,0 +1,194 @@+from __future__ import annotations+++import socket+import tempfile++import unittest+++from contextlib import ExitStack++from http.client import HTTPConnection, IncompleteRead+from pathlib import Path+from threading import Event++from unittest.mock import patch++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++DIRECTORY = "service-workers/service-worker"+RESOURCES = "/" + DIRECTORY + "/resources/"+++class ServiceWorkerRegistrationFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ for name in ("testharness.js", "testharnessreport.js"):+ (self.root / "resources" / name).write_text("// harness")+ resources = self.root / DIRECTORY / "resources"+ resources.mkdir(parents=True)+ for name in (+ "mime-type-worker.py", "import-mime-type-worker.py", "malformed-worker.py",+ "invalid-chunked-encoding.py", "invalid-chunked-encoding-with-flush.py",+ ):+ (resources / name).write_text("# Python source must not be sent as a script")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None+ ))++ def server(self) -> WptFixtureServer:+ return self.stack.enter_context(WptFixtureServer(self.root))+++ def request(+ self, port: int, resource: str, query: str = "", *, method: str = "GET",+ ) -> tuple[int, list[tuple[str, str]], bytes]:+ connection = HTTPConnection("127.0.0.1", port, timeout=5)+ try:+ path = resource if resource.startswith("/") else RESOURCES + resource+ connection.request(method, path + "?" + query)+ response = connection.getresponse()+ return (+ response.status,+ [(name.lower(), value) for name, value in response.getheaders()],+ response.read(),+ )+ finally:+ connection.close()+++ def test_registration_mime_handlers_preserve_missing_empty_and_raw_values(self) -> None:+ server = self.server()+ for query, mime in (+ ("", None), ("mime=", ""), ("mime=text%2Fjavascript", "text/javascript"),+ ("mime=text%2Fplain&mime=ignored", "text/plain"),+ ("mime=application%2Fjavascript%3B+charset%3Dutf-8", "application/javascript; charset=utf-8"),+ ("mime=%FF%27%26", "\xff'&"),+ ):+ for method in ("GET", "HEAD", "POST", "CUSTOM"):+ with self.subTest(query=query, method=method):+ status, headers, body = self.request(server.port, "mime-type-worker.py", query, method=method)+ self.assertEqual((status, body), (200, b""))+ self.assertEqual(dict(headers).get("content-type"), mime)+ self.assertNotIn("cache-control", dict(headers))+ status, headers, body = self.request(server.alternate_port, "import-mime-type-worker.py", query, method=method)+ suffix = b"?mime=" + mime.encode("latin-1") if mime is not None else b""+ expected = b"importScripts('./mime-type-worker.py" + suffix + b"');"+ self.assertEqual((status, body), (200, b"" if method == "HEAD" else expected))+ self.assertEqual(dict(headers)["content-type"], "application/javascript")+ self.assertEqual(dict(headers)["content-length"], str(len(expected)))+ self.assertNotIn("cache-control", dict(headers))+++ def test_malformed_worker_selects_on_the_complete_undecoded_query(self) -> None:+ server = self.server()+ for query, expected in (+ ("parse-error", b"var foo = function() {;"),+ ("caught-exception", b"try { throw new Error; } catch(e) {}"),+ ("import-malformed-script", b'importScripts("malformed-worker.py?parse-error");'),+ ("instantiation-error-and-top-level-await", b'import nonexistent from "./imported-module-script.js"; await Promise.resolve(1);'),+ ):+ for method in ("GET", "HEAD", "OPTIONS"):+ with self.subTest(query=query, method=method):+ status, headers, body = self.request(server.port, "%6dalformed-worker.py", query, method=method)+ self.assertEqual((status, body), (200, b"" if method == "HEAD" else expected))+ self.assertEqual(dict(headers)["content-type"], "application/javascript")+ self.assertEqual(dict(headers)["content-length"], str(len(expected)))+ self.assertNotIn("cache-control", dict(headers))+ for query in ("", "unknown", "parse%2Derror", "parse-error=", "parse-error&ignored"):+ with self.subTest(query=query):+ self.assertEqual(self.request(server.port, "malformed-worker.py", query)[0], 500)+++ def test_invalid_chunked_responses_keep_raw_bytes_and_distinct_head_behavior(self) -> None:+ server = self.server()+ with patch.object(server._stopping, "wait", return_value=False):+ for delayed in (False, True):+ resource = "invalid-chunked-encoding" + ("-with-flush" if delayed else "") + ".py"+ for method in ("GET", "HEAD", "CUSTOM"):+ with self.subTest(delayed=delayed, method=method), socket.create_connection(+ ("127.0.0.1", server.port), timeout=3,+ ) as connection:+ connection.sendall((+ f"{method} {RESOURCES}{resource} HTTP/1.1\r\n"+ f"Host: localhost:{server.port}\r\n\r\n"+ ).encode())+ with connection.makefile("rb") as response:+ head, body = response.read().split(b"\r\n\r\n", 1)+ fields = head.lower().split(b"\r\n")[1:]+ self.assertTrue(head.startswith(b"HTTP/1.1 200 "))+ self.assertIn(b"content-type: application/javascript", fields)+ self.assertIn(b"transfer-encoding: chunked", fields)+ self.assertEqual(b"content-length: 6" in fields, not delayed)+ self.assertEqual(body, b"" if method == "HEAD" and not delayed else b"XX\r\n\r\n")+ connection = HTTPConnection("127.0.0.1", server.port, timeout=3)+ try:+ connection.request("GET", RESOURCES + resource)+ response = connection.getresponse()+ self.assertTrue(response.chunked)+ with self.assertRaises(IncompleteRead):+ response.read()+ finally:+ connection.close()+++ def test_invalid_chunk_flushes_headers_before_waiting_and_does_not_read_upload(self) -> None:+ server = self.server()+ waiting, release = Event(), Event()+ def wait(timeout: float) -> bool:+ self.assertEqual(timeout, 1)+ waiting.set()+ release.wait(3)+ return False+ with patch.object(server._stopping, "wait", side_effect=wait), socket.create_connection(+ ("127.0.0.1", server.port), timeout=3,+ ) as connection:+ try:+ connection.sendall((+ f"POST {RESOURCES}invalid-chunked-encoding-with-flush.py HTTP/1.1\r\n"+ f"Host: localhost:{server.port}\r\nContent-Length: 1000000\r\n\r\n"+ ).encode())+ self.assertTrue(waiting.wait(2))+ with connection.makefile("rb") as response:+ headers = []+ while (line := response.readline()) != b"\r\n":+ self.assertTrue(line)+ headers.append(line)+ self.assertIn(b"Transfer-Encoding: chunked\r\n", headers)+ release.set()+ self.assertEqual(response.read(), b"XX\r\n\r\n")+ finally:+ release.set()+++ def test_discovery_only_accepts_supported_handler_locations(self) -> None:+ cases = {+ "mime.html": ("resources/mime-type-worker.py", True),+ "import-mime.html": ("resources/import-mime-type-worker.py", True),+ "malformed.html": ("resources/malformed-worker.py", True),+ "chunked.html": ("resources/invalid-chunked-encoding.py", True),+ "chunked-flush.html": ("resources/invalid-chunked-encoding-with-flush.py", True),+ "sub/malformed.html": ("../resources/malformed-worker.py", True),+ "sub/malformed-wrong.html": ("resources/malformed-worker.py", False),+ "malformed-suffix.html": ("resources/malformed-worker.py.extra", False),+ "unknown.html": ("resources/unknown.py", False),+ }+ for name, (reference, _) in cases.items():+ path = self.root / DIRECTORY / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(+ '<!doctype html><script src="/resources/testharness.js"></script>'+ '<script src="/resources/testharnessreport.js"></script>'+ f'<script src="{reference}?Key=ignored"></script>'+ )+ discovered = enumerate_cases(self.root, dir_prefixes=(DIRECTORY,))+ self.assertEqual(+ sorted(case.case_path for case in discovered),+ sorted(DIRECTORY + "/" + name for name, (_, allowed) in cases.items() if allowed),+ )