Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve Fetch Range streaming fixtures
ldm056cd4ddd1a63
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex b6fd56f7a..cf50cc8e4 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -395,6 +395,14 @@ SUPPORTED_FETCH_ABORT_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = tfor prefix in ("/fetch/api/resources/", "../resources/")for name in ("stash-put.py", "stash-take.py", "infinite-slow-response.py"))+SUPPORTED_FETCH_RANGE_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(prefix + name)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for prefix in ("/fetch/range/resources/", "resources/", "./resources/")+ for name in ("long-wav.py", "stash-take.py")+)SUPPORTED_WASM_WEBAPI_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = tuple(re.compile(rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"@@ -788,6 +796,8 @@ def _supported_wptserve_handler_references(supported += _script_content_type_handler_reference_patterns(posixpath.dirname(rel) or ".")if rel is not None and rel.rsplit("/", 1)[0] == "fetch/api/abort":supported += SUPPORTED_FETCH_ABORT_WPTSERVE_HANDLER_PATTERNS+ if rel is not None and rel.rsplit("/", 1)[0] == "fetch/range":+ supported += SUPPORTED_FETCH_RANGE_WPTSERVE_HANDLER_PATTERNSif rel is not None and rel.startswith("wasm/webapi/"):supported += SUPPORTED_WASM_WEBAPI_WPTSERVE_HANDLER_PATTERNSif rel is not None:diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 8566afbe0..f29e941fe 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -31,6 +31,7 @@ import mathimport mimetypesimport reimport socket+import structimport subprocessimport sysimport threading@@ -85,6 +86,10 @@ FETCH_ABORT_RESOURCE_PATHS = {"/fetch/api/resources/stash-take.py","/fetch/api/resources/infinite-slow-response.py",}+FETCH_RANGE_RESOURCE_PATHS = {+ "/fetch/range/resources/long-wav.py",+ "/fetch/range/resources/stash-take.py",+}LINK_STYLESHEET_COUNTER_PATH = ("/html/semantics/document-metadata/the-link-element/stylesheet.py")@@ -1608,6 +1613,7 @@ def _make_handler(stopping: threading.Event,) -> type[BaseHTTPRequestHandler]:link_stylesheet_stash = FetchStash()+ range_stash = FetchStash()class WptHandler(BaseHTTPRequestHandler):def do_GET(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API)@@ -1626,7 +1632,7 @@ def _make_handler(returnparsed = urlparse(self.path)path = unquote(parsed.path)- if path in FETCH_ABORT_RESOURCE_PATHS | {+ if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"}:self._serve_fetch_resource_method()@@ -1655,7 +1661,7 @@ def _make_handler(returnparsed = urlparse(self.path)path = unquote(parsed.path)- if path in FETCH_ABORT_RESOURCE_PATHS | {+ if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"}:self._serve_fetch_resource_method()@@ -1704,6 +1710,10 @@ def _make_handler(if self._serve_xhr_response_resource():returnparsed = urlparse(self.path)+ path = unquote(parsed.path)+ if path in FETCH_RANGE_RESOURCE_PATHS:+ self._serve_fetch_range_resource(path, parsed.query, emit_body=self.command != "HEAD")+ returnif unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS:self._serve_fetch_abort_resource(unquote(parsed.path), parsed.query, emit_body=True@@ -1731,7 +1741,7 @@ def _make_handler(self._serve_xhr_response_resource()returnparsed = urlparse(self.path)- if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | {+ if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"}:self._serve_fetch_resource_method()@@ -1862,6 +1872,9 @@ def _make_handler(returnparsed = urlparse(self.path)path = unquote(parsed.path)+ if path in FETCH_RANGE_RESOURCE_PATHS:+ self._serve_fetch_range_resource(path, parsed.query, emit_body=emit_body)+ returnif path == ("/html/semantics/scripting-1/the-script-element/""serve-with-content-type.py"@@ -2397,6 +2410,8 @@ def _make_handler(return self._serve_empty_location_resourceif name.startswith("do_") and unquote(urlparse(self.path).path) in XHR_RESPONSE_RESOURCE_PATHS:return self._serve_xhr_response_resource+ if name.startswith("do_") and unquote(urlparse(self.path).path) in FETCH_RANGE_RESOURCE_PATHS:+ return self._serve_fetch_resource_methodraise AttributeError(name)def _read_content_length_request_body(self) -> bytes | None:@@ -2513,6 +2528,85 @@ def _make_handler()return True+ def _serve_fetch_range_resource(+ self, path: str, query: str, *, emit_body: bool+ ) -> None:+ if not self._consume_request_body():+ return+ params = parse_qs(query, keep_blank_values=True, encoding="latin-1")+ try:+ if path.endswith("/stash-take.py"):+ body = json.dumps(range_stash.take(params["key"][0])).encode("ascii")+ self._send_bytes("application/json", body, emit_body=emit_body, cache_control=None)+ return+ if self.command == "OPTIONS":+ self._send_bytes("text/plain", b"Preflight not accepted",+ emit_body=emit_body, status_code=404, cache_control=None)+ return++ range_header = ", ".join(self.headers.get_all("Range", []))+ range_key = params.get("range-received-key", [""])[0]+ encoding_key = params.get("accept-encoding-key", [""])[0]+ if range_key and range_header:+ range_stash.put(range_key, "range-header-received", overwrite=True)+ if encoding_key:+ range_stash.put(encoding_key, ", ".join(self.headers.get_all("Accept-Encoding", [])),+ overwrite=True)++ # Match long-wav.py's 8 kHz, 8-bit mono, five-minute response,+ # including its header-inclusive Content-Length convention.+ total_length = 8000 * 300+ wav_header = struct.pack(+ "<4sI4s4sIHHIIHH4sI", b"RIFF", 36 + total_length, b"WAVE",+ b"fmt ", 16, 1, 1, 8000, 8000, 1, 8, b"data", total_length,+ )+ remaining = total_length+ initial = wav_header+ status = 200+ headers = [+ ("Content-Type", "audio/wav"),+ ("Accept-Ranges", "bytes"),+ ("Cache-Control", "no-cache"),+ ("Access-Control-Allow-Origin", ", ".join(self.headers.get_all("Origin", []))),+ ]+ match = re.search(r"^bytes=(\d*)-(\d*)$", range_header)+ if match:+ start = int(match[1])+ end = int(match[2]) if match[2] else 0+ remaining = end + 1 - start if end else total_length - start+ initial = wav_header[start:]+ if remaining < len(initial):+ initial = initial[:remaining]+ status = 206+ headers.append(("Content-Range", f"bytes {start}-{end or total_length - 1}/{total_length}"))+ except (KeyError, ValueError):+ self.send_error(500)+ return++ self.close_connection = True+ try:+ self.send_response(status)+ for name, value in headers:+ self.send_header(name, value)+ self.send_header("Content-Length", str(remaining))+ self.end_headers()+ self.wfile.flush()+ if not emit_body:+ return+ self.wfile.write(initial)+ self.wfile.flush()+ remaining -= len(initial)+ while remaining > 0 and not stopping.is_set():+ size = min(remaining, 8000)+ self.wfile.write(b"\0" * size)+ self.wfile.flush()+ remaining -= size+ if stopping.wait(0.5):+ break+ except OSError:+ pass # Upstream ends its stream when a write reports disconnect.++def _serve_fetch_abort_resource(self, path: str, query: str, *, emit_body: bool) -> None:diff --git a/moli-benchmark/tests/test_wpt_fetch_range_server.py b/moli-benchmark/tests/test_wpt_fetch_range_server.pynew file mode 100644index 000000000..9efd38c92--- /dev/null+++ b/moli-benchmark/tests/test_wpt_fetch_range_server.py@@ -0,0 +1,176 @@+from __future__ import annotations++import json+import tempfile+import unittest+import uuid+from contextlib import ExitStack+from http.client import HTTPConnection, IncompleteRead+from pathlib import Path+from unittest.mock import patch+from urllib.parse import urlencode++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++RESOURCE_DIR = "/fetch/range/resources/"+++class FetchRangeFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None+ ))+ self.server = self.stack.enter_context(WptFixtureServer(self.root))++ def request(self, resource="long-wav.py", *, method="GET", headers=(), query="", port=None):+ connection = HTTPConnection("127.0.0.1", port or self.server.port, timeout=3)+ self.stack.callback(connection.close)+ connection.putrequest(method, RESOURCE_DIR + resource + "?" + query,+ skip_accept_encoding=True)+ for name, value in headers:+ connection.putheader(name, value)+ connection.endheaders()+ response = connection.getresponse()+ self.stack.callback(response.close)+ return response++ def take(self, key, *, port=None):+ response = self.request("stash-take.py", query=urlencode({"key": key}), port=port)+ self.assertEqual(response.status, 200)+ self.assertEqual(response.getheader("Content-Type"), "application/json")+ self.assertIsNone(response.getheader("Access-Control-Allow-Origin"))+ self.assertIsNone(response.getheader("Cache-Control"))+ return json.loads(response.read())++ def test_closed_ranges_preserve_wav_bytes_and_response_metadata(self):+ response = self.request(headers=[("Range", "bytes=0-10"), ("Origin", "http://other.test")])+ self.assertEqual(response.status, 206)+ self.assertEqual(response.getheader("Content-Type"), "audio/wav")+ self.assertEqual(response.getheader("Content-Length"), "11")+ self.assertEqual(response.getheader("Content-Range"), "bytes 0-10/2400000")+ self.assertEqual(response.getheader("Accept-Ranges"), "bytes")+ self.assertEqual(response.getheader("Cache-Control"), "no-cache")+ self.assertEqual(response.getheader("Access-Control-Allow-Origin"), "http://other.test")+ self.assertEqual(response.read(), b"RIFF\x24\x9f\x24\0WAV")+ response = self.request(headers=[("Range", "bytes=100-103")])+ self.assertEqual(response.status, 206)+ self.assertEqual(response.read(), b"\0" * 4)++ def test_open_ranges_and_zero_end_match_upstream(self):+ for value, length, content_range in [+ ("bytes=11-", "2399989", "bytes 11-2399999/2400000"),+ ("bytes=0-0", "2400000", "bytes 0-2399999/2400000"),+ ("bytes=0000000000011-0000000000111", "101", "bytes 11-111/2400000"),+ ]:+ with self.subTest(value=value):+ response = self.request(method="HEAD", headers=[("Range", value)])+ self.assertEqual(response.status, 206)+ self.assertEqual(response.getheader("Content-Length"), length)+ self.assertEqual(response.getheader("Content-Range"), content_range)+ self.assertEqual(response.read(), b"")++ def test_invalid_units_empty_and_multiple_ranges_produce_full_response(self):+ for value in ["", "foo", "foo=0-10", "bytes=0-1, 4-5", "BYTES=0-10"]:+ with self.subTest(value=value):+ response = self.request(method="HEAD", headers=[("Range", value)])+ self.assertEqual(response.status, 200)+ self.assertEqual(response.getheader("Content-Length"), "2400000")+ self.assertIsNone(response.getheader("Content-Range"))+ response = self.request(method="HEAD", headers=[("Range", "bytes=0-1"), ("range", "4-5")])+ self.assertEqual(response.status, 200)++ def test_stash_preserves_header_bytes_duplicates_and_first_query_value(self):+ key, ignored = str(uuid.uuid4()), str(uuid.uuid4())+ self.server.fetch_stash.put(key, "separate abort namespace")+ query = urlencode({"accept-encoding-key": key, "range-received-key": ignored})+ response = self.request(method="HEAD", query=query + "&accept-encoding-key=" + ignored,+ headers=[("Range", "bytes=0-10"), ("Accept-Encoding", "gzip"),+ ("accept-encoding", "\xff")])+ self.assertEqual(response.status, 206)+ self.assertEqual(self.take(key.upper().replace("-", ""), port=self.server.alternate_port), "gzip, \xff")+ self.assertIsNone(self.take(key))+ self.assertEqual(self.take(ignored), "range-header-received")+ self.assertEqual(self.server.fetch_stash.take(key), "separate abort namespace")++ def test_stash_overwrites_encoding_and_only_records_nonempty_range(self):+ key, range_key = str(uuid.uuid4()), str(uuid.uuid4())+ query = urlencode({"accept-encoding-key": key, "range-received-key": range_key})+ self.request(method="HEAD", query=query, headers=[("Range", "bytes=1-2"), ("Accept-Encoding", "gzip")])+ self.request(method="HEAD", query=query, headers=[("Range", ""), ("Accept-Encoding", "identity")])+ self.assertEqual(self.take(key), "identity")+ self.assertEqual(self.take(range_key), "range-header-received")+ self.request(method="HEAD", query=query)+ self.assertEqual(self.take(key), "")+ self.assertIsNone(self.take(range_key))++ def test_preflight_rejection_does_not_record_headers(self):+ key = str(uuid.uuid4())+ response = self.request(method="OPTIONS", query=urlencode({"accept-encoding-key": key}),+ headers=[("Origin", "http://other.test"), ("Accept-Encoding", "gzip")])+ self.assertEqual(response.status, 404)+ self.assertEqual(response.read(), b"Preflight not accepted")+ self.assertIsNone(response.getheader("Access-Control-Allow-Origin"))+ self.assertIsNone(self.take(key))++ def test_invalid_keys_and_suffix_range_keep_upstream_errors(self):+ for resource, query, headers in [+ ("stash-take.py", "", []),+ ("stash-take.py", "key=not-a-uuid", []),+ ("long-wav.py", "accept-encoding-key=not-a-uuid", []),+ ("long-wav.py", "", [("Range", "bytes=-10")]),+ ]:+ with self.subTest(resource=resource, query=query, headers=headers):+ self.assertEqual(self.request(resource, query=query, headers=headers).status, 500)++ def test_streaming_and_shutdown_do_not_buffer_full_wav(self):+ with WptFixtureServer(self.root) as server:+ response = self.request(port=server.port)+ self.assertEqual(response.status, 200)+ self.assertEqual(response.getheader("Content-Length"), "2400000")+ prefix = response.read(8044)+ self.assertEqual(prefix[:4], b"RIFF")+ self.assertEqual(prefix[8:16], b"WAVEfmt ")+ self.assertEqual(prefix[36:40], b"data")+ self.assertEqual(prefix[44:], b"\0" * 8000)+ # Keep the client open: server shutdown must interrupt the slow stream.+ with self.assertRaises(IncompleteRead):+ response.read()++ def test_non_get_methods_use_the_same_resource_handlers(self):+ for method in ("POST", "PUT", "PATCH", "DELETE", "CUSTOM"):+ with self.subTest(method=method):+ key = str(uuid.uuid4())+ response = self.request(method=method, headers=[("Range", "bytes=0-10")],+ query=urlencode({"range-received-key": key}))+ self.assertEqual((response.status, response.read()), (206, b"RIFF\x24\x9f\x24\0WAV"))+ response = self.request("stash-take.py", method=method, query=urlencode({"key": key}))+ self.assertEqual(json.loads(response.read()), "range-header-received")+ self.assertIsNone(self.take(key))++ def test_case_selection_only_allows_supported_range_resource_paths(self):+ bodies = {+ "fetch/range/general.any.js": "fetch('resources/long-wav.py');fetch('resources/stash-take.py')",+ "fetch/range/absolute.any.js": "fetch('/fetch/range/resources/stash-take.py')",+ "fetch/range/dotted.any.js": "fetch('./resources/long-wav.py')",+ "fetch/range/unknown.any.js": "fetch('resources/other.py')",+ "fetch/range/suffix.any.js": "fetch('resources/long-wav.py2')",+ "fetch/range/parent.any.js": "fetch('../resources/long-wav.py')",+ "fetch/other/general.any.js": "fetch('resources/long-wav.py')",+ }+ for path, body in bodies.items():+ target = self.root / path+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_text("// META: global=window,worker\n" + body)+ cases = enumerate_cases(self.root, dir_prefixes=("fetch",), any_js_global="both")+ self.assertEqual([case.case_path for case in cases], [+ f"fetch/range/{name}.any.js?moli-wpt-any={realm}"+ for name in ["absolute", "dotted", "general"]+ for realm in ["dedicatedworker", "window"]+ ])