Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(wpt): serve Fetch Range streaming fixtures

ldm056cd4ddd1a63
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
index b6fd56f7a..cf50cc8e4 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
@@ -395,6 +395,14 @@ SUPPORTED_FETCH_ABORT_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = t
for prefix in ("/fetch/api/resources/", "../resources/")
for name in ("stash-put.py", "stash-take.py", "infinite-slow-response.py")
)
+SUPPORTED_FETCH_RANGE_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
+ re.compile(
+ rf"(?<![A-Za-z0-9_./-]){re.escape(prefix + name)}"
+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"
+ )
+ for prefix in ("/fetch/range/resources/", "resources/", "./resources/")
+ for name in ("long-wav.py", "stash-take.py")
+)
SUPPORTED_WASM_WEBAPI_WPTSERVE_HANDLER_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
re.compile(
rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"
@@ -788,6 +796,8 @@ def _supported_wptserve_handler_references(
supported += _script_content_type_handler_reference_patterns(posixpath.dirname(rel) or ".")
if rel is not None and rel.rsplit("/", 1)[0] == "fetch/api/abort":
supported += SUPPORTED_FETCH_ABORT_WPTSERVE_HANDLER_PATTERNS
+ if rel is not None and rel.rsplit("/", 1)[0] == "fetch/range":
+ supported += SUPPORTED_FETCH_RANGE_WPTSERVE_HANDLER_PATTERNS
if rel is not None and rel.startswith("wasm/webapi/"):
supported += SUPPORTED_WASM_WEBAPI_WPTSERVE_HANDLER_PATTERNS
if rel is not None:
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 8566afbe0..f29e941fe 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -31,6 +31,7 @@ import math
import mimetypes
import re
import socket
+import struct
import subprocess
import sys
import threading
@@ -85,6 +86,10 @@ FETCH_ABORT_RESOURCE_PATHS = {
"/fetch/api/resources/stash-take.py",
"/fetch/api/resources/infinite-slow-response.py",
}
+FETCH_RANGE_RESOURCE_PATHS = {
+ "/fetch/range/resources/long-wav.py",
+ "/fetch/range/resources/stash-take.py",
+}
LINK_STYLESHEET_COUNTER_PATH = (
"/html/semantics/document-metadata/the-link-element/stylesheet.py"
)
@@ -1608,6 +1613,7 @@ def _make_handler(
stopping: threading.Event,
) -> type[BaseHTTPRequestHandler]:
link_stylesheet_stash = FetchStash()
+ range_stash = FetchStash()
class WptHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API)
@@ -1626,7 +1632,7 @@ def _make_handler(
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
- if path in FETCH_ABORT_RESOURCE_PATHS | {
+ if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {
"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"
}:
self._serve_fetch_resource_method()
@@ -1655,7 +1661,7 @@ def _make_handler(
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
- if path in FETCH_ABORT_RESOURCE_PATHS | {
+ if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {
"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"
}:
self._serve_fetch_resource_method()
@@ -1704,6 +1710,10 @@ def _make_handler(
if self._serve_xhr_response_resource():
return
parsed = urlparse(self.path)
+ path = unquote(parsed.path)
+ if path in FETCH_RANGE_RESOURCE_PATHS:
+ self._serve_fetch_range_resource(path, parsed.query, emit_body=self.command != "HEAD")
+ return
if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS:
self._serve_fetch_abort_resource(
unquote(parsed.path), parsed.query, emit_body=True
@@ -1731,7 +1741,7 @@ def _make_handler(
self._serve_xhr_response_resource()
return
parsed = urlparse(self.path)
- if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | {
+ if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | {
"/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py"
}:
self._serve_fetch_resource_method()
@@ -1862,6 +1872,9 @@ def _make_handler(
return
parsed = urlparse(self.path)
path = unquote(parsed.path)
+ if path in FETCH_RANGE_RESOURCE_PATHS:
+ self._serve_fetch_range_resource(path, parsed.query, emit_body=emit_body)
+ return
if path == (
"/html/semantics/scripting-1/the-script-element/"
"serve-with-content-type.py"
@@ -2397,6 +2410,8 @@ def _make_handler(
return self._serve_empty_location_resource
if name.startswith("do_") and unquote(urlparse(self.path).path) in XHR_RESPONSE_RESOURCE_PATHS:
return self._serve_xhr_response_resource
+ if name.startswith("do_") and unquote(urlparse(self.path).path) in FETCH_RANGE_RESOURCE_PATHS:
+ return self._serve_fetch_resource_method
raise AttributeError(name)
def _read_content_length_request_body(self) -> bytes | None:
@@ -2513,6 +2528,85 @@ def _make_handler(
)
return True
+ def _serve_fetch_range_resource(
+ self, path: str, query: str, *, emit_body: bool
+ ) -> None:
+ if not self._consume_request_body():
+ return
+ params = parse_qs(query, keep_blank_values=True, encoding="latin-1")
+ try:
+ if path.endswith("/stash-take.py"):
+ body = json.dumps(range_stash.take(params["key"][0])).encode("ascii")
+ self._send_bytes("application/json", body, emit_body=emit_body, cache_control=None)
+ return
+ if self.command == "OPTIONS":
+ self._send_bytes("text/plain", b"Preflight not accepted",
+ emit_body=emit_body, status_code=404, cache_control=None)
+ return
+
+ range_header = ", ".join(self.headers.get_all("Range", []))
+ range_key = params.get("range-received-key", [""])[0]
+ encoding_key = params.get("accept-encoding-key", [""])[0]
+ if range_key and range_header:
+ range_stash.put(range_key, "range-header-received", overwrite=True)
+ if encoding_key:
+ range_stash.put(encoding_key, ", ".join(self.headers.get_all("Accept-Encoding", [])),
+ overwrite=True)
+
+ # Match long-wav.py's 8 kHz, 8-bit mono, five-minute response,
+ # including its header-inclusive Content-Length convention.
+ total_length = 8000 * 300
+ wav_header = struct.pack(
+ "<4sI4s4sIHHIIHH4sI", b"RIFF", 36 + total_length, b"WAVE",
+ b"fmt ", 16, 1, 1, 8000, 8000, 1, 8, b"data", total_length,
+ )
+ remaining = total_length
+ initial = wav_header
+ status = 200
+ headers = [
+ ("Content-Type", "audio/wav"),
+ ("Accept-Ranges", "bytes"),
+ ("Cache-Control", "no-cache"),
+ ("Access-Control-Allow-Origin", ", ".join(self.headers.get_all("Origin", []))),
+ ]
+ match = re.search(r"^bytes=(\d*)-(\d*)$", range_header)
+ if match:
+ start = int(match[1])
+ end = int(match[2]) if match[2] else 0
+ remaining = end + 1 - start if end else total_length - start
+ initial = wav_header[start:]
+ if remaining < len(initial):
+ initial = initial[:remaining]
+ status = 206
+ headers.append(("Content-Range", f"bytes {start}-{end or total_length - 1}/{total_length}"))
+ except (KeyError, ValueError):
+ self.send_error(500)
+ return
+
+ self.close_connection = True
+ try:
+ self.send_response(status)
+ for name, value in headers:
+ self.send_header(name, value)
+ self.send_header("Content-Length", str(remaining))
+ self.end_headers()
+ self.wfile.flush()
+ if not emit_body:
+ return
+ self.wfile.write(initial)
+ self.wfile.flush()
+ remaining -= len(initial)
+ while remaining > 0 and not stopping.is_set():
+ size = min(remaining, 8000)
+ self.wfile.write(b"\0" * size)
+ self.wfile.flush()
+ remaining -= size
+ if stopping.wait(0.5):
+ break
+ except OSError:
+ pass # Upstream ends its stream when a write reports disconnect.
+
+
def _serve_fetch_abort_resource(
self, path: str, query: str, *, emit_body: bool
) -> None:
diff --git a/moli-benchmark/tests/test_wpt_fetch_range_server.py b/moli-benchmark/tests/test_wpt_fetch_range_server.py
new file mode 100644
index 000000000..9efd38c92
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_fetch_range_server.py
@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+import uuid
+from contextlib import ExitStack
+from http.client import HTTPConnection, IncompleteRead
+from pathlib import Path
+from unittest.mock import patch
+from urllib.parse import urlencode
+
+from moli_benchmark.wpt_cross.case_set import enumerate_cases
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+RESOURCE_DIR = "/fetch/range/resources/"
+
+
+class FetchRangeFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None
+ ))
+ self.server = self.stack.enter_context(WptFixtureServer(self.root))
+
+ def request(self, resource="long-wav.py", *, method="GET", headers=(), query="", port=None):
+ connection = HTTPConnection("127.0.0.1", port or self.server.port, timeout=3)
+ self.stack.callback(connection.close)
+ connection.putrequest(method, RESOURCE_DIR + resource + "?" + query,
+ skip_accept_encoding=True)
+ for name, value in headers:
+ connection.putheader(name, value)
+ connection.endheaders()
+ response = connection.getresponse()
+ self.stack.callback(response.close)
+ return response
+
+ def take(self, key, *, port=None):
+ response = self.request("stash-take.py", query=urlencode({"key": key}), port=port)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.getheader("Content-Type"), "application/json")
+ self.assertIsNone(response.getheader("Access-Control-Allow-Origin"))
+ self.assertIsNone(response.getheader("Cache-Control"))
+ return json.loads(response.read())
+
+ def test_closed_ranges_preserve_wav_bytes_and_response_metadata(self):
+ response = self.request(headers=[("Range", "bytes=0-10"), ("Origin", "http://other.test")])
+ self.assertEqual(response.status, 206)
+ self.assertEqual(response.getheader("Content-Type"), "audio/wav")
+ self.assertEqual(response.getheader("Content-Length"), "11")
+ self.assertEqual(response.getheader("Content-Range"), "bytes 0-10/2400000")
+ self.assertEqual(response.getheader("Accept-Ranges"), "bytes")
+ self.assertEqual(response.getheader("Cache-Control"), "no-cache")
+ self.assertEqual(response.getheader("Access-Control-Allow-Origin"), "http://other.test")
+ self.assertEqual(response.read(), b"RIFF\x24\x9f\x24\0WAV")
+ response = self.request(headers=[("Range", "bytes=100-103")])
+ self.assertEqual(response.status, 206)
+ self.assertEqual(response.read(), b"\0" * 4)
+
+ def test_open_ranges_and_zero_end_match_upstream(self):
+ for value, length, content_range in [
+ ("bytes=11-", "2399989", "bytes 11-2399999/2400000"),
+ ("bytes=0-0", "2400000", "bytes 0-2399999/2400000"),
+ ("bytes=0000000000011-0000000000111", "101", "bytes 11-111/2400000"),
+ ]:
+ with self.subTest(value=value):
+ response = self.request(method="HEAD", headers=[("Range", value)])
+ self.assertEqual(response.status, 206)
+ self.assertEqual(response.getheader("Content-Length"), length)
+ self.assertEqual(response.getheader("Content-Range"), content_range)
+ self.assertEqual(response.read(), b"")
+
+ def test_invalid_units_empty_and_multiple_ranges_produce_full_response(self):
+ for value in ["", "foo", "foo=0-10", "bytes=0-1, 4-5", "BYTES=0-10"]:
+ with self.subTest(value=value):
+ response = self.request(method="HEAD", headers=[("Range", value)])
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.getheader("Content-Length"), "2400000")
+ self.assertIsNone(response.getheader("Content-Range"))
+ response = self.request(method="HEAD", headers=[("Range", "bytes=0-1"), ("range", "4-5")])
+ self.assertEqual(response.status, 200)
+
+ def test_stash_preserves_header_bytes_duplicates_and_first_query_value(self):
+ key, ignored = str(uuid.uuid4()), str(uuid.uuid4())
+ self.server.fetch_stash.put(key, "separate abort namespace")
+ query = urlencode({"accept-encoding-key": key, "range-received-key": ignored})
+ response = self.request(method="HEAD", query=query + "&accept-encoding-key=" + ignored,
+ headers=[("Range", "bytes=0-10"), ("Accept-Encoding", "gzip"),
+ ("accept-encoding", "\xff")])
+ self.assertEqual(response.status, 206)
+ self.assertEqual(self.take(key.upper().replace("-", ""), port=self.server.alternate_port), "gzip, \xff")
+ self.assertIsNone(self.take(key))
+ self.assertEqual(self.take(ignored), "range-header-received")
+ self.assertEqual(self.server.fetch_stash.take(key), "separate abort namespace")
+
+ def test_stash_overwrites_encoding_and_only_records_nonempty_range(self):
+ key, range_key = str(uuid.uuid4()), str(uuid.uuid4())
+ query = urlencode({"accept-encoding-key": key, "range-received-key": range_key})
+ self.request(method="HEAD", query=query, headers=[("Range", "bytes=1-2"), ("Accept-Encoding", "gzip")])
+ self.request(method="HEAD", query=query, headers=[("Range", ""), ("Accept-Encoding", "identity")])
+ self.assertEqual(self.take(key), "identity")
+ self.assertEqual(self.take(range_key), "range-header-received")
+ self.request(method="HEAD", query=query)
+ self.assertEqual(self.take(key), "")
+ self.assertIsNone(self.take(range_key))
+
+ def test_preflight_rejection_does_not_record_headers(self):
+ key = str(uuid.uuid4())
+ response = self.request(method="OPTIONS", query=urlencode({"accept-encoding-key": key}),
+ headers=[("Origin", "http://other.test"), ("Accept-Encoding", "gzip")])
+ self.assertEqual(response.status, 404)
+ self.assertEqual(response.read(), b"Preflight not accepted")
+ self.assertIsNone(response.getheader("Access-Control-Allow-Origin"))
+ self.assertIsNone(self.take(key))
+
+ def test_invalid_keys_and_suffix_range_keep_upstream_errors(self):
+ for resource, query, headers in [
+ ("stash-take.py", "", []),
+ ("stash-take.py", "key=not-a-uuid", []),
+ ("long-wav.py", "accept-encoding-key=not-a-uuid", []),
+ ("long-wav.py", "", [("Range", "bytes=-10")]),
+ ]:
+ with self.subTest(resource=resource, query=query, headers=headers):
+ self.assertEqual(self.request(resource, query=query, headers=headers).status, 500)
+
+ def test_streaming_and_shutdown_do_not_buffer_full_wav(self):
+ with WptFixtureServer(self.root) as server:
+ response = self.request(port=server.port)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(response.getheader("Content-Length"), "2400000")
+ prefix = response.read(8044)
+ self.assertEqual(prefix[:4], b"RIFF")
+ self.assertEqual(prefix[8:16], b"WAVEfmt ")
+ self.assertEqual(prefix[36:40], b"data")
+ self.assertEqual(prefix[44:], b"\0" * 8000)
+ # Keep the client open: server shutdown must interrupt the slow stream.
+ with self.assertRaises(IncompleteRead):
+ response.read()
+
+ def test_non_get_methods_use_the_same_resource_handlers(self):
+ for method in ("POST", "PUT", "PATCH", "DELETE", "CUSTOM"):
+ with self.subTest(method=method):
+ key = str(uuid.uuid4())
+ response = self.request(method=method, headers=[("Range", "bytes=0-10")],
+ query=urlencode({"range-received-key": key}))
+ self.assertEqual((response.status, response.read()), (206, b"RIFF\x24\x9f\x24\0WAV"))
+ response = self.request("stash-take.py", method=method, query=urlencode({"key": key}))
+ self.assertEqual(json.loads(response.read()), "range-header-received")
+ self.assertIsNone(self.take(key))
+
+ def test_case_selection_only_allows_supported_range_resource_paths(self):
+ bodies = {
+ "fetch/range/general.any.js": "fetch('resources/long-wav.py');fetch('resources/stash-take.py')",
+ "fetch/range/absolute.any.js": "fetch('/fetch/range/resources/stash-take.py')",
+ "fetch/range/dotted.any.js": "fetch('./resources/long-wav.py')",
+ "fetch/range/unknown.any.js": "fetch('resources/other.py')",
+ "fetch/range/suffix.any.js": "fetch('resources/long-wav.py2')",
+ "fetch/range/parent.any.js": "fetch('../resources/long-wav.py')",
+ "fetch/other/general.any.js": "fetch('resources/long-wav.py')",
+ }
+ for path, body in bodies.items():
+ target = self.root / path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text("// META: global=window,worker\n" + body)
+ cases = enumerate_cases(self.root, dir_prefixes=("fetch",), any_js_global="both")
+ self.assertEqual([case.case_path for case in cases], [
+ f"fetch/range/{name}.any.js?moli-wpt-any={realm}"
+ for name in ["absolute", "dotted", "general"]
+ for realm in ["dedicatedworker", "window"]
+ ])