Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(wpt): serve the preload request counter fixture

ldm0bde57e580c20
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
index 33259ac24..9e08c01b0 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
@@ -725,6 +725,19 @@ def _html_path_is_supported(
return not any(token in rel for token in excluded)
+@lru_cache(maxsize=None)
+def _preload_count_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
+ resource = "preload/resources/preload-count.py"
+ relative = posixpath.relpath(resource, directory)
+ return tuple(
+ re.compile(
+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"
+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"
+ )
+ for reference in ("/" + resource, relative, "./" + relative)
+ )
+
+
@lru_cache(maxsize=None)
def _common_echo_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
resource = "common/echo.py"
@@ -866,6 +879,7 @@ def _supported_wptserve_handler_references(
supported += _fetch_content_type_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _document_charset_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _iframe_stash_handler_reference_patterns(posixpath.dirname(rel) or ".")
+ supported += _preload_count_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _navigation_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _json_module_handler_reference_patterns(posixpath.dirname(rel) or ".")
if rel is not None:
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 7ae1aef8b..07a822c6d 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -81,6 +81,8 @@ from .case_set import (
http.client._MAXHEADERS = 512
+PRELOAD_COUNT_PATH = "/preload/resources/preload-count.py"
+PRELOAD_COUNT_KEY = "a8697ae7-c8cb-4dbd-a8ef-27111dc7042f"
DEFAULT_TESTHARNESS_TIMEOUT_SECONDS = 10.0
MAX_REQUEST_BODY_BYTES = 16 * 1024 * 1024
MAX_REQUEST_BODY_LINE_BYTES = 64 * 1024
@@ -1772,7 +1774,6 @@ class CspReportStore:
self._lock = threading.Lock()
self._cv = threading.Condition(self._lock)
self._reports: dict[str, list[dict]] = {}
- self._counts: dict[str, int] = {}
def append_reports(self, report_id: str, reports: list[dict]) -> None:
with self._cv:
@@ -1836,7 +1837,6 @@ class FetchStash:
def __init__(self) -> None:
self._lock = threading.Lock()
self._values: dict[tuple[str, uuid.UUID], object] = {}
- self._counts: dict[tuple[str, uuid.UUID], int] = {}
def put(
self, key: str, value: object, *, overwrite: bool = False,
@@ -1853,8 +1853,8 @@ class FetchStash:
def increment(self, key: str, *, path: str) -> int:
parsed_key = (path, uuid.UUID(key))
with self._lock:
- value = int(self._counts.get(parsed_key, 0)) + 1
- self._counts[parsed_key] = value
+ value = int(self._values.get(parsed_key, 0)) + 1
+ self._values[parsed_key] = value
return value
@@ -2143,6 +2143,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
return
if self._serve_common_redirect_resource():
return
+ if self._serve_preload_count_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:
@@ -2204,6 +2206,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
return
if self._serve_common_redirect_resource():
return
+ if self._serve_preload_count_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:
@@ -2297,6 +2301,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
return
if self._serve_common_redirect_resource():
return
+ if self._serve_preload_count_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:
@@ -2357,6 +2363,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
return
if self._serve_common_redirect_resource():
return
+ if self._serve_preload_count_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if unquote(urlparse(self.path).path) in {
@@ -2525,6 +2533,35 @@ requestExecutor("{executor_uuid}", {start_on_js});
escape_type=escape_type,
)
+ def _serve_preload_count_resource(self) -> bool:
+ parsed = urlsplit(self.path)
+ if unquote(parsed.path) != PRELOAD_COUNT_PATH:
+ return False
+ # The upstream handler only reads GET parameters, for every method.
+ self.close_connection = True
+ stash_path = PRELOAD_COUNT_PATH.rsplit("/", 1)[0] + "/"
+ params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")
+ try:
+ action = params["action"][0]
+ except KeyError:
+ # Upstream takes the counter before reading the required action.
+ fetch_stash.take(PRELOAD_COUNT_KEY, path=stash_path)
+ self.send_error(500)
+ return True
+ if action == "result":
+ count = fetch_stash.take(PRELOAD_COUNT_KEY, path=stash_path) or 0
+ status, content_type = 200, "text/javascript"
+ body = f"preloadCount = {count};".encode("ascii")
+ else:
+ fetch_stash.increment(PRELOAD_COUNT_KEY, path=stash_path)
+ status, content_type, body = 404, None, b"No entry is found"
+ self._send_bytes(
+ content_type, body, emit_body=self.command != "HEAD",
+ status_code=status, cache_control=None,
+ extra_headers=[("Connection", "close")],
+ )
+ return True
+
def _serve_common_echo_resource(self) -> bool:
parsed = urlsplit(self.path)
if unquote(parsed.path) != COMMON_ECHO_PATH:
@@ -2570,6 +2607,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
return
if self._serve_common_redirect_resource():
return
+ if self._serve_preload_count_resource():
+ return
if self._serve_empty_location_resource(emit_body=emit_body):
return
if unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:
@@ -3494,6 +3533,8 @@ requestExecutor("{executor_uuid}", {start_on_js});
)
def __getattr__(self, name: str):
+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == PRELOAD_COUNT_PATH:
+ return self._serve_preload_count_resource
if name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:
return self._serve_fetch_resource_method
if name.startswith("do_") and unquote(urlsplit(self.path).path) in REMOTE_CONTEXT_RESOURCE_PATHS:
diff --git a/moli-benchmark/tests/test_wpt_preload_count_server.py b/moli-benchmark/tests/test_wpt_preload_count_server.py
new file mode 100644
index 000000000..085bab2ea
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_preload_count_server.py
@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import ExitStack
+from http.client import HTTPConnection
+from pathlib import Path
+from unittest.mock import patch
+
+from moli_benchmark.wpt_cross.case_set import enumerate_cases
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+COUNT_PATH = "/preload/resources/preload-count.py"
+
+
+class PreloadCountFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ (self.root / "preload/resources").mkdir(parents=True)
+ (self.root / COUNT_PATH.lstrip("/")).write_text("# Do not serve Python source")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,
+ ))
+ self.server = self.stack.enter_context(WptFixtureServer(self.root))
+
+ def request(self, query: str, *, method="GET", path=COUNT_PATH, port=None, body=None):
+ connection = HTTPConnection("127.0.0.1", port or self.server.port, timeout=5)
+ try:
+ connection.request(method, path + "?" + query, body)
+ response = connection.getresponse()
+ return response.status, response.headers, response.read()
+ finally:
+ connection.close()
+
+ def assert_count(self, count: int, **kwargs) -> None:
+ status, headers, body = self.request("action=result", **kwargs)
+ expected = f"preloadCount = {count};".encode("ascii")
+ self.assertEqual(status, 200)
+ self.assertEqual(headers.get_all("Content-Type"), ["text/javascript"])
+ self.assertEqual(headers["Content-Length"], str(len(expected)))
+ self.assertIsNone(headers["Cache-Control"])
+ self.assertEqual(body, b"" if kwargs.get("method") == "HEAD" else expected)
+
+ def test_counts_failed_resource_requests_and_results_consume_the_count(self) -> None:
+ self.assert_count(0)
+ for action in ("image", "font", "", "RESULT"):
+ status, headers, body = self.request("action=" + action)
+ self.assertEqual((status, body), (404, b"No entry is found"))
+ self.assertIsNone(headers["Content-Type"])
+ self.assertIsNone(headers["Cache-Control"])
+ self.assert_count(4)
+ self.assert_count(0)
+
+ def test_query_keys_and_first_value_match_upstream(self) -> None:
+ for query in ("action=&action=result", "action=image&action=result", "act%69on=image"):
+ self.assertEqual(self.request(query)[0], 404)
+ status, _, body = self.request("action=result&action=image")
+ self.assertEqual((status, body), (200, b"preloadCount = 3;"))
+ self.assert_count(0)
+ for query in ("", "Action=result"):
+ self.request("action=image")
+ self.assertEqual(self.request(query)[0], 500)
+ self.assert_count(0)
+
+ def test_all_methods_use_query_parameters_and_head_still_consumes(self) -> None:
+ methods = ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "YO", "chicken")
+ for method in methods:
+ with self.subTest(method=method):
+ status, headers, body = self.request(
+ "action=image", method=method, body=b"action=result",
+ )
+ self.assertEqual(status, 404)
+ self.assertEqual(headers["Content-Length"], str(len(b"No entry is found")))
+ self.assertEqual(body, b"" if method == "HEAD" else b"No entry is found")
+ self.assert_count(len(methods), method="HEAD")
+ self.assert_count(0)
+ for method in methods:
+ self.request("action=image")
+ self.assert_count(1, method=method, body=b"action=image")
+ self.assert_count(0)
+
+ def test_parallel_requests_share_origins_but_not_fixture_servers(self) -> None:
+ with ThreadPoolExecutor(max_workers=32) as pool:
+ responses = list(pool.map(
+ lambda index: self.request(
+ "action=image", port=self.server.port if index % 2 else self.server.alternate_port,
+ ),
+ range(32),
+ ))
+ self.assertTrue(all(status == 404 for status, _, _ in responses))
+ other = self.stack.enter_context(WptFixtureServer(self.root))
+ self.assert_count(0, port=other.port)
+ self.assert_count(32, port=self.server.alternate_port)
+ self.assert_count(0)
+
+ def test_unused_uploads_do_not_delay_the_response(self) -> None:
+ for method in ("POST", "PUT", "OPTIONS", "chicken"):
+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)
+ try:
+ connection.putrequest(method, COUNT_PATH + "?action=image")
+ connection.putheader("Content-Length", "1000000")
+ connection.endheaders()
+ response = connection.getresponse()
+ self.assertEqual((response.status, response.read()), (404, b"No entry is found"))
+ self.assertEqual(response.headers["Connection"], "close")
+ finally:
+ connection.close()
+ self.assert_count(4)
+
+ def test_only_the_exact_counter_path_is_handled_and_discovered(self) -> None:
+ for path in (COUNT_PATH + "2", "/other" + COUNT_PATH, "/PRELOAD/resources/preload-count.py"):
+ self.assertEqual(self.request("action=result", path=path)[0], 404)
+ cases = {
+ "absolute.html": (COUNT_PATH, True),
+ "relative.html": ("resources/preload-count.py", True),
+ "dot-relative.html": ("./resources/preload-count.py", True),
+ "bare.html": ("preload-count.py", False),
+ "wrong-relative.html": ("../resources/preload-count.py", False),
+ "wrong-root.html": ("/other" + COUNT_PATH, False),
+ "suffix.html": (COUNT_PATH + ".extra", False),
+ }
+ for name, (reference, _) in cases.items():
+ (self.root / "preload" / name).write_text(
+ '<script src="/resources/testharness.js"></script>'
+ f'<script>fetch("{reference}?action=result")</script>'
+ )
+ (self.root / "preload/unknown.html").write_text(
+ '<script src="/resources/testharness.js"></script>'
+ f'<script>fetch("{COUNT_PATH}?action=result"); fetch("/unsupported.py")</script>'
+ )
+ discovered = enumerate_cases(self.root, dir_prefixes=("preload",))
+ self.assertEqual(
+ sorted(case.case_path for case in discovered),
+ sorted("preload/" + name for name, (_, allowed) in cases.items() if allowed),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()