Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(wpt): serve the preload request counter fixture
ldm0bde57e580c20
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.pyindex 33259ac24..9e08c01b0 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py@@ -725,6 +725,19 @@ def _html_path_is_supported(return not any(token in rel for token in excluded)+@lru_cache(maxsize=None)+def _preload_count_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:+ resource = "preload/resources/preload-count.py"+ relative = posixpath.relpath(resource, directory)+ return tuple(+ re.compile(+ rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"+ rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"+ )+ for reference in ("/" + resource, relative, "./" + relative)+ )++@lru_cache(maxsize=None)def _common_echo_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:resource = "common/echo.py"@@ -866,6 +879,7 @@ def _supported_wptserve_handler_references(supported += _fetch_content_type_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _document_charset_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _iframe_stash_handler_reference_patterns(posixpath.dirname(rel) or ".")+ supported += _preload_count_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _navigation_handler_reference_patterns(posixpath.dirname(rel) or ".")supported += _json_module_handler_reference_patterns(posixpath.dirname(rel) or ".")if rel is not None:diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 7ae1aef8b..07a822c6d 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -81,6 +81,8 @@ from .case_set import (http.client._MAXHEADERS = 512+PRELOAD_COUNT_PATH = "/preload/resources/preload-count.py"+PRELOAD_COUNT_KEY = "a8697ae7-c8cb-4dbd-a8ef-27111dc7042f"DEFAULT_TESTHARNESS_TIMEOUT_SECONDS = 10.0MAX_REQUEST_BODY_BYTES = 16 * 1024 * 1024MAX_REQUEST_BODY_LINE_BYTES = 64 * 1024@@ -1772,7 +1774,6 @@ class CspReportStore:self._lock = threading.Lock()self._cv = threading.Condition(self._lock)self._reports: dict[str, list[dict]] = {}- self._counts: dict[str, int] = {}def append_reports(self, report_id: str, reports: list[dict]) -> None:with self._cv:@@ -1836,7 +1837,6 @@ class FetchStash:def __init__(self) -> None:self._lock = threading.Lock()self._values: dict[tuple[str, uuid.UUID], object] = {}- self._counts: dict[tuple[str, uuid.UUID], int] = {}def put(self, key: str, value: object, *, overwrite: bool = False,@@ -1853,8 +1853,8 @@ class FetchStash:def increment(self, key: str, *, path: str) -> int:parsed_key = (path, uuid.UUID(key))with self._lock:- value = int(self._counts.get(parsed_key, 0)) + 1- self._counts[parsed_key] = value+ value = int(self._values.get(parsed_key, 0)) + 1+ self._values[parsed_key] = valuereturn value@@ -2143,6 +2143,8 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_common_redirect_resource():return+ if self._serve_preload_count_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:@@ -2204,6 +2206,8 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_common_redirect_resource():return+ if self._serve_preload_count_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:@@ -2297,6 +2301,8 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_common_redirect_resource():return+ if self._serve_preload_count_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:@@ -2357,6 +2363,8 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_common_redirect_resource():return+ if self._serve_preload_count_resource():+ returnif self._serve_empty_location_resource(emit_body=self.command != "HEAD"):returnif unquote(urlparse(self.path).path) in {@@ -2525,6 +2533,35 @@ requestExecutor("{executor_uuid}", {start_on_js});escape_type=escape_type,)+ def _serve_preload_count_resource(self) -> bool:+ parsed = urlsplit(self.path)+ if unquote(parsed.path) != PRELOAD_COUNT_PATH:+ return False+ # The upstream handler only reads GET parameters, for every method.+ self.close_connection = True+ stash_path = PRELOAD_COUNT_PATH.rsplit("/", 1)[0] + "/"+ params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")+ try:+ action = params["action"][0]+ except KeyError:+ # Upstream takes the counter before reading the required action.+ fetch_stash.take(PRELOAD_COUNT_KEY, path=stash_path)+ self.send_error(500)+ return True+ if action == "result":+ count = fetch_stash.take(PRELOAD_COUNT_KEY, path=stash_path) or 0+ status, content_type = 200, "text/javascript"+ body = f"preloadCount = {count};".encode("ascii")+ else:+ fetch_stash.increment(PRELOAD_COUNT_KEY, path=stash_path)+ status, content_type, body = 404, None, b"No entry is found"+ self._send_bytes(+ content_type, body, emit_body=self.command != "HEAD",+ status_code=status, cache_control=None,+ extra_headers=[("Connection", "close")],+ )+ return True+def _serve_common_echo_resource(self) -> bool:parsed = urlsplit(self.path)if unquote(parsed.path) != COMMON_ECHO_PATH:@@ -2570,6 +2607,8 @@ requestExecutor("{executor_uuid}", {start_on_js});returnif self._serve_common_redirect_resource():return+ if self._serve_preload_count_resource():+ returnif self._serve_empty_location_resource(emit_body=emit_body):returnif unquote(urlsplit(self.path).path) in SERVICE_WORKER_SCRIPT_RESOURCE_PATHS:@@ -3494,6 +3533,8 @@ requestExecutor("{executor_uuid}", {start_on_js});)def __getattr__(self, name: str):+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == PRELOAD_COUNT_PATH:+ return self._serve_preload_count_resourceif name.startswith("do_") and unquote(urlsplit(self.path).path) == FETCH_REDIRECT_RESOURCE_PATH:return self._serve_fetch_resource_methodif name.startswith("do_") and unquote(urlsplit(self.path).path) in REMOTE_CONTEXT_RESOURCE_PATHS:diff --git a/moli-benchmark/tests/test_wpt_preload_count_server.py b/moli-benchmark/tests/test_wpt_preload_count_server.pynew file mode 100644index 000000000..085bab2ea--- /dev/null+++ b/moli-benchmark/tests/test_wpt_preload_count_server.py@@ -0,0 +1,145 @@+from __future__ import annotations++import tempfile+import unittest+from concurrent.futures import ThreadPoolExecutor+from contextlib import ExitStack+from http.client import HTTPConnection+from pathlib import Path+from unittest.mock import patch++from moli_benchmark.wpt_cross.case_set import enumerate_cases+from moli_benchmark.wpt_cross.server import WptFixtureServer+++COUNT_PATH = "/preload/resources/preload-count.py"+++class PreloadCountFixtureTests(unittest.TestCase):+ def setUp(self) -> None:+ self.stack = ExitStack()+ self.addCleanup(self.stack.close)+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))+ (self.root / "resources").mkdir()+ (self.root / "resources/testharness.js").write_text("// testharness")+ (self.root / "preload/resources").mkdir(parents=True)+ (self.root / COUNT_PATH.lstrip("/")).write_text("# Do not serve Python source")+ self.stack.enter_context(patch(+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,+ ))+ self.server = self.stack.enter_context(WptFixtureServer(self.root))++ def request(self, query: str, *, method="GET", path=COUNT_PATH, port=None, body=None):+ connection = HTTPConnection("127.0.0.1", port or self.server.port, timeout=5)+ try:+ connection.request(method, path + "?" + query, body)+ response = connection.getresponse()+ return response.status, response.headers, response.read()+ finally:+ connection.close()++ def assert_count(self, count: int, **kwargs) -> None:+ status, headers, body = self.request("action=result", **kwargs)+ expected = f"preloadCount = {count};".encode("ascii")+ self.assertEqual(status, 200)+ self.assertEqual(headers.get_all("Content-Type"), ["text/javascript"])+ self.assertEqual(headers["Content-Length"], str(len(expected)))+ self.assertIsNone(headers["Cache-Control"])+ self.assertEqual(body, b"" if kwargs.get("method") == "HEAD" else expected)++ def test_counts_failed_resource_requests_and_results_consume_the_count(self) -> None:+ self.assert_count(0)+ for action in ("image", "font", "", "RESULT"):+ status, headers, body = self.request("action=" + action)+ self.assertEqual((status, body), (404, b"No entry is found"))+ self.assertIsNone(headers["Content-Type"])+ self.assertIsNone(headers["Cache-Control"])+ self.assert_count(4)+ self.assert_count(0)++ def test_query_keys_and_first_value_match_upstream(self) -> None:+ for query in ("action=&action=result", "action=image&action=result", "act%69on=image"):+ self.assertEqual(self.request(query)[0], 404)+ status, _, body = self.request("action=result&action=image")+ self.assertEqual((status, body), (200, b"preloadCount = 3;"))+ self.assert_count(0)+ for query in ("", "Action=result"):+ self.request("action=image")+ self.assertEqual(self.request(query)[0], 500)+ self.assert_count(0)++ def test_all_methods_use_query_parameters_and_head_still_consumes(self) -> None:+ methods = ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "YO", "chicken")+ for method in methods:+ with self.subTest(method=method):+ status, headers, body = self.request(+ "action=image", method=method, body=b"action=result",+ )+ self.assertEqual(status, 404)+ self.assertEqual(headers["Content-Length"], str(len(b"No entry is found")))+ self.assertEqual(body, b"" if method == "HEAD" else b"No entry is found")+ self.assert_count(len(methods), method="HEAD")+ self.assert_count(0)+ for method in methods:+ self.request("action=image")+ self.assert_count(1, method=method, body=b"action=image")+ self.assert_count(0)++ def test_parallel_requests_share_origins_but_not_fixture_servers(self) -> None:+ with ThreadPoolExecutor(max_workers=32) as pool:+ responses = list(pool.map(+ lambda index: self.request(+ "action=image", port=self.server.port if index % 2 else self.server.alternate_port,+ ),+ range(32),+ ))+ self.assertTrue(all(status == 404 for status, _, _ in responses))+ other = self.stack.enter_context(WptFixtureServer(self.root))+ self.assert_count(0, port=other.port)+ self.assert_count(32, port=self.server.alternate_port)+ self.assert_count(0)++ def test_unused_uploads_do_not_delay_the_response(self) -> None:+ for method in ("POST", "PUT", "OPTIONS", "chicken"):+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)+ try:+ connection.putrequest(method, COUNT_PATH + "?action=image")+ connection.putheader("Content-Length", "1000000")+ connection.endheaders()+ response = connection.getresponse()+ self.assertEqual((response.status, response.read()), (404, b"No entry is found"))+ self.assertEqual(response.headers["Connection"], "close")+ finally:+ connection.close()+ self.assert_count(4)++ def test_only_the_exact_counter_path_is_handled_and_discovered(self) -> None:+ for path in (COUNT_PATH + "2", "/other" + COUNT_PATH, "/PRELOAD/resources/preload-count.py"):+ self.assertEqual(self.request("action=result", path=path)[0], 404)+ cases = {+ "absolute.html": (COUNT_PATH, True),+ "relative.html": ("resources/preload-count.py", True),+ "dot-relative.html": ("./resources/preload-count.py", True),+ "bare.html": ("preload-count.py", False),+ "wrong-relative.html": ("../resources/preload-count.py", False),+ "wrong-root.html": ("/other" + COUNT_PATH, False),+ "suffix.html": (COUNT_PATH + ".extra", False),+ }+ for name, (reference, _) in cases.items():+ (self.root / "preload" / name).write_text(+ '<script src="/resources/testharness.js"></script>'+ f'<script>fetch("{reference}?action=result")</script>'+ )+ (self.root / "preload/unknown.html").write_text(+ '<script src="/resources/testharness.js"></script>'+ f'<script>fetch("{COUNT_PATH}?action=result"); fetch("/unsupported.py")</script>'+ )+ discovered = enumerate_cases(self.root, dir_prefixes=("preload",))+ self.assertEqual(+ sorted(case.case_path for case in discovered),+ sorted("preload/" + name for name, (_, allowed) in cases.items() if allowed),+ )+++if __name__ == "__main__":+ unittest.main()