Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(benchmark): route WPT permission changes through CDP
ldm0cf51cefb7d8f
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/native_input.py b/moli-benchmark/moli_benchmark/wpt_cross/native_input.pyindex 0bdb33b0e..9e7563f8f 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/native_input.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/native_input.py@@ -1,8 +1,10 @@-"""Trusted WPT input, served on a separate CDP connection from harness probes.+"""Native WPT automation on a separate CDP connection from harness probes.The input connection must remain runnable while the harness connection awaits aJavaScript promise. Each instance belongs to one case target and is disposedwith it; no pressed keys, pending bindings, or pointer state cross case borders.+Permission requests use frames exposed by the attached page session; out-of-+process iframe targets require a separate attachment and report an error here."""from __future__ import annotations@@ -14,6 +16,7 @@ import mathimport timefrom dataclasses import dataclass, fieldfrom typing import Any+from urllib.parse import urlsplitfrom ..raw_cdp import (RawCdpConnectionClosed,@@ -174,22 +177,42 @@ def validate_actions(sources: Any) -> None:raise ValueError("native pointer geometry properties are not supported")+def permission_origin(frame: dict[str, Any]) -> str:+ # Frame.securityOrigin preserves inherited about:blank origins and opaque+ # sandbox origins, which cannot be reconstructed from location.href.+ origin = frame.get("securityOrigin", frame.get("url"))+ if not isinstance(origin, str):+ raise ValueError("permission frame has no origin")+ parsed = urlsplit(origin)+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:+ raise ValueError("cannot set permission for an opaque or unsupported origin")+ return f"{parsed.scheme}://{parsed.netloc}"++class NativeInput:- def __init__(self, client: RoutedRawCdpClient, session_id: str) -> None:+ def __init__(+ self, client: RoutedRawCdpClient, session_id: str,+ browser_context_id: str | None = None,+ ) -> None:self.client = clientself.session_id = session_id+ self.browser_context_id = browser_context_idself.deadline: float | None = Noneself.task: asyncio.Task[None] | None = Noneself.keyboards: dict[str, dict[str, Key]] = {}self.pointers: dict[str, Pointer] = {}self._closed = False+ self._permissions_changed = False+ self._permission_cleanup_error: Exception | None = None@classmethod- async def attach(cls, endpoint: str, target_id: str) -> NativeInput:+ async def attach(+ cls, endpoint: str, target_id: str, browser_context_id: str | None = None,+ ) -> NativeInput:client = await connect_routed_raw_cdp(endpoint)try:result = await client.command("Target.attachToTarget", {"targetId": target_id, "flatten": True})- instance = cls(client, result.response["result"]["sessionId"])+ instance = cls(client, result.response["result"]["sessionId"], browser_context_id)await instance.command("Runtime.enable")await instance.command("Runtime.addBinding", {"name": BINDING_NAME})instance.task = asyncio.create_task(instance.run(), name="wpt-native-input")@@ -200,12 +223,20 @@ class NativeInput:async def close(self) -> None:if self._closed:+ if self._permission_cleanup_error is not None:+ raise RawCdpError("failed to reset WPT permissions") from self._permission_cleanup_errorreturnself._closed = Trueif self.task is not None:self.task.cancel()with contextlib.suppress(asyncio.CancelledError, Exception):await self.task+ if self._permissions_changed:+ params = {"browserContextId": self.browser_context_id} if self.browser_context_id else {}+ try:+ await self.client.command("Browser.resetPermissions", params, timeout=5)+ except Exception as error:+ self._permission_cleanup_error = error# The engine can exit before cleanup. Still detach when possible and# always close our own receiver and socket, even after a send fails.with contextlib.suppress(Exception):@@ -218,6 +249,8 @@ class NativeInput:"Target.detachFromTarget", {"sessionId": self.session_id}, timeout=5,)await self.client.close()+ if self._permission_cleanup_error is not None:+ raise RawCdpError("failed to reset WPT permissions") from self._permission_cleanup_errorasync def command(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:timeout = 10.0 if self.deadline is None else max(0.01, self.deadline - time.perf_counter())@@ -392,7 +425,9 @@ class NativeInput:request_id = request["id"]token = request["token"]kind = request["kind"]- if kind == "send_keys":+ if kind == "set_permission":+ await self.set_permission(context_id, request)+ elif kind == "send_keys":await self.evaluate(context_id, f"{STATE_NAME}.focus({request_id}, {json.dumps(token)})")await self.send_keys(request["keys"])elif kind == "click":@@ -435,6 +470,73 @@ class NativeInput:else:raise ValueError(f"unsupported native input request: {kind}")+ async def set_permission(self, context_id: int, request: dict[str, Any]) -> None:+ descriptor = request.get("descriptor")+ state = request.get("state")+ if not isinstance(descriptor, dict) or not isinstance(descriptor.get("name"), str) or not descriptor["name"]:+ raise ValueError("permission descriptor requires a name")+ if not isinstance(state, str) or state not in {"granted", "denied", "prompt"}:+ raise ValueError("invalid permission state")+ path = await self.evaluate(+ context_id,+ f"{STATE_NAME}.permissionFramePath({request['id']}, {json.dumps(request['token'])})",+ )+ if not isinstance(path, list) or any(type(index) is not int or index < 0 for index in path):+ raise ValueError("invalid permission frame path")+ tree = (await self.command("Page.getFrameTree")).get("frameTree")+ if not isinstance(tree, dict) or not isinstance(tree.get("frame"), dict):+ raise ValueError("permission target has no frame tree")+ embedding_origin = permission_origin(tree["frame"])+ for index in path:+ children = tree.get("childFrames", [])+ if not isinstance(children, list) or index >= len(children):+ raise ValueError("permission frame is unavailable in this CDP session (detached or out of process)")+ tree = children[index]+ embedded_origin = permission_origin(tree["frame"])+ if path:+ # A session's frame tree can omit out-of-process frames. Verify the+ # owner instead of letting an omitted sibling shift the indices.+ owner = await self.command("DOM.getFrameOwner", {"frameId": tree["frame"]["id"]})+ resolved = await self.command("DOM.resolveNode", {+ "backendNodeId": owner["backendNodeId"], "executionContextId": context_id,+ })+ object_id = resolved["object"]["objectId"]+ try:+ matches = await self.command("Runtime.callFunctionOn", {+ "executionContextId": context_id,+ "functionDeclaration": (+ "function(owner, id, token) { return "+ f"{STATE_NAME}.permissionFrameMatches(id, token, owner); }}"+ ),+ "arguments": [+ {"objectId": object_id}, {"value": request["id"]}, {"value": request["token"]},+ ],+ "returnByValue": True,+ })+ if matches.get("exceptionDetails") or matches.get("result", {}).get("value") is not True:+ raise ValueError("permission frame does not match the requested Window")+ finally:+ await self.command("Runtime.releaseObject", {"objectId": object_id})+ # Resolve the retained Window again after the protocol read, so a+ # detached or moved frame cannot grant a sibling's origin by index.+ current_path = await self.evaluate(+ context_id,+ f"{STATE_NAME}.permissionFramePath({request['id']}, {json.dumps(request['token'])})",+ )+ if current_path != path:+ raise ValueError("permission frame changed during the request")+ params = {+ "permission": descriptor, "setting": state,+ "origin": embedding_origin, "embeddedOrigin": embedded_origin,+ }+ if self.browser_context_id is not None:+ params["browserContextId"] = self.browser_context_id+ # A command can be applied even if its response is lost. Always reset+ # after attempting a mutation, and never attach a page sessionId to it.+ self._permissions_changed = True+ timeout = 10.0 if self.deadline is None else max(0.01, self.deadline - time.perf_counter())+ await self.client.command("Browser.setPermission", params, timeout=timeout)+async def action(self, context_id: int, request_id: int, token: str,source: dict[str, Any], action: dict[str, Any],diff --git a/moli-benchmark/moli_benchmark/wpt_cross/runner.py b/moli-benchmark/moli_benchmark/wpt_cross/runner.pyindex 2cdf1c475..01f9905ec 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/runner.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/runner.py@@ -277,8 +277,12 @@ async def _close_target(client: RawCdpClient, target_id: str) -> None:async def _close_page(client: RawCdpClient, page: _AttachedPage) -> None:"""Dispose one case's storage context and every target created inside it."""+ input_cleanup_error = Noneif page.native_input is not None:- await page.native_input.close()+ try:+ await page.native_input.close()+ except Exception as error:+ input_cleanup_error = errortry:before = await _target_infos(client)@@ -312,6 +316,8 @@ async def _close_page(client: RawCdpClient, page: _AttachedPage) -> None:try:after = await _target_infos(client)except (RawCdpError, asyncio.TimeoutError):+ if input_cleanup_error is not None and page.browser_context_id is None:+ raise input_cleanup_errorreturnresidual_ids = {@@ -346,6 +352,8 @@ async def _close_page(client: RawCdpClient, page: _AttachedPage) -> None:raise RawCdpError(f"case cleanup left auxiliary targets alive: {sorted(leaked)}")+ if input_cleanup_error is not None and page.browser_context_id is None:+ raise input_cleanup_errorasync def _attach_page(diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.pyindex 69769fb0c..b45e8544a 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py@@ -617,12 +617,6 @@ BENCH_TESTDRIVER_VENDOR_BRIDGE = (}window.test_driver_internal.in_automation = true;window.test_driver_internal.get_computed_label = getComputedLabel;- window.test_driver_internal.set_permission = async function(params) {- if (params && params.descriptor && params.descriptor.name === 'storage-access') {- return;- }- throw new Error("set_permission() is not implemented by the Moli WPT bridge");- };})();""")diff --git a/moli-benchmark/moli_benchmark/wpt_cross/testdriver_input.js b/moli-benchmark/moli_benchmark/wpt_cross/testdriver_input.jsindex d3ad0506b..675b17041 100644--- a/moli-benchmark/moli_benchmark/wpt_cross/testdriver_input.js+++ b/moli-benchmark/moli_benchmark/wpt_cross/testdriver_input.js@@ -62,6 +62,25 @@if (error !== null) item.reject(new Error(error));else item.resolve();},+ permissionFramePath(id, expectedToken) {+ let context = entry(id, expectedToken).context;+ if (!context || context.closed || context.top !== window.top) {+ throw new Error('Permission context is not in the current test target');+ }+ const path = [];+ while (context !== window.top) {+ const parent = context.parent;+ let index = 0;+ while (index < parent.length && parent[index] !== context) index++;+ if (index === parent.length) throw new Error('Permission frame is no longer attached');+ path.unshift(index);+ context = parent;+ }+ return path;+ },+ permissionFrameMatches(id, expectedToken, owner) {+ return owner.isConnected && owner.contentWindow === entry(id, expectedToken).context;+ },focus(id, expectedToken) {const element = entry(id, expectedToken).element;if (!element || !element.isConnected) throw new Error('stale element reference');@@ -110,6 +129,9 @@driver.in_automation = true;driver.click = (element, coords) => request({kind: 'click', x: coords.x, y: coords.y}, element);driver.send_keys = (element, keys) => request({kind: 'send_keys', keys}, element);+ driver.set_permission = async (params, context = null) => request({+ kind: 'set_permission', descriptor: params.descriptor, state: params.state,+ }, null, context || window);driver.action_sequence = (actions, context = null) => {const elements = [];const serialized = actions.map(source => ({...source, actions: source.actions.map(action => {diff --git a/moli-benchmark/tests/test_wpt_cross_result_reporting.py b/moli-benchmark/tests/test_wpt_cross_result_reporting.pyindex 89b303de6..135962de8 100644--- a/moli-benchmark/tests/test_wpt_cross_result_reporting.py+++ b/moli-benchmark/tests/test_wpt_cross_result_reporting.py@@ -391,15 +391,12 @@ class WptCrossResultReportingTests(WptCrossTestCase):self.assertNotIn(b"nativeGetClientRects.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)self.assertNotIn(b"nativeElementsFromPoint.apply", BENCH_TESTDRIVER_VENDOR_BRIDGE)- def test_testdriver_vendor_bridge_accepts_storage_access_permission_setup(self) -> None:- self.assertIn(- b"params.descriptor.name === 'storage-access'",- BENCH_TESTDRIVER_VENDOR_BRIDGE,- )- self.assertIn(- b"set_permission() is not implemented by the Moli WPT bridge",- BENCH_TESTDRIVER_VENDOR_BRIDGE,- )+ def test_testdriver_vendor_bridge_forwards_permission_descriptors(self) -> None:+ self.assertIn(b"driver.set_permission", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"kind: 'set_permission'", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertIn(b"descriptor: params.descriptor", BENCH_TESTDRIVER_VENDOR_BRIDGE)+ self.assertNotIn(b"descriptor.name === 'storage-access'", BENCH_TESTDRIVER_VENDOR_BRIDGE)+def test_testdriver_vendor_bridge_provides_computed_label(self) -> None:self.assertIn(b"get_computed_label", BENCH_TESTDRIVER_VENDOR_BRIDGE)self.assertIn(b"resolveReferenceTarget", BENCH_TESTDRIVER_VENDOR_BRIDGE)diff --git a/moli-benchmark/tests/test_wpt_native_input.py b/moli-benchmark/tests/test_wpt_native_input.pyindex 3dbbd0c49..9637d2f6a 100644--- a/moli-benchmark/tests/test_wpt_native_input.py+++ b/moli-benchmark/tests/test_wpt_native_input.py@@ -8,10 +8,11 @@ from pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import AsyncMock-from moli_benchmark.raw_cdp import RawCdpConnectionClosed+from moli_benchmark.raw_cdp import RawCdpConnectionClosed, RawCdpErrorfrom moli_benchmark.wpt_cross.__main__ import _case_references_testdriverfrom moli_benchmark.wpt_cross.case_set import WptCase-from moli_benchmark.wpt_cross.native_input import NativeInput, Pointer, key_description+from moli_benchmark.wpt_cross.native_input import NativeInput, Pointer, key_description, permission_origin+from moli_benchmark.wpt_cross.runner import _AttachedPage, _close_page, _close_targetclass RecordingClient:@@ -172,8 +173,151 @@ class NativeInputTests(unittest.IsolatedAsyncioTestCase):self.assertEqual(self.client.command.await_count, 2)self.client.close.assert_awaited_once()+ async def test_permissions_use_the_selected_frame_origin_and_case_browser_context(self):+ self.driver.browser_context_id = "case-context"+ self.driver.evaluate = AsyncMock(return_value=[1, 0])+ self.driver.command = AsyncMock(side_effect=[{"frameTree": {+ "frame": {"securityOrigin": "https://top.test"},+ "childFrames": [+ {"frame": {"securityOrigin": "https://sibling.test"}},+ {"frame": {"securityOrigin": "https://outer.test"}, "childFrames": [+ {"frame": {"id": "child-frame", "securityOrigin": "https://child.test", "url": "about:blank"}},+ ]},+ ],+ }}, {"backendNodeId": 42}, {"object": {"objectId": "owner-object"}},+ {"result": {"value": True}}, {}])+ descriptor = {"name": "clipboard-write", "allowWithoutSanitization": True}+ await self.driver.perform(9, {+ "id": 4, "token": "realm", "kind": "set_permission", "descriptor": descriptor, "state": "denied",+ })+ method, params, kwargs = self.client.commands[-1]+ self.assertEqual(method, "Browser.setPermission")+ self.assertEqual(params, {+ "permission": descriptor, "setting": "denied", "browserContextId": "case-context",+ "origin": "https://top.test", "embeddedOrigin": "https://child.test",+ })+ self.assertNotIn("session_id", kwargs)+ self.driver.command.assert_any_await("DOM.getFrameOwner", {"frameId": "child-frame"})+ self.driver.command.assert_any_await("Runtime.releaseObject", {"objectId": "owner-object"})+ self.driver.command.side_effect = None+ self.driver.command.return_value = {}+ await self.driver.close()+ reset = next(command for command in self.client.commands if command[0] == "Browser.resetPermissions")+ self.assertEqual(reset[1], {"browserContextId": "case-context"})+ self.assertNotIn("session_id", reset[2])++ async def test_permissions_reject_malformed_requests_before_any_protocol_mutation(self):+ for descriptor, state in [({}, "granted"), ({"name": ""}, "granted"), ({"name": 1}, "granted"),+ ({"name": "geolocation"}, "invalid"), ({"name": "geolocation"}, {})]:+ with self.subTest(descriptor=descriptor, state=state), self.assertRaises(ValueError):+ await self.driver.perform(1, {"id": 1, "token": "realm", "kind": "set_permission",+ "descriptor": descriptor, "state": state})+ self.assertEqual(self.client.commands, [])++ async def test_permissions_reject_detached_moved_and_opaque_frames(self):+ request = {"id": 1, "token": "realm", "kind": "set_permission", "descriptor": {"name": "geolocation"}, "state": "granted"}+ for paths, frame in [+ ([[0]], {"securityOrigin": "https://top.test"}),+ ([[], [0]], {"securityOrigin": "https://top.test"}),+ ([[]], {"securityOrigin": "null", "url": "https://opaque.test/"}),+ ]:+ with self.subTest(paths=paths, frame=frame):+ self.driver.evaluate = AsyncMock(side_effect=paths)+ self.driver.command = AsyncMock(return_value={"frameTree": {"frame": frame}})+ with self.assertRaises(ValueError):+ await self.driver.perform(1, request)+ self.assertEqual(self.client.commands, [])++ async def test_failed_permission_command_propagates_and_still_schedules_reset(self):+ self.driver.evaluate = AsyncMock(return_value=[])+ self.driver.command = AsyncMock(return_value={"frameTree": {"frame": {"securityOrigin": "https://top.test"}}})+ self.client.command = AsyncMock(side_effect=RawCdpError("unsupported permission"))+ with self.assertRaisesRegex(RawCdpError, "unsupported permission"):+ await self.driver.perform(1, {"id": 1, "token": "realm", "kind": "set_permission",+ "descriptor": {"name": "unsupported"}, "state": "granted"})+ with self.assertRaisesRegex(RawCdpError, "reset WPT permissions"):+ await self.driver.close()+ self.assertEqual([call.args[0] for call in self.client.command.await_args_list], [+ "Browser.setPermission", "Browser.resetPermissions", "Runtime.removeBinding", "Target.detachFromTarget",+ ])+ self.client.close.assert_awaited_once()+ with self.assertRaisesRegex(RawCdpError, "reset WPT permissions"):+ await self.driver.close()+ self.client.close.assert_awaited_once()++ async def test_omitted_frame_cannot_grant_permission_to_a_sibling(self):+ self.driver.evaluate = AsyncMock(return_value=[0])+ self.driver.command = AsyncMock(side_effect=[+ {"frameTree": {"frame": {"securityOrigin": "https://top.test"}, "childFrames": [+ {"frame": {"id": "sibling", "securityOrigin": "https://sibling.test"}},+ ]}},+ {"backendNodeId": 42}, {"object": {"objectId": "sibling-owner"}},+ {"result": {"value": False}}, {},+ ])+ with self.assertRaisesRegex(ValueError, "does not match"):+ await self.driver.perform(1, {"id": 1, "token": "realm", "kind": "set_permission",+ "descriptor": {"name": "geolocation"}, "state": "granted"})+ self.assertEqual(self.client.commands, [])+ self.driver.command.assert_awaited_with("Runtime.releaseObject", {"objectId": "sibling-owner"})++ async def test_failed_permission_cleanup_still_disposes_case_targets(self):+ class PageClient:+ def __init__(self):+ self.commands = []++ async def send(self, method, params=None, **kwargs):+ self.commands.append((method, params))+ return len(self.commands)++ async def recv_until_id(self, command_id, **kwargs):+ method, _ = self.commands[command_id - 1]+ result = {"targetInfos": []} if method == "Target.getTargets" else {"success": True}+ return {"result": result}, []++ for browser_context_id in (None, "isolated-context"):+ with self.subTest(browser_context_id=browser_context_id):+ client = PageClient()+ native = SimpleNamespace(close=AsyncMock(side_effect=RawCdpError("reset failed")))+ page = _AttachedPage(browser_context_id, "test-target", "test-session", frozenset(), native)+ if browser_context_id is None:+ with self.assertRaisesRegex(RawCdpError, "reset failed"):+ await _close_page(client, page)+ self.assertIn(("Target.closeTarget", {"targetId": "test-target"}), client.commands)+ else:+ await _close_page(client, page)+ self.assertIn(("Target.disposeBrowserContext", {"browserContextId": browser_context_id}), client.commands)+++ async def test_target_cleanup_accepts_a_concurrently_closed_target(self):+ client = SimpleNamespace(+ send=AsyncMock(side_effect=[1, 2]),+ recv_until_id=AsyncMock(side_effect=[+ RawCdpError("No target with given id found"),+ ({"result": {"targetInfos": [{"targetId": "unrelated"}]}}, []),+ ]),+ )+ await _close_target(client, "case-target")++ async def test_target_cleanup_preserves_errors_for_a_live_target(self):+ client = SimpleNamespace(+ send=AsyncMock(side_effect=[1, 2]),+ recv_until_id=AsyncMock(side_effect=[+ RawCdpError("close rejected"),+ ({"result": {"targetInfos": [{"targetId": "case-target"}]}}, []),+ ]),+ )+ with self.assertRaisesRegex(RawCdpError, "close rejected"):+ await _close_target(client, "case-target")+class NativeInputSelectionTests(unittest.TestCase):+ def test_permission_origins_keep_inherited_and_opaque_origin_semantics(self):+ self.assertEqual(permission_origin({"url": "about:blank", "securityOrigin": "https://parent.test:8443"}), "https://parent.test:8443")+ self.assertEqual(permission_origin({"url": "https://top.test/path?q=1"}), "https://top.test")+ for origin in ("null", "", "file://", "https://user:[email protected]"):+ with self.subTest(origin=origin), self.assertRaises(ValueError):+ permission_origin({"url": "https://top.test", "securityOrigin": origin})+def test_testdriver_detection_in_html_and_generated_variants(self):with tempfile.TemporaryDirectory() as directory:root = Path(directory)