Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Stage Browser Use CLI arm with pinned installation and fail-closed preflight (#5765)
injaneitycdda70795441
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/README.md b/libs/cua-driver-bench/automated-eval/macos_bench/README.mdindex 635efd71fc..e83637221c 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/README.md+++ b/libs/cua-driver-bench/automated-eval/macos_bench/README.md@@ -1,5 +1,10 @@# Bench v2 harness (macOS)+`cx-browser-use-cli` is staged and unavailable for scored execution: preflight always fails.+It requires `CDB_CODEX_AUTH=chatgpt` and browser-only tasks. Codex driving the CLI is not+like-for-like with Browser Use's own Agent loop. Its planned GUI-only exception covers+only recognized Browser Use CLI invocations, not general Bash or arbitrary Python.+The runner, test apps, BenchLab tasks and analysis of Cua Driver Bench v2. Startwith [`../../README.md`](../../README.md) for what the benchmark measures andhow to reproduce it, and [`../../METHOD.md`](../../METHOD.md) for the method.diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md b/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.mdindex 3b12b2cddc..8804173ecc 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md+++ b/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md@@ -7,6 +7,15 @@ invocation, isolation, schedule, gates and output. The method is in## Arms+`cx-browser-use-cli` (Amendment 24.2a) is staged: operational preflight always fails,+including offline, smoke and no-model modes. No execution policy or trial home is written.+It requires `CDB_CODEX_AUTH=chatgpt` and a nonempty selection of `browser: true` tasks;+no existing task is flagged here. This is not like-for-like with Browser Use's own Agent loop.+The planned GUI-only exception is recognized CLI invocations, not general Bash or arbitrary Python.+Install an already downloaded, pinned wheel with `python tools/install_browser_use.py /path/to/browser_use-0.13.11-py3-none-any.whl`.+Heredocs, skill discovery/provisioning, matcher-derived execution rules, browser lifecycle,+observed-pin comparison and doctor/CDP/page-info/deny probes remain prerequisites to enabling it.+| Arm | Public setup | Tools Claude Code gets || --- | --- | --- || `cc-cua-driver-script` | Cua Driver | Cua Driver built from the pinned main commit, MCP server `cua`, `run_script` on, the skill of that commit in the working directory, and a short `run_script` addendum in the system prompt |diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/arms.py b/libs/cua-driver-bench/automated-eval/macos_bench/arms.pyindex 639aa8c0d9..96d1f82116 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/arms.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/arms.py@@ -49,6 +49,7 @@ NATIVE_CODEX_ARMS = ("cx-codex-cu","cx-cua-driver","cx-claude-cu-helper",+ "cx-browser-use-cli",) # the last two: Amendment 23ALL_ARMS = CLAUDE_ARMS + CODEX_FALLBACK_ARMS + NATIVE_CODEX_ARMSdiff --git a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.pyindex dd3499cbbc..7f2def75db 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py@@ -73,14 +73,21 @@ def category_of(spec: dict[str, Any]) -> str:return str(category)-def validate_tasks(specs: dict[str, dict[str, Any]], task_ids: list[str]) -> list[str]:+def validate_tasks(+ specs: dict[str, dict[str, Any]], task_ids: list[str], arms: list[str] | None = None+) -> list[str]:"""Problems that stop a v2 run: unknown task, missing or unknown category, coding tools."""problems: list[str] = []+ browser_only = "cx-browser-use-cli" in (arms or [])+ if browser_only and not task_ids:+ problems.append("cx-browser-use-cli requires a nonempty browser task selection")for tid in task_ids:spec = specs.get(tid)if spec is None:problems.append(f"{tid}: unknown task")continue+ if browser_only and spec.get("browser") is not True:+ problems.append(f"{tid}: cx-browser-use-cli requires browser: true")try:category_of(spec)except ValueError as error:diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.py b/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.pynew file mode 100644index 0000000000..dabd5e69d6--- /dev/null+++ b/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.py@@ -0,0 +1,42 @@+from __future__ import annotations++import shlex+from dataclasses import dataclass+from pathlib import Path++import arms++ARM = "cx-browser-use-cli"+TOOL_DIR = Path.home() / ".cache/cua-bench-h2h/browser-use-cli"+TOOL_BIN_DIR = TOOL_DIR / "bin"+EXECUTABLE = TOOL_BIN_DIR / "browser-use"+++@dataclass(frozen=True)+class Invocation:+ argv: tuple[str, ...]+ python: str | None = None+++def require_chatgpt() -> None:+ if arms.codex_auth() != "chatgpt":+ raise ValueError(f"{ARM} requires CDB_CODEX_AUTH=chatgpt")+++def match(command: str, executable: Path = EXECUTABLE) -> Invocation | None:+ try:+ wrapper = shlex.split(command)+ if len(wrapper) == 3 and wrapper[:2] == ["/bin/zsh", "-lc"]:+ command = wrapper[2]+ if any(c in command for c in "\n\r\x00$`\\;|&<>(){}[]*?~#!^"):+ return None+ argv = shlex.split(command)+ except ValueError:+ return None+ if not argv or argv[0] not in ("browser-use", str(executable)):+ return None+ return Invocation((str(executable), *argv[1:]))+++def preflight() -> list[tuple[str, str, str]]:+ return [(ARM, "fail", "Browser Use CLI operational preflight is not yet implemented")]diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.pyindex 7d5b713654..c99b274e58 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py@@ -52,17 +52,19 @@ from pathlib import Pathfrom typing import Any, Callableimport arms+from browser_use_cli import InvocationARM = "cx-codex-cu"# cx-cua-driver and cx-claude-cu-helper: Amendment 23, Cua Driver's MCP server and Cua's helper adapter under the# same Codex host-ARMS = (ARM, "cx-cua-driver", "cx-claude-cu-helper")+ARMS = (ARM, "cx-cua-driver", "cx-claude-cu-helper", "cx-browser-use-cli")SERVER = "cua_repl" # the server name of the unified-computer-use pluginSERVER_OF = {"cx-codex-cu": SERVER,"cx-cua-driver": "cua","cx-claude-cu-helper": "claude-cu-helper",}+MCP_ARMS = tuple(SERVER_OF)# The Claude Code arm whose MCP config (and, for Cua Driver, whose daemon, build and system prompt) the Codex arm uses.SOURCE_ARM = {"cx-codex-cu": "cc-codex-cu",@@ -89,6 +91,7 @@ GATEWAY_ENV = "CDB_CODEX_GATEWAY"DEFAULT_GATEWAY = "http://127.0.0.1:4000/v1"CLOSED_PATH = "/usr/bin:/bin:/usr/sbin:/sbin"LABELS = {+ "cx-browser-use-cli": "Browser Use CLI, driven by Codex CLI + GPT-6.1 Sol (staged)","cx-codex-cu": f"Codex Computer-Use (Sky), driven by Codex CLI + {MODEL_DISPLAY.get(MODEL, MODEL)}","cx-cua-driver": f"Cua Driver, driven by Codex CLI + {MODEL_DISPLAY.get(MODEL, MODEL)}","cx-claude-cu-helper": "Claude Computer-Use (Desktop helper), driven by Codex CLI + "@@ -248,7 +251,12 @@ class Translator:result), so `bench_v2.GuiOnlyGuard`, `bench_v2.scan_events` and `claude_events.summarize` read this arm thesame way as the Claude arms."""- def __init__(self, server: str = SERVER) -> None:+ def __init__(+ self,+ server: str = SERVER,+ command_matcher: Callable[[str], Invocation | None] | None = None,+ ) -> None:+ self.command_matcher = command_matcherself.server = serverself.final_text = ""self.errors: list[str] = []@@ -305,16 +313,37 @@ class Translator:item = params.get("item") or {}itype = str(item.get("type"))item_id = str(item.get("id"))+ invocation = (+ self.command_matcher(str(item.get("command") or ""))+ if itype == "commandExecution" and self.command_matcher+ else None+ )if itype == "agentMessage" and method == "item/completed":self.final_text = str(item.get("text") or "")- elif itype == "mcpToolCall":- name = f"mcp__{item.get('server')}__{item.get('tool')}"+ elif itype == "mcpToolCall" or invocation:+ name = (+ "mcp__browser_use_cli__invoke"+ if invocation+ else f"mcp__{item.get('server')}__{item.get('tool')}"+ )+ tool_input = (+ {"argv": list(invocation.argv), "python": invocation.python}+ if invocation+ else item.get("arguments") or {}+ )if item_id not in self.started:self.started.add(item_id)- out.append(self._use(item_id, name, item.get("arguments") or {}))+ out.append(self._use(item_id, name, tool_input))if method == "item/completed":failed = item.get("status") != "completed" or item.get("error") is not None- text = json.dumps(item.get("error"))[:300] if failed else ""+ failed = failed or bool(invocation and item.get("exitCode") != 0)+ text = (+ str(item.get("aggregatedOutput") or "")[:2000]+ if invocation+ else json.dumps(item.get("error"))[:300]+ if failed+ else ""+ )out.append({"type": "user",diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json b/libs/cua-driver-bench/automated-eval/macos_bench/pins.jsonindex 4e94e81160..5bd1a6732b 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json+++ b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json@@ -115,6 +115,16 @@"type"]},+ "browser_use_cli": {+ "version": "0.13.11",+ "python": "3.12",+ "wheel_sha256": "50b0d16efc44215f99ed1c9197f05e2519bb79fa1501eed7371aff450cc56045",+ "skill_path": "browser_use/skills/browser-use/SKILL.md",+ "skill_sha256": "fee8f320d0bdcad3695393663b5defa6e5fc1e22ff7ac6ffb6f04f6997b93c36",+ "_requires_python": ">=3.11,<4.0",+ "_wheel": "browser_use-0.13.11-py3-none-any.whl",+ "_url": "https://files.pythonhosted.org/packages/cd/e6/8fb8ce81894afcc7626837232a4d284843d1c1fb4fd757aa00c54ce40d8e/browser_use-0.13.11-py3-none-any.whl"+ },"codex_native": {"_note": "Amendment 20 (CUA-1347): the Codex CLI bundled in ChatGPT 26.930.51102 (codex-cli/CodexCLI.app, Developer ID OpenAI OpCo, LLC, 2DC432GLL2), host protocol `codex app-server`, model gpt-6.1-sol (Amendment 23) at its catalog default effort (low); A20's smoke used gpt-6-astra (medium). Pinned from the host's identical app version; the smoke checks the VM copy.","codex_cli_version": "codex-cli 0.160.0",diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.pyindex bb1bb77ecb..14fa901808 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py@@ -52,6 +52,7 @@ import bench_core as core # noqa: E402import claude_arms as ca # noqa: E402import cdb_adapter # noqa: E402import codex_native as cxn # noqa: E402+import browser_use_cli as buimport claude_driver # noqa: E402import claude_events # noqa: E402import recorder as rec # noqa: E402@@ -826,7 +827,7 @@ def run_attempt(ctx: Ctx, entry: dict[str, Any], attempt: int) -> dict[str, Any]row["lab_unoccluded"] = row["lab_occlusion"].get("unoccluded")cwd = ca.prepare_cwd(arm)mcp_path, server = ctx.mcp[arm]- native = arm in cxn.ARMS+ native = arm in cxn.MCP_ARMSargv = (cxn.argv()if native@@ -1915,7 +1916,9 @@ def v2_static_checks(ctx: Ctx) -> list[tuple[str, str, str]]:"""Amendment 14: every task has one known category and is GUI-only, and no arm's argv offers ashell or file tool (the same surface for every setup)."""out: list[tuple[str, str, str]] = []- problems = v2.validate_tasks({t: task_spec(ctx.tasks[t]) for t in ctx.tasks}, ctx.task_ids)+ problems = v2.validate_tasks(+ {t: task_spec(ctx.tasks[t]) for t in ctx.tasks}, ctx.task_ids, ctx.arm_names+ )cats = {t: task_spec(ctx.tasks[t]).get("category") for t in ctx.task_ids}out.append((@@ -1951,6 +1954,8 @@ def v2_static_checks(ctx: Ctx) -> list[tuple[str, str, str]]:def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:args = ctx.args+ if bu.ARM in ctx.arm_names:+ return bu.preflight()checks: list[tuple[str, str, str]] = []def add(name: str, status: str, detail: str) -> None:@@ -2059,7 +2064,7 @@ def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:)except Exception as error: # noqa: BLE001add("arm B MCP server", "fail", f"{type(error).__name__}: {error}")- for native_arm in [a for a in cxn.ARMS if a in ctx.arm_names]:+ for native_arm in [a for a in cxn.MCP_ARMS if a in ctx.arm_names]:if not cxn.CODEX_CLI.is_file():add("Codex CLI in the ChatGPT app", "fail", str(cxn.CODEX_CLI))continue@@ -2363,6 +2368,9 @@ def build_parser() -> argparse.ArgumentParser:def make_ctx(args: argparse.Namespace) -> Ctx:+ arm_names = [args.only_arm] if args.only_arm else list(args.arms)+ if bu.ARM in arm_names:+ bu.require_chatgpt()run_id = args.run_id or datetime.now(timezone.utc).strftime("run-%Y%m%dT%H%M%SZ")run_dir = (args.runs_dir / run_id).resolve()if args.command == "dry-run":@@ -2378,14 +2386,13 @@ def make_ctx(args: argparse.Namespace) -> Ctx:unknown = [t for t in selected if t not in tasks]if unknown:raise SystemExit(f"unknown tasks {unknown}; known {sorted(tasks)}")- if bench_v2:- problems = v2.validate_tasks({t: task_spec(tasks[t]) for t in tasks}, selected)+ if bench_v2 or bu.ARM in arm_names:+ problems = v2.validate_tasks({t: task_spec(tasks[t]) for t in tasks}, selected, arm_names)if problems:raise SystemExit("bench v2: " + "; ".join(problems))explicit = bool(args.tasks) or bench_v2 # v2 keeps the registered order (A14.6)task_ids = core.order_tasks(selected, explicit=explicit)full_ids = task_ids if args.tasks else core.order_tasks(defaults, explicit=bench_v2)- arm_names = [args.only_arm] if args.only_arm else list(args.arms)codex = [a for a in arm_names if a in arms.CODEX_FALLBACK_ARMS]if codex and not args.allow_codex_arms:raise SystemExit(f"arms {codex} are fallback arms; pass --allow-codex-arms")@@ -2453,7 +2460,7 @@ def setup_arms(ctx: Ctx) -> None:except (FileNotFoundError, ValueError) as error:raise SystemExit(f"arm {arm} unavailable: {error}")elif (- arm in cxn.ARMS+ arm in cxn.MCP_ARMS): # A20/A23: the source Claude arm's MCP entry, copied unmodified, driven by Codex CLItry:path, _server = ca.mcp_config_for(diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.pyindex efbc62df0b..8b40b58bb2 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py@@ -26,6 +26,18 @@ def specs() -> dict[str, dict]:}+def test_browser_cli_requires_nonempty_browser_only_tasks():+ tasks = {+ "WEB": {"id": "WEB", "category": "web_forms", "browser": True},+ "OTHER": {"id": "OTHER", "category": "web_forms"},+ }+ arm = ["cx-browser-use-cli"]+ assert v2.validate_tasks(tasks, ["WEB"], arm) == []+ assert v2.validate_tasks(tasks, ["WEB", "OTHER"], arm)+ assert v2.validate_tasks(tasks, [], arm)+ assert v2.validate_tasks(tasks, ["OTHER"], ["cx-codex-cu"]) == []++class CategoriesTest(unittest.TestCase):def test_every_task_has_exactly_one_known_category(self) -> None:for tid, spec in specs().items():diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.pyindex a823406843..9bfcac3605 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py@@ -24,6 +24,114 @@ import bench_v2 as v2 # noqa: E402import claude_events # noqa: E402import codex_native as cxn # noqa: E402++def test_browser_cli_translation_and_rejections():+ import browser_use_cli as bu++ assert bu.ARM in arms.ALL_ARMS and bu.ARM in cxn.ARMS+ assert bu.ARM in cxn.LABELS and bu.ARM not in cxn.MCP_ARMS+ command = "browser-use open 'https://example.test/a b'"+ wrapped = "/bin/zsh -lc " + __import__("shlex").quote(command)+ for text in (command, wrapped, f"{bu.EXECUTABLE} open 'https://example.test/a b'"):+ tr = cxn.Translator(command_matcher=bu.match)+ item = {"id": "cli1", "type": "commandExecution", "command": text}+ events = tr.feed("item/started", {"item": item})+ assert tr.feed("item/started", {"item": item}) == []+ events += tr.feed(+ "item/completed",+ {+ "item": {+ **item,+ "status": "completed",+ "exitCode": 0,+ "aggregatedOutput": "OUTPUT" * 1000,+ }+ },+ )+ uses = [use for event in events for use in v2.tool_uses(event)]+ assert uses == [+ (+ "mcp__browser_use_cli__invoke",+ {"argv": [str(bu.EXECUTABLE), "open", "https://example.test/a b"], "python": None},+ )+ ]+ result = events[-1]["message"]["content"][0]+ assert result["tool_use_id"] == "cli1" and not result["is_error"]+ assert result["content"].startswith("OUTPUT") and len(result["content"]) == 2000+ assert tr.non_gui == []+ for text in (+ "browser-use open x; curl x",+ "browser-use open $(whoami)",+ "browser-use open `id`",+ "browser-use open x | cat",+ "browser-use open x > out",+ "browser-use open $HOME",+ "browser-use open x\necho y",+ "/tmp/browser-use open x",+ "curl x",+ "wget x",+ "browser-use python <<'PY'\nprint(1)\nPY",+ "browser-use open 'unterminated",+ ):+ assert bu.match(text) is None+ events = cxn.Translator(command_matcher=bu.match).feed(+ "item/started", {"item": {"id": "bad", "type": "commandExecution", "command": text}}+ )+ name, tool_input = v2.tool_uses(events[0])[0]+ assert v2.scan_tool_use(name, tool_input)[0] == ["builtin:Bash"]+++def test_browser_cli_python_and_argv_remain_scanned():+ import browser_use_cli as bu+ import run_bench as rb++ for argv, python in (+ (("browser-use", "subprocess"), None),+ (("browser-use", "python"), "subprocess evaluator-secret"),+ ):+ tr = cxn.Translator(command_matcher=lambda _: bu.Invocation(argv, python))+ events = tr.feed(+ "item/completed",+ {+ "item": {+ "id": "py",+ "type": "commandExecution",+ "command": "injected",+ "status": "completed",+ "exitCode": 1,+ "aggregatedOutput": "failure",+ }+ },+ )+ name, tool_input = v2.tool_uses(events[0])[0]+ assert v2.scan_tool_use(name, tool_input)[0]+ assert events[-1]["message"]["content"][0]["is_error"]+ if python:+ assert rb.evaluator_peeks(events, ["evaluator-secret"]) == [+ "mcp__browser_use_cli__invoke:evaluator-secret"+ ]+++def test_browser_cli_offline_installer(tmp_path):+ import hashlib+ import pytest+ import browser_use_cli as bu+ from tools import install_browser_use as installer++ pins = json.loads((HERE / "pins.json").read_text())["browser_use_cli"]+ wheel = tmp_path / "browser_use-0.13.11-py3-none-any.whl"+ wheel.write_bytes(b"test wheel")+ with mock.patch.object(installer.subprocess, "run") as run:+ with pytest.raises(ValueError):+ installer.install(wheel)+ run.assert_not_called()+ installer.install(wheel, expected_sha=hashlib.sha256(wheel.read_bytes()).hexdigest())+ cmd = run.call_args.args[0]+ assert cmd == ["uv", "tool", "install", "--python", "3.12", str(wheel)]+ assert run.call_args.kwargs["env"]["UV_TOOL_BIN_DIR"] == str(bu.EXECUTABLE.parent)+ assert pins["version"] == "0.13.11"++ARM_B = {"mcpServers": {"codex-cu": {@@ -60,7 +168,7 @@ class HomeTest(unittest.TestCase):source.chmod(0o000)before = source.stat().st_mtime_nsmcp = write_arm_b(root)- for arm in cxn.ARMS:+ for arm in cxn.MCP_ARMS:configs = {}texts = {}for mode in ("gateway", "chatgpt"):diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.pyindex 063b1a373d..7df72314a7 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py@@ -69,6 +69,65 @@ def make_ctx(tmp: str, **overrides: object) -> rb.Ctx:return ctx+def test_browser_cli_auth_rejected_before_provisioning(tmp_path):+ import pytest++ args = rb.build_parser().parse_args(["run", "--only-arm", "cx-browser-use-cli"])+ args.runs_dir = tmp_path+ args.ledger = tmp_path / "ledger.jsonl"+ with (+ mock.patch.dict("os.environ", {"CDB_CODEX_AUTH": "gateway"}),+ mock.patch.object(rb.subprocess, "run", side_effect=AssertionError("subprocess")),+ mock.patch.object(+ rb.arms, "link_chatgpt_auth", side_effect=AssertionError("credential link")+ ),+ ):+ with pytest.raises((ValueError, SystemExit), match="chatgpt"):+ rb.make_ctx(args)+++def test_browser_cli_run_fails_closed(tmp_path):+ with tempfile.TemporaryDirectory() as tmp:+ ctx = make_ctx(tmp, no_model_preflight=True, build_dir=None)+ ctx.args = rb.build_parser().parse_args(["run", "--only-arm", "cx-browser-use-cli"])+ ctx.arm_names = ["cx-browser-use-cli"]+ with (+ mock.patch.object(rb.ca, "observed_pins", return_value={}),+ mock.patch.object(rb.ca, "cua_version_string", return_value=None),+ mock.patch.object(rb.ca, "CUA_BUILDS", {}),+ ):+ rb.setup_arms(ctx)+ assert ctx.mcp == {}+ for offline in (False, True):+ for smoke in (False, True):+ ctx.args.offline, ctx.args.smoke = offline, smoke+ assert any(+ status == "fail" and "not yet implemented" in detail+ for _, status, detail in rb.preflight(ctx, with_models=False)+ )+ args = ctx.args+ args.no_model_preflight = True+ with contextlib.ExitStack() as stack:+ for name, value in (+ ("cmd_gate_check", 0),+ ("make_ctx", ctx),+ ("setup_arms", None),+ ("start_heartbeat", mock.Mock()),+ ("unregister_stale_copies", []),+ ("stop_agent_daemons", None),+ ("start_recorder", None),+ ("shutdown", None),+ ):+ stack.enter_context(mock.patch.object(rb, name, return_value=value))+ stack.enter_context(mock.patch.object(rb.ca, "stop_cua_daemon"))+ loop = stack.enter_context(+ mock.patch.object(rb, "run_loop", side_effect=AssertionError("trial started"))+ )+ assert rb.cmd_run(args) == 2+ assert (ctx.run_dir / "PREFLIGHT_FAILED").is_file()+ loop.assert_not_called()++class CodexAuthPreflightTest(unittest.TestCase):def test_chatgpt_checks_metadata_not_gateway_or_model_and_skips_offline(self) -> None:with tempfile.TemporaryDirectory() as tmp:@@ -289,7 +348,7 @@ class PreflightTest(unittest.TestCase):mock.patch.object(rb, "mcp_list_tools") as probe,mock.patch.object(rb, "ensure_agent_daemon") as daemon,):- for arm in rb.cxn.ARMS:+ for arm in rb.cxn.MCP_ARMS:with self.subTest(arm=arm):ctx.arm_names = [arm]ctx.mcp[arm] = (Path(tmp) / "mcp.json", rb.cxn.SERVER_OF[arm])diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.py b/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.pynew file mode 100644index 0000000000..a5a50bc718--- /dev/null+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.py@@ -0,0 +1,40 @@+from __future__ import annotations++import argparse+import hashlib+import json+import os+import subprocess+import sys+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(HERE))++import browser_use_cli as bu++PINS = json.loads((HERE / "pins.json").read_text())["browser_use_cli"]+++def install(wheel: Path, expected_sha: str = PINS["wheel_sha256"]) -> None:+ wheel = wheel.resolve()+ if wheel.name != f"browser_use-{PINS['version']}-py3-none-any.whl":+ raise ValueError("unexpected Browser Use wheel filename")+ if hashlib.sha256(wheel.read_bytes()).hexdigest() != expected_sha:+ raise ValueError("Browser Use wheel sha256 mismatch")+ subprocess.run(+ ["uv", "tool", "install", "--python", PINS["python"], str(wheel)],+ env={+ **os.environ,+ "UV_TOOL_DIR": str(bu.TOOL_DIR),+ "UV_TOOL_BIN_DIR": str(bu.TOOL_BIN_DIR),+ },+ check=True,+ timeout=120,+ )+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("wheel", type=Path)+ install(parser.parse_args().wheel)