Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Stage Browser Use CLI arm with pinned installation and fail-closed preflight (#5765)

injaneitycdda70795441
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/README.md b/libs/cua-driver-bench/automated-eval/macos_bench/README.md
index 635efd71fc..e83637221c 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/README.md
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/README.md
@@ -1,5 +1,10 @@
# Bench v2 harness (macOS)
+`cx-browser-use-cli` is staged and unavailable for scored execution: preflight always fails.
+It requires `CDB_CODEX_AUTH=chatgpt` and browser-only tasks. Codex driving the CLI is not
+like-for-like with Browser Use's own Agent loop. Its planned GUI-only exception covers
+only recognized Browser Use CLI invocations, not general Bash or arbitrary Python.
+
The runner, test apps, BenchLab tasks and analysis of Cua Driver Bench v2. Start
with [`../../README.md`](../../README.md) for what the benchmark measures and
how to reproduce it, and [`../../METHOD.md`](../../METHOD.md) for the method.
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md b/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md
index 3b12b2cddc..8804173ecc 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/RUNNER.md
@@ -7,6 +7,15 @@ invocation, isolation, schedule, gates and output. The method is in
## Arms
+`cx-browser-use-cli` (Amendment 24.2a) is staged: operational preflight always fails,
+including offline, smoke and no-model modes. No execution policy or trial home is written.
+It requires `CDB_CODEX_AUTH=chatgpt` and a nonempty selection of `browser: true` tasks;
+no existing task is flagged here. This is not like-for-like with Browser Use's own Agent loop.
+The planned GUI-only exception is recognized CLI invocations, not general Bash or arbitrary Python.
+Install an already downloaded, pinned wheel with `python tools/install_browser_use.py /path/to/browser_use-0.13.11-py3-none-any.whl`.
+Heredocs, skill discovery/provisioning, matcher-derived execution rules, browser lifecycle,
+observed-pin comparison and doctor/CDP/page-info/deny probes remain prerequisites to enabling it.
+
| Arm | Public setup | Tools Claude Code gets |
| --- | --- | --- |
| `cc-cua-driver-script` | Cua Driver | Cua Driver built from the pinned main commit, MCP server `cua`, `run_script` on, the skill of that commit in the working directory, and a short `run_script` addendum in the system prompt |
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/arms.py b/libs/cua-driver-bench/automated-eval/macos_bench/arms.py
index 639aa8c0d9..96d1f82116 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/arms.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/arms.py
@@ -49,6 +49,7 @@ NATIVE_CODEX_ARMS = (
"cx-codex-cu",
"cx-cua-driver",
"cx-claude-cu-helper",
+ "cx-browser-use-cli",
) # the last two: Amendment 23
ALL_ARMS = CLAUDE_ARMS + CODEX_FALLBACK_ARMS + NATIVE_CODEX_ARMS
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
index dd3499cbbc..7f2def75db 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
@@ -73,14 +73,21 @@ def category_of(spec: dict[str, Any]) -> str:
return str(category)
-def validate_tasks(specs: dict[str, dict[str, Any]], task_ids: list[str]) -> list[str]:
+def validate_tasks(
+ specs: dict[str, dict[str, Any]], task_ids: list[str], arms: list[str] | None = None
+) -> list[str]:
"""Problems that stop a v2 run: unknown task, missing or unknown category, coding tools."""
problems: list[str] = []
+ browser_only = "cx-browser-use-cli" in (arms or [])
+ if browser_only and not task_ids:
+ problems.append("cx-browser-use-cli requires a nonempty browser task selection")
for tid in task_ids:
spec = specs.get(tid)
if spec is None:
problems.append(f"{tid}: unknown task")
continue
+ if browser_only and spec.get("browser") is not True:
+ problems.append(f"{tid}: cx-browser-use-cli requires browser: true")
try:
category_of(spec)
except ValueError as error:
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.py b/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.py
new file mode 100644
index 0000000000..dabd5e69d6
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/browser_use_cli.py
@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import shlex
+from dataclasses import dataclass
+from pathlib import Path
+
+import arms
+
+ARM = "cx-browser-use-cli"
+TOOL_DIR = Path.home() / ".cache/cua-bench-h2h/browser-use-cli"
+TOOL_BIN_DIR = TOOL_DIR / "bin"
+EXECUTABLE = TOOL_BIN_DIR / "browser-use"
+
+
+@dataclass(frozen=True)
+class Invocation:
+ argv: tuple[str, ...]
+ python: str | None = None
+
+
+def require_chatgpt() -> None:
+ if arms.codex_auth() != "chatgpt":
+ raise ValueError(f"{ARM} requires CDB_CODEX_AUTH=chatgpt")
+
+
+def match(command: str, executable: Path = EXECUTABLE) -> Invocation | None:
+ try:
+ wrapper = shlex.split(command)
+ if len(wrapper) == 3 and wrapper[:2] == ["/bin/zsh", "-lc"]:
+ command = wrapper[2]
+ if any(c in command for c in "\n\r\x00$`\\;|&<>(){}[]*?~#!^"):
+ return None
+ argv = shlex.split(command)
+ except ValueError:
+ return None
+ if not argv or argv[0] not in ("browser-use", str(executable)):
+ return None
+ return Invocation((str(executable), *argv[1:]))
+
+
+def preflight() -> list[tuple[str, str, str]]:
+ return [(ARM, "fail", "Browser Use CLI operational preflight is not yet implemented")]
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
index 7d5b713654..c99b274e58 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
@@ -52,17 +52,19 @@ from pathlib import Path
from typing import Any, Callable
import arms
+from browser_use_cli import Invocation
ARM = "cx-codex-cu"
# cx-cua-driver and cx-claude-cu-helper: Amendment 23, Cua Driver's MCP server and Cua's helper adapter under the
# same Codex host
-ARMS = (ARM, "cx-cua-driver", "cx-claude-cu-helper")
+ARMS = (ARM, "cx-cua-driver", "cx-claude-cu-helper", "cx-browser-use-cli")
SERVER = "cua_repl" # the server name of the unified-computer-use plugin
SERVER_OF = {
"cx-codex-cu": SERVER,
"cx-cua-driver": "cua",
"cx-claude-cu-helper": "claude-cu-helper",
}
+MCP_ARMS = tuple(SERVER_OF)
# The Claude Code arm whose MCP config (and, for Cua Driver, whose daemon, build and system prompt) the Codex arm uses.
SOURCE_ARM = {
"cx-codex-cu": "cc-codex-cu",
@@ -89,6 +91,7 @@ GATEWAY_ENV = "CDB_CODEX_GATEWAY"
DEFAULT_GATEWAY = "http://127.0.0.1:4000/v1"
CLOSED_PATH = "/usr/bin:/bin:/usr/sbin:/sbin"
LABELS = {
+ "cx-browser-use-cli": "Browser Use CLI, driven by Codex CLI + GPT-6.1 Sol (staged)",
"cx-codex-cu": f"Codex Computer-Use (Sky), driven by Codex CLI + {MODEL_DISPLAY.get(MODEL, MODEL)}",
"cx-cua-driver": f"Cua Driver, driven by Codex CLI + {MODEL_DISPLAY.get(MODEL, MODEL)}",
"cx-claude-cu-helper": "Claude Computer-Use (Desktop helper), driven by Codex CLI + "
@@ -248,7 +251,12 @@ class Translator:
result), so `bench_v2.GuiOnlyGuard`, `bench_v2.scan_events` and `claude_events.summarize` read this arm the
same way as the Claude arms."""
- def __init__(self, server: str = SERVER) -> None:
+ def __init__(
+ self,
+ server: str = SERVER,
+ command_matcher: Callable[[str], Invocation | None] | None = None,
+ ) -> None:
+ self.command_matcher = command_matcher
self.server = server
self.final_text = ""
self.errors: list[str] = []
@@ -305,16 +313,37 @@ class Translator:
item = params.get("item") or {}
itype = str(item.get("type"))
item_id = str(item.get("id"))
+ invocation = (
+ self.command_matcher(str(item.get("command") or ""))
+ if itype == "commandExecution" and self.command_matcher
+ else None
+ )
if itype == "agentMessage" and method == "item/completed":
self.final_text = str(item.get("text") or "")
- elif itype == "mcpToolCall":
- name = f"mcp__{item.get('server')}__{item.get('tool')}"
+ elif itype == "mcpToolCall" or invocation:
+ name = (
+ "mcp__browser_use_cli__invoke"
+ if invocation
+ else f"mcp__{item.get('server')}__{item.get('tool')}"
+ )
+ tool_input = (
+ {"argv": list(invocation.argv), "python": invocation.python}
+ if invocation
+ else item.get("arguments") or {}
+ )
if item_id not in self.started:
self.started.add(item_id)
- out.append(self._use(item_id, name, item.get("arguments") or {}))
+ out.append(self._use(item_id, name, tool_input))
if method == "item/completed":
failed = item.get("status") != "completed" or item.get("error") is not None
- text = json.dumps(item.get("error"))[:300] if failed else ""
+ failed = failed or bool(invocation and item.get("exitCode") != 0)
+ text = (
+ str(item.get("aggregatedOutput") or "")[:2000]
+ if invocation
+ else json.dumps(item.get("error"))[:300]
+ if failed
+ else ""
+ )
out.append(
{
"type": "user",
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
index 4e94e81160..5bd1a6732b 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
@@ -115,6 +115,16 @@
"type"
]
},
+ "browser_use_cli": {
+ "version": "0.13.11",
+ "python": "3.12",
+ "wheel_sha256": "50b0d16efc44215f99ed1c9197f05e2519bb79fa1501eed7371aff450cc56045",
+ "skill_path": "browser_use/skills/browser-use/SKILL.md",
+ "skill_sha256": "fee8f320d0bdcad3695393663b5defa6e5fc1e22ff7ac6ffb6f04f6997b93c36",
+ "_requires_python": ">=3.11,<4.0",
+ "_wheel": "browser_use-0.13.11-py3-none-any.whl",
+ "_url": "https://files.pythonhosted.org/packages/cd/e6/8fb8ce81894afcc7626837232a4d284843d1c1fb4fd757aa00c54ce40d8e/browser_use-0.13.11-py3-none-any.whl"
+ },
"codex_native": {
"_note": "Amendment 20 (CUA-1347): the Codex CLI bundled in ChatGPT 26.930.51102 (codex-cli/CodexCLI.app, Developer ID OpenAI OpCo, LLC, 2DC432GLL2), host protocol `codex app-server`, model gpt-6.1-sol (Amendment 23) at its catalog default effort (low); A20's smoke used gpt-6-astra (medium). Pinned from the host's identical app version; the smoke checks the VM copy.",
"codex_cli_version": "codex-cli 0.160.0",
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
index bb1bb77ecb..14fa901808 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
@@ -52,6 +52,7 @@ import bench_core as core # noqa: E402
import claude_arms as ca # noqa: E402
import cdb_adapter # noqa: E402
import codex_native as cxn # noqa: E402
+import browser_use_cli as bu
import claude_driver # noqa: E402
import claude_events # noqa: E402
import recorder as rec # noqa: E402
@@ -826,7 +827,7 @@ def run_attempt(ctx: Ctx, entry: dict[str, Any], attempt: int) -> dict[str, Any]
row["lab_unoccluded"] = row["lab_occlusion"].get("unoccluded")
cwd = ca.prepare_cwd(arm)
mcp_path, server = ctx.mcp[arm]
- native = arm in cxn.ARMS
+ native = arm in cxn.MCP_ARMS
argv = (
cxn.argv()
if native
@@ -1915,7 +1916,9 @@ def v2_static_checks(ctx: Ctx) -> list[tuple[str, str, str]]:
"""Amendment 14: every task has one known category and is GUI-only, and no arm's argv offers a
shell or file tool (the same surface for every setup)."""
out: list[tuple[str, str, str]] = []
- problems = v2.validate_tasks({t: task_spec(ctx.tasks[t]) for t in ctx.tasks}, ctx.task_ids)
+ problems = v2.validate_tasks(
+ {t: task_spec(ctx.tasks[t]) for t in ctx.tasks}, ctx.task_ids, ctx.arm_names
+ )
cats = {t: task_spec(ctx.tasks[t]).get("category") for t in ctx.task_ids}
out.append(
(
@@ -1951,6 +1954,8 @@ def v2_static_checks(ctx: Ctx) -> list[tuple[str, str, str]]:
def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:
args = ctx.args
+ if bu.ARM in ctx.arm_names:
+ return bu.preflight()
checks: list[tuple[str, str, str]] = []
def add(name: str, status: str, detail: str) -> None:
@@ -2059,7 +2064,7 @@ def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:
)
except Exception as error: # noqa: BLE001
add("arm B MCP server", "fail", f"{type(error).__name__}: {error}")
- for native_arm in [a for a in cxn.ARMS if a in ctx.arm_names]:
+ for native_arm in [a for a in cxn.MCP_ARMS if a in ctx.arm_names]:
if not cxn.CODEX_CLI.is_file():
add("Codex CLI in the ChatGPT app", "fail", str(cxn.CODEX_CLI))
continue
@@ -2363,6 +2368,9 @@ def build_parser() -> argparse.ArgumentParser:
def make_ctx(args: argparse.Namespace) -> Ctx:
+ arm_names = [args.only_arm] if args.only_arm else list(args.arms)
+ if bu.ARM in arm_names:
+ bu.require_chatgpt()
run_id = args.run_id or datetime.now(timezone.utc).strftime("run-%Y%m%dT%H%M%SZ")
run_dir = (args.runs_dir / run_id).resolve()
if args.command == "dry-run":
@@ -2378,14 +2386,13 @@ def make_ctx(args: argparse.Namespace) -> Ctx:
unknown = [t for t in selected if t not in tasks]
if unknown:
raise SystemExit(f"unknown tasks {unknown}; known {sorted(tasks)}")
- if bench_v2:
- problems = v2.validate_tasks({t: task_spec(tasks[t]) for t in tasks}, selected)
+ if bench_v2 or bu.ARM in arm_names:
+ problems = v2.validate_tasks({t: task_spec(tasks[t]) for t in tasks}, selected, arm_names)
if problems:
raise SystemExit("bench v2: " + "; ".join(problems))
explicit = bool(args.tasks) or bench_v2 # v2 keeps the registered order (A14.6)
task_ids = core.order_tasks(selected, explicit=explicit)
full_ids = task_ids if args.tasks else core.order_tasks(defaults, explicit=bench_v2)
- arm_names = [args.only_arm] if args.only_arm else list(args.arms)
codex = [a for a in arm_names if a in arms.CODEX_FALLBACK_ARMS]
if codex and not args.allow_codex_arms:
raise SystemExit(f"arms {codex} are fallback arms; pass --allow-codex-arms")
@@ -2453,7 +2460,7 @@ def setup_arms(ctx: Ctx) -> None:
except (FileNotFoundError, ValueError) as error:
raise SystemExit(f"arm {arm} unavailable: {error}")
elif (
- arm in cxn.ARMS
+ arm in cxn.MCP_ARMS
): # A20/A23: the source Claude arm's MCP entry, copied unmodified, driven by Codex CLI
try:
path, _server = ca.mcp_config_for(
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
index efbc62df0b..8b40b58bb2 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
@@ -26,6 +26,18 @@ def specs() -> dict[str, dict]:
}
+def test_browser_cli_requires_nonempty_browser_only_tasks():
+ tasks = {
+ "WEB": {"id": "WEB", "category": "web_forms", "browser": True},
+ "OTHER": {"id": "OTHER", "category": "web_forms"},
+ }
+ arm = ["cx-browser-use-cli"]
+ assert v2.validate_tasks(tasks, ["WEB"], arm) == []
+ assert v2.validate_tasks(tasks, ["WEB", "OTHER"], arm)
+ assert v2.validate_tasks(tasks, [], arm)
+ assert v2.validate_tasks(tasks, ["OTHER"], ["cx-codex-cu"]) == []
+
+
class CategoriesTest(unittest.TestCase):
def test_every_task_has_exactly_one_known_category(self) -> None:
for tid, spec in specs().items():
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py
index a823406843..9bfcac3605 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_codex_native.py
@@ -24,6 +24,114 @@ import bench_v2 as v2 # noqa: E402
import claude_events # noqa: E402
import codex_native as cxn # noqa: E402
+
+def test_browser_cli_translation_and_rejections():
+ import browser_use_cli as bu
+
+ assert bu.ARM in arms.ALL_ARMS and bu.ARM in cxn.ARMS
+ assert bu.ARM in cxn.LABELS and bu.ARM not in cxn.MCP_ARMS
+ command = "browser-use open 'https://example.test/a b'"
+ wrapped = "/bin/zsh -lc " + __import__("shlex").quote(command)
+ for text in (command, wrapped, f"{bu.EXECUTABLE} open 'https://example.test/a b'"):
+ tr = cxn.Translator(command_matcher=bu.match)
+ item = {"id": "cli1", "type": "commandExecution", "command": text}
+ events = tr.feed("item/started", {"item": item})
+ assert tr.feed("item/started", {"item": item}) == []
+ events += tr.feed(
+ "item/completed",
+ {
+ "item": {
+ **item,
+ "status": "completed",
+ "exitCode": 0,
+ "aggregatedOutput": "OUTPUT" * 1000,
+ }
+ },
+ )
+ uses = [use for event in events for use in v2.tool_uses(event)]
+ assert uses == [
+ (
+ "mcp__browser_use_cli__invoke",
+ {"argv": [str(bu.EXECUTABLE), "open", "https://example.test/a b"], "python": None},
+ )
+ ]
+ result = events[-1]["message"]["content"][0]
+ assert result["tool_use_id"] == "cli1" and not result["is_error"]
+ assert result["content"].startswith("OUTPUT") and len(result["content"]) == 2000
+ assert tr.non_gui == []
+ for text in (
+ "browser-use open x; curl x",
+ "browser-use open $(whoami)",
+ "browser-use open `id`",
+ "browser-use open x | cat",
+ "browser-use open x > out",
+ "browser-use open $HOME",
+ "browser-use open x\necho y",
+ "/tmp/browser-use open x",
+ "curl x",
+ "wget x",
+ "browser-use python <<'PY'\nprint(1)\nPY",
+ "browser-use open 'unterminated",
+ ):
+ assert bu.match(text) is None
+ events = cxn.Translator(command_matcher=bu.match).feed(
+ "item/started", {"item": {"id": "bad", "type": "commandExecution", "command": text}}
+ )
+ name, tool_input = v2.tool_uses(events[0])[0]
+ assert v2.scan_tool_use(name, tool_input)[0] == ["builtin:Bash"]
+
+
+def test_browser_cli_python_and_argv_remain_scanned():
+ import browser_use_cli as bu
+ import run_bench as rb
+
+ for argv, python in (
+ (("browser-use", "subprocess"), None),
+ (("browser-use", "python"), "subprocess evaluator-secret"),
+ ):
+ tr = cxn.Translator(command_matcher=lambda _: bu.Invocation(argv, python))
+ events = tr.feed(
+ "item/completed",
+ {
+ "item": {
+ "id": "py",
+ "type": "commandExecution",
+ "command": "injected",
+ "status": "completed",
+ "exitCode": 1,
+ "aggregatedOutput": "failure",
+ }
+ },
+ )
+ name, tool_input = v2.tool_uses(events[0])[0]
+ assert v2.scan_tool_use(name, tool_input)[0]
+ assert events[-1]["message"]["content"][0]["is_error"]
+ if python:
+ assert rb.evaluator_peeks(events, ["evaluator-secret"]) == [
+ "mcp__browser_use_cli__invoke:evaluator-secret"
+ ]
+
+
+def test_browser_cli_offline_installer(tmp_path):
+ import hashlib
+ import pytest
+ import browser_use_cli as bu
+ from tools import install_browser_use as installer
+
+ pins = json.loads((HERE / "pins.json").read_text())["browser_use_cli"]
+ wheel = tmp_path / "browser_use-0.13.11-py3-none-any.whl"
+ wheel.write_bytes(b"test wheel")
+ with mock.patch.object(installer.subprocess, "run") as run:
+ with pytest.raises(ValueError):
+ installer.install(wheel)
+ run.assert_not_called()
+ installer.install(wheel, expected_sha=hashlib.sha256(wheel.read_bytes()).hexdigest())
+ cmd = run.call_args.args[0]
+ assert cmd == ["uv", "tool", "install", "--python", "3.12", str(wheel)]
+ assert run.call_args.kwargs["env"]["UV_TOOL_BIN_DIR"] == str(bu.EXECUTABLE.parent)
+ assert pins["version"] == "0.13.11"
+
+
ARM_B = {
"mcpServers": {
"codex-cu": {
@@ -60,7 +168,7 @@ class HomeTest(unittest.TestCase):
source.chmod(0o000)
before = source.stat().st_mtime_ns
mcp = write_arm_b(root)
- for arm in cxn.ARMS:
+ for arm in cxn.MCP_ARMS:
configs = {}
texts = {}
for mode in ("gateway", "chatgpt"):
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
index 063b1a373d..7df72314a7 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
@@ -69,6 +69,65 @@ def make_ctx(tmp: str, **overrides: object) -> rb.Ctx:
return ctx
+def test_browser_cli_auth_rejected_before_provisioning(tmp_path):
+ import pytest
+
+ args = rb.build_parser().parse_args(["run", "--only-arm", "cx-browser-use-cli"])
+ args.runs_dir = tmp_path
+ args.ledger = tmp_path / "ledger.jsonl"
+ with (
+ mock.patch.dict("os.environ", {"CDB_CODEX_AUTH": "gateway"}),
+ mock.patch.object(rb.subprocess, "run", side_effect=AssertionError("subprocess")),
+ mock.patch.object(
+ rb.arms, "link_chatgpt_auth", side_effect=AssertionError("credential link")
+ ),
+ ):
+ with pytest.raises((ValueError, SystemExit), match="chatgpt"):
+ rb.make_ctx(args)
+
+
+def test_browser_cli_run_fails_closed(tmp_path):
+ with tempfile.TemporaryDirectory() as tmp:
+ ctx = make_ctx(tmp, no_model_preflight=True, build_dir=None)
+ ctx.args = rb.build_parser().parse_args(["run", "--only-arm", "cx-browser-use-cli"])
+ ctx.arm_names = ["cx-browser-use-cli"]
+ with (
+ mock.patch.object(rb.ca, "observed_pins", return_value={}),
+ mock.patch.object(rb.ca, "cua_version_string", return_value=None),
+ mock.patch.object(rb.ca, "CUA_BUILDS", {}),
+ ):
+ rb.setup_arms(ctx)
+ assert ctx.mcp == {}
+ for offline in (False, True):
+ for smoke in (False, True):
+ ctx.args.offline, ctx.args.smoke = offline, smoke
+ assert any(
+ status == "fail" and "not yet implemented" in detail
+ for _, status, detail in rb.preflight(ctx, with_models=False)
+ )
+ args = ctx.args
+ args.no_model_preflight = True
+ with contextlib.ExitStack() as stack:
+ for name, value in (
+ ("cmd_gate_check", 0),
+ ("make_ctx", ctx),
+ ("setup_arms", None),
+ ("start_heartbeat", mock.Mock()),
+ ("unregister_stale_copies", []),
+ ("stop_agent_daemons", None),
+ ("start_recorder", None),
+ ("shutdown", None),
+ ):
+ stack.enter_context(mock.patch.object(rb, name, return_value=value))
+ stack.enter_context(mock.patch.object(rb.ca, "stop_cua_daemon"))
+ loop = stack.enter_context(
+ mock.patch.object(rb, "run_loop", side_effect=AssertionError("trial started"))
+ )
+ assert rb.cmd_run(args) == 2
+ assert (ctx.run_dir / "PREFLIGHT_FAILED").is_file()
+ loop.assert_not_called()
+
+
class CodexAuthPreflightTest(unittest.TestCase):
def test_chatgpt_checks_metadata_not_gateway_or_model_and_skips_offline(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
@@ -289,7 +348,7 @@ class PreflightTest(unittest.TestCase):
mock.patch.object(rb, "mcp_list_tools") as probe,
mock.patch.object(rb, "ensure_agent_daemon") as daemon,
):
- for arm in rb.cxn.ARMS:
+ for arm in rb.cxn.MCP_ARMS:
with self.subTest(arm=arm):
ctx.arm_names = [arm]
ctx.mcp[arm] = (Path(tmp) / "mcp.json", rb.cxn.SERVER_OF[arm])
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.py b/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.py
new file mode 100644
index 0000000000..a5a50bc718
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/install_browser_use.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(HERE))
+
+import browser_use_cli as bu
+
+PINS = json.loads((HERE / "pins.json").read_text())["browser_use_cli"]
+
+
+def install(wheel: Path, expected_sha: str = PINS["wheel_sha256"]) -> None:
+ wheel = wheel.resolve()
+ if wheel.name != f"browser_use-{PINS['version']}-py3-none-any.whl":
+ raise ValueError("unexpected Browser Use wheel filename")
+ if hashlib.sha256(wheel.read_bytes()).hexdigest() != expected_sha:
+ raise ValueError("Browser Use wheel sha256 mismatch")
+ subprocess.run(
+ ["uv", "tool", "install", "--python", PINS["python"], str(wheel)],
+ env={
+ **os.environ,
+ "UV_TOOL_DIR": str(bu.TOOL_DIR),
+ "UV_TOOL_BIN_DIR": str(bu.TOOL_BIN_DIR),
+ },
+ check=True,
+ timeout=120,
+ )
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("wheel", type=Path)
+ install(parser.parse_args().wheel)