Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): forward the desktop session to the jev-use Driver (#4176)
Francesco Bonacci2519b5113204
diff --git a/.github/workflows/ci-jev-use.yml b/.github/workflows/ci-jev-use.ymlindex 7e56fdcd34..87a503fba2 100644--- a/.github/workflows/ci-jev-use.yml+++ b/.github/workflows/ci-jev-use.yml@@ -205,14 +205,12 @@ jobs:set -euo pipefailproof="$1"driver_bin="$CUA_DRIVER_BIN"- driver_wrapper="$RUNNER_TEMP/jev-use-driver-with-x11"+ driver_wrapper="$RUNNER_TEMP/jev-use-driver-with-e2e-hooks"+ # The runners forward the display, session bus, and CUA_DRIVER_*+ # variables themselves. Only the CI-specific browser test hooks are+ # restored here, so this job fails if that forwarding regresses.{echo '#!/bin/bash'- printf 'export DISPLAY=%q\n' "$DISPLAY"- printf 'export XAUTHORITY=%q\n' "${XAUTHORITY:-}"- printf 'export DBUS_SESSION_BUS_ADDRESS=%q\n' "$DBUS_SESSION_BUS_ADDRESS"- echo 'export CUA_DRIVER_PERMISSION_MODE=unrestricted'- echo 'export CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS=1'echo 'export CUA_E2E_BROWSER_NO_SANDBOX=1'echo 'export CUA_E2E_BROWSER_STDERR=1'printf 'exec %q "$@"\n' "$driver_bin"@@ -226,8 +224,7 @@ jobs:uv run python verify_mcp_tools.pyuv run --frozen python verify_setup.py --typescript --output-dir "$proof"SH- # This focused recipe E2E reuses the repository's supported X11- # stack and restores display variables omitted by MCP stdio defaults.+ # This focused recipe E2E reuses the repository's supported X11 stack.xvfb-run -a --server-args="-screen 0 1920x1080x24" \dbus-run-session -- bash "$session_script" "$proof"- name: Audit redacted E2E evidencediff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.mdindex 061005220a..abac7a1f96 100644--- a/libs/cua-driver/examples/jev-use/README.md+++ b/libs/cua-driver/examples/jev-use/README.md@@ -65,6 +65,14 @@ Chromium-based browser that Cua Driver can prepare. Python with `uv` suppliesthe fixture and Python agent. Node.js 22 or later and npm are needed only forthe optional TypeScript agent.+The agents start `cua-driver mcp` with the MCP SDK's minimal default+environment plus the desktop-session variables (`DISPLAY`, `WAYLAND_DISPLAY`,+`XAUTHORITY`, `XDG_RUNTIME_DIR`, `DBUS_SESSION_BUS_ADDRESS`,+`AT_SPI_BUS_ADDRESS`, `XDG_SESSION_TYPE`, `XDG_CURRENT_DESKTOP`) and any+`CUA_DRIVER_*` variables. Other variables, including `TYPESAFE_API_KEY`, are+not passed to the Driver. On Linux, run the agents from the logged-in desktop+session so those variables are set.+From this directory, install the locked Python dependencies:```bashdiff --git a/libs/cua-driver/examples/jev-use/python/driver_env.py b/libs/cua-driver/examples/jev-use/python/driver_env.pynew file mode 100644index 0000000000..173415fa4e--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/driver_env.py@@ -0,0 +1,38 @@+"""Environment for the Cua Driver process started over MCP stdio.++The MCP SDK's stdio transport passes only a small default allowlist (HOME,+PATH, USER, ...) on POSIX. On Linux that drops the desktop session, so the+Driver cannot reach the display or session bus and the isolated browser exits+before exposing DevTools. Forward the SDK defaults plus the desktop-session and+Cua Driver variables only; provider credentials such as TYPESAFE_API_KEY stay+in the agent process.+"""++from __future__ import annotations++import os+from collections.abc import Mapping++from mcp.client.stdio import get_default_environment++DESKTOP_SESSION_VARS = (+ "DISPLAY",+ "WAYLAND_DISPLAY",+ "XAUTHORITY",+ "XDG_RUNTIME_DIR",+ "DBUS_SESSION_BUS_ADDRESS",+ "AT_SPI_BUS_ADDRESS",+ "XDG_SESSION_TYPE",+ "XDG_CURRENT_DESKTOP",+)+DRIVER_VAR_PREFIX = "CUA_DRIVER_"+++def driver_environment(source: Mapping[str, str] | None = None) -> dict[str, str]:+ """Return the SDK default environment plus desktop-session and Driver variables."""+ source = os.environ if source is None else source+ env = dict(get_default_environment())+ for key, value in source.items():+ if key in DESKTOP_SESSION_VARS or key.startswith(DRIVER_VAR_PREFIX):+ env[key] = value+ return envdiff --git a/libs/cua-driver/examples/jev-use/python/run.py b/libs/cua-driver/examples/jev-use/python/run.pyindex ef05eebefb..a969d967e4 100644--- a/libs/cua-driver/examples/jev-use/python/run.py+++ b/libs/cua-driver/examples/jev-use/python/run.py@@ -14,6 +14,7 @@ from urllib.request import Request, urlopenfrom mcp import ClientSession, StdioServerParametersfrom mcp.client.stdio import stdio_client+from driver_env import driver_environmentfrom core import (VisualObservation,VisualObservationError,@@ -160,7 +161,9 @@ async def run(args: argparse.Namespace) -> str:log_path.write_text("", encoding="utf-8")reset_fixture(args.fixture_url)- params = StdioServerParameters(command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"])+ params = StdioServerParameters(+ command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"], env=driver_environment()+ )async with stdio_client(params) as (read, write):async with ClientSession(read, write) as session:await session.initialize()diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_driver_env.py b/libs/cua-driver/examples/jev-use/python/tests/test_driver_env.pynew file mode 100644index 0000000000..6743a46351--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/test_driver_env.py@@ -0,0 +1,44 @@+from __future__ import annotations++import sys+import unittest+from pathlib import Path+from unittest import mock++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from driver_env import DESKTOP_SESSION_VARS, driver_environment+++class DriverEnvironmentTest(unittest.TestCase):+ def test_forwards_desktop_session_variables_when_set(self):+ source = {name: f"value-{name}" for name in DESKTOP_SESSION_VARS}+ env = driver_environment(source)+ for name in DESKTOP_SESSION_VARS:+ self.assertEqual(env[name], f"value-{name}")++ def test_omits_desktop_session_variables_when_unset(self):+ with mock.patch.dict("os.environ", {"PATH": "/usr/bin"}, clear=True):+ env = driver_environment()+ for name in DESKTOP_SESSION_VARS:+ self.assertNotIn(name, env)++ def test_forwards_driver_variables(self):+ env = driver_environment({"CUA_DRIVER_PERMISSION_MODE": "unrestricted"})+ self.assertEqual(env["CUA_DRIVER_PERMISSION_MODE"], "unrestricted")++ def test_does_not_forward_provider_credentials(self):+ with mock.patch.dict(+ "os.environ",+ {"PATH": "/usr/bin", "DISPLAY": ":99", "TYPESAFE_API_KEY": "secret"},+ clear=True,+ ):+ env = driver_environment()+ self.assertEqual(env["DISPLAY"], ":99")+ self.assertEqual(env["PATH"], "/usr/bin")+ self.assertNotIn("TYPESAFE_API_KEY", env)+ self.assertNotIn("secret", env.values())+++if __name__ == "__main__":+ unittest.main()diff --git a/libs/cua-driver/examples/jev-use/typescript/driver_env.test.ts b/libs/cua-driver/examples/jev-use/typescript/driver_env.test.tsnew file mode 100644index 0000000000..b023c61680--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/driver_env.test.ts@@ -0,0 +1,42 @@+import assert from 'node:assert/strict';+import test from 'node:test';++import { DESKTOP_SESSION_VARS, driverEnvironment } from './driver_env.js';++test('forwards desktop session variables when set', () => {+ const source = Object.fromEntries(DESKTOP_SESSION_VARS.map((name) => [name, `value-${name}`]));+ const env = driverEnvironment(source);+ for (const name of DESKTOP_SESSION_VARS) assert.equal(env[name], `value-${name}`);+});++test('omits desktop session variables when unset', () => {+ const saved = Object.fromEntries(DESKTOP_SESSION_VARS.map((name) => [name, process.env[name]]));+ try {+ for (const name of DESKTOP_SESSION_VARS) delete process.env[name];+ const env = driverEnvironment({ PATH: '/usr/bin' });+ for (const name of DESKTOP_SESSION_VARS) assert.equal(name in env, false, name);+ } finally {+ for (const [name, value] of Object.entries(saved)) {+ if (value !== undefined) process.env[name] = value;+ }+ }+});++test('forwards Cua Driver variables', () => {+ const env = driverEnvironment({ CUA_DRIVER_PERMISSION_MODE: 'unrestricted' });+ assert.equal(env.CUA_DRIVER_PERMISSION_MODE, 'unrestricted');+});++test('does not forward provider credentials', () => {+ const saved = process.env.TYPESAFE_API_KEY;+ process.env.TYPESAFE_API_KEY = 'secret';+ try {+ const env = driverEnvironment({ ...process.env, DISPLAY: ':99' });+ assert.equal(env.DISPLAY, ':99');+ assert.equal('TYPESAFE_API_KEY' in env, false);+ assert.equal(Object.values(env).includes('secret'), false);+ } finally {+ if (saved === undefined) delete process.env.TYPESAFE_API_KEY;+ else process.env.TYPESAFE_API_KEY = saved;+ }+});diff --git a/libs/cua-driver/examples/jev-use/typescript/driver_env.ts b/libs/cua-driver/examples/jev-use/typescript/driver_env.tsnew file mode 100644index 0000000000..a0d6676144--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/driver_env.ts@@ -0,0 +1,33 @@+import process from 'node:process';++import { getDefaultEnvironment } from '@modelcontextprotocol/sdk/client/stdio.js';++// The MCP SDK stdio transport passes only a small default allowlist (HOME,+// PATH, USER, ...) on POSIX. On Linux that drops the desktop session, so the+// Driver cannot reach the display or session bus and the isolated browser exits+// before exposing DevTools. Forward the SDK defaults plus the desktop-session+// and Cua Driver variables only; provider credentials such as TYPESAFE_API_KEY+// stay in the agent process.+export const DESKTOP_SESSION_VARS = [+ 'DISPLAY',+ 'WAYLAND_DISPLAY',+ 'XAUTHORITY',+ 'XDG_RUNTIME_DIR',+ 'DBUS_SESSION_BUS_ADDRESS',+ 'AT_SPI_BUS_ADDRESS',+ 'XDG_SESSION_TYPE',+ 'XDG_CURRENT_DESKTOP',+] as const;+const DRIVER_VAR_PREFIX = 'CUA_DRIVER_';++export function driverEnvironment(+ source: NodeJS.ProcessEnv = process.env,+): Record<string, string> {+ const env = getDefaultEnvironment();+ const desktop: ReadonlySet<string> = new Set(DESKTOP_SESSION_VARS);+ for (const [key, value] of Object.entries(source)) {+ if (value === undefined) continue;+ if (desktop.has(key) || key.startsWith(DRIVER_VAR_PREFIX)) env[key] = value;+ }+ return env;+}diff --git a/libs/cua-driver/examples/jev-use/typescript/run.ts b/libs/cua-driver/examples/jev-use/typescript/run.tsindex 4259e38493..da94af26d3 100644--- a/libs/cua-driver/examples/jev-use/typescript/run.ts+++ b/libs/cua-driver/examples/jev-use/typescript/run.ts@@ -14,6 +14,7 @@ import {type Outcome,type VisualObservation,} from './core.js';+import { driverEnvironment } from './driver_env.js';import { chooseLive, chooseMockAdapter } from './jev_adapter.js';type Arguments = {@@ -170,6 +171,7 @@ async function run(args: Arguments): Promise<Outcome> {const transport = new StdioClientTransport({command: process.env.CUA_DRIVER_BIN ?? 'cua-driver',args: ['mcp'],+ env: driverEnvironment(),});const client = new Client({ name: 'cua-driver-jev-use-example', version: '0.1.0' });const history: Record<string, unknown>[] = [];diff --git a/libs/cua-driver/examples/jev-use/verify_mcp_tools.py b/libs/cua-driver/examples/jev-use/verify_mcp_tools.pyindex e61dc43316..c0d5d34c6b 100644--- a/libs/cua-driver/examples/jev-use/verify_mcp_tools.py+++ b/libs/cua-driver/examples/jev-use/verify_mcp_tools.py@@ -2,10 +2,16 @@ from __future__ import annotationsimport asyncioimport os+import sys+from pathlib import Pathfrom mcp import ClientSession, StdioServerParametersfrom mcp.client.stdio import stdio_client+sys.path.insert(0, str(Path(__file__).resolve().parent / "python"))++from driver_env import driver_environment # noqa: E402+REQUIRED_TOOLS = {"browser_click",@@ -16,7 +22,9 @@ REQUIRED_TOOLS = {"list_windows",}async def verify() -> None:- params = StdioServerParameters(command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"])+ params = StdioServerParameters(+ command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"], env=driver_environment()+ )async with stdio_client(params) as (read, write):async with ClientSession(read, write) as session:initialized = await session.initialize()