Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(computer-server): fail closed for host tools in VNC mode
Francesco Bonacci78673aaa3134
diff --git a/libs/python/computer-server/computer_server/backend_policy.py b/libs/python/computer-server/computer_server/backend_policy.pynew file mode 100644index 0000000000..fad8085483--- /dev/null+++ b/libs/python/computer-server/computer_server/backend_policy.py@@ -0,0 +1,104 @@+"""Backend-specific exposure policy for computer-server ingress surfaces."""++import os+from typing import Any, Dict, Mapping, TypeVar++VNC_UNSUPPORTED_CODE = "unsupported_in_vnc"+VNC_UNSUPPORTED_ERROR = (+ "This operation is unavailable with the VNC backend because VNC only "+ "provides remote screen, pointer, scroll, and keyboard control."+)++# This is deliberately an allowlist. New commands and MCP tools stay hidden in+# VNC mode until they are proven to operate through the remote VNC connection.+VNC_REMOTE_COMMANDS = frozenset(+ {+ "version",+ "mouse_down",+ "mouse_up",+ "left_click",+ "right_click",+ "double_click",+ "move_cursor",+ "drag_to",+ "drag",+ "key_down",+ "key_up",+ "type_text",+ "press_key",+ "hotkey",+ "scroll",+ "scroll_down",+ "scroll_up",+ "scroll_direction",+ "screenshot",+ "get_cursor_position",+ "get_screen_size",+ }+)++VNC_REMOTE_MCP_TOOLS = frozenset(+ {+ "computer_screenshot",+ "computer_get_screen_size",+ "computer_get_cursor_position",+ "computer_click",+ "computer_double_click",+ "computer_move",+ "computer_drag",+ "computer_scroll",+ "computer_mouse_down",+ "computer_mouse_up",+ "computer_type",+ "computer_press_key",+ "computer_hotkey",+ "computer_key_down",+ "computer_key_up",+ }+)+++def configured_backend() -> str:+ """Return the effective backend using the same selection rule as the factory."""++ backend = os.environ.get("CUA_BACKEND", "native").strip().lower()+ if backend == "vnc" or os.environ.get("CUA_VNC_HOST"):+ return "vnc"+ return backend+++def is_vnc_backend() -> bool:+ return configured_backend() == "vnc"+++def vnc_unsupported_result() -> Dict[str, Any]:+ """Return the stable refusal shared by defensive handler fallbacks."""++ return {+ "success": False,+ "code": VNC_UNSUPPORTED_CODE,+ "error": VNC_UNSUPPORTED_ERROR,+ }+++RegistryValue = TypeVar("RegistryValue")+++def exposed_command_registry(+ registry: Mapping[str, RegistryValue],+) -> Dict[str, RegistryValue]:+ """Return the commands safe to advertise for the configured backend."""++ if not is_vnc_backend():+ return dict(registry)+ return {name: value for name, value in registry.items() if name in VNC_REMOTE_COMMANDS}+++class VNCUnavailableHandler:+ """Fail-closed placeholder for capabilities VNC cannot address remotely."""++ def __getattr__(self, _name: str):+ async def refuse(*_args, **_kwargs) -> Dict[str, Any]:+ return vnc_unsupported_result()++ return refusediff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.pyindex 0ef4bfcbe5..ef53987016 100644--- a/libs/python/computer-server/computer_server/handlers/factory.py+++ b/libs/python/computer-server/computer_server/handlers/factory.py@@ -5,6 +5,7 @@ from typing import Optional, Tuplefrom computer_server.diorama.base import BaseDioramaHandler+from ..backend_policy import VNCUnavailableHandler, configured_backendfrom ..utils.helpers import get_current_osfrom .base import (BaseAccessibilityHandler,@@ -64,9 +65,9 @@ class HandlerFactory:NotImplementedError: If the current OS is not supportedRuntimeError: If unable to determine the current OS"""- backend = os.environ.get("CUA_BACKEND", "native").strip().lower()+ backend = configured_backend()vnc_host = os.environ.get("CUA_VNC_HOST")- if backend == "vnc" or vnc_host:+ if backend == "vnc":if not vnc_host:raise RuntimeError("CUA_VNC_HOST must be set when using VNC backend "@@ -76,14 +77,15 @@ class HandlerFactory:vnc_port = int(os.environ.get("CUA_VNC_PORT", "5900"))vnc_password = os.environ.get("CUA_VNC_PASSWORD", "")+ unavailable = VNCUnavailableHandler()logger.info(f"Using VNC backend → {vnc_host}:{vnc_port}")return (VNCAccessibilityHandler(),VNCAutomationHandler(host=vnc_host, port=vnc_port, password=vnc_password),BaseDioramaHandler(),- GenericFileHandler(),- GenericDesktopHandler(),- GenericWindowHandler(),+ unavailable,+ unavailable,+ unavailable,)if backend not in {"native", "cua-driver"}:raise RuntimeError("CUA_BACKEND must be native, vnc, or cua-driver")diff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.pyindex 7285907b82..9b0de5da99 100644--- a/libs/python/computer-server/computer_server/handlers/vnc.py+++ b/libs/python/computer-server/computer_server/handlers/vnc.py@@ -24,6 +24,7 @@ import loggingfrom io import BytesIOfrom typing import Any, Dict, List, Optional, Tuple+from ..backend_policy import vnc_unsupported_resultfrom .base import BaseAccessibilityHandler, BaseAutomationHandlerlogger = logging.getLogger(__name__)@@ -563,7 +564,17 @@ class VNCAutomationHandler(BaseAutomationHandler):except Exception as e:return {"success": False, "error": str(e)}- # Clipboard and run_command inherited from BaseAutomationHandler+ # VNC has no clipboard or shell channel. Override the base class's local+ # fallbacks so even a stale/direct caller cannot mutate the server host.++ async def copy_to_clipboard(self) -> Dict[str, Any]:+ return vnc_unsupported_result()++ async def set_clipboard(self, text: str) -> Dict[str, Any]:+ return vnc_unsupported_result()++ async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:+ return vnc_unsupported_result()# ---------------------------------------------------------------------------diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex c068f53090..25f2d0ebeb 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -26,6 +26,11 @@ from fastapi import (from fastapi.middleware.cors import CORSMiddlewarefrom fastapi.responses import JSONResponse, StreamingResponse+from .backend_policy import (+ exposed_command_registry,+ is_vnc_backend,+ vnc_unsupported_result,+)from .browser import get_browser_managerfrom .handlers.factory import OS_TYPE, HandlerFactory@@ -193,6 +198,54 @@ class UnavailableWithoutContainerMiddleware:app.add_middleware(UnavailableWithoutContainerMiddleware)++class VNCBackendScopeGuard:+ """Refuse host-only HTTP and WebSocket surfaces in remote VNC mode."""++ def __init__(self, app):+ self.app = app++ async def __call__(self, scope, receive, send):+ path = scope.get("path", "")+ host_scoped = path in {"/playwright_exec", "/responses", "/pty"} or path.startswith("/pty/")+ if not is_vnc_backend() or not host_scoped:+ await self.app(scope, receive, send)+ return++ if scope.get("type") == "websocket":+ event = await receive()+ if event.get("type") != "websocket.connect":+ return+ await send({"type": "websocket.accept"})+ await send(+ {+ "type": "websocket.send",+ "text": json.dumps(vnc_unsupported_result()),+ }+ )+ await send({"type": "websocket.close", "code": 1008})+ return++ if scope.get("type") == "http":+ body = json.dumps(vnc_unsupported_result()).encode()+ await send(+ {+ "type": "http.response.start",+ "status": 409,+ "headers": [+ (b"content-type", b"application/json"),+ (b"content-length", str(len(body)).encode()),+ ],+ }+ )+ await send({"type": "http.response.body", "body": body})+ return++ await self.app(scope, receive, send)+++app.add_middleware(VNCBackendScopeGuard)+# CORS configurationorigins = ["*"]app.add_middleware(@@ -417,6 +470,12 @@ if hasattr(automation_handler, "get_capture_scope_state"):if hasattr(automation_handler, "escalate_capture_scope"):handlers["escalate_capture_scope"] = automation_handler.escalate_capture_scope+handlers = exposed_command_registry(handlers)+if is_vnc_backend():+ COMMAND_ALIASES = {+ alias: canonical for alias, canonical in COMMAND_ALIASES.items() if canonical in handlers+ }+class AuthenticationManager:def __init__(self):@@ -829,6 +888,27 @@ async def _require_auth(raise HTTPException(status_code=401, detail="Authentication failed")+class DirectComputerInterface:+ """BrowserTool-compatible interface without host leakage in VNC mode."""++ def __init__(self, automation_handler, browser_manager):+ self._auto = automation_handler+ self._browser = None if is_vnc_backend() else browser_manager++ @property+ def interface(self):+ """Return the target-scoped automation handler."""++ return self._auto++ async def playwright_exec(self, command: str, params: dict) -> dict:+ """Execute a browser command only when it addresses the local backend."""++ if self._browser is None:+ return vnc_unsupported_result()+ return await self._browser.execute_command(command, params)++# ---------------------------------------------------------------------------# PTY endpoints# ---------------------------------------------------------------------------@@ -1141,31 +1221,10 @@ async def agent_response_endpoint(# and delegates to our existing automation/file/accessibility handlers.from cua_agent.computers import AsyncComputerHandler # runtime-checkable Protocol- class DirectComputerInterface:- """Interface wrapper providing BrowserTool compatibility.-- Matches the same interface shape as Computer.interface so BrowserTool- works identically with both Computer (cloud) and DirectComputer (local).- """-- def __init__(self, automation_handler, browser_manager):- self._auto = automation_handler- self._browser = browser_manager-- @property- def interface(self):- """Return automation handler for hotkey, move_cursor, etc."""- return self._auto-- async def playwright_exec(self, command: str, params: dict) -> dict:- """Execute browser command via browser_manager."""- return await self._browser.execute_command(command, params)-class DirectComputer(AsyncComputerHandler):def __init__(self):# use module-scope handler singletons created by HandlerFactoryself._auto = automation_handler- self._file = file_handlerself._access = accessibility_handler# Create interface for BrowserTool compatibilityself._interface = DirectComputerInterface(automation_handler, get_browser_manager())diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.pyindex 57bff0aa2e..92132cca19 100644--- a/libs/python/computer-server/computer_server/mcp_server.py+++ b/libs/python/computer-server/computer_server/mcp_server.py@@ -15,6 +15,7 @@ import loggingimport sysfrom typing import Any, Dict, List, Optional, Tuple+from computer_server.backend_policy import VNC_REMOTE_MCP_TOOLS, is_vnc_backendfrom fastmcp import FastMCPfrom fastmcp.utilities.types import Image@@ -117,19 +118,45 @@ def create_mcp_server() -> FastMCP:"""mcp = FastMCP(name="cua-computer-server",- instructions="""You are connected to a computer control server that provides low-level+ instructions=(+ """You are connected to a remote VNC computer control server. It provides only+ screen, pointer, scroll, and keyboard operations that act on the VNC target.++ Always take a screenshot first to see the current state before performing actions.+ After performing actions, take another screenshot to verify the result."""+ if is_vnc_backend()+ else """You are connected to a computer control server that provides low-levelprimitives for interacting with a desktop computer. You can take screenshots, click,type text, press keys, scroll, manage windows, read/write files, and run commands.Always take a screenshot first to see the current state before performing actions.- After performing actions, take another screenshot to verify the result.""",+ After performing actions, take another screenshot to verify the result."""+ ),)+ original_tool = mcp.tool++ def register_tool(function=None, **kwargs):+ """Register only tools proven to address the remote target in VNC mode."""++ def register(candidate):+ if not is_vnc_backend() or candidate.__name__ in VNC_REMOTE_MCP_TOOLS:+ return original_tool(candidate, **kwargs)+ return candidate++ if function is None:+ return register+ return register(function)++ # Keep every decorator below on the same fail-closed registration path,+ # including future tools added to this function.+ mcp.tool = register_tool+# ============================================================# SCREEN & MOUSE ACTIONS# ============================================================- @mcp.tool+ @register_toolasync def computer_screenshot() -> Image:"""Capture a screenshot of the current screen.diff --git a/libs/python/computer-server/tests/test_vnc_backend_scope.py b/libs/python/computer-server/tests/test_vnc_backend_scope.pynew file mode 100644index 0000000000..9aa4f6926a--- /dev/null+++ b/libs/python/computer-server/tests/test_vnc_backend_scope.py@@ -0,0 +1,216 @@+"""Fail-closed transport coverage for the remote VNC backend."""++import os+from unittest.mock import AsyncMock, Mock++import pytest+from computer_server.backend_policy import (+ VNC_REMOTE_COMMANDS,+ VNC_REMOTE_MCP_TOOLS,+ VNC_UNSUPPORTED_CODE,+ VNCUnavailableHandler,+ exposed_command_registry,+)+from computer_server.mcp_server import create_mcp_server+from fastapi.testclient import TestClient+from fastmcp.exceptions import NotFoundError+++def vnc_backend(monkeypatch):+ monkeypatch.setenv("CUA_BACKEND", "vnc")+ monkeypatch.setenv("CUA_VNC_HOST", "127.0.0.1")+ monkeypatch.setenv("PYNPUT_BACKEND", "dummy")+++async def test_vnc_factory_never_constructs_host_file_desktop_or_window_handlers(+ vnc_backend, tmp_path+):+ from computer_server.handlers.factory import HandlerFactory++ handlers = HandlerFactory.create_handlers()++ assert all(isinstance(handler, VNCUnavailableHandler) for handler in handlers[3:])++ marker = tmp_path / "must-not-exist"+ file_result = await handlers[3].write_text(str(marker), "host mutation")+ shell_result = await handlers[1].run_command(f"touch {marker}")++ assert file_result["code"] == VNC_UNSUPPORTED_CODE+ assert shell_result["code"] == VNC_UNSUPPORTED_CODE+ assert not marker.exists()+++async def test_vnc_mcp_registry_contains_only_remote_target_tools(vnc_backend, tmp_path):+ server = create_mcp_server()+ names = {tool.name for tool in await server.list_tools()}++ assert names == VNC_REMOTE_MCP_TOOLS++ marker = tmp_path / "must-not-exist"+ with pytest.raises(NotFoundError, match="computer_file_write"):+ await server.call_tool(+ "computer_file_write",+ {"path": str(marker), "content": "host mutation"},+ )+ assert not marker.exists()+++async def test_non_vnc_mcp_registry_keeps_existing_host_tools(monkeypatch):+ monkeypatch.setenv("CUA_BACKEND", "native")+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)++ names = {tool.name for tool in await create_mcp_server().list_tools()}++ assert {+ "computer_screenshot",+ "computer_run_command",+ "computer_file_write",+ "computer_set_wallpaper",+ "computer_close_window",+ } <= names+++def test_vnc_command_registry_is_an_allowlist(vnc_backend):+ candidate = {name: object() for name in VNC_REMOTE_COMMANDS}+ candidate["future_host_operation"] = object()++ exposed = exposed_command_registry(candidate)++ assert set(exposed) == VNC_REMOTE_COMMANDS+++def test_non_vnc_command_registry_is_unchanged(monkeypatch):+ monkeypatch.setenv("CUA_BACKEND", "native")+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)+ candidate = {"run_command": object(), "screenshot": object()}++ assert exposed_command_registry(candidate) == candidate+++def test_vnc_http_and_websocket_commands_refuse_before_host_mutation(+ vnc_backend, monkeypatch, tmp_path+):+ from computer_server import main++ marker = tmp_path / "must-not-exist"++ async def write_text(path: str, content: str):+ marker.write_text(content)+ return {"success": True}++ monkeypatch.setattr(+ main,+ "handlers",+ exposed_command_registry(+ {+ "write_text": write_text,+ "screenshot": AsyncMock(return_value={"success": True}),+ }+ ),+ )+ monkeypatch.setattr(main, "COMMAND_ALIASES", {})++ with TestClient(main.app) as client:+ advertised = client.get("/commands").json()+ response = client.post(+ "/cmd",+ json={"command": "write_text", "params": {"path": str(marker), "content": "x"}},+ )+ assert response.status_code == 400++ with client.websocket_connect("/ws") as websocket:+ websocket.send_json(+ {+ "command": "write_text",+ "params": {"path": str(marker), "content": "x"},+ }+ )+ result = websocket.receive_json()++ assert set(advertised["commands"]) == {"screenshot"}+ assert advertised["aliases"] == {}+ assert result["success"] is False+ assert "Unknown command" in result["error"]+ assert not marker.exists()+++def test_vnc_host_scoped_http_surfaces_are_refused(vnc_backend, monkeypatch):+ from computer_server import main++ create_pty = AsyncMock()+ browser_command = AsyncMock()+ monkeypatch.setattr(main.pty_manager, "create", create_pty)+ monkeypatch.setattr(main.get_browser_manager(), "execute_command", browser_command)++ with TestClient(main.app) as client:+ pty_response = client.post("/pty", json={"command": "echo wrong-host"})+ browser_response = client.post(+ "/playwright_exec",+ json={"command": "visit_url", "params": {"url": "https://example.com"}},+ )+ agent_response = client.post(+ "/responses",+ json={+ "model": "test-model",+ "input": "inspect the remote target",+ "env": {"CUA_BACKEND": "native", "CUA_VNC_HOST": ""},+ },+ )++ assert pty_response.status_code == 409+ assert pty_response.json()["code"] == VNC_UNSUPPORTED_CODE+ assert browser_response.status_code == 409+ assert browser_response.json()["code"] == VNC_UNSUPPORTED_CODE+ assert agent_response.status_code == 409+ assert agent_response.json()["code"] == VNC_UNSUPPORTED_CODE+ assert os.environ["CUA_BACKEND"] == "vnc"+ assert os.environ["CUA_VNC_HOST"] == "127.0.0.1"+ create_pty.assert_not_awaited()+ browser_command.assert_not_awaited()+++def test_vnc_pty_websocket_is_refused_before_subscription(vnc_backend, monkeypatch):+ from computer_server import main++ subscribe = Mock()+ monkeypatch.setattr(main.pty_manager, "subscribe", subscribe)++ with TestClient(main.app) as client:+ with client.websocket_connect("/pty/123/ws") as websocket:+ result = websocket.receive_json()++ assert result["code"] == VNC_UNSUPPORTED_CODE+ subscribe.assert_not_called()+++async def test_vnc_direct_agent_interface_refuses_host_browser(vnc_backend):+ from computer_server.main import DirectComputerInterface++ browser = AsyncMock()+ interface = DirectComputerInterface(automation_handler=object(), browser_manager=browser)++ result = await interface.playwright_exec("visit_url", {"url": "https://example.com"})++ assert result["code"] == VNC_UNSUPPORTED_CODE+ browser.execute_command.assert_not_awaited()+++async def test_non_vnc_direct_agent_interface_keeps_browser_behavior(monkeypatch):+ from computer_server.main import DirectComputerInterface++ monkeypatch.setenv("CUA_BACKEND", "native")+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)+ browser = AsyncMock()+ browser.execute_command.return_value = {"success": True}+ interface = DirectComputerInterface(automation_handler=object(), browser_manager=browser)++ result = await interface.playwright_exec("visit_url", {"url": "https://example.com"})++ assert result == {"success": True}+ browser.execute_command.assert_awaited_once()