Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(computer-server): fail closed for host tools in VNC mode

Francesco Bonacci78673aaa3134
diff --git a/libs/python/computer-server/computer_server/backend_policy.py b/libs/python/computer-server/computer_server/backend_policy.py
new file mode 100644
index 0000000000..fad8085483
--- /dev/null
+++ b/libs/python/computer-server/computer_server/backend_policy.py
@@ -0,0 +1,104 @@
+"""Backend-specific exposure policy for computer-server ingress surfaces."""
+
+import os
+from typing import Any, Dict, Mapping, TypeVar
+
+VNC_UNSUPPORTED_CODE = "unsupported_in_vnc"
+VNC_UNSUPPORTED_ERROR = (
+ "This operation is unavailable with the VNC backend because VNC only "
+ "provides remote screen, pointer, scroll, and keyboard control."
+)
+
+# This is deliberately an allowlist. New commands and MCP tools stay hidden in
+# VNC mode until they are proven to operate through the remote VNC connection.
+VNC_REMOTE_COMMANDS = frozenset(
+ {
+ "version",
+ "mouse_down",
+ "mouse_up",
+ "left_click",
+ "right_click",
+ "double_click",
+ "move_cursor",
+ "drag_to",
+ "drag",
+ "key_down",
+ "key_up",
+ "type_text",
+ "press_key",
+ "hotkey",
+ "scroll",
+ "scroll_down",
+ "scroll_up",
+ "scroll_direction",
+ "screenshot",
+ "get_cursor_position",
+ "get_screen_size",
+ }
+)
+
+VNC_REMOTE_MCP_TOOLS = frozenset(
+ {
+ "computer_screenshot",
+ "computer_get_screen_size",
+ "computer_get_cursor_position",
+ "computer_click",
+ "computer_double_click",
+ "computer_move",
+ "computer_drag",
+ "computer_scroll",
+ "computer_mouse_down",
+ "computer_mouse_up",
+ "computer_type",
+ "computer_press_key",
+ "computer_hotkey",
+ "computer_key_down",
+ "computer_key_up",
+ }
+)
+
+
+def configured_backend() -> str:
+ """Return the effective backend using the same selection rule as the factory."""
+
+ backend = os.environ.get("CUA_BACKEND", "native").strip().lower()
+ if backend == "vnc" or os.environ.get("CUA_VNC_HOST"):
+ return "vnc"
+ return backend
+
+
+def is_vnc_backend() -> bool:
+ return configured_backend() == "vnc"
+
+
+def vnc_unsupported_result() -> Dict[str, Any]:
+ """Return the stable refusal shared by defensive handler fallbacks."""
+
+ return {
+ "success": False,
+ "code": VNC_UNSUPPORTED_CODE,
+ "error": VNC_UNSUPPORTED_ERROR,
+ }
+
+
+RegistryValue = TypeVar("RegistryValue")
+
+
+def exposed_command_registry(
+ registry: Mapping[str, RegistryValue],
+) -> Dict[str, RegistryValue]:
+ """Return the commands safe to advertise for the configured backend."""
+
+ if not is_vnc_backend():
+ return dict(registry)
+ return {name: value for name, value in registry.items() if name in VNC_REMOTE_COMMANDS}
+
+
+class VNCUnavailableHandler:
+ """Fail-closed placeholder for capabilities VNC cannot address remotely."""
+
+ def __getattr__(self, _name: str):
+ async def refuse(*_args, **_kwargs) -> Dict[str, Any]:
+ return vnc_unsupported_result()
+
+ return refuse
diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.py
index 0ef4bfcbe5..ef53987016 100644
--- a/libs/python/computer-server/computer_server/handlers/factory.py
+++ b/libs/python/computer-server/computer_server/handlers/factory.py
@@ -5,6 +5,7 @@ from typing import Optional, Tuple
from computer_server.diorama.base import BaseDioramaHandler
+from ..backend_policy import VNCUnavailableHandler, configured_backend
from ..utils.helpers import get_current_os
from .base import (
BaseAccessibilityHandler,
@@ -64,9 +65,9 @@ class HandlerFactory:
NotImplementedError: If the current OS is not supported
RuntimeError: If unable to determine the current OS
"""
- backend = os.environ.get("CUA_BACKEND", "native").strip().lower()
+ backend = configured_backend()
vnc_host = os.environ.get("CUA_VNC_HOST")
- if backend == "vnc" or vnc_host:
+ if backend == "vnc":
if not vnc_host:
raise RuntimeError(
"CUA_VNC_HOST must be set when using VNC backend "
@@ -76,14 +77,15 @@ class HandlerFactory:
vnc_port = int(os.environ.get("CUA_VNC_PORT", "5900"))
vnc_password = os.environ.get("CUA_VNC_PASSWORD", "")
+ unavailable = VNCUnavailableHandler()
logger.info(f"Using VNC backend → {vnc_host}:{vnc_port}")
return (
VNCAccessibilityHandler(),
VNCAutomationHandler(host=vnc_host, port=vnc_port, password=vnc_password),
BaseDioramaHandler(),
- GenericFileHandler(),
- GenericDesktopHandler(),
- GenericWindowHandler(),
+ unavailable,
+ unavailable,
+ unavailable,
)
if backend not in {"native", "cua-driver"}:
raise RuntimeError("CUA_BACKEND must be native, vnc, or cua-driver")
diff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.py
index 7285907b82..9b0de5da99 100644
--- a/libs/python/computer-server/computer_server/handlers/vnc.py
+++ b/libs/python/computer-server/computer_server/handlers/vnc.py
@@ -24,6 +24,7 @@ import logging
from io import BytesIO
from typing import Any, Dict, List, Optional, Tuple
+from ..backend_policy import vnc_unsupported_result
from .base import BaseAccessibilityHandler, BaseAutomationHandler
logger = logging.getLogger(__name__)
@@ -563,7 +564,17 @@ class VNCAutomationHandler(BaseAutomationHandler):
except Exception as e:
return {"success": False, "error": str(e)}
- # Clipboard and run_command inherited from BaseAutomationHandler
+ # VNC has no clipboard or shell channel. Override the base class's local
+ # fallbacks so even a stale/direct caller cannot mutate the server host.
+
+ async def copy_to_clipboard(self) -> Dict[str, Any]:
+ return vnc_unsupported_result()
+
+ async def set_clipboard(self, text: str) -> Dict[str, Any]:
+ return vnc_unsupported_result()
+
+ async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+ return vnc_unsupported_result()
# ---------------------------------------------------------------------------
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index c068f53090..25f2d0ebeb 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -26,6 +26,11 @@ from fastapi import (
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, StreamingResponse
+from .backend_policy import (
+ exposed_command_registry,
+ is_vnc_backend,
+ vnc_unsupported_result,
+)
from .browser import get_browser_manager
from .handlers.factory import OS_TYPE, HandlerFactory
@@ -193,6 +198,54 @@ class UnavailableWithoutContainerMiddleware:
app.add_middleware(UnavailableWithoutContainerMiddleware)
+
+class VNCBackendScopeGuard:
+ """Refuse host-only HTTP and WebSocket surfaces in remote VNC mode."""
+
+ def __init__(self, app):
+ self.app = app
+
+ async def __call__(self, scope, receive, send):
+ path = scope.get("path", "")
+ host_scoped = path in {"/playwright_exec", "/responses", "/pty"} or path.startswith("/pty/")
+ if not is_vnc_backend() or not host_scoped:
+ await self.app(scope, receive, send)
+ return
+
+ if scope.get("type") == "websocket":
+ event = await receive()
+ if event.get("type") != "websocket.connect":
+ return
+ await send({"type": "websocket.accept"})
+ await send(
+ {
+ "type": "websocket.send",
+ "text": json.dumps(vnc_unsupported_result()),
+ }
+ )
+ await send({"type": "websocket.close", "code": 1008})
+ return
+
+ if scope.get("type") == "http":
+ body = json.dumps(vnc_unsupported_result()).encode()
+ await send(
+ {
+ "type": "http.response.start",
+ "status": 409,
+ "headers": [
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(body)).encode()),
+ ],
+ }
+ )
+ await send({"type": "http.response.body", "body": body})
+ return
+
+ await self.app(scope, receive, send)
+
+
+app.add_middleware(VNCBackendScopeGuard)
+
# CORS configuration
origins = ["*"]
app.add_middleware(
@@ -417,6 +470,12 @@ if hasattr(automation_handler, "get_capture_scope_state"):
if hasattr(automation_handler, "escalate_capture_scope"):
handlers["escalate_capture_scope"] = automation_handler.escalate_capture_scope
+handlers = exposed_command_registry(handlers)
+if is_vnc_backend():
+ COMMAND_ALIASES = {
+ alias: canonical for alias, canonical in COMMAND_ALIASES.items() if canonical in handlers
+ }
+
class AuthenticationManager:
def __init__(self):
@@ -829,6 +888,27 @@ async def _require_auth(
raise HTTPException(status_code=401, detail="Authentication failed")
+class DirectComputerInterface:
+ """BrowserTool-compatible interface without host leakage in VNC mode."""
+
+ def __init__(self, automation_handler, browser_manager):
+ self._auto = automation_handler
+ self._browser = None if is_vnc_backend() else browser_manager
+
+ @property
+ def interface(self):
+ """Return the target-scoped automation handler."""
+
+ return self._auto
+
+ async def playwright_exec(self, command: str, params: dict) -> dict:
+ """Execute a browser command only when it addresses the local backend."""
+
+ if self._browser is None:
+ return vnc_unsupported_result()
+ return await self._browser.execute_command(command, params)
+
+
# ---------------------------------------------------------------------------
# PTY endpoints
# ---------------------------------------------------------------------------
@@ -1141,31 +1221,10 @@ async def agent_response_endpoint(
# and delegates to our existing automation/file/accessibility handlers.
from cua_agent.computers import AsyncComputerHandler # runtime-checkable Protocol
- class DirectComputerInterface:
- """Interface wrapper providing BrowserTool compatibility.
-
- Matches the same interface shape as Computer.interface so BrowserTool
- works identically with both Computer (cloud) and DirectComputer (local).
- """
-
- def __init__(self, automation_handler, browser_manager):
- self._auto = automation_handler
- self._browser = browser_manager
-
- @property
- def interface(self):
- """Return automation handler for hotkey, move_cursor, etc."""
- return self._auto
-
- async def playwright_exec(self, command: str, params: dict) -> dict:
- """Execute browser command via browser_manager."""
- return await self._browser.execute_command(command, params)
-
class DirectComputer(AsyncComputerHandler):
def __init__(self):
# use module-scope handler singletons created by HandlerFactory
self._auto = automation_handler
- self._file = file_handler
self._access = accessibility_handler
# Create interface for BrowserTool compatibility
self._interface = DirectComputerInterface(automation_handler, get_browser_manager())
diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.py
index 57bff0aa2e..92132cca19 100644
--- a/libs/python/computer-server/computer_server/mcp_server.py
+++ b/libs/python/computer-server/computer_server/mcp_server.py
@@ -15,6 +15,7 @@ import logging
import sys
from typing import Any, Dict, List, Optional, Tuple
+from computer_server.backend_policy import VNC_REMOTE_MCP_TOOLS, is_vnc_backend
from fastmcp import FastMCP
from fastmcp.utilities.types import Image
@@ -117,19 +118,45 @@ def create_mcp_server() -> FastMCP:
"""
mcp = FastMCP(
name="cua-computer-server",
- instructions="""You are connected to a computer control server that provides low-level
+ instructions=(
+ """You are connected to a remote VNC computer control server. It provides only
+ screen, pointer, scroll, and keyboard operations that act on the VNC target.
+
+ Always take a screenshot first to see the current state before performing actions.
+ After performing actions, take another screenshot to verify the result."""
+ if is_vnc_backend()
+ else """You are connected to a computer control server that provides low-level
primitives for interacting with a desktop computer. You can take screenshots, click,
type text, press keys, scroll, manage windows, read/write files, and run commands.
Always take a screenshot first to see the current state before performing actions.
- After performing actions, take another screenshot to verify the result.""",
+ After performing actions, take another screenshot to verify the result."""
+ ),
)
+ original_tool = mcp.tool
+
+ def register_tool(function=None, **kwargs):
+ """Register only tools proven to address the remote target in VNC mode."""
+
+ def register(candidate):
+ if not is_vnc_backend() or candidate.__name__ in VNC_REMOTE_MCP_TOOLS:
+ return original_tool(candidate, **kwargs)
+ return candidate
+
+ if function is None:
+ return register
+ return register(function)
+
+ # Keep every decorator below on the same fail-closed registration path,
+ # including future tools added to this function.
+ mcp.tool = register_tool
+
# ============================================================
# SCREEN & MOUSE ACTIONS
# ============================================================
- @mcp.tool
+ @register_tool
async def computer_screenshot() -> Image:
"""
Capture a screenshot of the current screen.
diff --git a/libs/python/computer-server/tests/test_vnc_backend_scope.py b/libs/python/computer-server/tests/test_vnc_backend_scope.py
new file mode 100644
index 0000000000..9aa4f6926a
--- /dev/null
+++ b/libs/python/computer-server/tests/test_vnc_backend_scope.py
@@ -0,0 +1,216 @@
+"""Fail-closed transport coverage for the remote VNC backend."""
+
+import os
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from computer_server.backend_policy import (
+ VNC_REMOTE_COMMANDS,
+ VNC_REMOTE_MCP_TOOLS,
+ VNC_UNSUPPORTED_CODE,
+ VNCUnavailableHandler,
+ exposed_command_registry,
+)
+from computer_server.mcp_server import create_mcp_server
+from fastapi.testclient import TestClient
+from fastmcp.exceptions import NotFoundError
+
+
+def vnc_backend(monkeypatch):
+ monkeypatch.setenv("CUA_BACKEND", "vnc")
+ monkeypatch.setenv("CUA_VNC_HOST", "127.0.0.1")
+ monkeypatch.setenv("PYNPUT_BACKEND", "dummy")
+
+
+async def test_vnc_factory_never_constructs_host_file_desktop_or_window_handlers(
+ vnc_backend, tmp_path
+):
+ from computer_server.handlers.factory import HandlerFactory
+
+ handlers = HandlerFactory.create_handlers()
+
+ assert all(isinstance(handler, VNCUnavailableHandler) for handler in handlers[3:])
+
+ marker = tmp_path / "must-not-exist"
+ file_result = await handlers[3].write_text(str(marker), "host mutation")
+ shell_result = await handlers[1].run_command(f"touch {marker}")
+
+ assert file_result["code"] == VNC_UNSUPPORTED_CODE
+ assert shell_result["code"] == VNC_UNSUPPORTED_CODE
+ assert not marker.exists()
+
+
+async def test_vnc_mcp_registry_contains_only_remote_target_tools(vnc_backend, tmp_path):
+ server = create_mcp_server()
+ names = {tool.name for tool in await server.list_tools()}
+
+ assert names == VNC_REMOTE_MCP_TOOLS
+
+ marker = tmp_path / "must-not-exist"
+ with pytest.raises(NotFoundError, match="computer_file_write"):
+ await server.call_tool(
+ "computer_file_write",
+ {"path": str(marker), "content": "host mutation"},
+ )
+ assert not marker.exists()
+
+
+async def test_non_vnc_mcp_registry_keeps_existing_host_tools(monkeypatch):
+ monkeypatch.setenv("CUA_BACKEND", "native")
+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)
+
+ names = {tool.name for tool in await create_mcp_server().list_tools()}
+
+ assert {
+ "computer_screenshot",
+ "computer_run_command",
+ "computer_file_write",
+ "computer_set_wallpaper",
+ "computer_close_window",
+ } <= names
+
+
+def test_vnc_command_registry_is_an_allowlist(vnc_backend):
+ candidate = {name: object() for name in VNC_REMOTE_COMMANDS}
+ candidate["future_host_operation"] = object()
+
+ exposed = exposed_command_registry(candidate)
+
+ assert set(exposed) == VNC_REMOTE_COMMANDS
+
+
+def test_non_vnc_command_registry_is_unchanged(monkeypatch):
+ monkeypatch.setenv("CUA_BACKEND", "native")
+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)
+ candidate = {"run_command": object(), "screenshot": object()}
+
+ assert exposed_command_registry(candidate) == candidate
+
+
+def test_vnc_http_and_websocket_commands_refuse_before_host_mutation(
+ vnc_backend, monkeypatch, tmp_path
+):
+ from computer_server import main
+
+ marker = tmp_path / "must-not-exist"
+
+ async def write_text(path: str, content: str):
+ marker.write_text(content)
+ return {"success": True}
+
+ monkeypatch.setattr(
+ main,
+ "handlers",
+ exposed_command_registry(
+ {
+ "write_text": write_text,
+ "screenshot": AsyncMock(return_value={"success": True}),
+ }
+ ),
+ )
+ monkeypatch.setattr(main, "COMMAND_ALIASES", {})
+
+ with TestClient(main.app) as client:
+ advertised = client.get("/commands").json()
+ response = client.post(
+ "/cmd",
+ json={"command": "write_text", "params": {"path": str(marker), "content": "x"}},
+ )
+ assert response.status_code == 400
+
+ with client.websocket_connect("/ws") as websocket:
+ websocket.send_json(
+ {
+ "command": "write_text",
+ "params": {"path": str(marker), "content": "x"},
+ }
+ )
+ result = websocket.receive_json()
+
+ assert set(advertised["commands"]) == {"screenshot"}
+ assert advertised["aliases"] == {}
+ assert result["success"] is False
+ assert "Unknown command" in result["error"]
+ assert not marker.exists()
+
+
+def test_vnc_host_scoped_http_surfaces_are_refused(vnc_backend, monkeypatch):
+ from computer_server import main
+
+ create_pty = AsyncMock()
+ browser_command = AsyncMock()
+ monkeypatch.setattr(main.pty_manager, "create", create_pty)
+ monkeypatch.setattr(main.get_browser_manager(), "execute_command", browser_command)
+
+ with TestClient(main.app) as client:
+ pty_response = client.post("/pty", json={"command": "echo wrong-host"})
+ browser_response = client.post(
+ "/playwright_exec",
+ json={"command": "visit_url", "params": {"url": "https://example.com"}},
+ )
+ agent_response = client.post(
+ "/responses",
+ json={
+ "model": "test-model",
+ "input": "inspect the remote target",
+ "env": {"CUA_BACKEND": "native", "CUA_VNC_HOST": ""},
+ },
+ )
+
+ assert pty_response.status_code == 409
+ assert pty_response.json()["code"] == VNC_UNSUPPORTED_CODE
+ assert browser_response.status_code == 409
+ assert browser_response.json()["code"] == VNC_UNSUPPORTED_CODE
+ assert agent_response.status_code == 409
+ assert agent_response.json()["code"] == VNC_UNSUPPORTED_CODE
+ assert os.environ["CUA_BACKEND"] == "vnc"
+ assert os.environ["CUA_VNC_HOST"] == "127.0.0.1"
+ create_pty.assert_not_awaited()
+ browser_command.assert_not_awaited()
+
+
+def test_vnc_pty_websocket_is_refused_before_subscription(vnc_backend, monkeypatch):
+ from computer_server import main
+
+ subscribe = Mock()
+ monkeypatch.setattr(main.pty_manager, "subscribe", subscribe)
+
+ with TestClient(main.app) as client:
+ with client.websocket_connect("/pty/123/ws") as websocket:
+ result = websocket.receive_json()
+
+ assert result["code"] == VNC_UNSUPPORTED_CODE
+ subscribe.assert_not_called()
+
+
+async def test_vnc_direct_agent_interface_refuses_host_browser(vnc_backend):
+ from computer_server.main import DirectComputerInterface
+
+ browser = AsyncMock()
+ interface = DirectComputerInterface(automation_handler=object(), browser_manager=browser)
+
+ result = await interface.playwright_exec("visit_url", {"url": "https://example.com"})
+
+ assert result["code"] == VNC_UNSUPPORTED_CODE
+ browser.execute_command.assert_not_awaited()
+
+
+async def test_non_vnc_direct_agent_interface_keeps_browser_behavior(monkeypatch):
+ from computer_server.main import DirectComputerInterface
+
+ monkeypatch.setenv("CUA_BACKEND", "native")
+ monkeypatch.delenv("CUA_VNC_HOST", raising=False)
+ browser = AsyncMock()
+ browser.execute_command.return_value = {"success": True}
+ interface = DirectComputerInterface(automation_handler=object(), browser_manager=browser)
+
+ result = await interface.playwright_exec("visit_url", {"url": "https://example.com"})
+
+ assert result == {"success": True}
+ browser.execute_command.assert_awaited_once()