The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
feat(terminal): let plugin backends set their start dir and host Bot Desktop
unsupportedpastelse166392fec12
diff --git a/agent/terminal_env_provider.py b/agent/terminal_env_provider.pyindex 9f3d84c9afc..44c53a68a8f 100644--- a/agent/terminal_env_provider.py+++ b/agent/terminal_env_provider.py@@ -38,11 +38,26 @@ class TerminalEnvironmentProvider(ProviderBase):spawns so a model-authored command can never read them.* ``session_isolated_when_nonpersistent`` — non-persistent mode gives each session its ownsandbox identity; opt in when a shared name would let two ephemeral runs destroy each other.+ * ``default_cwd`` — where commands start when ``terminal.cwd`` is unset or unusable there: an+ absolute in-sandbox path, or ``"~"`` for each sandbox's own home. With ``"~"``,+ ``create_environment`` receives ``cwd="~"`` (never expanded on the host) and the environment sets+ ``self.cwd`` to the absolute home it resolved; commands may also arrive with ``cwd="~"``.+ ``None`` keeps the generic ``/root``.+ * ``supports_streams`` — environments implement ``exec_prefix(*, user=None, interactive=True)``:+ a local argv that runs its remainder (``bash -c <script>``, passed through as separate words)+ inside the sandbox with stdin/stdout open, or ``None`` when it cannot right now. Bot Desktop,+ ``computer_use`` and the browser then follow the terminal into this backend, as they do for+ docker/ssh/singularity, instead of refusing.++ ``default_cwd`` home resolution and ``exec_prefix`` need the environment object itself, so they+ work for in-process plugins only; under ``plugins.isolation: host`` the environment reaches Hermes+ as an opaque placeholder and Bot Desktop placement refuses a ``supports_streams`` backend."""is_remote: bool = Trueis_container: bool = Truesession_isolated_when_nonpersistent: bool = False+ supports_streams: bool = False@propertydef description(self) -> str:@@ -57,6 +72,10 @@ class TerminalEnvironmentProvider(ProviderBase):def cache_path_base(self) -> Optional[str]:return None+ @property+ def default_cwd(self) -> Optional[str]:+ return None+@propertydef strip_env_keys(self) -> frozenset:return frozenset()diff --git a/tests/tools/test_terminal_plugin_cwd_and_streams.py b/tests/tools/test_terminal_plugin_cwd_and_streams.pynew file mode 100644index 00000000000..747e7fe55e4--- /dev/null+++ b/tests/tools/test_terminal_plugin_cwd_and_streams.py@@ -0,0 +1,133 @@+"""Plugin terminal backends declare where commands start (``default_cwd``) and whether Bot Desktop,+computer_use and the browser can follow the terminal into them (``supports_streams`` plus the+environment's ``exec_prefix``), instead of the core assuming ``/root`` and refusing every plugin."""++import subprocess+from types import SimpleNamespace++import pytest++from agent import terminal_env_registry as reg+from agent.terminal_env_provider import TerminalEnvironmentProvider+from tools import file_tools, file_tools_paths, terminal_tool+from tools.bot_desktop import placement+from tools.environments import streams+from tools.terminal_tool_backends import _create_environment+++class _SpaceEnv:+ def __init__(self, cwd="/", home="/home/cua"):+ # Like a sandbox that learns its home only once connected.+ self.cwd = home if cwd == "~" else cwd++ def execute(self, command, **kwargs):+ return {"output": "", "exit_code": 0}++ def cleanup(self):+ pass++ def exec_prefix(self, *, user=None, interactive=True):+ return ["spacectl", "exec", "box-1", "--"]+++class _PlainProvider(TerminalEnvironmentProvider):+ name = "plainbox"+ display_name = "PlainBox"++ def is_available(self):+ return True++ def create_environment(self, *, cwd, timeout, task_id="default", image=None, container_config=None, **kwargs):+ return _SpaceEnv(cwd, home=f"/home/{task_id}") # every sandbox has its own home+++class _SpaceProvider(_PlainProvider):+ name = "spacebox"+ supports_streams = True++ @property+ def default_cwd(self):+ return "/home/space"+++class _HomeProvider(_PlainProvider):+ name = "homebox"++ @property+ def default_cwd(self):+ return "~"++[email protected](autouse=True)+def _providers(monkeypatch):+ reg._reset_for_tests()+ for provider in (_PlainProvider(), _SpaceProvider(), _HomeProvider()):+ reg.register_provider(provider)+ monkeypatch.delenv("TERMINAL_CWD", raising=False)+ yield+ reg._reset_for_tests()+++def _create(backend, task_id, cwd="~"):+ return _create_environment(backend, image="", cwd=cwd, timeout=10, task_id=task_id)+++def test_plugin_default_cwd_replaces_root_and_absorbs_host_paths(monkeypatch):+ assert terminal_tool._resolve_config_cwd("spacebox", False) == ("/home/space", None)+ monkeypatch.setenv("TERMINAL_CWD", "/Users/me/project") # host-shaped: never valid in the sandbox+ assert terminal_tool._resolve_config_cwd("spacebox", False) == ("/home/space", None)+ monkeypatch.delenv("TERMINAL_CWD")+ assert terminal_tool._resolve_config_cwd("plainbox", False) == ("/root", None)+++def test_home_default_stays_per_sandbox_and_is_never_expanded_on_the_host():+ assert terminal_tool._resolve_config_cwd("homebox", False) == ("~", None)+ alice, bob = _create("homebox", "alice"), _create("homebox", "bob")+ # A second sandbox still gets "~" and resolves its OWN home, not the first one's.+ assert terminal_tool._resolve_config_cwd("homebox", False) == ("~", None)+ assert (alice._hermes_sandbox_home, bob._hermes_sandbox_home) == ("/home/alice", "/home/bob")+++def test_a_cd_inside_the_sandbox_home_survives_while_host_workspaces_are_still_dropped(monkeypatch):+ env = _create("homebox", "alice")+ def resolve():+ return terminal_tool._resolve_command_cwd(+ workdir=None, default_cwd="~", session_key="alice", env_type="homebox", env=env)++ monkeypatch.setattr(terminal_tool, "get_session_cwd", lambda _key: "/home/alice/project")+ assert resolve() == "/home/alice/project"+ monkeypatch.setattr(terminal_tool, "get_session_cwd", lambda _key: "/home/mark/workspace")+ assert resolve() == "~"+++def test_file_tools_anchor_relative_paths_at_their_own_sandbox_home(monkeypatch):+ monkeypatch.setenv("TERMINAL_ENV", "homebox")+ monkeypatch.setattr(file_tools, "_get_file_ops",+ lambda task_id="default": SimpleNamespace(env=_create("homebox", task_id)))+ assert file_tools_paths._container_default_cwd("sess") == "/home/sess"++[email protected](("backend", "expected"), [("spacebox", placement.TERMINAL), ("plainbox", placement.REFUSED)])+def test_placement_follows_a_plugin_backend_only_when_it_declares_streams(monkeypatch, backend, expected):+ monkeypatch.setattr(placement, "_setting", lambda: "auto")+ monkeypatch.setattr(placement, "_terminal_backend", lambda: backend)+ monkeypatch.setattr(placement, "_plugin_host_isolated", lambda: False)+ assert placement.resolve().where == expected+++def test_placement_refuses_a_streaming_plugin_under_host_isolation(monkeypatch):+ # The plugin host returns the environment as an opaque placeholder, so exec_prefix() is unreachable.+ monkeypatch.setattr(placement, "_setting", lambda: "auto")+ monkeypatch.setattr(placement, "_terminal_backend", lambda: "spacebox")+ monkeypatch.setattr(placement, "_plugin_host_isolated", lambda: True)+ where = placement.resolve()+ assert where.where == placement.REFUSED+ assert "plugins.isolation" in where.reason and "placement: gateway" in where.reason+++def test_streams_spawn_through_the_plugin_environments_exec_prefix(monkeypatch):+ spawned: list[list[str]] = []+ monkeypatch.setattr(subprocess, "Popen", lambda argv, **kw: spawned.append(argv) or object())+ streams.open_stream(_SpaceEnv(), ["cua-driver", "mcp"], child_env={"DISPLAY": ":20"})+ assert spawned[0][:6] == ["spacectl", "exec", "box-1", "--", "bash", "-c"]+ assert "export DISPLAY=:20;" in spawned[0][6] and "exec cua-driver mcp" in spawned[0][6]diff --git a/tools/bot_desktop/placement.py b/tools/bot_desktop/placement.pyindex 86c50dd4131..0b4cb5de5a6 100644--- a/tools/bot_desktop/placement.py+++ b/tools/bot_desktop/placement.py@@ -2,7 +2,8 @@``bot_desktop.placement``:``auto`` (default) follow the terminal backend when it is a sandbox that can host a stream- (docker / ssh / singularity); the gateway host when ``terminal.backend`` is local. A+ (docker / ssh / singularity, or a plugin backend declaring ``supports_streams``); the+ gateway host when ``terminal.backend`` is local. Asandbox backend that CANNOT host one (modal, daytona, vercel) resolves to ``refused``:the user chose a sandbox for the agent's actions, so quietly running the screen, cua-driverand the browser on the host beside it would hand the agent a desktop outside that sandbox.@@ -47,6 +48,19 @@ def _terminal_backend() -> str:return str(_get_env_config().get("env_type") or "local")+def _plugin_streams(backend: str) -> bool:+ from tools.terminal_tool_config import _plugin_env_flag+ return _plugin_env_flag(backend, "supports_streams") is True+++def _plugin_host_isolated() -> bool:+ # Under host isolation a plugin's environment object reaches Hermes as an opaque placeholder, so its+ # exec_prefix() is unreachable. (No bundled plugin backend declares supports_streams, so bundled+ # plugins, which stay in-process, are not a case here.)+ from hermes_cli.plugin_isolation import ISOLATION_HOST, isolation_mode+ return isolation_mode() == ISOLATION_HOST++def resolve() -> Placement:# Backend first (env only): a local terminal IS the gateway host whatever the setting says, and that is# the common case — it must not cost a config load (this runs on every browser / cua-driver spawn).@@ -58,6 +72,14 @@ def resolve() -> Placement:return Placement(GATEWAY, backend)if backend in _STREAM_BACKENDS:return Placement(TERMINAL, backend)+ if _plugin_streams(backend):+ if not _plugin_host_isolated():+ return Placement(TERMINAL, backend)+ return Placement(REFUSED, backend, (+ f"terminal.backend is {backend}, a plugin that runs in the plugin host (plugins.isolation: host), "+ f"and the host cannot hand its sandbox's exec channel to Hermes, so the screen cannot follow it. "+ f"Set plugins.isolation: in_process, or bot_desktop.placement: gateway to run the desktop on the "+ f"gateway host."))reason = (f"terminal.backend is {backend}, which cannot host a screen yet, and running the desktop on the "f"gateway host would put the agent's screen, browser and computer_use outside the sandbox you chose "f"for it. Set bot_desktop.placement: gateway to allow that explicitly.")diff --git a/tools/environments/streams.py b/tools/environments/streams.pyindex f4dc87f0450..cc5092acab5 100644--- a/tools/environments/streams.py+++ b/tools/environments/streams.py@@ -8,10 +8,12 @@ stdin/stdout as pipes". Each spawn-per-call backend has a local argv prefix that(``docker exec -i``, ``ssh``, ``apptainer exec``); this module is the one place that knows it.SDK backends (modal, daytona, vercel) have no argv prefix: ``exec_prefix`` returns None there and-the desktop stays on the gateway host (or refuses, per ``bot_desktop.placement``).+the desktop stays on the gateway host (or refuses, per ``bot_desktop.placement``). Plugin backends+that declare ``supports_streams`` supply the prefix from their environment's own ``exec_prefix()``."""from __future__ import annotations+import loggingimport shleximport socketimport subprocess@@ -20,6 +22,8 @@ from typing import Optional, Sequencefrom tools.environments.base import BaseEnvironment+logger = logging.getLogger(__name__)+def exec_prefix(env: BaseEnvironment, *, user: Optional[str] = None, interactive: bool = True) -> Optional[list[str]]:"""Local argv that runs its remainder inside ``env``, or None for backends without one.@@ -47,7 +51,21 @@ def exec_prefix(env: BaseEnvironment, *, user: Optional[str] = None, interactiveif not getattr(env, "_instance_started", False):return Nonereturn [env.executable, "exec", f"instance://{env.instance_id}"]- return None+ return _plugin_exec_prefix(env, user=user, interactive=interactive)+++def _plugin_exec_prefix(env: BaseEnvironment, *, user: Optional[str], interactive: bool) -> Optional[list[str]]:+ """A plugin environment's own ``exec_prefix()`` (``TerminalEnvironmentProvider.supports_streams``)."""+ hook = getattr(env, "exec_prefix", None)+ if not callable(hook):+ return None+ try:+ prefix = hook(user=user, interactive=interactive)+ except Exception:+ logger.warning("%s.exec_prefix() failed; treating the backend as unable to stream",+ type(env).__name__, exc_info=True)+ return None+ return [str(part) for part in prefix] if prefix else Nonedef supports_streams(env: BaseEnvironment) -> bool:diff --git a/tools/file_tools_paths.py b/tools/file_tools_paths.pyindex a12e40fb50e..ebcfd1c50b8 100644--- a/tools/file_tools_paths.py+++ b/tools/file_tools_paths.py@@ -276,6 +276,14 @@ def _container_default_cwd(task_id: str = "default") -> str | None:from tools.terminal_tool import _get_env_configcwd = _get_env_config().get("cwd")+ if cwd == "~":+ # A plugin backend's sandbox home is known once its environment connects: bring it up+ # through the file tools' own creator, as the call that follows would, and use the home+ # THAT environment resolved.+ from tools.file_tools import _get_file_ops+ from tools.terminal_tool_config import plugin_sandbox_home++ cwd = plugin_sandbox_home(_get_file_ops(task_id).env)if isinstance(cwd, str) and posixpath.isabs(cwd):return cwdexcept Exception:diff --git a/tools/terminal_tool.py b/tools/terminal_tool.pyindex 38c32c5fcf3..f1624611b6b 100644--- a/tools/terminal_tool.py+++ b/tools/terminal_tool.py@@ -46,7 +46,7 @@ from tools.terminal_tool_lifecycle import (from tools.terminal_tool_config import (_is_container_backend, _is_host_cwd, _is_mounted_host_cwd, _is_unusable_container_cwd,_is_windows_drive_path, _parse_env_var, _plugin_env_flag, _quiet, _safe_getcwd, _tenv, _tenv_bool,- coerce_ssh_remote_cwd, translate_mounted_host_path,+ _in_sandbox_home, coerce_ssh_remote_cwd, translate_mounted_host_path,)from tools.terminal_tool_backends import (_REQUIREMENT_CHECKERS, _VERCEL_SANDBOX_DEFAULT_CWD, _check_plugin_requirements,@@ -702,10 +702,23 @@ def _ensure_terminal_env_bridged() -> None:apply_terminal_config_to_env(env=None, override=False)-# Default cwd per backend; anything else (container backends, plugins) is "/root".+# Default cwd per backend; plugins may declare ``default_cwd``; anything else (container backends) is "/root"._DEFAULT_CWD_BY_BACKEND = {"ssh": "~", "vercel_sandbox": _VERCEL_SANDBOX_DEFAULT_CWD}+def _backend_default_cwd(env_type: str) -> str:+ if env_type == "local":+ return _safe_getcwd()+ if env_type in _DEFAULT_CWD_BY_BACKEND:+ return _DEFAULT_CWD_BY_BACKEND[env_type]+ declared = _plugin_env_flag(env_type, "default_cwd", None)+ # "~" is each sandbox's own home, expanded inside it; otherwise only an absolute in-sandbox path+ # (a relative value would be anchored on the host).+ if isinstance(declared, str) and (declared == "~" or declared.startswith("/")):+ return declared+ return "/root"++def _resolve_config_cwd(env_type: str, mount_docker_cwd: bool) -> tuple:"""``(cwd, host_cwd)`` from TERMINAL_CWD for *env_type*.@@ -713,10 +726,11 @@ def _resolve_config_cwd(env_type: str, mount_docker_cwd: bool) -> tuple:path is remapped to /workspace and tracked as host_cwd; otherwise host pathsare discarded in favor of the backend default."""- default_cwd = _safe_getcwd() if env_type == "local" else _DEFAULT_CWD_BY_BACKEND.get(env_type, "/root")+ default_cwd = _backend_default_cwd(env_type)cwd = _tenv("TERMINAL_CWD", default_cwd)from hermes_cli.config import _is_ssh_remote_tilde_cwd- if cwd and not _is_ssh_remote_tilde_cwd(env_type, cwd):+ # The backend's own default is an in-sandbox path: its ``~`` is the sandbox home, not ours.+ if cwd and cwd != default_cwd and not _is_ssh_remote_tilde_cwd(env_type, cwd):cwd = os.path.expanduser(cwd)host_cwd = Noneif env_type == "docker" and mount_docker_cwd:@@ -982,7 +996,7 @@ def _resolve_command_cwd(recorded = get_session_cwd(session_key)if recorded and _is_container_backend(env_type) and _is_unusable_container_cwd(recorded, mounted_host=mounted_host- ):+ ) and not _in_sandbox_home(recorded, env):visible = _container_visible_cwd(recorded, env_type, env)if visible != recorded:return visiblediff --git a/tools/terminal_tool_backends.py b/tools/terminal_tool_backends.pyindex c3230af339a..999b2c1e693 100644--- a/tools/terminal_tool_backends.py+++ b/tools/terminal_tool_backends.py@@ -214,6 +214,28 @@ def _build_ssh_env(*, cwd, timeout, ssh_config, probe_only=False, **_):persistent=ssh_config.get("persistent", True))+def _stamp_sandbox_home(env_obj, provider, cwd) -> None:+ """Record the home this environment started in when it started at the provider's ``default_cwd``: the+ path ``~`` resolved to inside THIS sandbox, or the declared absolute path. Per environment, never per+ backend, so a second sandbox with a different home gets its own."""+ try:+ declared = provider.default_cwd+ except Exception:+ # A plugin's property must not take environment creation down; it just gets no home.+ logger.warning("%s.default_cwd raised; not recording a sandbox home", type(provider).__name__,+ exc_info=True)+ declared = None+ if cwd == "~" and declared == "~":+ home = getattr(env_obj, "cwd", None)+ else:+ home = cwd if isinstance(declared, str) and cwd == declared else None+ if isinstance(home, str) and home.startswith("/") and home != "/":+ try:+ env_obj._hermes_sandbox_home = home+ except AttributeError:+ pass++def _build_plugin_env(*, env_type, image, cwd, timeout, cc, task_id, **_):provider = _get_plugin_env_provider(env_type)if provider is not None:@@ -225,6 +247,7 @@ def _build_plugin_env(*, env_type, image, cwd, timeout, cc, task_id, **_):env_obj._hermes_backend_name = provider.name.strip().lower()except AttributeError:pass+ _stamp_sandbox_home(env_obj, provider, cwd)return env_objtry:from agent.terminal_env_registry import plugin_backend_namesdiff --git a/tools/terminal_tool_config.py b/tools/terminal_tool_config.pyindex e9cede76601..a294a909a2a 100644--- a/tools/terminal_tool_config.py+++ b/tools/terminal_tool_config.py@@ -151,6 +151,23 @@ def _get_plugin_env_provider(env_type: str):return _plugin_registry_lookup(env_type, "get_provider", None)+def plugin_sandbox_home(env: Any) -> str | None:+ """The home a plugin environment started in (``_build_plugin_env`` stamps it per environment, so two+ sandboxes of one backend never share it), or None."""+ home = getattr(env, "_hermes_sandbox_home", None)+ return home if isinstance(home, str) and posixpath.isabs(home) and home != "/" else None+++def _in_sandbox_home(cwd: Any, env: Any) -> bool:+ """True for *cwd* at or under *env*'s sandbox home: a ``cd`` inside that sandbox, not a host workspace,+ even when it looks like one (``/home/...``)."""+ home = plugin_sandbox_home(env)+ if not home or not isinstance(cwd, str) or not posixpath.isabs(cwd):+ return False+ home, cwd = posixpath.normpath(home), posixpath.normpath(cwd)+ return cwd == home or cwd.startswith(home + "/")++@overloaddef coerce_ssh_remote_cwd(cwd: str, env_type: str | None) -> str: ...@overloaddiff --git a/website/docs/developer-guide/terminal-environment-plugin.md b/website/docs/developer-guide/terminal-environment-plugin.mdindex aca7424bdd4..89745d30277 100644--- a/website/docs/developer-guide/terminal-environment-plugin.md+++ b/website/docs/developer-guide/terminal-environment-plugin.md@@ -23,6 +23,8 @@ A registered backend automatically participates in every core surface:| System-prompt environment hints | `is_remote`, `env_description` || Dangerous-command approval skipping | `skip_container_guards` || Container path/cwd handling | `is_container` |+| Where commands start when `terminal.cwd` is unset or unusable | `default_cwd` |+| Bot Desktop, `computer_use` and the browser following the terminal | `supports_streams` + the environment's `exec_prefix()` || Synced cache-file path translation | `cache_path_base` || Secret stripping from spawned subprocesses | `strip_env_keys` || Per-session sandbox isolation (`container_persistent: false`) | `session_isolated_when_nonpersistent` |@@ -130,6 +132,73 @@ interface as `tools.environments.base.BaseEnvironment`:Subclassing `BaseEnvironment` is recommended (you inherit the shared file-syncand background-process plumbing) but not required.+## Starting directory++Without `default_cwd`, a plugin backend starts in `/root`, and a host-looking+`terminal.cwd` is replaced by `/root` when `is_container` is set. Return an+absolute in-sandbox path to start somewhere else:++```python+ @property+ def default_cwd(self):+ return "/home/acme"+```++When the home directory is only known once the sandbox is up, return `"~"`+instead. Hermes then passes `cwd="~"` to `create_environment()` (it never+expands it on the host), and your environment sets `self.cwd` to the absolute+home it resolved:++```python+class AcmeBoxEnvironment(BaseEnvironment):+ def __init__(self, *, cwd, timeout, sandbox):+ if cwd == "~":+ cwd = sandbox.home() # e.g. "/home/acme"+ super().__init__(cwd=cwd, timeout=timeout)+```++Hermes keeps `~` as the backend default, so every new sandbox resolves its+own home, and records the path each environment resolved on that environment.+Commands can also arrive with `cwd="~"`; `BaseEnvironment` already `cd`s+there correctly. A `cd` anywhere under that home is kept for later commands+and used as the base for relative paths in the file tools. Without this, a+sandbox path like `/home/acme/project` would be discarded as if it were a+host path.++`default_cwd` is read on every command, so keep it cheap (no network calls).+Relative paths are ignored, because Hermes would anchor them on the host.++## Bot Desktop and `computer_use`++The screen, `computer_use` and the browser follow the terminal into Docker, SSH+and Singularity sandboxes by spawning long-lived processes there with stdin and+stdout left open. A plugin backend can opt in the same way:++1. Set `supports_streams = True` on the provider. `bot_desktop.placement: auto`+ then places the screen inside your backend instead of refusing.+2. Give the environment an `exec_prefix(*, user=None, interactive=True)` method.+ It returns a local argv that runs whatever follows it inside the sandbox,+ with stdin and stdout open (the shape of `docker exec -i <container>`), or+ `None` when it can't right now. Hermes appends `bash -c <script>` as+ separate words, so the client must pass them through unchanged and the+ sandbox needs `bash`. `user` names the account to run as; ignore it if your+ backend can't switch users.++```python+class AcmeBoxEnvironment:+ def exec_prefix(self, *, user=None, interactive=True):+ return ["acmebox", "exec", "--stdin", self.sandbox_id, "--"]+```++The sandbox image still needs the desktop stack (Xvnc, Xfce, Chromium,+`cua-driver`); `nousresearch/hermes-sandbox:desktop` has it.++Both `exec_prefix()` and the `~` home resolution read the environment object+itself, so they work only for plugins loaded in-process. Under+`plugins.isolation: host` the environment reaches Hermes as an opaque+placeholder: Bot Desktop refuses a `supports_streams` backend with a message+naming the setting, and `default_cwd = "~"` stays a literal `~`.+## Session isolation semanticsIf your sandbox is **resumed by name** (a durable VM the backend re-attachesdiff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.mdindex 1f90b8f7dcf..35af80df88e 100644--- a/website/docs/user-guide/features/bot-screen.md+++ b/website/docs/user-guide/features/bot-screen.md@@ -235,7 +235,8 @@ same place. `bot_desktop.placement` decides where:|---|---|---|| `local` | gateway host | The terminal, the screen, the browser and `computer_use` all share the machine running the gateway. || `docker`, `ssh`, `singularity` | **inside the sandbox** | Xvnc + Xfce, Chromium and cua-driver run in the container / on the SSH host, spawned through the same `docker exec` / `ssh` channel the terminal uses. The pane streams the sandbox's screen; nothing of the host desktop is reachable. |-| `modal`, `daytona`, `vercel_sandbox` | **refused** | These backends cannot host a display yet. Rather than quietly running the screen on the host beside the sandbox you chose for the agent, `Start` explains and points at `placement: gateway`. |+| A plugin backend that declares `supports_streams` | **inside the sandbox** | Same as above, through the plugin's own exec channel. |+| `modal`, `daytona`, `vercel_sandbox`, other plugin backends | **refused** | These backends cannot host a display yet. Rather than quietly running the screen on the host beside the sandbox you chose for the agent, `Start` explains and points at `placement: gateway`. |`placement: gateway` forces the pre-existing behaviour (screen on the gatewayhost even with a sandboxed terminal) as an explicit opt-in; `placement:diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.mdindex d54fd618791..6fa01b69f71 100644--- a/website/docs/user-guide/features/computer-use.md+++ b/website/docs/user-guide/features/computer-use.md@@ -44,7 +44,8 @@ no-foreground invariant, click-dispatch internals — see`computer_use` acts on the same machine the bot's screen lives on, never onthe machine running Hermes Desktop. On a gateway with `terminal.backend:local` that is the gateway host. With a sandboxed terminal (`docker`, `ssh`,-`singularity`) the driver runs **inside the sandbox** on the sandbox's own+`singularity`, or a plugin backend that declares `supports_streams`) the+driver runs **inside the sandbox** on the sandbox's owndisplay, so it can only ever touch what the terminal can; the sandbox imagemust carry `cua-driver` (`nousresearch/hermes-sandbox:desktop` does). Modal,Daytona and Vercel sandboxes cannot host a display yet, so with those