AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(platform): one box per owner — the desktop is the shell's screen (#14382)
Nicholas Tindlefcaa5e12c702
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 9bd00ce2e1..5f0618ca1d 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -870,7 +870,7 @@ async def get_session_computer() -> ComputerInfo:"""The computer behind this chat.- A plain chat has its own boxes; a chat that runs as a hired expert reports+ A plain chat has its own box; a chat that runs as a hired expert reportsthe expert's persistent computer instead. Listing never wakes a paused box."""# Metadata only: the panel polls this, and the history is not needed.diff --git a/autogpt_platform/backend/backend/api/features/experts/routes.py b/autogpt_platform/backend/backend/api/features/experts/routes.pyindex 55effddbb2..3adaf52e06 100644--- a/autogpt_platform/backend/backend/api/features/experts/routes.py+++ b/autogpt_platform/backend/backend/api/features/experts/routes.py@@ -42,7 +42,7 @@ from backend.copilot.computer import (open_desktop,)from backend.copilot.config import ChatConfig-from backend.copilot.tools.e2b_sandbox import SandboxOwner, kill_expert_sandboxes+from backend.copilot.tools.e2b_sandbox import SandboxOwner, kill_expert_sandboxfrom backend.util import product_analyticsfrom backend.util.exceptions import NotFoundError@@ -379,7 +379,9 @@ async def get_expert_computer(expert_id: str,user_id: str = Security(autogpt_auth_lib.get_user_id),) -> ComputerInfo:- """The expert's own computer: its shell and desktop boxes as E2B lists them.+ """The expert's own computer: its box as E2B lists it, and whether its+ screen is on. E2B knows nothing about the screen; that flag is ours,+ kept beside the box id, because asking the box would wake it.Listing never wakes a paused box, so the Computer tab can refresh freely."""@@ -674,10 +676,10 @@ async def archive_expert(# deliberately kept — files outlive the machine.if api_key := ChatConfig().active_e2b_api_key:try:- await kill_expert_sandboxes(expert_id, api_key)+ await kill_expert_sandbox(expert_id, api_key)except Exception:logger.warning(- "[E2B] Failed to kill sandboxes for archived expert %s",+ "[E2B] Failed to kill the sandbox for archived expert %s",expert_id[:12],exc_info=True,)diff --git a/autogpt_platform/backend/backend/blocks/desktop/_api.py b/autogpt_platform/backend/backend/blocks/desktop/_api.pyindex d0b3b07fb4..43046c8879 100644--- a/autogpt_platform/backend/backend/blocks/desktop/_api.py+++ b/autogpt_platform/backend/backend/blocks/desktop/_api.py@@ -29,9 +29,15 @@ PERSISTENT_HOME_DIRS = ("Downloads", "Desktop", "Documents")DISPLAY = ":0"VNC_PORT = 5900STREAM_PORT = 6080-# The stream password lives next to x11vnc's hashed one so a resume can hand-# back the URL the user already holds instead of restarting the proxy.-STREAM_PASSWORD_PATH = f"{HOME_PATH}/.vnc/agpt_stream_password"+# The VNC stack runs as root. The model's shell runs as ``user``, and a+# password that shell can read is a password it can print into a transcript,+# where the chat's share link would then carry a working desktop URL. x11vnc+# takes its password from a root-only file that it deletes the moment it has+# read it, so once the stream is up nothing on the box holds the credential.+# What a resume needs to hand back the same URL is kept by the caller, off the+# box (see ``backend.copilot.computer``).+VNC_USER = "root"+VNC_PASSWORD_PATH = "/root/.vnc/stream_password"# Bound on the E2B volumes API (private beta) so a slow create cannot stall# sandbox creation; the by-name mount fallback is the normal path anyway.VOLUME_API_TIMEOUT_SECONDS = 10@@ -123,63 +129,66 @@ class DesktopSession:)return cls(sandbox)- async def start_stream(self) -> DesktopStream:- """Return the live stream URL, starting the VNC stack only if needed.-- A resume (or a second ``start_desktop`` call) must not restart x11vnc- and noVNC: that would sever the stream the user is watching and rotate- the password baked into the URL they already hold. When the proxy is- still serving — E2B's pause/resume restores processes — the saved- password is reused and the same URL comes back.+ async def start_stream(+ self, password: Optional[str] = None+ ) -> tuple[DesktopStream, str]:+ """Return the live stream URL and its password, starting the VNC stack+ only if needed.++ *password* is the one this caller issued last time. While noVNC is+ still serving it (E2B's pause/resume restores processes) the same URL+ comes back: restarting x11vnc and noVNC would sever the stream the+ user is watching. Without it, or once the proxy is gone, the stack is+ (re)started under a fresh password, and whoever held the old URL is+ locked out. The caller decides when to forget the password (after a+ pause, say) and so when a URL that may have leaked stops working."""- password = await self._running_stream_password()- if password is None:+ if password is None or not await self._stream_listening():password = "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(16))- await self.run_command(+ await self._vnc_command("pkill -f '[n]ovnc_proxy' || true; pkill -x x11vnc || true")- await self.run_command(- f"mkdir -p ~/.vnc && x11vnc -storepasswd {password} ~/.vnc/passwd"+ await self._vnc_command(+ f"umask 077 && mkdir -p {shlex.quote(VNC_PASSWORD_PATH.rsplit('/', 1)[0])}"+ f" && printf %s {shlex.quote(password)} > {shlex.quote(VNC_PASSWORD_PATH)}")- await self.run_command(+ await self._vnc_command(f"x11vnc -bg -display {DISPLAY} -forever -wait 50 -shared "- f"-rfbport {VNC_PORT} -usepw >/tmp/x11vnc.log 2>/tmp/x11vnc_stderr.log"+ f"-rfbport {VNC_PORT} -passwdfile rm:{VNC_PASSWORD_PATH} "+ ">/tmp/x11vnc.log 2>/tmp/x11vnc_stderr.log")await self.sandbox.commands.run(f"cd /opt/noVNC/utils && ./novnc_proxy --vnc localhost:{VNC_PORT} "f"--listen {STREAM_PORT} --web /opt/noVNC > /tmp/novnc.log 2>&1",background=True,+ user=VNC_USER,)await self._wait_for(f'netstat -tuln | grep ":{STREAM_PORT} "')- await self.run_command(- f"umask 077 && printf %s {shlex.quote(password)} "- f"> {shlex.quote(STREAM_PASSWORD_PATH)}"- )host = self.sandbox.get_host(STREAM_PORT)url = (f"https://{host}/vnc.html"f"?autoconnect=true&resize=scale&password={password}")- return DesktopStream(url=url, sandbox_id=self.sandbox_id)+ return DesktopStream(url=url, sandbox_id=self.sandbox_id), password- async def _running_stream_password(self) -> Optional[str]:- """The saved stream password, only while noVNC is actually listening."""- if not await self._check(f'netstat -tuln | grep -q ":{STREAM_PORT} "'):- return None- try:- saved = await self.sandbox.files.read(STREAM_PASSWORD_PATH)- except Exception:- return None- saved = saved.strip() if isinstance(saved, str) else ""- return saved or None+ async def _stream_listening(self) -> bool:+ """Whether noVNC is still serving the stream (it survives a pause)."""+ return await self._check(f'netstat -tuln | grep -q ":{STREAM_PORT} "')++ async def _vnc_command(self, command: str):+ return await self.run_command(command, user=VNC_USER)async def run_command(- self, command: str, cwd: Optional[str] = None, timeout: int = 60+ self,+ command: str,+ cwd: Optional[str] = None,+ timeout: int = 60,+ user: Optional[str] = None,):return await self.sandbox.commands.run(- command, cwd=cwd, timeout=timeout, envs={"DISPLAY": DISPLAY}+ command, cwd=cwd, timeout=timeout, envs={"DISPLAY": DISPLAY}, user=user)async def is_workspace_mounted(self) -> bool:diff --git a/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.py b/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.pynew file mode 100644index 0000000000..9b0a200aa6--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.py@@ -0,0 +1,91 @@+"""The live stream: its password never rests on the box, and a re-open hands+back the URL the user holds only while the box has kept running."""++from unittest.mock import AsyncMock, MagicMock++import pytest++from backend.blocks.desktop._api import (+ HOME_PATH,+ VNC_PASSWORD_PATH,+ VNC_USER,+ DesktopSession,+)+++def _session(listening: bool) -> tuple[DesktopSession, AsyncMock]:+ """A box where noVNC is (not) serving; ``run`` records every command."""+ run = AsyncMock()++ async def fake_run(command: str, **kwargs):+ if command.startswith("netstat") and "grep -q" in command:+ if listening:+ return MagicMock()+ raise RuntimeError("nothing on the stream port")+ return MagicMock()++ run.side_effect = fake_run+ sandbox = MagicMock()+ sandbox.sandbox_id = "sb-1"+ sandbox.commands.run = run+ sandbox.get_host = MagicMock(return_value="6080-sb-1.e2b.app")+ return DesktopSession(sandbox), run+++def _commands(run: AsyncMock) -> list[tuple[str, str | None]]:+ return [(c.args[0], c.kwargs.get("user")) for c in run.await_args_list]+++async def test_a_fresh_stream_keeps_its_password_out_of_the_shells_reach():+ session, run = _session(listening=False)++ stream, password = await session.start_stream(None)++ assert len(password) == 16 and f"password={password}" in stream.url+ commands = _commands(run)+ vnc = [(cmd, user) for cmd, user in commands if "x11vnc" in cmd or "novnc" in cmd]+ # The whole VNC stack runs as root, not as the model's user.+ assert vnc and all(user == VNC_USER for _, user in vnc)+ # The password is handed to x11vnc through a root-only file it deletes+ # once read; nothing under the model's home ever holds it.+ (write,) = [cmd for cmd, _ in commands if password in cmd]+ assert "umask 077" in write and VNC_PASSWORD_PATH in write+ assert HOME_PATH not in write+ (x11vnc,) = [cmd for cmd, _ in commands if cmd.startswith("x11vnc")]+ assert f"-passwdfile rm:{VNC_PASSWORD_PATH}" in x11vnc+ assert "-storepasswd" not in x11vnc and password not in x11vnc+++async def test_reopen_reuses_the_issued_password_while_novnc_still_serves():+ session, run = _session(listening=True)++ stream, password = await session.start_stream("issued-before")++ assert password == "issued-before" and "password=issued-before" in stream.url+ # Restarting the stack would sever the stream the user is watching.+ assert not any("x11vnc" in cmd for cmd, _ in _commands(run))+++async def test_without_a_remembered_password_the_stack_is_restarted():+ """noVNC still serving under a password nobody remembers (the box was+ paused since) is restarted: whoever held the old URL is locked out."""+ session, run = _session(listening=True)++ _, password = await session.start_stream(None)++ assert len(password) == 16+ assert any(cmd.startswith("pkill") for cmd, _ in _commands(run))+ assert any(cmd.startswith("x11vnc") for cmd, _ in _commands(run))+++async def test_a_remembered_password_is_dropped_once_the_proxy_is_gone():+ session, run = _session(listening=False)++ _, password = await session.start_stream("issued-before")++ assert password != "issued-before"+ assert any(cmd.startswith("x11vnc") for cmd, _ in _commands(run))diff --git a/autogpt_platform/backend/backend/copilot/computer.py b/autogpt_platform/backend/backend/copilot/computer.pyindex a3f299fd83..a92c2859c4 100644--- a/autogpt_platform/backend/backend/copilot/computer.py+++ b/autogpt_platform/backend/backend/copilot/computer.py@@ -1,25 +1,28 @@-"""An owner's computer on E2B: what exists, and how to get its desktop up.--Both the expert page's Computer tab and the copilot side panel's Computer-view read from here, so they agree on what "the box" is: the owner's shell-sandbox and its on-demand desktop, found through the same E2B metadata the-lifecycle module stamps on every sandbox (``copilot/tools/e2b_sandbox``).--``describe_computer`` only lists — it never connects, so a paused box stays-paused (connecting is what E2B's auto-resume reacts to). ``open_desktop`` is-the one write: it creates or resumes the owner's desktop and hands back the-live stream, exactly as the ``start_desktop`` tool does from inside a turn.+"""An owner's computer on E2B: what exists, and how to turn its screen on.++One box per owner: the same sandbox ``bash_exec`` runs in, on our desktop+image (``backend.util.e2b_template``). Nothing graphical runs until someone+asks for the screen. ``open_desktop`` starts X, XFCE and the VNC stream *in+that box* and hands back the live stream, whether the ask comes from the+``start_desktop`` tool inside a turn or from the Computer tab and side+panel. The box pauses at turn end like any other and comes back with the+screen exactly as it was, but under a fresh stream password: the password+is kept here, off the box, and forgotten whenever the box pauses, so a+stream URL that may have leaked is good for one running stretch only.++``describe_computer`` only lists: it never connects, so a paused box stays+paused (connecting is what E2B's auto-resume reacts to). Whether the screen+is on is remembered in Redis next to the box id, because asking the box+would wake it."""import asyncio-import contextlibimport loggingimport uuidfrom datetime import datetime-from typing import Any, Literal, Mapping, Optional+from typing import Literal, Mapping, Optionalfrom e2b import SandboxState-from e2b.exceptions import NotFoundExceptionfrom pydantic import BaseModelfrom backend.blocks.desktop._api import DesktopSession, DesktopStream@@ -31,11 +34,9 @@ from backend.blocks.desktop._common import (from backend.copilot.sdk.env import config as chat_configfrom backend.copilot.tools.e2b_sandbox import (METADATA_MOUNTS,- SandboxKind,- SandboxNotOwnedError,+ SandboxLookupError,SandboxOwner,- connect_owned,- find_owned_sandbox_id,+ get_or_create_owner_sandbox,list_owned_sandboxes,)from backend.data.redis_client import get_redis_async@@ -44,13 +45,12 @@ from backend.util.desktop_preview import create_preview_linklogger = logging.getLogger(__name__)_DESKTOP_RESOLUTION = (1280, 720)-_KILL_TIMEOUT_SECONDS = 10-_RECONNECT_RETRY_DELAY_SECONDS = 1.0-# Opening a desktop can be volume resolution, two create attempts, the-# display coming up and the home setup: about four minutes at the very worst.-# The open is cut off before the lock can lapse, so a second opener can-# never slip in under a still-running first one, and it waits about as long.+# Turning the screen on can be a box creation (volume resolution, up to three+# create attempts), the display coming up and the home setup: about four+# minutes at the very worst. The open is cut off before the lock can lapse,+# so a second opener can never slip in under a still-running first one, and+# it waits about as long._DESKTOP_LOCK_TTL_SECONDS = 300_DESKTOP_OPEN_DEADLINE_SECONDS = _DESKTOP_LOCK_TTL_SECONDS - 15_DESKTOP_LOCK_WAIT_SECONDS = 300@@ -62,9 +62,8 @@ _UNLOCK_SCRIPT = (class SandboxSummary(BaseModel):- """One of the owner's sandboxes as E2B reports it, without waking it."""+ """The owner's box as E2B reports it, without waking it."""- kind: SandboxKindsandbox_id: strstate: Literal["running", "paused"]started_at: datetime@@ -75,13 +74,13 @@ class SandboxSummary(BaseModel):class ComputerInfo(BaseModel):- """The owner's computer: its boxes and the volumes they mount."""+ """The owner's computer: its one box, whether its screen is on, its volumes."""owner_kind: Literal["session", "expert"]owner_id: stre2b_active: bool- shell: Optional[SandboxSummary] = None- desktop: Optional[SandboxSummary] = None+ box: Optional[SandboxSummary] = None+ screen_on: bool = False# path -> volume name; empty when the owner has no volumes (no user).mounts: dict[str, str] = {}workspace_path: str = WORKSPACE_PATH@@ -95,7 +94,7 @@ def computer_owner(session_id: str, expert_id: Optional[str]) -> SandboxOwner:async def describe_computer(owner: SandboxOwner, mounts: Mapping[str, str]) -> ComputerInfo:- """List the owner's shell and desktop boxes without resuming either."""+ """List the owner's box and screen state without resuming anything."""api_key = chat_config.active_e2b_api_keyinfo = ComputerInfo(owner_kind=owner.kind,@@ -105,34 +104,25 @@ async def describe_computer()if not api_key:return info- kinds: tuple[SandboxKind, ...] = ("shell", "desktop")- # Two independent E2B round-trips on an endpoint polled every 15 s.- listings = await asyncio.gather(- *(list_owned_sandboxes(owner, kind, api_key) for kind in kinds),- return_exceptions=True,+ try:+ boxes = await list_owned_sandboxes(owner, api_key)+ except SandboxLookupError as exc:+ # A listing is informational; show nothing rather than fail the page.+ logger.warning("[E2B] describe_computer: %s", exc)+ return info+ if not boxes:+ return info+ box = boxes[0]+ info.box = SandboxSummary(+ sandbox_id=box.sandbox_id,+ state="running" if box.state == SandboxState.RUNNING else "paused",+ started_at=box.started_at,+ cpu_count=box.cpu_count,+ memory_mb=box.memory_mb,+ template_id=box.template_id,+ mounts_attached=(box.metadata or {}).get(METADATA_MOUNTS) == "attached",)- for kind, boxes in zip(kinds, listings):- if isinstance(boxes, BaseException):- # A listing is informational; show nothing rather than fail the page.- logger.warning("[E2B] describe_computer: %s", boxes)- continue- if not boxes:- continue- box = boxes[0]- summary = SandboxSummary(- kind=kind,- sandbox_id=box.sandbox_id,- state="running" if box.state == SandboxState.RUNNING else "paused",- started_at=box.started_at,- cpu_count=box.cpu_count,- memory_mb=box.memory_mb,- template_id=box.template_id,- mounts_attached=(box.metadata or {}).get(METADATA_MOUNTS) == "attached",- )- if kind == "shell":- info.shell = summary- else:- info.desktop = summary+ info.screen_on = await screen_is_on(owner, box.sandbox_id)return info@@ -144,25 +134,24 @@ async def open_desktop(user_id: Optional[str],session_id: Optional[str] = None,) -> tuple[DesktopStream, bool, bool]:- """Return ``(stream, created, shared)`` — resuming the owner's desktop if it exists.+ """Turn the screen on in the owner's box; return ``(stream, first_time, shared)``.- Shared by the ``start_desktop`` tool and the HTTP endpoints, so a desktop- opened from the expert page is the same box the expert's next turn finds.+ Finds or creates the owner's box the same way a turn does, then starts+ the display and stream inside it. Opening from outside a turn does not+ count as one, so the turn-end pause still fires when the agent finishes;+ an idle box opened from the UI is paused by the lifecycle timeout instead.*user_id* is who the stream link is issued to (see ``_owner_bound``) and,- with *session_id*, provenance stamped on a newly created box. A cached- or recovered sandbox id is only reattached after E2B confirms the box is- the owner's.-- One opener at a time per owner: the panel's Start and the model's- ``start_desktop`` (or two tabs) can both miss the cache, and without the- lock each would create a box, one of which nothing would ever find again.- A second opener waits for the first and then reattaches to its box.+ with *session_id*, provenance stamped on a newly created box.++ One opener at a time per owner: the panel's button and the model's+ ``start_desktop`` (or two tabs) would otherwise race to start a second+ display and VNC stack in the same box. A second opener waits for the+ first and then reuses its stream."""if not user_id:raise ValueError("A desktop needs an authenticated user to issue its link to")redis = await get_redis_async()- key = owner.key("desktop")- lock_key = f"{key}:lock"+ lock_key = f"{owner.display_key()}:lock"token = uuid.uuid4().hexwaited = 0.0while not await redis.set(lock_key, token, nx=True, ex=_DESKTOP_LOCK_TTL_SECONDS):@@ -173,13 +162,7 @@ async def open_desktop(try:return await asyncio.wait_for(_open_desktop_locked(- owner,- mounts,- api_key,- redis,- key,- user_id=user_id,- session_id=session_id,+ owner, mounts, api_key, user_id=user_id, session_id=session_id),timeout=_DESKTOP_OPEN_DEADLINE_SECONDS,)@@ -191,111 +174,62 @@ async def _open_desktop_locked(owner: SandboxOwner,mounts: Mapping[str, str],api_key: str,- redis: Any,- key: str,*,user_id: str,session_id: Optional[str],) -> tuple[DesktopStream, bool, bool]:- raw = await redis.get(key)- sandbox_id = raw.decode() if isinstance(raw, bytes) else raw- if not sandbox_id:- # An expert's desktop outlives the Redis cache; E2B metadata is the record.- sandbox_id = await find_owned_sandbox_id(owner, "desktop", api_key)- if sandbox_id:- desktop = await _reconnect_desktop(sandbox_id, owner, api_key, redis, key)- if desktop is not None:- # From here on a failure is a real error on a live box, not a- # reason to abandon it and create another. The reconnect woke- # the box, so a failed setup pauses it again rather than leaving- # it on the meter until the lifecycle timeout.- try:- await desktop.ensure_display(*_DESKTOP_RESOLUTION)- await redis.set(key, sandbox_id, ex=owner.ttl)- stream = _owner_bound(await desktop.start_stream(), user_id)- mounted = await desktop.is_workspace_mounted()- except Exception:- await _pause_quietly(desktop)- raise- return stream, False, mounted-- desktop, persistence = await DesktopSession.create(- api_key=api_key,- timeout_seconds=chat_config.e2b_desktop_timeout,- width=_DESKTOP_RESOLUTION[0],- height=_DESKTOP_RESOLUTION[1],- volume_mounts=dict(mounts) or None,- template=chat_config.e2b_desktop_template,- metadata=owner.creation_metadata(- "desktop",- user_id=user_id,- session_id=session_id,- template=chat_config.e2b_desktop_template,- mounts="attached" if mounts else "none",- ),+ sandbox = await get_or_create_owner_sandbox(+ owner,+ api_key,+ timeout=chat_config.e2b_sandbox_timeout,+ template=chat_config.e2b_sandbox_template,+ on_timeout=chat_config.e2b_sandbox_on_timeout,+ volume_mounts=mounts,+ user_id=user_id,+ session_id=session_id,+ count_turn=False,)- try:- await redis.set(key, desktop.sandbox_id, ex=owner.ttl)- except Exception:- if not owner.is_expert:- # Nothing else can find a session desktop: no metadata recovery,- # no archive path. Kill it rather than bill it until timeout.- with contextlib.suppress(Exception):- await asyncio.wait_for(desktop.kill(), timeout=_KILL_TIMEOUT_SECONDS)- raise- try:- stream = _owner_bound(await desktop.start_stream(), user_id)- except Exception:- # The id is cached, so the next open resumes this box; pause it now- # rather than bill it until the lifecycle timeout.- await _pause_quietly(desktop)- raise- return stream, True, persistence.volume_mounted---async def _pause_quietly(desktop: DesktopSession) -> None:- """Best-effort pause of a box a failed open would otherwise leave running."""- with contextlib.suppress(Exception):- await asyncio.wait_for(desktop.pause(), timeout=_KILL_TIMEOUT_SECONDS)---async def _reconnect_desktop(- sandbox_id: str, owner: SandboxOwner, api_key: str, redis: Any, key: str-) -> Optional[DesktopSession]:- """Reattach to a cached or recovered desktop, or ``None`` if it is gone.-- Reattaches only once E2B confirms the box is the owner's. A box E2B no- longer has, or one stamped for someone else, is given up on and dropped- from the cache so the owner gets one of their own. A transient failure- is retried once and then raised: forking an expert's desktop over a- network blip is worse than asking the user to try again.+ was_on = await screen_is_on(owner, sandbox.sandbox_id)+ desktop = DesktopSession(sandbox)+ await desktop.ensure_display(*_DESKTOP_RESOLUTION)+ shared = await desktop.is_workspace_mounted()+ if shared and not was_on:+ # Browser downloads and saved files land in the durable home.+ await desktop.ensure_persistent_home()+ # The password issued last time, if the box has run without a pause+ # since; otherwise the stack restarts under a new one.+ stream, password = await desktop.start_stream(+ await _stream_password(owner) if was_on else None+ )+ await _remember_screen(owner, sandbox.sandbox_id, password)+ return _owner_bound(stream, user_id), not was_on, shared+++async def screen_is_on(owner: SandboxOwner, sandbox_id: str) -> bool:+ """Whether ``open_desktop`` has run on *this* box (a replaced box starts off)."""+ redis = await get_redis_async()+ raw = await redis.get(owner.display_key())+ value = raw.decode() if isinstance(raw, bytes) else raw+ return value == sandbox_id+++async def _stream_password(owner: SandboxOwner) -> Optional[str]:+ redis = await get_redis_async()+ raw = await redis.get(owner.stream_key())+ value = raw.decode() if isinstance(raw, bytes) else raw+ return value or None+++async def _remember_screen(owner: SandboxOwner, sandbox_id: str, password: str) -> None:+ """Record the screen as on in this box, and the password its stream uses.++ The password outlives a pause only as long as the box could have kept+ running: its expiry is the box's running-time limit, and the turn-end+ pause drops it outright (``e2b_sandbox._forget_stream``)."""- for attempt in (1, 2):- try:- return DesktopSession(- await connect_owned(- sandbox_id,- owner,- "desktop",- api_key,- timeout=chat_config.e2b_desktop_timeout,- )- )- except (NotFoundException, SandboxNotOwnedError) as exc:- logger.warning("[E2B] Desktop %.12s replaced: %s", sandbox_id, exc)- await redis.delete(key)- return None- except Exception as exc:- if attempt == 1:- logger.warning(- "[E2B] Desktop %.12s reconnect failed (%s); retrying once",- sandbox_id,- exc,- )- await asyncio.sleep(_RECONNECT_RETRY_DELAY_SECONDS)- continue- raise- raise AssertionError("unreachable")+ redis = await get_redis_async()+ await redis.set(owner.display_key(), sandbox_id, ex=owner.ttl)+ await redis.set(owner.stream_key(), password, ex=chat_config.e2b_sandbox_timeout)def _owner_bound(stream: DesktopStream, user_id: str) -> DesktopStream:@@ -311,7 +245,7 @@ def _owner_bound(stream: DesktopStream, user_id: str) -> DesktopStream:def mounts_for(user_id: Optional[str], expert_id: Optional[str]) -> dict[str, str]:- """The desktop mounts exactly what the owner's shell mounts.+ """The screen is on the same box as the shell, so it mounts what the shell mounts.Same rule as ``workspace_volume_mounts``: an expert always gets its ownhome, the user's shared volume only when there is a user.diff --git a/autogpt_platform/backend/backend/copilot/computer_test.py b/autogpt_platform/backend/backend/copilot/computer_test.pyindex 30f5ab876b..018f73eb3e 100644--- a/autogpt_platform/backend/backend/copilot/computer_test.py+++ b/autogpt_platform/backend/backend/copilot/computer_test.py@@ -1,4 +1,4 @@-"""Tests for backend.copilot.computer: describe without waking, open by owner."""+"""Tests for backend.copilot.computer: describe without waking, screen on in place."""import asynciofrom datetime import datetime, timezone@@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom e2b import SandboxState-from backend.blocks.desktop._api import DesktopStream, PersistenceInfo+from backend.blocks.desktop._api import DesktopStreamfrom backend.blocks.desktop._common import SHARED_PATH, WORKSPACE_PATHfrom backend.copilot.computer import (ComputerInfo,@@ -16,26 +16,67 @@ from backend.copilot.computer import (describe_computer,mounts_for,open_desktop,+ screen_is_on,)-from backend.copilot.tools.e2b_sandbox import SandboxOwner-from backend.util.sandbox_metadata import deployment_env+from backend.copilot.tools.e2b_sandbox import SandboxLookupError, SandboxOwner_C = "backend.copilot.computer"_USER, _EXPERT, _SESSION = "user-1", "exp-1", "sess-1"-def _info(kind: str, sandbox_id: str, state: SandboxState, mounts: str = "attached"):+def _info(sandbox_id: str, state: SandboxState, mounts: str = "attached"):return SimpleNamespace(sandbox_id=sandbox_id,state=state,started_at=datetime(2026, 9, 5, 12, 0, tzinfo=timezone.utc),- cpu_count=2,- memory_mb=4096,- template_id="desktop" if kind == "desktop" else "base",- metadata={"autogpt_kind": kind, "autogpt_mounts": mounts},+ cpu_count=1,+ memory_mb=2048,+ template_id="agpt-desktop-1x2",+ metadata={"autogpt_kind": "shell", "autogpt_mounts": mounts},)+def _redis(display: str | None, lock_free: bool = True, stream: str | None = None):+ r = MagicMock()+ r.get = AsyncMock(+ side_effect=lambda key: stream if key.endswith(":stream") else display+ )+ r.set = AsyncMock(return_value=lock_free)+ r.delete = AsyncMock()+ r.eval = AsyncMock(return_value=1)+ return r+++def _sandbox(sandbox_id: str = "sb-1"):+ sb = MagicMock()+ sb.sandbox_id = sandbox_id+ return sb+++def _desktop(sandbox_id: str = "sb-1", mounted: bool = True):+ d = MagicMock()+ d.ensure_display = AsyncMock()+ d.ensure_persistent_home = AsyncMock()+ d.is_workspace_mounted = AsyncMock(return_value=mounted)+ d.start_stream = AsyncMock(return_value=(_LIVE_STREAM(sandbox_id), "secret"))+ return d+++def _LIVE_STREAM(sandbox_id: str) -> DesktopStream:+ return DesktopStream(+ url="https://6080-x.e2b.app/vnc.html?password=secret", sandbox_id=sandbox_id+ )++[email protected](autouse=True)+def _owner_bound_links():+ with patch(+ f"{_C}.create_preview_link",+ side_effect=lambda user_id, url: f"preview://{user_id}/token",+ ) as link:+ yield link++class TestComputerOwner:def test_expert_session_is_the_experts_computer(self):assert computer_owner(_SESSION, _EXPERT) == SandboxOwner(@@ -55,18 +96,15 @@ class TestComputerOwner:class TestDescribeComputer:@pytest.mark.asyncio- async def test_reports_both_boxes_without_connecting(self):+ async def test_reports_the_box_and_its_screen_without_connecting(self):owner = SandboxOwner(kind="expert", id=_EXPERT)- listed = {- "shell": [_info("shell", "sb-shell", SandboxState.PAUSED)],- "desktop": [- _info("desktop", "sb-desk", SandboxState.RUNNING, mounts="none")- ],- }- list_mock = AsyncMock(side_effect=lambda o, kind, key: listed[kind])with (patch(f"{_C}.chat_config") as cfg,- patch(f"{_C}.list_owned_sandboxes", list_mock),+ patch(+ f"{_C}.list_owned_sandboxes",+ AsyncMock(return_value=[_info("sb-1", SandboxState.PAUSED)]),+ ) as list_mock,+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=_redis("sb-1"))),patch(f"{_C}.DesktopSession") as desktop_cls,):cfg.active_e2b_api_key = "k"@@ -74,19 +112,33 @@ class TestDescribeComputer:assert isinstance(info, ComputerInfo)assert info.owner_kind == "expert" and info.e2b_active- assert (- info.shell and info.shell.state == "paused" and info.shell.mounts_attached- )- assert info.desktop and info.desktop.state == "running"- assert info.desktop.mounts_attached is False+ assert info.box and info.box.state == "paused" and info.box.mounts_attached+ assert info.box.cpu_count == 1 and info.box.memory_mb == 2048+ assert info.screen_on is Trueassert info.mounts[SHARED_PATH].startswith("autogpt-user-")+ list_mock.assert_awaited_once_with(owner, "k")# Describing must never resume a paused box.- desktop_cls.connect.assert_not_called()+ desktop_cls.assert_not_called()@pytest.mark.asyncio- async def test_a_failed_listing_shows_nothing_rather_than_failing(self):- from backend.copilot.tools.e2b_sandbox import SandboxLookupError+ async def test_screen_flag_for_a_replaced_box_does_not_count(self):+ owner = SandboxOwner(kind="session", id=_SESSION)+ with (+ patch(f"{_C}.chat_config") as cfg,+ patch(+ f"{_C}.list_owned_sandboxes",+ AsyncMock(return_value=[_info("sb-new", SandboxState.RUNNING)]),+ ),+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=_redis("sb-old"))),+ ):+ cfg.active_e2b_api_key = "k"+ info = await describe_computer(owner, {})+ assert info.box and info.box.state == "running"+ assert info.screen_on is False+ @pytest.mark.asyncio+ async def test_a_failed_listing_shows_nothing_rather_than_failing(self):+ owner = SandboxOwner(kind="expert", id=_EXPERT)with (patch(f"{_C}.chat_config") as cfg,patch(@@ -95,8 +147,8 @@ class TestDescribeComputer:),):cfg.active_e2b_api_key = "k"- info = await describe_computer(SandboxOwner(kind="expert", id=_EXPERT), {})- assert info.e2b_active and info.shell is None and info.desktop is None+ info = await describe_computer(owner, {})+ assert info.e2b_active and info.box is None and info.screen_on is False@pytest.mark.asyncioasync def test_without_e2b_it_says_so_and_lists_nothing(self):@@ -107,79 +159,174 @@ class TestDescribeComputer:):cfg.active_e2b_api_key = Noneinfo = await describe_computer(owner, {})- assert info.e2b_active is False and info.shell is None and info.desktop is None+ assert info.e2b_active is False and info.box is None+ assert info.screen_on is Falselist_mock.assert_not_awaited()-def _redis(stored: str | None, lock_free: bool = True):- r = MagicMock()- r.get = AsyncMock(return_value=stored)- r.set = AsyncMock(return_value=lock_free)- r.delete = AsyncMock()- r.eval = AsyncMock(return_value=1)- return r+class TestOpenDesktop:+ def _patches(self, redis, sandbox, desktop):+ desktop_cls = MagicMock(return_value=desktop)+ return (+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),+ patch(f"{_C}.get_or_create_owner_sandbox", AsyncMock(return_value=sandbox)),+ patch(f"{_C}.DesktopSession", desktop_cls),+ patch(f"{_C}.chat_config"),+ )+ @pytest.mark.asyncio+ async def test_needs_a_user_to_issue_the_link_to(self):+ with pytest.raises(ValueError, match="authenticated user"):+ await open_desktop(+ SandboxOwner(kind="expert", id=_EXPERT), {}, "k", user_id=None+ )-def _desktop(sandbox_id="sb-desk"):- d = MagicMock()- d.sandbox_id = sandbox_id- d.ensure_display = AsyncMock()- d.start_stream = AsyncMock(- return_value=DesktopStream(- url="https://6080-x.e2b.app/vnc.html", sandbox_id=sandbox_id+ @pytest.mark.asyncio+ async def test_turns_the_screen_on_in_the_owners_own_box(self):+ owner = SandboxOwner(kind="expert", id=_EXPERT)+ mounts = mounts_for(_USER, _EXPERT)+ redis, sandbox, desktop = _redis(None), _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p as get_mock, cls_p as desktop_cls, cfg_p as cfg:+ cfg.e2b_sandbox_timeout = 420+ cfg.e2b_sandbox_template = "agpt-desktop-1x2"+ cfg.e2b_sandbox_on_timeout = "pause"+ stream, first_time, shared = await open_desktop(+ owner, mounts, "k", user_id=_USER, session_id=_SESSION+ )++ assert first_time and shared and stream.sandbox_id == "sb-1"+ # The same box a turn would use, found or created the same way, but+ # not counted as a turn so the agent's turn-end pause still fires.+ get_mock.assert_awaited_once_with(+ owner,+ "k",+ timeout=420,+ template="agpt-desktop-1x2",+ on_timeout="pause",+ volume_mounts=mounts,+ user_id=_USER,+ session_id=_SESSION,+ count_turn=False,+ )+ desktop_cls.assert_called_once_with(sandbox)+ desktop.ensure_display.assert_awaited_once()+ desktop.ensure_persistent_home.assert_awaited_once()+ desktop.start_stream.assert_awaited_once()+ # The screen flag is remembered against this box's id, under the owner.+ flag = [c for c in redis.set.await_args_list if c.args[0].endswith(":display")]+ assert flag and flag[0].args[:2] == (+ f"copilot:e2b:expert:{_EXPERT}:shell:display",+ "sb-1",)- )- d.is_workspace_mounted = AsyncMock(return_value=True)- return d+ @pytest.mark.asyncio+ async def test_hands_out_an_owner_bound_link_never_the_password_url(+ self, _owner_bound_links+ ):+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis, sandbox, desktop = _redis(None), _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ stream, _, _ = await open_desktop(owner, {}, "k", user_id=_USER)+ # The live URL went into the link for this user, and only the link+ # comes out.+ _owner_bound_links.assert_called_once_with(_USER, _LIVE_STREAM("sb-1").url)+ assert stream.url == f"preview://{_USER}/token"+ assert stream.requires_auth is True+ assert "secret" not in stream.model_dump_json()[email protected](autouse=True)-def _owner_bound_links():- with patch(- f"{_C}.create_preview_link",- side_effect=lambda user_id, url: f"preview://{user_id}/{url}",+ @pytest.mark.asyncio+ async def test_first_open_starts_a_stream_under_a_new_password(self):+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis, sandbox, desktop = _redis(None), _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p as cfg:+ cfg.e2b_sandbox_timeout = 420+ await open_desktop(owner, {}, "k", user_id=_USER)+ desktop.start_stream.assert_awaited_once_with(None)+ # Remembered off the box, for as long as the box could keep running.+ redis.set.assert_any_await(+ f"copilot:e2b:sandbox:{_SESSION}:stream", "secret", ex=420+ )++ @pytest.mark.asyncio+ async def test_reopen_hands_back_the_same_stream_while_the_box_kept_running(+ self,):- yield+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis = _redis("sb-1", stream="issued-before")+ sandbox, desktop = _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ await open_desktop(owner, {}, "k", user_id=_USER)+ desktop.start_stream.assert_awaited_once_with("issued-before")+ @pytest.mark.asyncio+ async def test_a_password_left_over_from_a_replaced_box_is_not_reused(self):+ """The screen flag names another box: whatever password Redis still+ holds belonged to that one."""+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis = _redis("sb-old", stream="issued-before")+ sandbox, desktop = _sandbox("sb-new"), _desktop("sb-new")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ await open_desktop(owner, {}, "k", user_id=_USER)+ desktop.start_stream.assert_awaited_once_with(None)-class TestOpenDesktop:@pytest.mark.asyncio- async def test_needs_a_user_to_issue_the_link_to(self):- with pytest.raises(ValueError, match="authenticated user"):- await open_desktop(- SandboxOwner(kind="expert", id=_EXPERT), {}, "k", user_id=None+ async def test_second_open_only_refreshes_the_stream(self):+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis, sandbox, desktop = _redis("sb-1"), _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ stream, first_time, shared = await open_desktop(+ owner, {}, "k", user_id=_USER+ )++ assert not first_time and stream.sandbox_id == "sb-1"+ # ensure_display is idempotent and cheap; the home redirect is not+ # repeated once the screen has been on.+ desktop.ensure_display.assert_awaited_once()+ desktop.ensure_persistent_home.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_no_volume_means_nothing_to_redirect(self):+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis, sandbox = _redis(None), _sandbox("sb-1")+ desktop = _desktop("sb-1", mounted=False)+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ _stream, first_time, shared = await open_desktop(+ owner, {}, "k", user_id=_USER)+ assert first_time and not shared+ desktop.ensure_persistent_home.assert_not_awaited()@pytest.mark.asyncio- async def test_a_second_opener_waits_and_reattaches(self):- """Start from the panel and start_desktop from the model can both miss- the cache; only one may create, the other reattaches to its box."""+ async def test_a_second_opener_waits_for_the_first(self):+ """The panel's button and the model's start_desktop can race; only one+ may start the display stack, the other waits and reuses the stream."""owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis(None)- # Lock taken on the first attempt; free on the second, by which time- # the first opener has cached its box.- redis.set = AsyncMock(side_effect=[False, True, True])- redis.get = AsyncMock(side_effect=[b"sb-first"])- desktop = _desktop("sb-first")+ redis, sandbox, desktop = _redis("sb-1"), _sandbox("sb-1"), _desktop("sb-1")+ # Lock taken on the first attempt; free on the second; then the+ # screen flag and the stream password.+ redis.set = AsyncMock(side_effect=[False, True, True, True])+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),+ redis_p,+ get_p,+ cls_p,+ cfg_p,patch(f"{_C}.asyncio.sleep", AsyncMock()) as sleep,- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned") as connect_owned,- patch(f"{_C}.chat_config") as cfg,):- cfg.e2b_desktop_timeout = 900- connect_owned.return_value = MagicMock()- desktop_cls.return_value = desktop- desktop_cls.create = AsyncMock()- stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)+ stream, first_time, _ = await open_desktop(owner, {}, "k", user_id=_USER)- assert not created and stream.sandbox_id == "sb-first"+ assert not first_time and stream.sandbox_id == "sb-1"sleep.assert_awaited_once()- desktop_cls.create.assert_not_awaited()# The lock is released by token, never a bare delete of the key.script, _, lock_key, token = redis.eval.await_args.args- assert lock_key == f"copilot:e2b:expert:{_EXPERT}:desktop:lock"+ assert lock_key == f"copilot:e2b:expert:{_EXPERT}:shell:display:lock"assert token == redis.set.await_args_list[1].args[1]@pytest.mark.asyncio@@ -187,19 +334,15 @@ class TestOpenDesktop:owner = SandboxOwner(kind="expert", id=_EXPERT)redis = _redis(None)- async def never(**_kwargs):+ async def never(*_args, **_kwargs):await asyncio.Event().wait()with (patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.chat_config") as cfg,+ patch(f"{_C}.get_or_create_owner_sandbox", AsyncMock(side_effect=never)),+ patch(f"{_C}.chat_config"),patch(f"{_C}._DESKTOP_OPEN_DEADLINE_SECONDS", 0.01),):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- desktop_cls.create = AsyncMock(side_effect=never)with pytest.raises(asyncio.TimeoutError):await open_desktop(owner, {}, "k", user_id=_USER)from backend.copilot import computer@@ -210,127 +353,6 @@ class TestOpenDesktop:# The lock is still released on the way out.redis.eval.assert_awaited_once()- @pytest.mark.asyncio- async def test_a_gone_desktop_is_replaced(self):- from e2b.exceptions import NotFoundException-- owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis("sb-gone")- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned") as connect_owned,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- connect_owned.side_effect = NotFoundException("gone")- desktop_cls.create = AsyncMock(- return_value=(_desktop("sb-new"), PersistenceInfo())- )- stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)- assert created and stream.sandbox_id == "sb-new"- redis.delete.assert_awaited_once()-- @pytest.mark.asyncio- async def test_a_transient_reconnect_failure_is_retried_then_raised(self):- """A network blip must not fork an expert's desktop: one retry, then- the error surfaces and the cached id is kept for next time."""- owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis("sb-live")- desktop = _desktop("sb-live")- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.asyncio.sleep", AsyncMock()),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned") as connect_owned,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- connect_owned.side_effect = [RuntimeError("502"), MagicMock()]- desktop_cls.return_value = desktop- desktop_cls.create = AsyncMock()- stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)- assert not created and stream.sandbox_id == "sb-live"- assert connect_owned.await_count == 2-- connect_owned.side_effect = RuntimeError("502")- with pytest.raises(RuntimeError, match="502"):- await open_desktop(owner, {}, "k", user_id=_USER)- desktop_cls.create.assert_not_awaited()- redis.delete.assert_not_awaited()-- @pytest.mark.asyncio- async def test_a_failure_after_reconnect_is_an_error_not_a_new_box(self):- """Only a failed connect means the box is gone. A display or stream- failure on a live box must surface, not abandon the box and bill for- a second one."""- owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis("sb-live")- desktop = _desktop("sb-live")- desktop.ensure_display = AsyncMock(side_effect=RuntimeError("no display"))- desktop.pause = AsyncMock()- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned", AsyncMock(return_value=MagicMock())),- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- desktop_cls.return_value = desktop- desktop_cls.create = AsyncMock()- with pytest.raises(RuntimeError, match="no display"):- await open_desktop(owner, {}, "k", user_id=_USER)- desktop_cls.create.assert_not_awaited()- redis.delete.assert_not_awaited()- # The reconnect woke the box; a failed setup does not leave it running.- desktop.pause.assert_awaited_once()-- @pytest.mark.asyncio- async def test_a_session_desktop_whose_id_cannot_be_saved_is_killed(self):- """Nothing can find a session desktop later; an unsaved one would bill- until timeout. An expert's is recoverable by metadata and is kept."""- session_owner = SandboxOwner(kind="session", id=_SESSION)- redis = _redis(None)- redis.set = AsyncMock(side_effect=[True, ConnectionError("redis down")])- desktop = _desktop("sb-new")- desktop.kill = AsyncMock()- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- desktop_cls.create = AsyncMock(return_value=(desktop, PersistenceInfo()))- with pytest.raises(ConnectionError):- await open_desktop(session_owner, {}, "k", user_id=_USER)- desktop.kill.assert_awaited_once()-- @pytest.mark.asyncio- async def test_a_new_desktop_whose_stream_fails_is_paused_not_lost(self):- """Its id is cached, so the next open resumes it; meanwhile it must- not sit running on the meter."""- owner = SandboxOwner(kind="session", id=_SESSION)- redis = _redis(None)- desktop = _desktop("sb-new")- desktop.start_stream = AsyncMock(side_effect=RuntimeError("no novnc"))- desktop.pause = AsyncMock()- desktop.kill = AsyncMock()- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- desktop_cls.create = AsyncMock(return_value=(desktop, PersistenceInfo()))- with pytest.raises(RuntimeError, match="no novnc"):- await open_desktop(owner, {}, "k", user_id=_USER)- desktop.pause.assert_awaited_once()- desktop.kill.assert_not_awaited()- assert any(c.args[1] == "sb-new" for c in redis.set.await_args_list)-@pytest.mark.asyncioasync def test_gives_up_when_the_lock_never_frees(self):owner = SandboxOwner(kind="expert", id=_EXPERT)@@ -338,102 +360,18 @@ class TestOpenDesktop:with (patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),patch(f"{_C}.asyncio.sleep", AsyncMock()),- patch(f"{_C}.DesktopSession") as desktop_cls,+ patch(f"{_C}.get_or_create_owner_sandbox", AsyncMock()) as get_mock,):- desktop_cls.create = AsyncMock()with pytest.raises(RuntimeError, match="still opening"):await open_desktop(owner, {}, "k", user_id=_USER)- desktop_cls.create.assert_not_awaited()+ get_mock.assert_not_awaited()redis.eval.assert_not_awaited()- @pytest.mark.asyncio- async def test_creates_under_the_owner_key_with_the_owner_mounts(self):- owner = SandboxOwner(kind="expert", id=_EXPERT)- mounts = mounts_for(_USER, _EXPERT)- redis = _redis(None)- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- desktop_cls.create = AsyncMock(- return_value=(_desktop(), PersistenceInfo(volume_mounted=True))- )- stream, created, shared = await open_desktop(- owner, mounts, "k", user_id=_USER- )-- assert created and shared and stream.sandbox_id == "sb-desk"- # The password-bearing URL stays here; callers get an owner-bound link.- assert stream.url == f"preview://{_USER}/https://6080-x.e2b.app/vnc.html"- assert stream.requires_auth is True- kwargs = desktop_cls.create.await_args.kwargs- assert kwargs["volume_mounts"] == mounts- assert kwargs["metadata"] == {- "service": "autogpt-platform",- "autogpt_owner": f"expert:{_EXPERT}",- "autogpt_kind": "desktop",- "autogpt_source": "copilot",- "autogpt_env": deployment_env(),- "autogpt_user": _USER,- "autogpt_expert": _EXPERT,- "autogpt_template": "desktop",- "autogpt_mounts": "attached",- }- assert redis.set.await_args.args[0] == f"copilot:e2b:expert:{_EXPERT}:desktop"-- @pytest.mark.asyncio- async def test_resumes_the_recovered_box_and_reuses_its_stream(self):- owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis(None)- desktop = _desktop("sb-old")- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value="sb-old")),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned") as connect_owned,- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- connect_owned.return_value = box = MagicMock()- desktop_cls.return_value = desktop- desktop_cls.create = AsyncMock()- stream, created, shared = await open_desktop(owner, {}, "k", user_id=_USER)-- assert not created and shared and stream.sandbox_id == "sb-old"- # Reattached only once E2B confirms the box is the owner's, and with- # the same running-time limit as a new one.- connect_owned.assert_awaited_once_with(- "sb-old", owner, "desktop", "k", timeout=900- )- desktop_cls.assert_called_once_with(box)- desktop_cls.create.assert_not_awaited()- desktop.start_stream.assert_awaited_once()+class TestScreenIsOn:@pytest.mark.asyncio- async def test_a_recovered_id_that_is_not_the_owners_box_is_replaced(self):- """The cached id said sb-old; E2B says sb-old is someone else's. The- stale cache entry goes and the owner gets a box of their own."""- from backend.copilot.tools.e2b_sandbox import SandboxNotOwnedError-- owner = SandboxOwner(kind="expert", id=_EXPERT)- redis = _redis("sb-old")- with (- patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),- patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),- patch(f"{_C}.DesktopSession") as desktop_cls,- patch(f"{_C}.connect_owned", side_effect=SandboxNotOwnedError("nope")),- patch(f"{_C}.chat_config") as cfg,- ):- cfg.e2b_desktop_timeout = 900- cfg.e2b_desktop_template = "desktop"- desktop_cls.create = AsyncMock(- return_value=(_desktop("sb-new"), PersistenceInfo())- )- stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)-- assert created and stream.sandbox_id == "sb-new"- redis.delete.assert_awaited_once()+ async def test_matches_only_the_current_box(self):+ owner = SandboxOwner(kind="session", id=_SESSION)+ with patch(f"{_C}.get_redis_async", AsyncMock(return_value=_redis(b"sb-1"))):+ assert await screen_is_on(owner, "sb-1") is True+ assert await screen_is_on(owner, "sb-2") is Falsediff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex bf8ad4abae..f90630f4e7 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -701,7 +701,10 @@ class ChatConfig(BaseSettings):"own image (E2B's desktop image at 1 vCPU / 2 GiB, ~$0.08/h running, ""no display started), built on the team automatically the first time ""it is needed; see backend.util.e2b_template. Any other value is used "- "as-is and must already exist on the team.",+ "as-is and must already exist on the team, and it must carry what the "+ "desktop needs (Xvfb, XFCE, x11vnc and noVNC, as E2B's desktop image "+ "does): the screen is turned on inside this same box, so a plain "+ "image such as 'base' makes every start_desktop fail.",)e2b_sandbox_timeout: int = Field(default=420, # 7 min safety net — allows headroom for compaction retries@@ -713,21 +716,6 @@ class ChatConfig(BaseSettings):default="pause",description="E2B lifecycle action on timeout: 'pause' (default, free) or 'kill'.",)- e2b_desktop_template: str = Field(- default="desktop",- description="E2B template for the on-demand start_desktop sandbox. E2B's "- "public 'desktop' is 8 vCPU / 8 GiB (~$0.53/h running); build a smaller "- "one on the team with `poetry run build-desktop-template` "- "(agpt-desktop-1x1: 1 vCPU / 1 GiB, ~$0.07/h) and set this to its alias.",- )- e2b_desktop_timeout: int = Field(- default=900,- description="Running-time timeout (seconds) for the on-demand desktop "- "sandbox. Unlike the bash sandbox it is NOT paused at turn end — the "- "user may still be watching the stream — so this is the only thing "- "stopping the meter; it auto-pauses (free) when the timeout lapses "- "and resumes on the next start_desktop call.",- )@propertydef openrouter_active(self) -> bool:diff --git a/autogpt_platform/backend/backend/copilot/expert_context.py b/autogpt_platform/backend/backend/copilot/expert_context.pyindex 2fe35d5c00..d387fedb29 100644--- a/autogpt_platform/backend/backend/copilot/expert_context.py+++ b/autogpt_platform/backend/backend/copilot/expert_context.py@@ -305,9 +305,10 @@ def _expert_computer_block() -> str:"their other sessions. Trust the tool output on whether it is ""mounted: without the mount, say where the file really is instead ""of calling it shared.\n"- "- Use start_desktop when a task needs a browser or GUI app. The "- "desktop is shared with the user, not private from either of you: "- "you can see everything on it, and so can they.\n"+ "- Use start_desktop to turn your screen on when a task needs a browser "+ "or GUI app; it is the same machine your commands run in. The desktop "+ "is shared with the user, not private from either of you: you can see "+ "everything on it, and so can they.\n""- Never ask the user to sign into personal accounts on this ""desktop; use their connected integrations instead.\n""</expert_computer>\n\n"diff --git a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py b/autogpt_platform/backend/backend/copilot/tools/bash_exec.pyindex cd4b6e01ae..0228f65d54 100644--- a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py+++ b/autogpt_platform/backend/backend/copilot/tools/bash_exec.py@@ -74,11 +74,11 @@ class BashExecTool(BaseTool):return ("Execute a Bash command or script. Shares filesystem with SDK file tools. ""Useful for scripts, data processing, and package installation. "- "Killed after `timeout` seconds. Anything that should persist across "- "sessions or show on the user's desktop belongs in ~/workspace (a "- "durable volume shared live with start_desktop); other paths are "- "scratch. Expert sessions: ~/workspace is the expert's own machine, "- "~/shared is the user's workspace."+ "Killed after `timeout` seconds. Anything that should persist belongs "+ "in ~/workspace (a durable volume); other paths are scratch. The "+ "desktop (start_desktop) is this same machine, all paths included. "+ "Expert sessions: ~/workspace is the expert's own machine, ~/shared "+ "is the user's workspace.")@propertydiff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.pyindex d7dfdb8a6e..f7da254ac5 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py@@ -18,10 +18,14 @@ A sandbox belongs to a :class:`SandboxOwner`:model has a root shell in that same VM, so nothing on the box, browsersessions included, is private from it. Deleting achat never touches it; archiving the expert does- (``kill_expert_sandboxes``). The box mounts the expert's own durable volume+ (``kill_expert_sandbox``). The box mounts the expert's own durable volumeat ``~/workspace`` and the owning user's volume at ``~/shared`` (see``workspace_volume_mounts``).+There is exactly one sandbox per owner. It runs our desktop image+(``backend.util.e2b_template``), and ``backend.copilot.computer`` turns its+screen on *in place* when a desktop is wanted: no second box.+Lifecycle---------1. **Turn start** – connect to the existing sandbox (sandbox_id in Redis) or@@ -80,6 +84,7 @@ from e2b import (SandboxQuery,SandboxState,)+from e2b.exceptions import NotFoundExceptionfrom pydantic import BaseModel, ConfigDictfrom backend.blocks.desktop._api import resolve_volume@@ -90,7 +95,6 @@ from backend.util.sandbox_metadata import MountState, SandboxMetadatalogger = logging.getLogger(__name__)_SANDBOX_KEY_PREFIX = "copilot:e2b:sandbox:"-_DESKTOP_KEY_PREFIX = "copilot:e2b:desktop:"_EXPERT_KEY_PREFIX = "copilot:e2b:expert:"_CREATING_SENTINEL = "creating"@@ -101,8 +105,6 @@ METADATA_KIND = "autogpt_kind"# to fall back to a volume-less box — visible in the E2B dashboard and API.METADATA_MOUNTS = "autogpt_mounts"-SandboxKind = Literal["shell", "desktop"]-# Per-attempt timeout for AsyncSandbox.create(). E2B normally provisions a# sandbox in 5-15 s; 30 s gives generous headroom while ensuring a slow/hung# E2B API call fails fast rather than blocking an executor goroutine for hours.@@ -166,10 +168,19 @@ _RELEASE_TURN_SCRIPT = ("return n")-# One more reconnect attempt before a listed expert box is given up on: a-# durable box that fails once on a transient error must not be forked.+# One more reconnect attempt before a transient error surfaces: a box that+# fails once on a blip must not be replaced, whoever owns it._RECONNECT_RETRY_DELAY_SECONDS = 1.0+# Boxes from before one box per owner: a separate desktop, stamped+# ``autogpt_kind=desktop`` and cached under its own key, that this code no+# longer opens. Left alone it would sit paused on E2B, browser profile and+# all, for as long as E2B keeps paused boxes. The kill paths sweep them so a+# deleted chat or an archived expert takes its old desktop with it; once no+# box from before the switch can exist any more this can go.+_LEGACY_DESKTOP_KIND = "desktop"+_LEGACY_DESKTOP_KEY_PREFIX = "copilot:e2b:desktop:"+class SandboxOwner(BaseModel):"""Who a CoPilot sandbox belongs to — and therefore how long it lives.@@ -198,24 +209,42 @@ class SandboxOwner(BaseModel):def is_expert(self) -> bool:return self.kind == "expert"- def key(self, sandbox_kind: SandboxKind = "shell") -> str:+ def key(self) -> str:"""Redis key caching this owner's sandbox id (doubles as creation lock)."""if self.is_expert:- return f"{_EXPERT_KEY_PREFIX}{self.id}:{sandbox_kind}"- prefix = _SANDBOX_KEY_PREFIX if sandbox_kind == "shell" else _DESKTOP_KEY_PREFIX- return f"{prefix}{self.id}"+ return f"{_EXPERT_KEY_PREFIX}{self.id}:shell"+ return f"{_SANDBOX_KEY_PREFIX}{self.id}"++ def display_key(self) -> str:+ """Redis key remembering which box ``open_desktop`` turned the screen on in."""+ return f"{self.key()}:display"++ def stream_key(self) -> str:+ """Redis key holding the screen's current stream password.++ The password never rests on the box (see ``DesktopSession.start_stream``);+ this is what lets a re-open hand back the URL the user already holds.+ It is dropped whenever the box pauses, so a URL that may have leaked+ is good for one running stretch only.+ """+ return f"{self.key()}:stream"++ def legacy_desktop_key(self) -> str:+ """Where the pre-one-box desktop's id was cached; swept on kill."""+ if self.is_expert:+ return f"{_EXPERT_KEY_PREFIX}{self.id}:desktop"+ return f"{_LEGACY_DESKTOP_KEY_PREFIX}{self.id}"@propertydef ttl(self) -> int:return _EXPERT_ID_TTL if self.is_expert else _SANDBOX_ID_TTL- def metadata(self, sandbox_kind: SandboxKind = "shell") -> dict[str, str]:+ def metadata(self) -> dict[str, str]:"""The identity keys a lookup filters on; a subset of ``creation_metadata``."""- return {METADATA_OWNER: f"{self.kind}:{self.id}", METADATA_KIND: sandbox_kind}+ return {METADATA_OWNER: f"{self.kind}:{self.id}", METADATA_KIND: "shell"}def creation_metadata(self,- sandbox_kind: SandboxKind = "shell",*,user_id: str | None = None,session_id: str | None = None,@@ -225,7 +254,7 @@ class SandboxOwner(BaseModel):"""Identity plus provenance, stamped on the box when it is created."""return SandboxMetadata.for_copilot(f"{self.kind}:{self.id}",- sandbox_kind,+ "shell",user_id=user_id,session_id=session_id,expert_id=self.id if self.is_expert else None,@@ -244,7 +273,6 @@ class SandboxNotOwnedError(Exception):async def connect_owned(sandbox_id: str,owner: SandboxOwner,- sandbox_kind: SandboxKind,api_key: str,*,timeout: int | None = None,@@ -259,16 +287,14 @@ async def connect_owned(The stamp is read *before* connecting: a connect is not passive, itresumes a paused box and re-arms its running-time limit, so a foreignid must be refused without ever waking someone else's box. *timeout*- is that limit for the owner's box (a resumed desktop would otherwise get- the SDK's default).+ is that limit for the owner's box (a resumed box would otherwise get the+ SDK's default)."""info = await AsyncSandbox.get_info(sandbox_id, api_key=api_key)- expected = owner.metadata(sandbox_kind)+ expected = owner.metadata()stamped = info.metadata or {}if any(stamped.get(key) != value for key, value in expected.items()):- raise SandboxNotOwnedError(- f"Sandbox {sandbox_id[:12]} is not {owner}'s {sandbox_kind} box"- )+ raise SandboxNotOwnedError(f"Sandbox {sandbox_id[:12]} is not {owner}'s box")return await AsyncSandbox.connect(sandbox_id, api_key=api_key, timeout=timeout)@@ -283,27 +309,21 @@ def _sandbox_key(session_id: str) -> str:return SandboxOwner(kind="session", id=session_id).key()-async def _get_stored_sandbox_id(- owner: SandboxOwner, sandbox_kind: SandboxKind = "shell"-) -> str | None:+async def _get_stored_sandbox_id(owner: SandboxOwner) -> str | None:redis = await get_redis_async()- raw = await redis.get(owner.key(sandbox_kind))+ raw = await redis.get(owner.key())value = raw.decode() if isinstance(raw, bytes) else rawreturn None if value == _CREATING_SENTINEL else value-async def _set_stored_sandbox_id(- owner: SandboxOwner, sandbox_id: str, sandbox_kind: SandboxKind = "shell"-) -> None:+async def _set_stored_sandbox_id(owner: SandboxOwner, sandbox_id: str) -> None:redis = await get_redis_async()- await redis.set(owner.key(sandbox_kind), sandbox_id, ex=owner.ttl)+ await redis.set(owner.key(), sandbox_id, ex=owner.ttl)-async def _clear_stored_sandbox_id(- owner: SandboxOwner, sandbox_kind: SandboxKind = "shell"-) -> None:+async def _clear_stored_sandbox_id(owner: SandboxOwner) -> None:redis = await get_redis_async()- await redis.delete(owner.key(sandbox_kind))+ await redis.delete(owner.key())class SandboxLookupError(Exception):@@ -314,10 +334,8 @@ class SandboxLookupError(Exception):"""-async def list_owned_sandboxes(- owner: SandboxOwner, sandbox_kind: SandboxKind, api_key: str-) -> list[SandboxInfo]:- """The owner's boxes of one kind as E2B lists them, newest first.+async def list_owned_sandboxes(owner: SandboxOwner, api_key: str) -> list[SandboxInfo]:+ """The owner's boxes as E2B lists them, newest first.A running box sorts before a paused one. Listing never connects, so apaused box stays paused — connecting is what auto-resume reacts to.@@ -326,7 +344,7 @@ async def list_owned_sandboxes(try:paginator = AsyncSandbox.list(query=SandboxQuery(- metadata=owner.metadata(sandbox_kind),+ metadata=owner.metadata(),state=[SandboxState.RUNNING, SandboxState.PAUSED],),limit=10,@@ -337,7 +355,7 @@ async def list_owned_sandboxes()except Exception as exc:raise SandboxLookupError(- f"E2B lookup of {owner}'s {sandbox_kind} boxes failed: {exc}"+ f"E2B lookup of {owner}'s boxes failed: {exc}") from excinfos = sorted(infos, key=lambda info: info.started_at, reverse=True)running = [info for info in infos if info.state == SandboxState.RUNNING]@@ -345,9 +363,7 @@ async def list_owned_sandboxes(return running + paused-async def find_owned_sandbox_id(- owner: SandboxOwner, sandbox_kind: SandboxKind, api_key: str-) -> str | None:+async def find_owned_sandbox_id(owner: SandboxOwner, api_key: str) -> str | None:"""Recover an expert's box through the E2B API when Redis has forgotten it.Session sandboxes are Redis-only: losing that key just means a fresh@@ -360,36 +376,49 @@ async def find_owned_sandbox_id("""if not owner.is_expert:return None- infos = await list_owned_sandboxes(owner, sandbox_kind, api_key)+ infos = await list_owned_sandboxes(owner, api_key)if not infos:return Nonechosen = infos[0]if len(infos) > 1:logger.warning(- "[E2B] %s has %d %s sandboxes; using %.12s",+ "[E2B] %s has %d sandboxes; using %.12s",owner,len(infos),- sandbox_kind,chosen.sandbox_id,)return chosen.sandbox_idasync def _try_reconnect(- sandbox_id: str, owner: "SandboxOwner | str", api_key: str+ sandbox_id: str,+ owner: "SandboxOwner | str",+ api_key: str,+ *,+ timeout: int | None = None,) -> "AsyncSandbox | None":- """Try to reconnect to an existing sandbox. Returns None on failure."""+ """Reconnect to the owner's box, or ``None`` if it is gone.++ Gone means E2B no longer has it, it is stamped for someone else, or it+ came back not running: the cached id is dropped so a replacement can be+ created. Anything else (a 5xx, a network blip) is raised, not swallowed.+ The box may be perfectly fine, and replacing it on a guess would fork+ everything on it that is not in a volume: the screen, running processes,+ installed tools. *timeout* re-arms the box's running-time limit.+ """owner = _as_owner(owner)try:- sandbox = await connect_owned(sandbox_id, owner, "shell", api_key)+ sandbox = await connect_owned(sandbox_id, owner, api_key, timeout=timeout)+ except SandboxNotOwnedError as exc:+ logger.warning("[E2B] Refusing reconnect: %s", exc)+ except NotFoundException as exc:+ logger.warning("[E2B] Box %.12s is gone: %s", sandbox_id, exc)+ else:if await sandbox.is_running():# Refresh TTL so an active owner cannot lose its sandbox_id at expiry.await _set_stored_sandbox_id(owner, sandbox_id)return sandbox- except SandboxNotOwnedError as exc:- logger.warning("[E2B] Refusing reconnect: %s", exc)- except Exception as exc:- logger.warning("[E2B] Reconnect to %.12s failed: %s", sandbox_id, exc)+ logger.warning("[E2B] Box %.12s came back not running", sandbox_id)# Stale — clear the sandbox_id from Redis so a new one can be created.await _clear_stored_sandbox_id(owner)@@ -421,7 +450,7 @@ async def _resolve_volume_mounts(def _active_turns_key(owner: SandboxOwner) -> str:- return f"{owner.key('shell')}:active"+ return f"{owner.key()}:active"async def _acquire_turn(owner: SandboxOwner) -> None:@@ -476,25 +505,23 @@ async def _release_turn(owner: SandboxOwner) -> bool:return False-async def get_or_create_sandbox(- session_id: str,+async def get_or_create_owner_sandbox(+ owner: SandboxOwner,api_key: str,timeout: int,template: str = "base",on_timeout: Literal["kill", "pause"] = "pause",volume_mounts: Mapping[str, str] | None = None,*,- expert_id: str | None = None,user_id: str | None = None,+ session_id: str | None = None,count_turn: bool = True,) -> AsyncSandbox:- """Return the existing E2B sandbox for this turn's owner or create a new one.+ """Return the owner's E2B sandbox, creating it if needed.- The owner is the session, or — when *expert_id* is set — the expert, whose- box every one of its sessions shares. The owner's key in Redis serves a- dual purpose: it stores the sandbox_id and acts as a creation lock via a- ``"creating"`` sentinel value. This removes the need for a separate lock- key.+ The owner's key in Redis serves a dual purpose: it stores the sandbox_id+ and acts as a creation lock via a ``"creating"`` sentinel value. This+ removes the need for a separate lock key.*timeout* controls how long the e2b sandbox may run continuously beforethe ``on_timeout`` lifecycle rule fires (default: 5 min).@@ -502,24 +529,25 @@ async def get_or_create_sandbox(or ``"kill"``. When ``"pause"``, ``auto_resume`` is enabled so pausedsandboxes wake transparently on SDK activity.*volume_mounts* maps mount paths to durable volume names (see- ``workspace_volume_mounts``) so the agent shell shares a persistent- ``~/workspace`` with the on-demand desktop and, for an expert, the owning- user's ``~/shared``. A mount failure degrades to a volume-less sandbox- rather than failing the session.- *count_turn* records this turn on an expert's box so another turn's end- cannot pause it; pass ``False`` and call ``count_expert_turn`` later when- the release is not yet guaranteed to run.+ ``workspace_volume_mounts``) so the box's ``~/workspace`` and, for an+ expert, the owning user's ``~/shared`` persist across boxes. A mount+ failure degrades to a volume-less sandbox rather than failing the session.+ *count_turn* records this caller as an active turn on an expert's box so+ another turn's end cannot pause it; pass ``False`` when opening the box+ from outside a turn (turning its screen on from the UI), or when the+ release is not yet guaranteed to run and ``count_expert_turn`` follows.+ *user_id* / *session_id* are provenance only, stamped on a newly created box.Raises :class:`SandboxLookupError` when E2B cannot say whether an expertalready has a box: a fresh box would fork the expert's durable state."""- owner = SandboxOwner.for_session(session_id, expert_id)redis = await get_redis_async()- key = owner.key("shell")- # Boxes E2B still lists but we could not reconnect to twice (mid-teardown,- # an unresumable snapshot). Without this an expert owner would re-find the+ key = owner.key()+ # Boxes E2B still lists but that are gone by the time we connect (a+ # teardown in progress). Without this an expert owner would re-find the# same id on every iteration and never fall through to creating a fresh one.failed_ids: set[str] = set()+ # Boxes that already had their one retry after a transient error.retried_ids: set[str] = set()for _ in range(_MAX_WAIT_ATTEMPTS):@@ -528,7 +556,7 @@ async def get_or_create_sandbox(if not value and owner.is_expert:# Redis is only a cache for an expert's box; E2B is the record.- value = await find_owned_sandbox_id(owner, "shell", api_key)+ value = await find_owned_sandbox_id(owner, api_key)if value in failed_ids:value = Noneelif value:@@ -536,19 +564,27 @@ async def get_or_create_sandbox(if value and value != _CREATING_SENTINEL:# Existing sandbox ID — try to reconnect (auto-resumes if paused).- sandbox = await _try_reconnect(value, owner, api_key)- if sandbox is None and owner.is_expert and value not in retried_ids:- # A durable box gets one more chance before it is given up- # on; a single transient failure must not fork it.+ try:+ sandbox = await _try_reconnect(value, owner, api_key, timeout=timeout)+ except Exception as exc:+ if value in retried_ids:+ raise+ # One more chance before the error surfaces: a single blip+ # must not replace a box, and asking the caller to try+ # again is better than forking what is on it.retried_ids.add(value)+ logger.warning(+ "[E2B] Reconnect to %.12s failed (%s); retrying once", value, exc+ )await asyncio.sleep(_RECONNECT_RETRY_DELAY_SECONDS)- sandbox = await _try_reconnect(value, owner, api_key)+ continueif sandbox:logger.info("[E2B] Reconnected to %.12s for %s", value, owner)if count_turn:await _acquire_turn(owner)return sandbox- # _try_reconnect cleared the key — loop to create a new sandbox.+ # The box is gone and _try_reconnect cleared the key — loop to+ # create a new sandbox.failed_ids.add(value)continue@@ -602,7 +638,6 @@ async def get_or_create_sandbox(lifecycle=lifecycle,volume_mounts=mounts,metadata=owner.creation_metadata(- "shell",user_id=user_id,session_id=session_id,template=template,@@ -688,55 +723,79 @@ async def get_or_create_sandbox(raise RuntimeError(f"Could not acquire E2B sandbox for {owner}")+async def get_or_create_sandbox(+ session_id: str,+ api_key: str,+ timeout: int,+ template: str = "base",+ on_timeout: Literal["kill", "pause"] = "pause",+ volume_mounts: Mapping[str, str] | None = None,+ *,+ expert_id: str | None = None,+ user_id: str | None = None,+ count_turn: bool = True,+) -> AsyncSandbox:+ """The sandbox for this turn (the session's, or its expert's), counting the turn.++ Thin wrapper over ``get_or_create_owner_sandbox`` for the chat engines,+ which know a session and maybe an expert rather than an owner.+ """+ return await get_or_create_owner_sandbox(+ SandboxOwner.for_session(session_id, expert_id),+ api_key,+ timeout=timeout,+ template=template,+ on_timeout=on_timeout,+ volume_mounts=volume_mounts,+ user_id=user_id,+ session_id=session_id,+ count_turn=count_turn,+ )++async def _act_on_sandbox(owner: SandboxOwner,api_key: str,action: str,fn: Callable[[AsyncSandbox], Awaitable[Any]],*,- sandbox_kind: SandboxKind = "shell",sandbox_id: str | None = None,clear_stored_id: bool = False,) -> bool:"""Connect to the owner's sandbox and run *fn* on it.Shared by ``pause_sandbox``, ``kill_sandbox`` and- ``kill_expert_sandboxes``. Returns ``True`` on success, ``False`` when no+ ``kill_expert_sandbox``. Returns ``True`` on success, ``False`` when nosandbox is found or the action fails. If *clear_stored_id* is ``True``,the sandbox_id is removed from Redis only after the action succeeds so afailed kill can be retried."""if sandbox_id is None:- sandbox_id = await _get_stored_sandbox_id(owner, sandbox_kind)+ sandbox_id = await _get_stored_sandbox_id(owner)if not sandbox_id:return Falseasync def _run() -> None:- await fn(await connect_owned(sandbox_id, owner, sandbox_kind, api_key))+ await fn(await connect_owned(sandbox_id, owner, api_key))try:await asyncio.wait_for(_run(), timeout=_E2B_API_TIMEOUT_SECONDS)if clear_stored_id:- await _clear_stored_sandbox_id(owner, sandbox_kind)+ await _clear_stored_sandbox_id(owner)logger.info(- "[E2B] %s %s sandbox %.12s for %s",- action.capitalize(),- sandbox_kind,- sandbox_id,- owner,+ "[E2B] %s sandbox %.12s for %s", action.capitalize(), sandbox_id, owner)return Trueexcept SandboxNotOwnedError as exc:# A cached id that is not the owner's box any more is never going to# be: forget it, or every later pause and kill fails the same way.logger.warning("[E2B] Refusing to %s: %s", action, exc)- await _clear_stored_sandbox_id(owner, sandbox_kind)+ await _clear_stored_sandbox_id(owner)return Falseexcept Exception as exc:logger.warning(- "[E2B] Failed to %s %s sandbox %.12s for %s: %s",+ "[E2B] Failed to %s sandbox %.12s for %s: %s",action,- sandbox_kind,sandbox_id,owner,exc,@@ -763,7 +822,10 @@ async def pause_sandbox(owner = SandboxOwner.for_session(session_id, expert_id)if not await _release_turn(owner):return False- return await _act_on_sandbox(owner, api_key, "pause", lambda sb: sb.pause())+ paused = await _act_on_sandbox(owner, api_key, "pause", lambda sb: sb.pause())+ if paused:+ await _forget_stream(owner)+ return pausedasync def pause_sandbox_direct(@@ -785,6 +847,7 @@ async def pause_sandbox_direct(try:await asyncio.wait_for(sandbox.pause(), timeout=_E2B_API_TIMEOUT_SECONDS)logger.info("[E2B] Paused sandbox %.12s for %s", sandbox.sandbox_id, owner)+ await _forget_stream(owner)return Trueexcept Exception as exc:logger.warning(@@ -796,68 +859,132 @@ async def pause_sandbox_direct(return False-async def kill_sandbox(- session_id: str,- api_key: str,-) -> bool:- """Kill the session's E2B sandboxes (shell and on-demand desktop).+async def kill_sandbox(session_id: str, api_key: str) -> bool:+ """Kill a session's box: the chat is gone, so is its scratch computer.- Only ever touches *session* sandboxes: an expert session has none under- its own id, so deleting an expert chat leaves the expert's computer alone- (see ``kill_expert_sandboxes`` for the archive path). The desktop is- included because nothing else ever stops it: it is deliberately not paused- at turn end, so an orphaned one would bill until its timeout and then sit- paused forever.+ Only ever touches a *session* box: an expert session has none under its+ own id, so deleting an expert chat leaves the expert's computer alone+ (see ``kill_expert_sandbox`` for the archive path).- Returns ``True`` if at least one sandbox was found and killed, ``False``- otherwise. Safe to call even when no sandbox exists for the session.+ Returns ``True`` if a sandbox was found and killed, ``False`` otherwise.+ Safe to call even when no sandbox exists for the session."""owner = SandboxOwner(kind="session", id=session_id)- killed = False- for sandbox_kind in ("shell", "desktop"):- if await _act_on_sandbox(- owner,- api_key,- "kill",- lambda sb: sb.kill(),- sandbox_kind=sandbox_kind,- clear_stored_id=True,- ):- killed = True+ sandbox_id = await _get_stored_sandbox_id(owner)+ if not sandbox_id:+ return await _kill_legacy_desktops(owner, api_key) > 0+ killed = await _act_on_sandbox(+ owner,+ api_key,+ "kill",+ lambda sb: sb.kill(),+ sandbox_id=sandbox_id,+ clear_stored_id=True,+ )+ if killed:+ await _forget_owner_state(owner)+ await _kill_legacy_desktops(owner, api_key)+ # The current box is the verdict: a swept old desktop must not report a+ # failed kill of it as done.return killed-async def kill_expert_sandboxes(expert_id: str, api_key: str) -> int:- """Kill an expert's shell and desktop boxes — its computer goes on archive.+async def kill_expert_sandbox(expert_id: str, api_key: str) -> bool:+ """Kill an expert's box: its computer goes on archive.The expert's volume is deliberately kept: files outlive the machine, anddestroying user data is not something a best-effort cleanup should do.- Falls back to the E2B metadata lookup for each box so a stale Redis cache- cannot leave a paused machine behind. Returns the number killed.+ Falls back to the E2B metadata lookup so a stale Redis cache cannot leave+ a paused machine behind. Returns ``True`` if a box was killed."""owner = SandboxOwner(kind="expert", id=expert_id)+ swept = await _kill_legacy_desktops(owner, api_key)+ sandbox_id = await _get_stored_sandbox_id(owner)+ if not sandbox_id:+ try:+ sandbox_id = await find_owned_sandbox_id(owner, api_key)+ except SandboxLookupError as exc:+ # Unknown whether a box exists: not done, whatever was swept.+ logger.warning("[E2B] Archive of %s: %s", owner, exc)+ return False+ if not sandbox_id:+ return swept > 0+ killed = await _act_on_sandbox(+ owner,+ api_key,+ "kill",+ lambda sb: sb.kill(),+ sandbox_id=sandbox_id,+ clear_stored_id=True,+ )+ if killed:+ await _forget_owner_state(owner)+ # The current box is the verdict: a swept old desktop must not report a+ # failed kill of it as done.+ return killed+++async def _kill_legacy_desktops(owner: SandboxOwner, api_key: str) -> int:+ """Kill the owner's pre-one-box desktop boxes, if any are still around.++ Found by stamp, not by cache, so a desktop whose key expired is swept+ too. Killed by id without connecting: a paused desktop must not be+ resumed (and billed) just to be destroyed. Returns how many were killed.+ """+ try:+ paginator = AsyncSandbox.list(+ query=SandboxQuery(+ metadata={**owner.metadata(), METADATA_KIND: _LEGACY_DESKTOP_KIND},+ state=[SandboxState.RUNNING, SandboxState.PAUSED],+ ),+ limit=10,+ api_key=api_key,+ )+ infos = await asyncio.wait_for(+ paginator.next_items(), timeout=_E2B_API_TIMEOUT_SECONDS+ )+ except Exception as exc:+ logger.warning("[E2B] Could not list %s's old desktop boxes: %s", owner, exc)+ return 0killed = 0- for sandbox_kind in ("shell", "desktop"):- sandbox_id = await _get_stored_sandbox_id(owner, sandbox_kind)- if not sandbox_id:- try:- sandbox_id = await find_owned_sandbox_id(owner, sandbox_kind, api_key)- except SandboxLookupError as exc:- logger.warning("[E2B] Archive of %s: %s", owner, exc)- continue- if not sandbox_id:+ for info in infos:+ try:+ await asyncio.wait_for(+ AsyncSandbox.kill(info.sandbox_id, api_key=api_key),+ timeout=_E2B_API_TIMEOUT_SECONDS,+ )+ except Exception as exc:+ logger.warning(+ "[E2B] Failed to kill old desktop %.12s for %s: %s",+ info.sandbox_id,+ owner,+ exc,+ )continue- if await _act_on_sandbox(- owner,- api_key,- "kill",- lambda sb: sb.kill(),- sandbox_kind=sandbox_kind,- sandbox_id=sandbox_id,- clear_stored_id=True,- ):- killed += 1+ killed += 1+ logger.info("[E2B] Killed old desktop %.12s for %s", info.sandbox_id, owner)with contextlib.suppress(Exception):redis = await get_redis_async()- await redis.delete(_active_turns_key(owner))+ await redis.delete(owner.legacy_desktop_key())return killed+++async def _forget_owner_state(owner: SandboxOwner) -> None:+ """Drop the screen flag, stream password and turn counter once their box+ is really gone."""+ with contextlib.suppress(Exception):+ redis = await get_redis_async()+ await redis.delete(+ owner.display_key(), owner.stream_key(), _active_turns_key(owner)+ )+++async def _forget_stream(owner: SandboxOwner) -> None:+ """Drop the stream password: the screen's next open issues a fresh one.++ Called whenever the box pauses, so the window in which a stream URL+ works is one running stretch of the box (``SandboxOwner.stream_key``).+ """+ with contextlib.suppress(Exception):+ redis = await get_redis_async()+ await redis.delete(owner.stream_key())diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.pyindex b8c2c74a7a..543090b470 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py@@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom e2b import SandboxState+from e2b.exceptions import SandboxNotFoundExceptionfrom backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATHfrom backend.blocks.desktop._common import (@@ -39,7 +40,7 @@ from .e2b_sandbox import (count_expert_turn,find_owned_sandbox_id,get_or_create_sandbox,- kill_expert_sandboxes,+ kill_expert_sandbox,kill_sandbox,pause_sandbox,pause_sandbox_direct,@@ -56,7 +57,6 @@ def _mock_sandbox(running: bool = True,*,owner: SandboxOwner | None = None,- kind: str = "shell",) -> MagicMock:"""A connected box stamped as *owner*'s (the session's shell box by default).@@ -69,9 +69,7 @@ def _mock_sandbox(sb.is_running = AsyncMock(return_value=running)sb.pause = AsyncMock()sb.kill = AsyncMock()- stamped = (owner or SandboxOwner(kind="session", id=_SESSION_ID)).metadata(- kind # type: ignore[arg-type]- )+ stamped = (owner or SandboxOwner(kind="session", id=_SESSION_ID)).metadata()sb.get_info = AsyncMock(return_value=MagicMock(metadata=stamped))_STAMPS[sandbox_id] = sb.get_info.return_valuereturn sb@@ -79,7 +77,7 @@ def _mock_sandbox(# sandbox_id -> the info the SDK reports for it, filled by ``_mock_sandbox``._STAMPS: dict[str, MagicMock] = {}-_SESSION_SHELL_STAMP = SandboxOwner(kind="session", id=_SESSION_ID).metadata("shell")+_SESSION_SHELL_STAMP = SandboxOwner(kind="session", id=_SESSION_ID).metadata()@pytest.fixture(autouse=True)@@ -103,6 +101,10 @@ def _patch_sdk():return _STAMPS.get(sandbox_id) or MagicMock(metadata=_SESSION_SHELL_STAMP)mock_cls.get_info = AsyncMock(side_effect=_get_info)+ # Nothing listed and kills by id succeed, unless a test says otherwise;+ # every kill path now sweeps for pre-one-box desktops through these.+ mock_cls.list = _mock_list([])+ mock_cls.kill = AsyncMock(return_value=True)yield mock_cls@@ -121,7 +123,7 @@ def _mock_redis(r = AsyncMock()raw = stored_sandbox_id.encode() if stored_sandbox_id else Noneshell_key = f"copilot:e2b:sandbox:{_SESSION_ID}"- # Only the session's shell key holds the id; its desktop key is empty.+ # Only the session's box key holds the id.r.get = AsyncMock(side_effect=lambda key: raw if key == shell_key else None)r.set = AsyncMock(return_value=set_nx_result)r.delete = AsyncMock()@@ -189,19 +191,45 @@ class TestTryReconnect:assert result is Noneredis.delete.assert_awaited_once()- def test_reconnect_exception_clears_redis(self):- """Clears sandbox_id in Redis when connect raises an exception."""+ def test_reconnect_gone_box_clears_redis(self):+ """A box E2B no longer has is given up on before any connect."""redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)with (_patch_sdk() as mock_cls,_patch_redis(redis),):- mock_cls.connect = AsyncMock(side_effect=ConnectionError("gone"))+ mock_cls.get_info = AsyncMock(side_effect=SandboxNotFoundException("404"))+ mock_cls.connect = AsyncMock()result = asyncio.run(_try_reconnect(_SANDBOX_ID, _SESSION_ID, _API_KEY))assert result is None+ mock_cls.connect.assert_not_awaited()redis.delete.assert_awaited_once()+ def test_reconnect_transient_error_surfaces_and_keeps_redis(self):+ """A 5xx says nothing about the box; it must not be replaced over one."""+ redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)+ with (+ _patch_sdk() as mock_cls,+ _patch_redis(redis),+ ):+ mock_cls.connect = AsyncMock(side_effect=ConnectionError("blip"))+ with pytest.raises(ConnectionError):+ asyncio.run(_try_reconnect(_SANDBOX_ID, _SESSION_ID, _API_KEY))++ redis.delete.assert_not_awaited()++ def test_reconnect_rearms_the_running_time_limit(self):+ sb = _mock_sandbox()+ with _patch_sdk() as mock_cls, _patch_redis(_mock_redis()):+ mock_cls.connect = AsyncMock(return_value=sb)+ asyncio.run(+ _try_reconnect(_SANDBOX_ID, _SESSION_ID, _API_KEY, timeout=_TIMEOUT)+ )+ mock_cls.connect.assert_awaited_once_with(+ _SANDBOX_ID, api_key=_API_KEY, timeout=_TIMEOUT+ )+# ---------------------------------------------------------------------------# get_or_create_sandbox@@ -568,8 +596,16 @@ class TestKillSandbox:assert result is Truesb.kill.assert_awaited_once()- # Redis key cleared after successful kill- redis.delete.assert_awaited_once()+ # The cached id is cleared after a successful kill, then the screen+ # flag and turn counter that belonged to the box.+ deleted = [key for call in redis.delete.await_args_list for key in call.args]+ assert deleted[0] == f"copilot:e2b:sandbox:{_SESSION_ID}"+ assert set(deleted[1:]) == {+ f"copilot:e2b:sandbox:{_SESSION_ID}:display",+ f"copilot:e2b:sandbox:{_SESSION_ID}:stream",+ f"copilot:e2b:sandbox:{_SESSION_ID}:active",+ f"copilot:e2b:desktop:{_SESSION_ID}",+ }def test_kill_refuses_a_foreign_box_and_forgets_it(self):"""A cached id stamped for someone else is dropped so the next@@ -583,12 +619,14 @@ class TestKillSandbox:assert result is Falsemock_cls.connect.assert_not_awaited()foreign.kill.assert_not_awaited()- redis.delete.assert_awaited_once_with(f"copilot:e2b:sandbox:{_SESSION_ID}")+ deleted = {key for call in redis.delete.await_args_list for key in call.args}+ assert f"copilot:e2b:sandbox:{_SESSION_ID}" in deleted+ assert f"copilot:e2b:sandbox:{_SESSION_ID}:display" not in deleteddef test_kill_no_sandbox(self):"""No-op when Redis has no sandbox_id."""redis = _mock_redis(stored_sandbox_id=None)- with _patch_redis(redis):+ with _patch_sdk(), _patch_redis(redis):result = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))assert result is False@@ -607,7 +645,8 @@ class TestKillSandbox:result = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))assert result is False- redis.delete.assert_not_awaited()+ deleted = {key for call in redis.delete.await_args_list for key in call.args}+ assert f"copilot:e2b:sandbox:{_SESSION_ID}" not in deleteddef test_kill_timeout_keeps_redis(self):"""Returns False and leaves Redis entry intact when the E2B call times out."""@@ -628,7 +667,7 @@ class TestKillSandbox:def test_kill_creating_sentinel_returns_false(self):"""No-op when the key holds the 'creating' sentinel (no real sandbox yet)."""redis = _mock_redis(stored_sandbox_id=_CREATING_SENTINEL)- with _patch_redis(redis):+ with _patch_sdk(), _patch_redis(redis):result = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))assert result is False@@ -653,8 +692,10 @@ class TestPauseSandbox:assert result is Truesb.pause.assert_awaited_once()- # sandbox_id should remain in Redis (not cleared on pause)- redis.delete.assert_not_awaited()+ # The id stays cached; only the stream password goes, so the next+ # open issues a new one and a URL that leaked stops working.+ deleted = {key for call in redis.delete.await_args_list for key in call.args}+ assert deleted == {f"copilot:e2b:sandbox:{_SESSION_ID}:stream"}def test_pause_no_sandbox(self):"""No-op when Redis has no sandbox_id."""@@ -739,26 +780,39 @@ class TestPauseSandboxDirect:def test_pause_direct_success(self):"""Pauses the sandbox directly without a Redis lookup or reconnect."""sb = _mock_sandbox()- result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))+ redis = _mock_redis()+ with _patch_redis(redis):+ result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))assert result is Truesb.pause.assert_awaited_once()+ redis.get.assert_not_awaited()+ # The turn-end pause is where the stream password is dropped.+ redis.delete.assert_awaited_once_with(+ f"copilot:e2b:sandbox:{_SESSION_ID}:stream"+ )def test_pause_direct_failure_returns_false(self):- """Returns False when sandbox.pause() raises."""+ """Returns False when sandbox.pause() raises; the password stays."""sb = _mock_sandbox()sb.pause = AsyncMock(side_effect=RuntimeError("e2b error"))- result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))+ redis = _mock_redis()+ with _patch_redis(redis):+ result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))assert result is False+ redis.delete.assert_not_awaited()def test_pause_direct_timeout_returns_false(self):"""Returns False when sandbox.pause() exceeds the 10s timeout."""sb = _mock_sandbox()- with patch(- "backend.copilot.tools.e2b_sandbox.asyncio.wait_for",- new_callable=AsyncMock,- side_effect=asyncio.TimeoutError,+ with (+ _patch_redis(_mock_redis()),+ patch(+ "backend.copilot.tools.e2b_sandbox.asyncio.wait_for",+ new_callable=AsyncMock,+ side_effect=asyncio.TimeoutError,+ ),):result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))@@ -772,25 +826,48 @@ class TestPauseSandboxDirect:_EXPERT_ID = "exp-777"_USER_ID = "user-42"_EXPERT_SHELL_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:shell"-_EXPERT_DESKTOP_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:desktop"+_EXPERT_DISPLAY_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:shell:display"+_EXPERT_STREAM_KEY = f"{_EXPERT_SHELL_KEY}:stream"_EXPERT_ACTIVE_KEY = f"{_EXPERT_SHELL_KEY}:active"+_EXPERT_LEGACY_DESKTOP_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:desktop"def _info(- sandbox_id: str, state: SandboxState, age_seconds: int = 0+ sandbox_id: str, state: SandboxState, age_seconds: int = 0, kind: str = "shell") -> SimpleNamespace:return SimpleNamespace(sandbox_id=sandbox_id,state=state,started_at=datetime.now(timezone.utc) - timedelta(seconds=age_seconds),+ metadata={"autogpt_kind": kind},)def _mock_list(infos: list) -> MagicMock:- """``AsyncSandbox.list`` is sync and returns an async paginator."""- paginator = MagicMock()- paginator.next_items = AsyncMock(return_value=infos)- return MagicMock(return_value=paginator)+ """``AsyncSandbox.list`` is sync and returns an async paginator.++ Answers per query kind: the owner's shell box and its pre-one-box+ desktop are looked up through the same call with different stamps.+ """++ def _list(query=None, **_):+ kind = (query.metadata or {}).get("autogpt_kind") if query else None+ paginator = MagicMock()+ paginator.next_items = AsyncMock(+ return_value=[+ i for i in infos if kind in (None, i.metadata["autogpt_kind"])+ ]+ )+ return paginator++ return MagicMock(side_effect=_list)+++def _listed_kinds(mock_cls: MagicMock) -> list[str]:+ """Which stamps ``AsyncSandbox.list`` was asked for, in order."""+ return [+ c.kwargs["query"].metadata["autogpt_kind"] for c in mock_cls.list.call_args_list+ ]def _keyed_redis(values: dict[str, str | None], decr_result: int = 0) -> AsyncMock:@@ -817,10 +894,10 @@ def _turn_acquires(redis: AsyncMock) -> list[str]:class TestConnectOwned:def test_checks_the_stamp_then_connects(self):owner = SandboxOwner(kind="expert", id=_EXPERT_ID)- sb = _mock_sandbox("sb-expert", owner=owner, kind="desktop")+ sb = _mock_sandbox("sb-expert", owner=owner)with _patch_sdk() as mock_cls:mock_cls.connect = AsyncMock(return_value=sb)- result = asyncio.run(connect_owned("sb-expert", owner, "desktop", _API_KEY))+ result = asyncio.run(connect_owned("sb-expert", owner, _API_KEY))assert result is sbmock_cls.get_info.assert_awaited_once_with("sb-expert", api_key=_API_KEY)mock_cls.connect.assert_awaited_once_with(@@ -828,14 +905,12 @@ class TestConnectOwned:)def test_timeout_reaches_the_connect(self):- """A resumed desktop gets its own running-time limit, not the SDK's."""+ """A resumed box gets its own running-time limit, not the SDK's."""owner = SandboxOwner(kind="expert", id=_EXPERT_ID)- sb = _mock_sandbox("sb-expert", owner=owner, kind="desktop")+ sb = _mock_sandbox("sb-expert", owner=owner)with _patch_sdk() as mock_cls:mock_cls.connect = AsyncMock(return_value=sb)- asyncio.run(- connect_owned("sb-expert", owner, "desktop", _API_KEY, timeout=900)- )+ asyncio.run(connect_owned("sb-expert", owner, _API_KEY, timeout=900))mock_cls.connect.assert_awaited_once_with("sb-expert", api_key=_API_KEY, timeout=900)@@ -860,7 +935,7 @@ class TestConnectOwned:mock_cls.get_info = AsyncMock(return_value=MagicMock(metadata=stamp))mock_cls.connect = AsyncMock()with pytest.raises(SandboxNotOwnedError):- asyncio.run(connect_owned("sb-x", owner, "shell", _API_KEY))+ asyncio.run(connect_owned("sb-x", owner, _API_KEY))mock_cls.connect.assert_not_awaited()@@ -870,13 +945,13 @@ class TestSandboxOwner:assert owner == SandboxOwner(kind="expert", id=_EXPERT_ID)assert owner.is_expertassert owner.key() == _EXPERT_SHELL_KEY- assert owner.key("desktop") == _EXPERT_DESKTOP_KEY+ assert owner.display_key() == _EXPERT_DISPLAY_KEYdef test_plain_session_keys_are_unchanged(self):owner = SandboxOwner.for_session(_SESSION_ID, None)assert not owner.is_expertassert owner.key() == f"copilot:e2b:sandbox:{_SESSION_ID}"- assert owner.key("desktop") == f"copilot:e2b:desktop:{_SESSION_ID}"+ assert owner.display_key() == f"copilot:e2b:sandbox:{_SESSION_ID}:display"def test_expert_cache_outlives_session_cache(self):assert (@@ -885,9 +960,9 @@ class TestSandboxOwner:)def test_metadata_identifies_owner_and_kind(self):- assert SandboxOwner(kind="expert", id=_EXPERT_ID).metadata("desktop") == {+ assert SandboxOwner(kind="expert", id=_EXPERT_ID).metadata() == {"autogpt_owner": f"expert:{_EXPERT_ID}",- "autogpt_kind": "desktop",+ "autogpt_kind": "shell",}@@ -896,7 +971,7 @@ class TestFindOwnedSandboxId:with _patch_sdk() as mock_cls:result = asyncio.run(find_owned_sandbox_id(- SandboxOwner(kind="session", id=_SESSION_ID), "shell", _API_KEY+ SandboxOwner(kind="session", id=_SESSION_ID), _API_KEY))assert result is None@@ -911,7 +986,7 @@ class TestFindOwnedSandboxId:mock_cls.list = _mock_list(infos)result = asyncio.run(find_owned_sandbox_id(- SandboxOwner(kind="expert", id=_EXPERT_ID), "shell", _API_KEY+ SandboxOwner(kind="expert", id=_EXPERT_ID), _API_KEY))assert result == "sb-running-old"@@ -931,7 +1006,7 @@ class TestFindOwnedSandboxId:mock_cls.list = _mock_list(infos)result = asyncio.run(find_owned_sandbox_id(- SandboxOwner(kind="expert", id=_EXPERT_ID), "desktop", _API_KEY+ SandboxOwner(kind="expert", id=_EXPERT_ID), _API_KEY))assert result == "sb-new"@@ -944,7 +1019,7 @@ class TestFindOwnedSandboxId:with pytest.raises(SandboxLookupError, match="e2b down"):asyncio.run(find_owned_sandbox_id(- SandboxOwner(kind="expert", id=_EXPERT_ID), "shell", _API_KEY+ SandboxOwner(kind="expert", id=_EXPERT_ID), _API_KEY))@@ -970,8 +1045,9 @@ class TestExpertShellBox:)assert result is sb+ # Resumed under the box's own running-time limit, not the SDK's.mock_cls.connect.assert_awaited_once_with(- "sb-expert", api_key=_API_KEY, timeout=None+ "sb-expert", api_key=_API_KEY, timeout=_TIMEOUT)mock_cls.create.assert_not_awaited()# Re-cached under the expert key, never the session key.@@ -1216,126 +1292,173 @@ class TestExpertKill:assert ok is Falsemock_cls.connect.assert_not_awaited()- def test_archive_kills_shell_and_desktop_and_clears_cache(self):+ def test_archive_kills_the_box_and_clears_its_state(self):expert = SandboxOwner(kind="expert", id=_EXPERT_ID)- shell = _mock_sandbox("sb-shell", owner=expert)- desktop = _mock_sandbox("sb-desktop", owner=expert, kind="desktop")- redis = _keyed_redis(- {_EXPERT_SHELL_KEY: "sb-shell", _EXPERT_DESKTOP_KEY: "sb-desktop"}- )+ box = _mock_sandbox("sb-box", owner=expert)+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-box"})with (_patch_sdk() as mock_cls,_patch_redis(redis),):- mock_cls.connect = AsyncMock(- side_effect=lambda sid, api_key, timeout=None: {- "sb-shell": shell,- "sb-desktop": desktop,- }[sid]- )- killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))-- assert killed == 2- shell.kill.assert_awaited_once()- desktop.kill.assert_awaited_once()- deleted = {call.args[0] for call in redis.delete.await_args_list}- assert deleted == {_EXPERT_SHELL_KEY, _EXPERT_DESKTOP_KEY, _EXPERT_ACTIVE_KEY}- mock_cls.list.assert_not_called()+ mock_cls.connect = AsyncMock(return_value=box)+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))++ assert killed is True+ box.kill.assert_awaited_once()+ deleted = {key for call in redis.delete.await_args_list for key in call.args}+ # The cached id, and with it the screen flag, stream password and+ # turn counter; the pre-one-box desktop key is swept along.+ assert deleted == {+ _EXPERT_SHELL_KEY,+ _EXPERT_DISPLAY_KEY,+ _EXPERT_STREAM_KEY,+ _EXPERT_ACTIVE_KEY,+ _EXPERT_LEGACY_DESKTOP_KEY,+ }+ # The cached id was enough; no lookup for the shell box.+ assert "shell" not in _listed_kinds(mock_cls)- def test_archive_falls_back_to_e2b_metadata_for_forgotten_boxes(self):- shell = _mock_sandbox(- "sb-shell", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)- )+ def test_archive_falls_back_to_e2b_metadata_for_a_forgotten_box(self):+ box = _mock_sandbox("sb-box", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))redis = _keyed_redis({})- lists = {- "shell": [_info("sb-shell", SandboxState.PAUSED)],- "desktop": [],- }with (_patch_sdk() as mock_cls,_patch_redis(redis),):+ mock_cls.list = _mock_list([_info("sb-box", SandboxState.PAUSED)])+ mock_cls.connect = AsyncMock(return_value=box)+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))- def _list(query, **_):- paginator = MagicMock()- paginator.next_items = AsyncMock(- return_value=lists[query.metadata["autogpt_kind"]]- )- return paginator-- mock_cls.list = MagicMock(side_effect=_list)- mock_cls.connect = AsyncMock(return_value=shell)- killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))-- assert killed == 1- shell.kill.assert_awaited_once()- assert mock_cls.list.call_count == 2+ assert killed is True+ box.kill.assert_awaited_once()+ assert _listed_kinds(mock_cls).count("shell") == 1def test_failed_kill_keeps_cache_for_retry(self):- _mock_sandbox("sb-shell", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))- redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-shell"})+ _mock_sandbox("sb-box", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-box"})with (_patch_sdk() as mock_cls,_patch_redis(redis),):mock_cls.list = _mock_list([])mock_cls.connect = AsyncMock(side_effect=RuntimeError("boom"))- killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))- assert killed == 0- deleted = {call.args[0] for call in redis.delete.await_args_list}+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))+ assert killed is False+ deleted = {key for call in redis.delete.await_args_list for key in call.args}assert _EXPERT_SHELL_KEY not in deleted-class TestKillSandboxDesktop:- def test_deleting_a_chat_kills_its_desktop_too(self):- """start_desktop's box is never paused at turn end, so the session- delete is the only thing that ever stops it."""- shell, desktop = _mock_sandbox("sb-shell"), _mock_sandbox(- "sb-desk", kind="desktop"- )- redis = _keyed_redis(- {- f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-shell",- f"copilot:e2b:desktop:{_SESSION_ID}": "sb-desk",- }- )+class TestKillSandboxForgetsScreen:+ def test_deleting_a_chat_drops_its_screen_flag_and_turn_counter(self):+ """The screen lives in the same box, so nothing else needs killing,+ but the flag that says it was on must not outlive the box."""+ box = _mock_sandbox("sb-box")+ redis = _keyed_redis({f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-box"})with (_patch_sdk() as mock_cls,_patch_redis(redis),):- mock_cls.connect = AsyncMock(- side_effect=lambda sid, api_key, timeout=None: {- "sb-shell": shell,- "sb-desk": desktop,- }[sid]- )+ mock_cls.connect = AsyncMock(return_value=box)ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))assert ok is True- shell.kill.assert_awaited_once()- desktop.kill.assert_awaited_once()- deleted = {call.args[0] for call in redis.delete.await_args_list}+ box.kill.assert_awaited_once()+ deleted = {key for call in redis.delete.await_args_list for key in call.args}assert deleted == {f"copilot:e2b:sandbox:{_SESSION_ID}",+ f"copilot:e2b:sandbox:{_SESSION_ID}:display",+ f"copilot:e2b:sandbox:{_SESSION_ID}:stream",+ f"copilot:e2b:sandbox:{_SESSION_ID}:active",f"copilot:e2b:desktop:{_SESSION_ID}",}- def test_desktop_only_session_still_reports_a_kill(self):- desktop = _mock_sandbox("sb-desk", kind="desktop")- redis = _keyed_redis({f"copilot:e2b:desktop:{_SESSION_ID}": "sb-desk"})- with (- _patch_sdk() as mock_cls,- _patch_redis(redis),- ):- mock_cls.connect = AsyncMock(return_value=desktop)++class TestLegacyDesktopSweep:+ """Boxes from before one box per owner: a separate desktop, stamped+ ``autogpt_kind=desktop``, that nothing opens any more. The kill paths+ take it along so it does not sit paused on E2B, browser profile and all."""++ def test_deleting_a_chat_kills_its_old_desktop_by_id_without_waking_it(self):+ box = _mock_sandbox("sb-box")+ redis = _keyed_redis({f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-box"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = _mock_list(+ [_info("sb-old-desktop", SandboxState.PAUSED, kind="desktop")]+ )+ mock_cls.connect = AsyncMock(return_value=box)+ ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))+ assert ok is True+ box.kill.assert_awaited_once()+ mock_cls.kill.assert_awaited_once_with("sb-old-desktop", api_key=_API_KEY)+ # Connected once, to the shell box; the desktop was killed by id.+ mock_cls.connect.assert_awaited_once()+ query = mock_cls.list.call_args.kwargs["query"].metadata+ assert query["autogpt_kind"] == "desktop"+ assert query["autogpt_owner"] == f"session:{_SESSION_ID}"++ def test_archiving_an_expert_sweeps_its_old_desktop_too(self):+ expert = SandboxOwner(kind="expert", id=_EXPERT_ID)+ box = _mock_sandbox("sb-box", owner=expert)+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-box"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = _mock_list(+ [_info("sb-old-desktop", SandboxState.PAUSED, kind="desktop")]+ )+ mock_cls.connect = AsyncMock(return_value=box)+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))+ assert killed is True+ box.kill.assert_awaited_once()+ mock_cls.kill.assert_awaited_once_with("sb-old-desktop", api_key=_API_KEY)+ deleted = {key for call in redis.delete.await_args_list for key in call.args}+ assert _EXPERT_LEGACY_DESKTOP_KEY in deleted++ def test_an_expert_with_only_an_old_desktop_still_counts_as_cleaned(self):+ redis = _keyed_redis({})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = _mock_list(+ [_info("sb-old-desktop", SandboxState.PAUSED, kind="desktop")]+ )+ mock_cls.connect = AsyncMock()+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))+ assert killed is True+ mock_cls.kill.assert_awaited_once_with("sb-old-desktop", api_key=_API_KEY)+ mock_cls.connect.assert_not_awaited()++ def test_a_swept_old_desktop_does_not_hide_a_failed_shell_kill(self):+ """The current box is what the caller asked about; reporting it+ killed because an old desktop went would end its retries."""+ redis = _keyed_redis({f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-box"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = _mock_list(+ [_info("sb-old-desktop", SandboxState.PAUSED, kind="desktop")]+ )+ mock_cls.connect = AsyncMock(side_effect=ConnectionError("gone"))+ ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))+ assert ok is False+ mock_cls.kill.assert_awaited_once_with("sb-old-desktop", api_key=_API_KEY)++ def test_an_expert_lookup_failure_is_not_done_whatever_was_swept(self):+ redis = _keyed_redis({})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = MagicMock(side_effect=RuntimeError("e2b down"))+ killed = asyncio.run(kill_expert_sandbox(_EXPERT_ID, _API_KEY))+ assert killed is False++ def test_a_failed_sweep_does_not_stop_the_shell_kill(self):+ box = _mock_sandbox("sb-box")+ redis = _keyed_redis({f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-box"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.list = MagicMock(side_effect=RuntimeError("e2b down"))+ mock_cls.connect = AsyncMock(return_value=box)ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))assert ok is True- desktop.kill.assert_awaited_once()+ box.kill.assert_awaited_once()+ mock_cls.kill.assert_not_awaited()class TestExpertBoxRecovery:- def test_unconnectable_listed_box_falls_through_to_create(self):- """E2B may keep listing a box we cannot reconnect to; the loop must- create a replacement instead of spinning on the same id."""+ def test_listed_box_that_is_gone_falls_through_to_create(self):+ """E2B may still list a box mid-teardown; the loop must create a+ replacement instead of spinning on the same id."""fresh = _mock_sandbox("sb-fresh")# Stamped as the expert's, so the refusal is not what stops the reconnect._mock_sandbox("sb-dead", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))@@ -1346,7 +1469,7 @@ class TestExpertBoxRecovery:_patch_redis(redis),):mock_cls.list = _mock_list([_info("sb-dead", SandboxState.PAUSED)])- mock_cls.connect = AsyncMock(side_effect=RuntimeError("unresumable"))+ mock_cls.connect = AsyncMock(side_effect=SandboxNotFoundException("gone"))mock_cls.create = AsyncMock(return_value=fresh)result = asyncio.run(get_or_create_sandbox(@@ -1354,11 +1477,48 @@ class TestExpertBoxRecovery:))assert result is fresh- # Exactly one retry of the listed box, then a replacement.- assert mock_cls.connect.await_count == 2- mock_cls.connect.assert_awaited_with("sb-dead", api_key=_API_KEY, timeout=None)+ mock_cls.connect.assert_awaited_once_with(+ "sb-dead", api_key=_API_KEY, timeout=_TIMEOUT+ )mock_cls.create.assert_awaited_once()- assert mock_cls.list.call_count <= 2+ assert _listed_kinds(mock_cls).count("shell") <= 2++ def test_a_transient_error_twice_surfaces_instead_of_replacing_the_box(self):+ """Whoever owns it: after the one retry the caller hears about it,+ and the box, with everything on it, is still theirs next time."""+ redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)+ with (+ _patch_sdk() as mock_cls,+ patch("backend.copilot.tools.e2b_sandbox.asyncio.sleep", AsyncMock()),+ _patch_redis(redis),+ ):+ mock_cls.connect = AsyncMock(side_effect=RuntimeError("502"))+ mock_cls.create = AsyncMock()+ with pytest.raises(RuntimeError, match="502"):+ asyncio.run(+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)+ )+ assert mock_cls.connect.await_count == 2+ mock_cls.create.assert_not_awaited()+ redis.delete.assert_not_awaited()++ def test_a_session_box_also_gets_its_one_retry(self):+ """The session's box carries its screen now, so a blip must not+ cost it the box any more than it would an expert."""+ sb = _mock_sandbox()+ redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)+ with (+ _patch_sdk() as mock_cls,+ patch("backend.copilot.tools.e2b_sandbox.asyncio.sleep", AsyncMock()),+ _patch_redis(redis),+ ):+ mock_cls.connect = AsyncMock(side_effect=[RuntimeError("502"), sb])+ mock_cls.create = AsyncMock()+ result = asyncio.run(+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)+ )+ assert result is sb+ mock_cls.create.assert_not_awaited()def test_mounts_survive_transient_create_failures(self):"""Only the final attempt goes volume-less; a slow first attempt mustdiff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop.pyindex 5c4dc2805c..28415e2bb6 100644--- a/autogpt_platform/backend/backend/copilot/tools/start_desktop.py+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop.py@@ -1,29 +1,19 @@-"""On-demand interactive desktop for CoPilot sessions (E2B).--The session's bash sandbox stays on the cheap headless ``base`` template —-E2B bills actual vCPU+RAM per running second, so putting every session on the-GUI-capable ``desktop`` template would tax the majority of sessions that never-need a screen. Instead this tool manages a SEPARATE desktop-template sandbox-per owner: created on first use, kept running after the turn ends so the-user can watch and control the live stream, auto-paused when idle (free), and-resumed in about a second on the next call.--The owner is the session — or, in an expert session, the expert itself, so-the desktop is the expert's own persistent computer: the same box (browser-profile, installed apps and all) comes back for every chat, delegation and-scheduled run that happens as that expert. The model has a root shell in-that same VM, so the desktop is shared with it, never private from it: any-account a person signs into on it is readable by the model. The expert page's-Computer tab and the copilot side panel open the very same box through-``backend.copilot.computer``.--Both the desktop and the owner's bash sandbox mount the SAME durable volumes-(see ``workspace_volume_mounts``): ``/home/user/workspace`` is the owner's-persistent home and, for an expert, ``/home/user/shared`` is the owning-user's workspace — so files the agent writes appear on the desktop and vice-versa, and both survive across sessions. The desktop additionally redirects-Downloads/Desktop/Documents into the home volume, so a person's browser-downloads and saved files persist too.+"""Turn the screen on in the session's computer (E2B) and stream it.++There is one box per owner: the session, or in an expert session the+expert itself, and it is the same sandbox ``bash_exec`` runs in. It runs+our desktop image with nothing graphical started, so a screen costs nothing+until this tool starts X, XFCE and the VNC stream *in that box*+(``backend.copilot.computer.open_desktop``). The expert page's Computer tab+and the copilot side panel open the very same screen.++Everything is shared because it is one machine: files, processes, installed+tools, and the durable volumes (``/home/user/workspace`` is the owner's+persistent home; for an expert ``/home/user/shared`` is the owning user's+workspace). The model has a root shell in that same VM, so the desktop is+shared with it, never private from it: any account a person signs into on+it is readable by the model. The box pauses at turn end like always and+the screen comes back exactly as it was on the next resume."""import logging@@ -40,7 +30,7 @@ logger = logging.getLogger(__name__)class StartDesktopTool(BaseTool):- """Start (or resume) the owner's on-demand desktop and return its stream."""+ """Turn on the screen of the session's computer and return its live stream."""@propertydef name(self) -> str:@@ -49,12 +39,13 @@ class StartDesktopTool(BaseTool):@propertydef description(self) -> str:return (- "Start or resume an interactive cloud desktop and return a live "- "stream the user can watch and control. It shares your persistent "- "~/workspace with bash_exec (in an expert session it is the expert's "- "own persistent computer; ~/shared is the user's workspace). Stays "- "up after your turn, suspends when idle. Use when a task needs a "- "browser or GUI app."+ "Turn on the screen of your computer and return a live stream the "+ "user can watch and control. It is the same machine bash_exec runs "+ "in, so files, ~/workspace and running processes are shared (in an "+ "expert session it is the expert's own persistent computer; "+ "~/shared is the user's workspace). The machine suspends between "+ "turns and the screen comes back as you left it. Use when a task "+ "needs a browser or GUI app.")@property@@ -80,7 +71,7 @@ class StartDesktopTool(BaseTool):session_id = session.session_id if session else Noneif not session_id:return ErrorResponse(- message="No active session to attach a desktop to.",+ message="No active session to turn a screen on for.",error="session_unavailable",)api_key = chat_config.active_e2b_api_key@@ -107,7 +98,7 @@ class StartDesktopTool(BaseTool):owner = computer_owner(session_id, expert_id)try:- stream, created, shared = await open_desktop(+ stream, first_time, shared = await open_desktop(owner,mounts_for(user_id, expert_id),api_key,@@ -124,14 +115,14 @@ class StartDesktopTool(BaseTool):)return DesktopStreamToolResponse(- message=_build_message(created, shared, expert=owner.is_expert),+ message=_build_message(first_time, shared, expert=owner.is_expert),desktop_stream=stream.model_dump(),session_id=session_id,)-def _build_message(created: bool, shared: bool, *, expert: bool = False) -> str:- state = "started" if created else "resumed"+def _build_message(first_time: bool, shared: bool, *, expert: bool = False) -> str:+ state = "Screen is on" if first_time else "Screen was already on; stream refreshed"if shared and expert:files = (f"This is your own persistent computer. {WORKSPACE_PATH} is your "@@ -142,16 +133,17 @@ def _build_message(created: bool, shared: bool, *, expert: bool = False) -> str:)elif shared:files = (- f"It shares your persistent {WORKSPACE_PATH} with bash_exec live, "- "so put anything the user should see (or that should persist) there."+ f"Your persistent {WORKSPACE_PATH} is on it, so put anything the "+ "user should see (or that should persist) there.")else:files = (- "No per-user volume is attached (no user context), so this desktop "- "is ephemeral and not shared with bash_exec."+ "No per-user volume is attached (no user context), so nothing on "+ "this machine persists beyond the sandbox.")return (- f"Desktop {state}. The user can watch and control it via the live "- f"stream. It stays up after this turn and suspends itself when idle; "- f"calling start_desktop again resumes it with state intact. {files}"+ f"{state}. This is the same machine bash_exec runs in, so files, "+ "processes and installed tools are shared. The user can watch and "+ "control it via the live stream. The machine suspends between turns "+ f"and the screen comes back as you left it. {files}")diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.pyindex 02f3cb97a3..5b8e72d919 100644--- a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py@@ -1,61 +1,94 @@-"""Tests for StartDesktopTool — session-linked on-demand E2B desktop."""+"""Tests for StartDesktopTool: the screen goes on in the owner's own box."""from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from backend.blocks.desktop._api import (- SHARED_PATH,- WORKSPACE_PATH,- DesktopStream,- PersistenceInfo,-)+from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH, DesktopStreamfrom backend.blocks.desktop._common import expert_volume_name, user_volume_name-from backend.copilot.tools.e2b_sandbox import SandboxOwner-from backend.util.sandbox_metadata import deployment_envfrom ._test_data import make_sessionfrom .models import DesktopStreamToolResponse, ErrorResponsefrom .start_desktop import StartDesktopTool_USER = "user-start-desktop-test"+_BOX = "sbx-1"_STREAM = DesktopStream(- url="https://6080-sbx.e2b.app/vnc.html?autoconnect=true",- sandbox_id="sbx-desktop-1",+ url="https://6080-sbx.e2b.app/vnc.html?autoconnect=true&password=secret",+ sandbox_id=_BOX,)+_C = "backend.copilot.computer"def _preview_link(user_id: str, live_url: str) -> str:- return f"https://platform.example/api/proxy/api/desktop-preview?for={user_id}"+ return f"/api/proxy/api/desktop-preview?for={user_id}"@pytest.fixture(autouse=True)def _owner_bound_links():- """The real link needs FRONTEND_BASE_URL and a Fernet key; stub the wrapper."""- with patch(- "backend.copilot.computer.create_preview_link", side_effect=_preview_link- ):+ """The real link needs a Fernet key; stub the wrapper."""+ with patch(f"{_C}.create_preview_link", side_effect=_preview_link):yield-def _make_redis(stored: str | None = None) -> MagicMock:+def _make_redis(display: str | None = None) -> MagicMock:redis = MagicMock()- redis.get = AsyncMock(return_value=stored)+ redis.get = AsyncMock(return_value=display)redis.set = AsyncMock(return_value=True)redis.delete = AsyncMock()redis.eval = AsyncMock(return_value=1)return redis-def _make_desktop() -> MagicMock:+def _make_desktop(mounted: bool = True) -> MagicMock:desktop = MagicMock()- desktop.sandbox_id = "sbx-desktop-1"desktop.ensure_display = AsyncMock()- desktop.start_stream = AsyncMock(return_value=_STREAM)- desktop.is_workspace_mounted = AsyncMock(return_value=True)+ desktop.ensure_persistent_home = AsyncMock()+ desktop.is_workspace_mounted = AsyncMock(return_value=mounted)+ desktop.start_stream = AsyncMock(return_value=(_STREAM, "pw"))return desktop+def _sandbox() -> MagicMock:+ sb = MagicMock()+ sb.sandbox_id = _BOX+ return sb+++class _Box:+ """Patches that stand in for the owner's box and the display on it."""++ def __init__(self, display: str | None = None, mounted: bool = True):+ self.redis = _make_redis(display)+ self.sandbox = _sandbox()+ self.desktop = _make_desktop(mounted)+ self.get_box = AsyncMock(return_value=self.sandbox)++ def patches(self):+ return (+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=self.redis)),+ patch(f"{_C}.get_or_create_owner_sandbox", self.get_box),+ patch(f"{_C}.DesktopSession", MagicMock(return_value=self.desktop)),+ patch(f"{_C}.chat_config"),+ patch("backend.copilot.tools.start_desktop.chat_config"),+ )++ def display_flag(self):+ return [+ c for c in self.redis.set.await_args_list if c.args[0].endswith(":display")+ ]+++async def _run(tool, box: _Box, *, user_id, session):+ redis_p, get_p, cls_p, computer_cfg, tool_cfg = box.patches()+ with redis_p, get_p, cls_p, computer_cfg as ccfg, tool_cfg as tcfg:+ tcfg.active_e2b_api_key = "e2b_test_key"+ ccfg.e2b_sandbox_timeout = 420+ ccfg.e2b_sandbox_template = "agpt-desktop-1x2"+ ccfg.e2b_sandbox_on_timeout = "pause"+ return await tool._execute(user_id=user_id, session=session)++class TestStartDesktop:def test_is_only_offered_with_e2b_and_only_to_signed_in_users(self):tool = StartDesktopTool()@@ -77,253 +110,116 @@ class TestStartDesktop:assert result.error == "e2b_unconfigured"@pytest.mark.asyncio(loop_scope="session")- async def test_creates_desktop_on_shared_user_volume(self):+ async def test_turns_the_screen_on_in_the_sessions_box(self):tool = StartDesktopTool()session = make_session(user_id=_USER)- desktop = _make_desktop()- redis = _make_redis(stored=None)-- with (- patch(- "backend.copilot.computer.get_redis_async",- new=AsyncMock(return_value=redis),- ),- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- mock_config.active_e2b_api_key = "e2b_test_key"- mock_config.e2b_desktop_timeout = 900- mock_config.e2b_desktop_template = "desktop"- mock_session_cls.create = AsyncMock(- return_value=(desktop, PersistenceInfo(volume_mounted=True))- )+ box = _Box()- result = await tool._execute(user_id=_USER, session=session)+ result = await _run(tool, box, user_id=_USER, session=session)assert isinstance(result, DesktopStreamToolResponse)assert result.desktop_stream["kind"] == "desktop_stream"- # The tool result, and so the stored message, carries an owner-bound- # link, never the stream URL with the desktop's password in it.+ # The very box bash_exec runs in: same owner, same volume, found the+ # same way, but not counted as a turn.+ kwargs = box.get_box.await_args.kwargs+ assert box.get_box.await_args.args[0].key() == (+ f"copilot:e2b:sandbox:{session.session_id}"+ )+ assert kwargs["volume_mounts"] == {WORKSPACE_PATH: user_volume_name(_USER)}+ assert kwargs["template"] == "agpt-desktop-1x2"+ assert kwargs["count_turn"] is False+ assert kwargs["user_id"] == _USER+ assert kwargs["session_id"] == session.session_id+ box.desktop.ensure_display.assert_awaited_once()+ box.desktop.ensure_persistent_home.assert_awaited_once()+ # The screen flag is remembered against this box's id.+ assert box.display_flag()[0].args[:2] == (+ f"copilot:e2b:sandbox:{session.session_id}:display",+ _BOX,+ )+ assert "Screen is on" in result.message+ assert "same machine" in result.message+ assert WORKSPACE_PATH in result.message++ @pytest.mark.asyncio(loop_scope="session")+ async def test_stored_result_carries_the_owner_link_not_the_password(self):+ tool = StartDesktopTool()+ session = make_session(user_id=_USER)+ box = _Box()++ result = await _run(tool, box, user_id=_USER, session=session)++ assert isinstance(result, DesktopStreamToolResponse)assert result.desktop_stream["url"] == _preview_link(_USER, _STREAM.url)assert result.desktop_stream["requires_auth"] is True- assert _STREAM.url not in result.model_dump_json()- # The desktop mounts the SAME per-user volume as the agent shell, and- # is tagged so its owner can be found through the E2B API.- create_kwargs = mock_session_cls.create.await_args.kwargs- assert create_kwargs["volume_mounts"] == {- WORKSPACE_PATH: user_volume_name(_USER)- }- assert create_kwargs["metadata"] == {- "service": "autogpt-platform",- "autogpt_owner": f"session:{session.session_id}",- "autogpt_kind": "desktop",- "autogpt_source": "copilot",- "autogpt_env": deployment_env(),- "autogpt_user": _USER,- "autogpt_session": session.session_id,- "autogpt_template": "desktop",- "autogpt_mounts": "attached",- }- redis.set.assert_awaited()- assert redis.set.await_args.args[0] == (- f"copilot:e2b:desktop:{session.session_id}"- )- assert "started" in result.message- assert "workspace" in result.message+ assert "vnc.html" not in result.model_dump_json()@pytest.mark.asyncio(loop_scope="session")- async def test_resumes_existing_desktop_without_creating(self):+ async def test_screen_already_on_just_refreshes_the_stream(self):tool = StartDesktopTool()session = make_session(user_id=_USER)- desktop = _make_desktop()- redis = _make_redis(stored="sbx-desktop-1")-- with (- patch(- "backend.copilot.computer.get_redis_async",- new=AsyncMock(return_value=redis),- ),- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.computer.connect_owned") as connect_owned,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- mock_config.active_e2b_api_key = "e2b_test_key"- mock_config.e2b_desktop_timeout = 900- connect_owned.return_value = box = MagicMock()- mock_session_cls.return_value = desktop- mock_session_cls.create = AsyncMock()+ box = _Box(display=_BOX)- result = await tool._execute(user_id=_USER, session=session)+ result = await _run(tool, box, user_id=_USER, session=session)assert isinstance(result, DesktopStreamToolResponse)- # Reattached only after E2B confirmed the box is this session's desktop,- # with its running-time limit re-armed.- connect_owned.assert_awaited_once_with(- "sbx-desktop-1",- SandboxOwner(kind="session", id=session.session_id),- "desktop",- "e2b_test_key",- timeout=900,- )- mock_session_cls.assert_called_once_with(box)- mock_session_cls.create.assert_not_awaited()- desktop.ensure_display.assert_awaited_once()- assert "resumed" in result.message+ assert "already on" in result.message+ box.desktop.ensure_display.assert_awaited_once()+ box.desktop.ensure_persistent_home.assert_not_awaited()+ box.desktop.start_stream.assert_awaited_once()@pytest.mark.asyncio(loop_scope="session")async def test_no_user_id_gets_no_desktop(self):"""The stream link is issued to a user; without one there is nobody to- issue it to, so no box is created."""+ issue it to, so the box is not even looked up."""tool = StartDesktopTool()session = make_session(user_id=_USER)+ box = _Box(mounted=False)- with (- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- mock_config.active_e2b_api_key = "e2b_test_key"- mock_session_cls.create = AsyncMock()-- result = await tool._execute(user_id=None, session=session)+ result = await _run(tool, box, user_id=None, session=session)assert isinstance(result, ErrorResponse)assert result.error == "user_required"- mock_session_cls.create.assert_not_awaited()+ box.get_box.assert_not_awaited()++ @pytest.mark.asyncio(loop_scope="session")+ async def test_failure_is_reported_not_raised(self):+ tool = StartDesktopTool()+ session = make_session(user_id=_USER)+ box = _Box()+ box.get_box.side_effect = RuntimeError("e2b down")++ result = await _run(tool, box, user_id=_USER, session=session)++ assert isinstance(result, ErrorResponse)+ assert result.error == "desktop_start_failed"class TestExpertDesktop:- """An expert session's desktop is the expert's own persistent computer."""+ """An expert session's screen goes on in the expert's own computer."""_EXPERT = "exp-desktop-1"- def _patches(self, redis, mock_session_cls, mock_config, found: str | None):- mock_config.active_e2b_api_key = "e2b_test_key"- mock_config.e2b_desktop_timeout = 900- mock_config.e2b_desktop_template = "desktop"- return (- patch(- "backend.copilot.computer.get_redis_async",- new=AsyncMock(return_value=redis),- ),- patch(- "backend.copilot.computer.find_owned_sandbox_id",- new=AsyncMock(return_value=found),- ),- )-@pytest.mark.asyncio(loop_scope="session")- async def test_expert_desktop_is_keyed_by_expert_with_home_and_shared(self):+ async def test_expert_screen_is_keyed_by_expert_with_home_and_shared(self):tool = StartDesktopTool()session = make_session(user_id=_USER, expert_id=self._EXPERT)- desktop = _make_desktop()- redis = _make_redis(stored=None)-- with (- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- redis_patch, find_patch = self._patches(- redis, mock_session_cls, mock_config, found=None- )- mock_session_cls.create = AsyncMock(- return_value=(- desktop,- PersistenceInfo(- volume_mounted=True,- mounted_paths=[WORKSPACE_PATH, SHARED_PATH],- ),- )- )- with redis_patch, find_patch:- result = await tool._execute(user_id=_USER, session=session)+ box = _Box()++ result = await _run(tool, box, user_id=_USER, session=session)assert isinstance(result, DesktopStreamToolResponse)- create_kwargs = mock_session_cls.create.await_args.kwargs+ owner = box.get_box.await_args.args[0]+ assert owner.kind == "expert" and owner.id == self._EXPERT# Own home first, then the user's shared workspace.- assert create_kwargs["volume_mounts"] == {+ assert box.get_box.await_args.kwargs["volume_mounts"] == {WORKSPACE_PATH: expert_volume_name(self._EXPERT),SHARED_PATH: user_volume_name(_USER),}- assert create_kwargs["metadata"] == {- "service": "autogpt-platform",- "autogpt_owner": f"expert:{self._EXPERT}",- "autogpt_kind": "desktop",- "autogpt_source": "copilot",- "autogpt_env": deployment_env(),- "autogpt_user": _USER,- "autogpt_session": session.session_id,- "autogpt_expert": self._EXPERT,- "autogpt_template": "desktop",- "autogpt_mounts": "attached",- }- # Cached under the expert, not the session: every session of this- # expert must come back to the same desktop.- assert redis.set.await_args.args[0] == (- f"copilot:e2b:expert:{self._EXPERT}:desktop"- )- assert session.session_id not in redis.set.await_args.args[0]+ # Remembered under the expert, not the session: every session of this+ # expert must come back to the same screen.+ flag_key = box.display_flag()[0].args[0]+ assert flag_key == f"copilot:e2b:expert:{self._EXPERT}:shell:display"+ assert session.session_id not in flag_keyassert "own persistent computer" in result.messageassert SHARED_PATH in result.message-- @pytest.mark.asyncio(loop_scope="session")- async def test_expert_desktop_recovered_from_e2b_when_cache_is_empty(self):- tool = StartDesktopTool()- session = make_session(user_id=_USER, expert_id=self._EXPERT)- desktop = _make_desktop()- redis = _make_redis(stored=None)-- with (- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.computer.connect_owned") as connect_owned,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- redis_patch, find_patch = self._patches(- redis, mock_session_cls, mock_config, found="sbx-desktop-1"- )- connect_owned.return_value = MagicMock()- mock_session_cls.return_value = desktop- mock_session_cls.create = AsyncMock()- with redis_patch, find_patch:- result = await tool._execute(user_id=_USER, session=session)-- assert isinstance(result, DesktopStreamToolResponse)- connect_owned.assert_awaited_once_with(- "sbx-desktop-1",- SandboxOwner(kind="expert", id=self._EXPERT),- "desktop",- "e2b_test_key",- timeout=900,- )- mock_session_cls.create.assert_not_awaited()- # The recovered id is re-cached under the expert key.- assert redis.set.await_args.args[0] == (- f"copilot:e2b:expert:{self._EXPERT}:desktop"- )- assert "resumed" in result.message-- @pytest.mark.asyncio(loop_scope="session")- async def test_plain_session_never_asks_e2b_for_a_lost_desktop(self):- tool = StartDesktopTool()- session = make_session(user_id=_USER)- desktop = _make_desktop()- redis = _make_redis(stored=None)-- with (- patch(- "backend.copilot.computer.get_redis_async",- new=AsyncMock(return_value=redis),- ),- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_sandbox,- patch("backend.copilot.computer.DesktopSession") as mock_session_cls,- patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,- ):- mock_config.active_e2b_api_key = "e2b_test_key"- mock_config.e2b_desktop_timeout = 900- mock_config.e2b_desktop_template = "desktop"- mock_session_cls.create = AsyncMock(- return_value=(desktop, PersistenceInfo(volume_mounted=True))- )- result = await tool._execute(user_id=_USER, session=session)-- assert isinstance(result, DesktopStreamToolResponse)- mock_sandbox.list.assert_not_called()diff --git a/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py b/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.pyindex b89baf3f9b..f7da1c3116 100644--- a/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py@@ -137,6 +137,12 @@ from backend.copilot.tools import TOOL_REGISTRY# registry measures 68,503 with start_desktop removed, so start_desktop's# delta is +493 once merged (the +670 above was against an older dev).# Merged registry measures 68,996; the ceiling is that plus one.+# Bumped 68_997 -> 69_063 for #14382 (one box per owner): start_desktop's+# description now says it is the same machine bash_exec runs in, +66.+# Merged registry measures 69,062.+# Lowered 69_063 -> 69_056 on the same PR: bash_exec's description no longer+# tells the model that only ~/workspace shows on the desktop, -7. Merged+# registry measures 69,055.# There is NO margin on top, deliberately. This limit is a brake: it exists to# make every increase in what Otto pays per turn a decision someone took,# so slack for growth nobody has measured is the one thing it must not carry.@@ -159,7 +165,7 @@ from backend.copilot.tools import TOOL_REGISTRY# counts against this ceiling. The tip reads 68,235 and `refs/pull/14416/merge`# 68,237 — dev widened raise_expert by two characters after this line was first# set, which reddened three interpreters on a branch that had added nothing.-_CHAR_BUDGET = 68_997+_CHAR_BUDGET = 69_056@pytest.fixture(scope="module")diff --git a/autogpt_platform/backend/scripts/desktop_smoke_test.py b/autogpt_platform/backend/scripts/desktop_smoke_test.pyindex bca49e5700..efe26879a7 100644--- a/autogpt_platform/backend/scripts/desktop_smoke_test.py+++ b/autogpt_platform/backend/scripts/desktop_smoke_test.py@@ -49,7 +49,7 @@ async def main() -> None:try:print("2) start stream")t = time.monotonic()- stream = await session.start_stream()+ stream, _password = await session.start_stream()timed("stream_start", t, timings)# The URL carries the VNC password; log where it points, not the secret.print(diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsxindex 7a70f00871..3185929fbb 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx@@ -97,8 +97,7 @@ describe("ComputerPanelContent", () => {owner_kind: "expert",owner_id: "exp-1",e2b_active: true,- shell: {- kind: "shell",+ box: {sandbox_id: "sb",state: "paused",started_at: new Date("2026-09-05T12:00:00Z"),@@ -107,7 +106,7 @@ describe("ComputerPanelContent", () => {template_id: "base",mounts_attached: true,},- desktop: null,+ screen_on: false,mounts: {},workspace_path: "/home/user/workspace",shared_path: "/home/user/shared",@@ -124,7 +123,7 @@ describe("ComputerPanelContent", () => {expect(await screen.findByText("Suspended")).toBeDefined();expect(screen.getByText(/expert's own computer/)).toBeDefined();await userEvent.click(- screen.getByRole("button", { name: "Start desktop" }),+ screen.getByRole("button", { name: "Turn on screen" }),);await waitFor(() =>diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsxindex 298b56a8e0..56c92ac724 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx@@ -18,23 +18,24 @@ interface Props {sessionId: string;}-function StatePill({ state }: { state: "running" | "paused" | null }) {+interface PillProps {+ tone: "on" | "off" | "none";+ label: string;+}++function Pill({ tone, label }: PillProps) {return (<spanclassName={cn("rounded-full px-2 py-0.5 text-xs font-medium",- state === "running"+ tone === "on"? "bg-emerald-50 text-emerald-700"- : state === "paused"+ : tone === "off"? "bg-zinc-100 text-zinc-600": "bg-zinc-50 text-zinc-500",)}>- {state === "running"- ? "Running"- : state === "paused"- ? "Suspended"- : "None"}+ {label}</span>);}@@ -81,19 +82,34 @@ export function ComputerPanelContent({ sessionId }: Props) {});const isExpert = computer?.owner_kind === "expert";- const hasDesktop = computer?.desktop != null;+ const box = computer?.box ?? null;+ const machineLabel = !box+ ? "None"+ : box.state === "running"+ ? "Running"+ : "Suspended";+ const machineTone = !box ? "none" : box.state === "running" ? "on" : "off";+ const screenOn = computer?.screen_on === true;+ const actionLabel = stream+ ? "Refresh"+ : screenOn+ ? "Open desktop"+ : "Turn on screen";return (<div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto p-3"><div className="flex items-center justify-between gap-3 rounded-lg border border-zinc-200 bg-white px-3 py-2"><div className="flex items-center gap-3 text-sm"><span className="flex items-center gap-1.5">- <span className="text-zinc-500">Shell</span>- <StatePill state={computer?.shell?.state ?? null} />+ <span className="text-zinc-500">Machine</span>+ <Pill tone={machineTone} label={machineLabel} /></span><span className="flex items-center gap-1.5">- <span className="text-zinc-500">Desktop</span>- <StatePill state={computer?.desktop?.state ?? null} />+ <span className="text-zinc-500">Screen</span>+ <Pill+ tone={!box ? "none" : screenOn ? "on" : "off"}+ label={!box ? "None" : screenOn ? "On" : "Off"}+ /></span></div><Button@@ -103,7 +119,7 @@ export function ComputerPanelContent({ sessionId }: Props) {disabled={computer != null && !computer.e2b_active}onClick={() => startDesktop({ sessionId })}>- {stream ? "Refresh" : hasDesktop ? "Resume desktop" : "Start desktop"}+ {actionLabel}</Button></div>@@ -113,11 +129,11 @@ export function ComputerPanelContent({ sessionId }: Props) {</div>) : (<div className="flex flex-1 flex-col items-center justify-center gap-1 rounded-lg border border-dashed border-zinc-200 p-6 text-center">- <Text variant="small-medium">No desktop on screen yet</Text>+ <Text variant="small-medium">Screen is not on this panel yet</Text><Text variant="small" className="max-w-xs text-zinc-500">{isExpert- ? "This chat runs on the expert's own computer. Start the desktop to watch it work, or ask for one in the chat."- : "Start the desktop to watch this chat work in a real browser, or ask for one in the chat."}+ ? "This chat runs on the expert's own computer. Turn on its screen to watch it work in a real browser, or ask for one in the chat."+ : "Turn on this chat's screen to watch it work in a real browser, or ask for one in the chat. It is the same machine your commands run in."}</Text></div>)}diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsxindex acc5432118..59c75b4c9b 100644--- a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx@@ -1,6 +1,5 @@"use client";-import type { SandboxSummary } from "@/app/api/__generated__/models/sandboxSummary";import { ACTION_BUTTON_CLASS } from "@/app/(platform)/team/helpers";import { Button } from "@/components/atoms/Button/Button";import { Icon } from "@/components/atoms/Icon/Icon";@@ -10,12 +9,12 @@ import { desktopStreamRenderer } from "@/components/contextual/OutputRenderers/rimport { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";import { cn } from "@/lib/utils";import { ComputerIcon, ComputerTerminalIcon } from "@hugeicons/core-free-icons";-import type { IconSvgElement } from "@hugeicons/react";import {describeMount,desktopActionLabel,formatResources,formatSandboxState,+ formatScreen,} from "./helpers";import { useExpertComputerSection } from "./useExpertComputerSection";@@ -25,49 +24,6 @@ interface Props {enabled: boolean;}-interface SandboxCardProps {- title: string;- icon: IconSvgElement;- summary: SandboxSummary | null | undefined;- idleHint: string;-}--function SandboxCard({ title, icon, summary, idleHint }: SandboxCardProps) {- const running = summary?.state === "running";- const resources = formatResources(summary);- return (- <div className="rounded-lg border border-zinc-200 bg-white p-4">- <div className="flex items-center justify-between gap-2">- <div className="flex items-center gap-2">- <Icon icon={icon} size={16} className="text-zinc-500" />- <Text variant="small-medium">{title}</Text>- </div>- <span- className={cn(- "rounded-full px-2 py-0.5 text-xs font-medium",- running- ? "bg-emerald-50 text-emerald-700"- : summary- ? "bg-zinc-100 text-zinc-600"- : "bg-zinc-50 text-zinc-500",- )}- >- {running ? "Running" : summary ? "Suspended" : "None"}- </span>- </div>- <Text variant="small" className="mt-2 text-zinc-600">- {summary ? formatSandboxState(summary) : idleHint}- </Text>- {resources ? (- <Text variant="small" className="mt-1 text-zinc-500">- {resources}- {summary?.mounts_attached === false ? " · no volumes attached" : ""}- </Text>- ) : null}- </div>- );-}-export function ExpertComputerSection({expertId,expertName,@@ -97,6 +53,9 @@ export function ExpertComputerSection({);}+ const box = computer.box;+ const running = box?.state === "running";+ const resources = formatResources(box);const mounts = Object.entries(computer.mounts ?? {});return (@@ -105,11 +64,12 @@ export function ExpertComputerSection({<div><Text variant="large-medium">{`${expertName}'s computer`}</Text><Text variant="small" className="mt-1 max-w-prose text-zinc-500">- A persistent cloud machine only {expertName} uses. It is suspended- when idle and costs nothing while suspended; installed tools and- files stay put between chats. {expertName} has full access to it, so- anything signed in on its desktop is visible to the AI. Do not sign- into personal accounts there.+ One persistent cloud machine only {expertName} uses: the shell it+ runs commands in and, when needed, its screen. It is suspended+ between turns and costs nothing while suspended; installed tools and+ files stay put. {expertName} has full access to it, so anything+ signed in on its desktop is visible to the AI. Do not sign into+ personal accounts there.</Text></div><Button@@ -132,19 +92,46 @@ export function ExpertComputerSection({</div>) : null}- <div className="grid gap-3 sm:grid-cols-2">- <SandboxCard- title="Shell"- icon={ComputerTerminalIcon}- summary={computer.shell}- idleHint="Created on the first message that runs a command."- />- <SandboxCard- title="Desktop"- icon={ComputerIcon}- summary={computer.desktop}- idleHint="Created the first time a task needs a browser or GUI app."- />+ <div className="rounded-lg border border-zinc-200 bg-white p-4">+ <div className="flex items-center justify-between gap-2">+ <div className="flex items-center gap-2">+ <Icon+ icon={ComputerTerminalIcon}+ size={16}+ className="text-zinc-500"+ />+ <Text variant="small-medium">Machine</Text>+ </div>+ <span+ className={cn(+ "rounded-full px-2 py-0.5 text-xs font-medium",+ running+ ? "bg-emerald-50 text-emerald-700"+ : box+ ? "bg-zinc-100 text-zinc-600"+ : "bg-zinc-50 text-zinc-500",+ )}+ >+ {running ? "Running" : box ? "Suspended" : "None"}+ </span>+ </div>+ <Text variant="small" className="mt-2 text-zinc-600">+ {box+ ? formatSandboxState(box)+ : "Created on the first message that runs a command."}+ </Text>+ {resources ? (+ <Text variant="small" className="mt-1 text-zinc-500">+ {resources}+ {box?.mounts_attached === false ? " · no volumes attached" : ""}+ </Text>+ ) : null}+ <div className="mt-3 flex items-center gap-2 border-t border-zinc-100 pt-3">+ <Icon icon={ComputerIcon} size={16} className="text-zinc-500" />+ <Text variant="small" className="text-zinc-600">+ {formatScreen(computer)}+ </Text>+ </div></div>{mounts.length > 0 ? (diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsxindex 78d9dc4064..ff134ec162 100644--- a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx@@ -13,8 +13,7 @@ const computer: ComputerInfo = {owner_kind: "expert",owner_id: "expert-1",e2b_active: true,- shell: {- kind: "shell",+ box: {sandbox_id: "sb-shell",state: "paused",started_at: new Date("2026-09-05T12:00:00Z"),@@ -23,7 +22,7 @@ const computer: ComputerInfo = {template_id: "base",mounts_attached: true,},- desktop: null,+ screen_on: false,mounts: {"/home/user/workspace": "autogpt-expert-expert-1","/home/user/shared": "autogpt-user-u1",@@ -45,7 +44,9 @@ describe("ExpertComputerSection", () => {expect(screen.getByText("2 vCPU · 512 MiB")).toBeDefined();expect(screen.getByText("/home/user/shared")).toBeDefined();expect(screen.getByText("autogpt-expert-expert-1")).toBeDefined();- expect(screen.getByRole("button", { name: "Start desktop" })).toBeDefined();+ expect(+ screen.getByRole("button", { name: "Turn on screen" }),+ ).toBeDefined();});it("starts the desktop and embeds its stream", async () => {@@ -65,7 +66,7 @@ describe("ExpertComputerSection", () => {);await userEvent.click(- await screen.findByRole("button", { name: "Start desktop" }),+ await screen.findByRole("button", { name: "Turn on screen" }),);await waitFor(() =>@@ -91,7 +92,7 @@ describe("ExpertComputerSection", () => {<ExpertComputerSection expertId="expert-1" expertName="Maria" enabled />,);await userEvent.click(- await screen.findByRole("button", { name: "Start desktop" }),+ await screen.findByRole("button", { name: "Turn on screen" }),);await waitFor(() =>expect(@@ -115,7 +116,7 @@ describe("ExpertComputerSection", () => {getGetV2GetExpertComputerMockHandler({...computer,e2b_active: false,- shell: null,+ box: null,mounts: {},}),);@@ -130,7 +131,7 @@ describe("ExpertComputerSection", () => {),).toBeDefined();expect(- screen.getByRole("button", { name: "Start desktop" }),+ screen.getByRole("button", { name: "Turn on screen" }),).toHaveProperty("disabled", true);});});diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/helpers.test.tsnew file mode 100644index 0000000000..5d51ac0b95--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/helpers.test.ts@@ -0,0 +1,57 @@+import type { ComputerInfo } from "@/app/api/__generated__/models/computerInfo";+import { describe, expect, it } from "vitest";+import { desktopActionLabel, formatScreen } from "../helpers";++function computer(+ box: "running" | "paused" | null,+ screenOn: boolean,+): ComputerInfo {+ return {+ owner_kind: "expert",+ owner_id: "expert-1",+ e2b_active: true,+ box: box+ ? {+ sandbox_id: "sb-1",+ state: box,+ started_at: new Date("2026-09-05T12:00:00Z"),+ cpu_count: 1,+ memory_mb: 2048,+ template_id: "agpt-desktop-1x2",+ mounts_attached: true,+ }+ : null,+ screen_on: screenOn,+ mounts: {},+ workspace_path: "/home/user/workspace",+ shared_path: "/home/user/shared",+ };+}++describe("desktopActionLabel", () => {+ it("offers to turn the screen on while it is off, box or no box", () => {+ expect(desktopActionLabel(null)).toBe("Turn on screen");+ expect(desktopActionLabel(computer(null, false))).toBe("Turn on screen");+ expect(desktopActionLabel(computer("running", false))).toBe(+ "Turn on screen",+ );+ expect(desktopActionLabel(computer("paused", false))).toBe(+ "Turn on screen",+ );+ });++ it("opens a running box's screen and resumes a suspended one's", () => {+ expect(desktopActionLabel(computer("running", true))).toBe("Open desktop");+ expect(desktopActionLabel(computer("paused", true))).toBe("Resume desktop");+ });+});++describe("formatScreen", () => {+ it("describes the screen for every box and screen state", () => {+ expect(formatScreen(computer(null, false))).toMatch(/comes on the first/);+ expect(formatScreen(computer("running", false))).toMatch(+ /inside this same machine/,+ );+ expect(formatScreen(computer("paused", true))).toMatch(/Screen on/);+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.tsindex c6cab33f2b..964b8d2e2c 100644--- a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts@@ -23,6 +23,17 @@ export function formatResources(summary: SandboxSummary | null | undefined) {return `${summary.cpu_count} vCPU · ${memory}`;}+export function formatScreen(+ computer: Pick<ComputerInfo, "box" | "screen_on">,+) {+ if (!computer.box) {+ return "Screen off. It comes on the first time a task needs a browser or GUI app.";+ }+ return computer.screen_on+ ? "Screen on. Open the desktop to watch or take over."+ : "Screen off. Turning it on happens inside this same machine.";+}+export function describeMount(path: string,computer: Pick<ComputerInfo, "workspace_path" | "shared_path" | "owner_kind">,@@ -39,8 +50,6 @@ export function describeMount(}export function desktopActionLabel(computer: ComputerInfo | null | undefined) {- if (!computer?.desktop) return "Start desktop";- return computer.desktop.state === "running"- ? "Open desktop"- : "Resume desktop";+ if (!computer?.box || !computer.screen_on) return "Turn on screen";+ return computer.box.state === "running" ? "Open desktop" : "Resume desktop";}diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 26c56dfba2..21e6a3d3ac 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -4232,7 +4232,7 @@"get": {"tags": ["v2", "chat", "chat"],"summary": "Get Session Computer",- "description": "The computer behind this chat.\n\nA plain chat has its own boxes; a chat that runs as a hired expert reports\nthe expert's persistent computer instead. Listing never wakes a paused box.",+ "description": "The computer behind this chat.\n\nA plain chat has its own box; a chat that runs as a hired expert reports\nthe expert's persistent computer instead. Listing never wakes a paused box.","operationId": "getV2GetSessionComputer","security": [{ "HTTPBearerJWT": [] }],"parameters": [@@ -6965,7 +6965,7 @@"get": {"tags": ["v2", "experts", "experts", "private"],"summary": "Get Expert Computer",- "description": "The expert's own computer: its shell and desktop boxes as E2B lists them.\n\nListing never wakes a paused box, so the Computer tab can refresh freely.",+ "description": "The expert's own computer: its box as E2B lists it, and whether its\nscreen is on. E2B knows nothing about the screen; that flag is ours,\nkept beside the box id, because asking the box would wake it.\n\nListing never wakes a paused box, so the Computer tab can refresh freely.","operationId": "getV2GetExpertComputer","security": [{ "HTTPBearerJWT": [] }],"parameters": [@@ -19155,17 +19155,16 @@},"owner_id": { "type": "string", "title": "Owner Id" },"e2b_active": { "type": "boolean", "title": "E2B Active" },- "shell": {+ "box": {"anyOf": [{ "$ref": "#/components/schemas/SandboxSummary" },{ "type": "null" }]},- "desktop": {- "anyOf": [- { "$ref": "#/components/schemas/SandboxSummary" },- { "type": "null" }- ]+ "screen_on": {+ "type": "boolean",+ "title": "Screen On",+ "default": false},"mounts": {"additionalProperties": { "type": "string" },@@ -19187,7 +19186,7 @@"type": "object","required": ["owner_kind", "owner_id", "e2b_active"],"title": "ComputerInfo",- "description": "The owner's computer: its boxes and the volumes they mount."+ "description": "The owner's computer: its one box, whether its screen is on, its volumes."},"ConfirmLinkRequest": {"properties": {@@ -27960,11 +27959,6 @@},"SandboxSummary": {"properties": {- "kind": {- "type": "string",- "enum": ["shell", "desktop"],- "title": "Kind"- },"sandbox_id": { "type": "string", "title": "Sandbox Id" },"state": {"type": "string",@@ -27983,7 +27977,6 @@},"type": "object","required": [- "kind","sandbox_id","state","started_at",@@ -27993,7 +27986,7 @@"mounts_attached"],"title": "SandboxSummary",- "description": "One of the owner's sandboxes as E2B reports it, without waking it."+ "description": "The owner's box as E2B reports it, without waking it."},"ScheduleCleanupRequest": {"properties": {