Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(platform): interactive E2B desktop sandboxes with live embedded streams (#14365)

Nicholas Tindleb85b73c8f70b
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 9ddd707bdf..9bd00ce2e1 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -13,6 +13,7 @@ from fastapi.responses import StreamingResponse
from pydantic import BaseModel, ConfigDict, Field, field_validator
from backend.api.features.experts import experts_db
+from backend.blocks.desktop._api import DesktopStream
from backend.copilot import active_turns
from backend.copilot import service as chat_service
from backend.copilot import stream_registry, turn_queue
@@ -22,6 +23,13 @@ from backend.copilot.active_turns import (
inflight_turn_limit_message,
)
from backend.copilot.builder_context import resolve_session_permissions
+from backend.copilot.computer import (
+ ComputerInfo,
+ computer_owner,
+ describe_computer,
+ mounts_for,
+ open_desktop,
+)
from backend.copilot.config import ChatConfig, CopilotLlmAuthProvider, CopilotLLMModel
from backend.copilot.db import (
chat_message_has_assistant_reply,
@@ -851,6 +859,76 @@ async def create_session(
)
+ "/sessions/{session_id}/computer",
+ dependencies=[Security(auth.requires_user)],
+ responses={404: {"description": "Session not found or access denied"}},
+)
+async def get_session_computer(
+ session_id: str,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+) -> ComputerInfo:
+ """The computer behind this chat.
+
+ A plain chat has its own boxes; a chat that runs as a hired expert reports
+ the expert's persistent computer instead. Listing never wakes a paused box.
+ """
+ # Metadata only: the panel polls this, and the history is not needed.
+ session = await _validate_and_get_session(session_id, user_id)
+ return await describe_computer(
+ computer_owner(session_id, session.expert_id),
+ mounts_for(user_id, session.expert_id),
+ )
+
+
+ "/sessions/{session_id}/desktop",
+ dependencies=[Security(auth.requires_user)],
+ responses={
+ 404: {"description": "Session not found or access denied"},
+ 502: {"description": "The desktop could not be started"},
+ 503: {"description": "E2B is not configured"},
+ },
+)
+async def start_session_desktop(
+ session_id: str,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+) -> DesktopStream:
+ """Start or resume the desktop behind this chat and return its live stream.
+
+ Same box the ``start_desktop`` tool uses from inside a turn, so the side
+ panel and the model always look at one screen.
+ """
+ # Same gate as a turn: an archived expert's chat must not resurrect the
+ # expert's box (the archive already killed it, and nothing would kill
+ # the new one).
+ session = await _validate_and_get_writable_session(session_id, user_id)
+ api_key = ChatConfig().active_e2b_api_key
+ if not api_key:
+ raise HTTPException(
+ status_code=503, detail="E2B is not configured on this deployment."
+ )
+ try:
+ stream, _created, _shared = await open_desktop(
+ computer_owner(session_id, session.expert_id),
+ mounts_for(user_id, session.expert_id),
+ api_key,
+ user_id=user_id,
+ session_id=session_id,
+ )
+ except Exception as exc:
+ logger.error(
+ "[E2B] start_session_desktop failed for %s: %s",
+ session_id[:12],
+ exc,
+ exc_info=True,
+ )
+ # The cause is in the server log; provider errors can carry sandbox
+ # ids and infrastructure detail that the client has no use for.
+ raise HTTPException(status_code=502, detail="Failed to start the desktop.")
+ return stream
+
+
@router.delete(
"/sessions/{session_id}",
dependencies=[Security(auth.requires_user)],
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
index 22ce6f6abf..126a2b131b 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
@@ -4246,3 +4246,135 @@ def test_standard_tier_is_not_gated(
# Balanced. The request goes on to fail for unrelated reasons in this
# harness; what matters is that it was not refused for entitlement.
allowed.assert_not_called()
+
+
+# --- Computer -----------------------------------------------------------------
+
+
+def _session_like(expert_id: str | None):
+ return MagicMock(session_id="sess-1", user_id="u1", expert_id=expert_id)
+
+
+def test_get_session_computer_uses_the_expert_owner_for_expert_chats(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ from backend.copilot.computer import ComputerInfo
+
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=_session_like("exp-9"),
+ )
+ describe = mocker.patch(
+ "backend.api.features.chat.routes.describe_computer",
+ new_callable=AsyncMock,
+ return_value=ComputerInfo(
+ owner_kind="expert", owner_id="exp-9", e2b_active=True
+ ),
+ )
+
+ response = client.get("/sessions/sess-1/computer")
+
+ assert response.status_code == 200
+ assert response.json()["owner_kind"] == "expert"
+ owner, mounts = describe.await_args.args
+ assert owner.kind == "expert" and owner.id == "exp-9"
+ assert "/home/user/shared" in mounts
+
+
+def test_get_session_computer_plain_chat_is_session_owned(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ from backend.copilot.computer import ComputerInfo
+
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=_session_like(None),
+ )
+ describe = mocker.patch(
+ "backend.api.features.chat.routes.describe_computer",
+ new_callable=AsyncMock,
+ return_value=ComputerInfo(
+ owner_kind="session", owner_id="sess-1", e2b_active=True
+ ),
+ )
+ assert client.get("/sessions/sess-1/computer").status_code == 200
+ owner, _mounts = describe.await_args.args
+ assert owner.kind == "session" and owner.id == "sess-1"
+
+
+def test_get_session_computer_unknown_session_is_404(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=None,
+ )
+ assert client.get("/sessions/nope/computer").status_code == 404
+
+
+def test_start_session_desktop_returns_the_stream(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ from backend.blocks.desktop._api import DesktopStream
+
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=_session_like(None),
+ )
+ config = mocker.patch("backend.api.features.chat.routes.ChatConfig")
+ config.return_value.active_e2b_api_key = "e2b-key"
+ mocker.patch(
+ "backend.api.features.chat.routes.open_desktop",
+ new_callable=AsyncMock,
+ return_value=(
+ DesktopStream(url="https://6080-sbx.e2b.app/vnc.html", sandbox_id="sbx"),
+ False,
+ True,
+ ),
+ )
+ response = client.post("/sessions/sess-1/desktop")
+ assert response.status_code == 200
+ assert response.json()["sandbox_id"] == "sbx"
+
+
+def test_start_session_desktop_without_e2b_is_503(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=_session_like(None),
+ )
+ config = mocker.patch("backend.api.features.chat.routes.ChatConfig")
+ config.return_value.active_e2b_api_key = None
+ assert client.post("/sessions/sess-1/desktop").status_code == 503
+
+
+def test_start_session_desktop_is_refused_for_an_archived_experts_chat(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """The archive killed the expert's boxes; a Start from an old chat must not
+ create a new one that nothing will ever kill."""
+ mocker.patch(
+ "backend.api.features.chat.routes.get_chat_session_metadata",
+ new_callable=AsyncMock,
+ return_value=_session_like("exp-archived"),
+ )
+ owns_active = mocker.patch(
+ "backend.api.features.chat.routes.experts_db.owns_active_expert",
+ new_callable=AsyncMock,
+ return_value=False,
+ )
+ open_desktop = mocker.patch(
+ "backend.api.features.chat.routes.open_desktop", new_callable=AsyncMock
+ )
+ response = client.post("/sessions/sess-1/desktop")
+ # Same non-enumerable 404 a turn gets for a fired or foreign expert.
+ assert response.status_code == 404
+ owns_active.assert_awaited_once()
+ assert owns_active.await_args.args[1] == "exp-archived"
+ open_desktop.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/api/features/desktop_preview.py b/autogpt_platform/backend/backend/api/features/desktop_preview.py
new file mode 100644
index 0000000000..a22c4b560b
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/desktop_preview.py
@@ -0,0 +1,36 @@
+"""Authenticated gateway to desktop streams; never cache the credential redirect."""
+
+from typing import Annotated
+
+from autogpt_libs.auth import get_user_id
+from fastapi import APIRouter, HTTPException, Query, Security
+from fastapi.responses import RedirectResponse
+
+from backend.util.desktop_preview import resolve_preview_link
+
+router = APIRouter()
+
+
+ "/desktop-preview",
+ response_class=RedirectResponse,
+ responses={404: {"description": "Preview unavailable"}},
+)
+async def open_desktop_preview(
+ token: Annotated[str, Query(max_length=8192)],
+ user_id: Annotated[str, Security(get_user_id)],
+) -> RedirectResponse:
+ """Redirect the desktop's owner to its live stream.
+
+ The stream URL carries the desktop's password, so it is only ever
+ disclosed here, to the user the token was issued to, and marked
+ ``no-store`` so no cache or referrer keeps a copy.
+ """
+ url = resolve_preview_link(user_id, token)
+ if not url:
+ raise HTTPException(status_code=404, detail="Preview unavailable")
+ return RedirectResponse(
+ url,
+ status_code=307,
+ headers={"Cache-Control": "no-store", "Referrer-Policy": "no-referrer"},
+ )
diff --git a/autogpt_platform/backend/backend/api/features/desktop_preview_test.py b/autogpt_platform/backend/backend/api/features/desktop_preview_test.py
new file mode 100644
index 0000000000..3bd03bda93
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/desktop_preview_test.py
@@ -0,0 +1,86 @@
+"""The preview gateway checks ownership before disclosing the desktop URL."""
+
+import json
+from unittest.mock import patch
+
+import pytest
+from autogpt_libs.auth import get_user_id
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from httpx import Response
+from pytest_snapshot.plugin import Snapshot
+
+from backend.api.features.desktop_preview import router
+
+
+def client():
+ app = FastAPI()
+ app.include_router(router, prefix="/api")
+ app.dependency_overrides[get_user_id] = lambda: "owner"
+ with TestClient(app) as client:
+ yield client
+
+
+def test_owner_redirect_is_private(client, configured_snapshot: Snapshot):
+ with patch(
+ "backend.api.features.desktop_preview.resolve_preview_link",
+ return_value="https://6080-sandbox.e2b.app/vnc.html?password=private",
+ ) as resolve:
+ response = client.get(
+ "/api/desktop-preview?token=encrypted", follow_redirects=False
+ )
+ resolve.assert_called_once_with("owner", "encrypted")
+ assert response.status_code == 307
+ assert response.headers["location"].endswith("?password=private")
+ assert response.headers["cache-control"] == "no-store"
+ assert response.headers["referrer-policy"] == "no-referrer"
+ assert "private" not in response.text
+ configured_snapshot.assert_match(
+ _response_json(response), "desktop_preview_owner_redirect"
+ )
+
+
+def test_wrong_owner_or_invalid_link_returns_no_credential(
+ client, configured_snapshot: Snapshot
+):
+ with patch(
+ "backend.api.features.desktop_preview.resolve_preview_link", return_value=None
+ ):
+ response = client.get(
+ "/api/desktop-preview?token=encrypted", follow_redirects=False
+ )
+ assert response.status_code == 404
+ assert "location" not in response.headers
+ configured_snapshot.assert_match(
+ _response_json(response), "desktop_preview_unavailable"
+ )
+
+
+def test_unauthenticated_request_is_rejected(configured_snapshot: Snapshot):
+ app = FastAPI()
+ app.include_router(router, prefix="/api")
+ with (
+ TestClient(app) as client,
+ patch("backend.api.features.desktop_preview.resolve_preview_link") as resolve,
+ ):
+ response = client.get(
+ "/api/desktop-preview?token=encrypted", follow_redirects=False
+ )
+ assert response.status_code == 401
+ resolve.assert_not_called()
+ configured_snapshot.assert_match(
+ _response_json(response), "desktop_preview_unauthenticated"
+ )
+
+
+def _response_json(response: Response) -> str:
+ return json.dumps(
+ {
+ "status": response.status_code,
+ "headers": dict(response.headers),
+ "body": response.text,
+ },
+ indent=2,
+ sort_keys=True,
+ )
diff --git a/autogpt_platform/backend/backend/api/features/experts/routes.py b/autogpt_platform/backend/backend/api/features/experts/routes.py
index 6e2c4f1ea5..55effddbb2 100644
--- a/autogpt_platform/backend/backend/api/features/experts/routes.py
+++ b/autogpt_platform/backend/backend/api/features/experts/routes.py
@@ -1,3 +1,5 @@
+import logging
+
import autogpt_libs.auth as autogpt_auth_lib
import fastapi
from fastapi import APIRouter, Security
@@ -32,9 +34,20 @@ from backend.api.features.experts.models import (
RaiseResult,
validate_avatar_url,
)
+from backend.blocks.desktop._api import DesktopStream
+from backend.copilot.computer import (
+ ComputerInfo,
+ describe_computer,
+ mounts_for,
+ open_desktop,
+)
+from backend.copilot.config import ChatConfig
+from backend.copilot.tools.e2b_sandbox import SandboxOwner, kill_expert_sandboxes
from backend.util import product_analytics
from backend.util.exceptions import NotFoundError
+logger = logging.getLogger(__name__)
+
router = APIRouter(
prefix="/experts",
tags=["experts", "private"],
@@ -357,6 +370,79 @@ async def get_expert_activity(
raise fastapi.HTTPException(status_code=404, detail=str(e))
+ "/{expert_id}/computer",
+ operation_id="getV2GetExpertComputer",
+ responses={404: {"description": "Expert not found"}},
+)
+async def get_expert_computer(
+ expert_id: str,
+ user_id: str = Security(autogpt_auth_lib.get_user_id),
+) -> ComputerInfo:
+ """The expert's own computer: its shell and desktop boxes as E2B lists them.
+
+ Listing never wakes a paused box, so the Computer tab can refresh freely.
+ """
+ expert = await experts_db.get_expert(user_id, expert_id, include_workflows=False)
+ if expert is None:
+ raise fastapi.HTTPException(
+ status_code=404, detail=f"Expert {expert_id} not found"
+ )
+ return await describe_computer(
+ SandboxOwner(kind="expert", id=expert_id), mounts_for(user_id, expert_id)
+ )
+
+
+ "/{expert_id}/computer/desktop",
+ operation_id="postV2StartExpertDesktop",
+ responses={
+ 404: {"description": "Expert not found"},
+ 502: {"description": "The desktop could not be started"},
+ 503: {"description": "E2B is not configured"},
+ },
+)
+async def start_expert_desktop(
+ expert_id: str,
+ user_id: str = Security(autogpt_auth_lib.get_user_id),
+) -> DesktopStream:
+ """Start or resume the expert's desktop and return its live stream.
+
+ This is the same box the expert's next ``start_desktop`` turn reconnects
+ to, so what the user does here is what the expert sees.
+ """
+ expert = await experts_db.get_expert(user_id, expert_id, include_workflows=False)
+ if expert is None:
+ raise fastapi.HTTPException(
+ status_code=404, detail=f"Expert {expert_id} not found"
+ )
+ api_key = ChatConfig().active_e2b_api_key
+ if not api_key:
+ raise fastapi.HTTPException(
+ status_code=503, detail="E2B is not configured on this deployment."
+ )
+ try:
+ stream, _created, _shared = await open_desktop(
+ SandboxOwner(kind="expert", id=expert_id),
+ mounts_for(user_id, expert_id),
+ api_key,
+ user_id=user_id,
+ )
+ except Exception as exc:
+ logger.error(
+ "[E2B] start_expert_desktop failed for %s: %s",
+ expert_id[:12],
+ exc,
+ exc_info=True,
+ )
+ # The cause is in the server log; provider errors can carry sandbox
+ # ids and infrastructure detail that the client has no use for.
+ raise fastapi.HTTPException(
+ status_code=502, detail="Failed to start the desktop."
+ )
+ return stream
+
+
class GrantCredentialsRequest(BaseModel):
credential_ids: list[str] = Field(min_length=1, max_length=50)
@@ -583,4 +669,16 @@ async def archive_expert(
await experts_db.archive_expert(user_id, expert_id)
except experts_db.ExpertNotFoundError as e:
raise fastapi.HTTPException(status_code=404, detail=str(e))
+ # The expert's computer goes with it. Best-effort: the archive is already
+ # committed and a slow E2B call must not turn it into a 5xx. Its volume is
+ # deliberately kept — files outlive the machine.
+ if api_key := ChatConfig().active_e2b_api_key:
+ try:
+ await kill_expert_sandboxes(expert_id, api_key)
+ except Exception:
+ logger.warning(
+ "[E2B] Failed to kill sandboxes for archived expert %s",
+ expert_id[:12],
+ exc_info=True,
+ )
return fastapi.Response(status_code=204)
diff --git a/autogpt_platform/backend/backend/api/features/experts/routes_test.py b/autogpt_platform/backend/backend/api/features/experts/routes_test.py
index 24a1979875..4218f08b95 100644
--- a/autogpt_platform/backend/backend/api/features/experts/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/experts/routes_test.py
@@ -1855,3 +1855,110 @@ def test_list_expert_setup_items_is_not_swallowed_by_the_expert_route(
assert response.json()[0]["resolution"] == "connect"
assert response.json()[0]["providers"] == ["notion"]
mock_list.assert_awaited_once()
+
+
+# --- Computer -----------------------------------------------------------------
+
+
+def test_get_expert_computer_describes_the_experts_boxes(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+) -> None:
+ from backend.copilot.computer import ComputerInfo
+
+ mocker.patch(
+ "backend.api.features.experts.routes.experts_db.get_expert",
+ new_callable=AsyncMock,
+ return_value=_make_expert(),
+ )
+ describe = mocker.patch(
+ "backend.api.features.experts.routes.describe_computer",
+ new_callable=AsyncMock,
+ return_value=ComputerInfo(
+ owner_kind="expert", owner_id="expert-1", e2b_active=True
+ ),
+ )
+
+ response = client.get("/experts/expert-1/computer")
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["owner_kind"] == "expert" and body["owner_id"] == "expert-1"
+ owner, mounts = describe.await_args.args
+ assert owner.kind == "expert" and owner.id == "expert-1"
+ assert set(mounts) == {"/home/user/workspace", "/home/user/shared"}
+
+
+def test_get_expert_computer_unknown_expert_returns_404(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocker.patch(
+ "backend.api.features.experts.routes.experts_db.get_expert",
+ new_callable=AsyncMock,
+ return_value=None,
+ )
+ assert client.get("/experts/nope/computer").status_code == 404
+
+
+def test_start_expert_desktop_returns_the_stream(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ from backend.blocks.desktop._api import DesktopStream
+
+ mocker.patch(
+ "backend.api.features.experts.routes.experts_db.get_expert",
+ new_callable=AsyncMock,
+ return_value=_make_expert(),
+ )
+ config = mocker.patch("backend.api.features.experts.routes.ChatConfig")
+ config.return_value.active_e2b_api_key = "e2b-key"
+ open_desktop = mocker.patch(
+ "backend.api.features.experts.routes.open_desktop",
+ new_callable=AsyncMock,
+ return_value=(
+ DesktopStream(url="https://6080-sbx.e2b.app/vnc.html", sandbox_id="sbx"),
+ True,
+ True,
+ ),
+ )
+
+ response = client.post("/experts/expert-1/computer/desktop")
+
+ assert response.status_code == 200
+ assert response.json()["url"] == "https://6080-sbx.e2b.app/vnc.html"
+ owner = open_desktop.await_args.args[0]
+ assert owner.kind == "expert" and owner.id == "expert-1"
+
+
+def test_start_expert_desktop_failure_is_a_502_with_a_fixed_message(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocker.patch(
+ "backend.api.features.experts.routes.experts_db.get_expert",
+ new_callable=AsyncMock,
+ return_value=_make_expert(),
+ )
+ config = mocker.patch("backend.api.features.experts.routes.ChatConfig")
+ config.return_value.active_e2b_api_key = "e2b-key"
+ mocker.patch(
+ "backend.api.features.experts.routes.open_desktop",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("sandbox sbx-123 rejected by api.e2b.app"),
+ )
+ response = client.post("/experts/expert-1/computer/desktop")
+ assert response.status_code == 502
+ # The provider's words stay in the log, never in the response.
+ assert response.json()["detail"] == "Failed to start the desktop."
+
+
+def test_start_expert_desktop_without_e2b_is_503(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocker.patch(
+ "backend.api.features.experts.routes.experts_db.get_expert",
+ new_callable=AsyncMock,
+ return_value=_make_expert(),
+ )
+ config = mocker.patch("backend.api.features.experts.routes.ChatConfig")
+ config.return_value.active_e2b_api_key = None
+ assert client.post("/experts/expert-1/computer/desktop").status_code == 503
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index e4c913b148..30fc2cac37 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -37,6 +37,7 @@ import backend.api.features.builder.routes
import backend.api.features.chat.routes as chat_routes
import backend.api.features.chat.share as chat_share
import backend.api.features.chat.speech as chat_speech
+import backend.api.features.desktop_preview
import backend.api.features.experiments
import backend.api.features.experts.routes as experts_routes
import backend.api.features.graph_executions.review.routes
@@ -390,6 +391,9 @@ app.add_exception_handler(PreconditionFailed, handle_internal_http_error(428))
app.add_exception_handler(Exception, handle_internal_http_error(500))
app.include_router(backend.api.features.v1.v1_router, tags=["v1"], prefix="/api")
+app.include_router(
+ backend.api.features.desktop_preview.router, tags=["v1"], prefix="/api"
+)
app.include_router(subscription_trial_routes.router, prefix="/api")
app.include_router(
api_keys_routes.router,
diff --git a/autogpt_platform/backend/backend/blocks/claude_code.py b/autogpt_platform/backend/backend/blocks/claude_code.py
index 8e1720e19a..9a1209d13a 100644
--- a/autogpt_platform/backend/backend/blocks/claude_code.py
+++ b/autogpt_platform/backend/backend/blocks/claude_code.py
@@ -25,6 +25,7 @@ from backend.util.sandbox_files import (
SandboxFileOutput,
extract_and_store_sandbox_files,
)
+from backend.util.sandbox_metadata import SandboxMetadata, owned_by_user
if TYPE_CHECKING:
from backend.executor.utils import ExecutionContext
@@ -323,7 +324,19 @@ class ClaudeCodeBlock(Block):
try:
# Either reconnect to existing sandbox or create a new one
if existing_sandbox_id:
- # Reconnect to existing sandbox for conversation continuation
+ # Reconnect to existing sandbox for conversation continuation.
+ # The id is caller-supplied and any id connects under our key,
+ # so the box must be stamped with this user before it is used.
+ # The stamp is read before connecting: a connect resumes a
+ # paused box on its owner's bill, so a foreign id is refused
+ # without waking it.
+ info = await BaseAsyncSandbox.get_info(
+ existing_sandbox_id, api_key=e2b_api_key
+ )
+ if not owned_by_user(info.metadata, execution_context.user_id):
+ raise PermissionError(
+ f"Sandbox {existing_sandbox_id} does not belong to this user"
+ )
sandbox = await BaseAsyncSandbox.connect(
sandbox_id=existing_sandbox_id,
api_key=e2b_api_key,
@@ -335,6 +348,9 @@ class ClaudeCodeBlock(Block):
api_key=e2b_api_key,
timeout=timeout,
envs={"ANTHROPIC_API_KEY": anthropic_api_key},
+ metadata=SandboxMetadata.for_block(
+ execution_context, "claude_code", self.id, self.DEFAULT_TEMPLATE
+ ).as_e2b(),
)
# Install Claude Code from npm (ensures we get the latest version)
diff --git a/autogpt_platform/backend/backend/blocks/claude_code_test.py b/autogpt_platform/backend/backend/blocks/claude_code_test.py
new file mode 100644
index 0000000000..d0c97063d1
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/claude_code_test.py
@@ -0,0 +1,49 @@
+"""Claude Code only resumes a sandbox that was created for the same user."""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from backend.blocks.claude_code import ClaudeCodeBlock, ClaudeCodeExecutionError
+from backend.executor.utils import ExecutionContext
+
+
+async def test_resuming_another_users_sandbox_is_refused():
+ block = ClaudeCodeBlock()
+ theirs = AsyncMock()
+ theirs.get_info = AsyncMock(
+ return_value=AsyncMock(
+ metadata={
+ "service": "autogpt-platform",
+ "autogpt_owner": "user:user-a",
+ "autogpt_kind": "claude_code",
+ "autogpt_source": "block",
+ "autogpt_env": "dev",
+ "autogpt_user": "user-a",
+ }
+ )
+ )
+ with patch("backend.blocks.claude_code.BaseAsyncSandbox") as cls:
+ # The stamp is read through the static ``get_info`` before any connect.
+ cls.get_info = AsyncMock(return_value=theirs.get_info.return_value)
+ cls.connect = AsyncMock(return_value=theirs)
+ # The block wraps every failure in its own error; the reason survives.
+ with pytest.raises(ClaudeCodeExecutionError, match="does not belong"):
+ await block.execute_claude_code(
+ e2b_api_key="k",
+ anthropic_api_key="a",
+ prompt="hi",
+ timeout=60,
+ setup_commands=[],
+ working_directory="/home/user",
+ session_id="s1",
+ existing_sandbox_id="sb-theirs",
+ conversation_history="",
+ dispose_sandbox=True,
+ execution_context=ExecutionContext(user_id="user-b"),
+ )
+ cls.get_info.assert_awaited_once_with("sb-theirs", api_key="k")
+ # Connecting would resume the other user's box on their bill.
+ cls.connect.assert_not_awaited()
+ theirs.commands.run.assert_not_awaited()
+ theirs.kill.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/blocks/code_executor.py b/autogpt_platform/backend/backend/blocks/code_executor.py
index 9ae29e798d..35e83e003a 100644
--- a/autogpt_platform/backend/backend/blocks/code_executor.py
+++ b/autogpt_platform/backend/backend/blocks/code_executor.py
@@ -27,6 +27,7 @@ from backend.util.sandbox_files import (
SandboxFileOutput,
extract_and_store_sandbox_files,
)
+from backend.util.sandbox_metadata import SandboxMetadata, owned_by_user
if TYPE_CHECKING:
from backend.executor.utils import ExecutionContext
@@ -129,6 +130,7 @@ class BaseE2BExecutorMixin:
execution_context: Optional["ExecutionContext"] = None,
extract_files: bool = False,
envs: Optional[dict[str, str]] = None,
+ metadata: Optional[dict[str, str]] = None,
):
"""
Unified code execution method that handles all three use cases:
@@ -144,14 +146,28 @@ class BaseE2BExecutorMixin:
files: list[SandboxFileOutput] = []
try:
if sandbox_id:
- # Connect to existing sandbox (ExecuteCodeStepBlock case)
+ # Connect to existing sandbox (ExecuteCodeStepBlock case). The
+ # id is caller-supplied and any id connects under our key, so
+ # the box must be stamped with this user before it is used.
+ # The stamp is read before connecting: a connect resumes a
+ # paused box on its owner's bill, so a foreign id is refused
+ # without waking it.
+ info = await AsyncSandbox.get_info(sandbox_id, api_key=api_key)
+ user_id = execution_context.user_id if execution_context else None
+ if not owned_by_user(info.metadata, user_id):
+ raise PermissionError(
+ f"Sandbox {sandbox_id} does not belong to this user"
+ )
sandbox = await AsyncSandbox.connect(
sandbox_id=sandbox_id, api_key=api_key
)
else:
# Create new sandbox (ExecuteCodeBlock/InstantiateCodeSandboxBlock case)
sandbox = await AsyncSandbox.create(
- api_key=api_key, template=template_id, timeout=timeout
+ api_key=api_key,
+ template=template_id,
+ timeout=timeout,
+ metadata=metadata,
)
if setup_commands:
for cmd in setup_commands:
@@ -376,6 +392,9 @@ class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):
execution_context=execution_context,
extract_files=True,
envs=envs,
+ metadata=SandboxMetadata.for_block(
+ execution_context, "code", self.id, input_data.template_id
+ ).as_e2b(),
)
# Determine result object shape & filter out empty formats
@@ -485,7 +504,12 @@ class InstantiateCodeSandboxBlock(Block, BaseE2BExecutorMixin):
)
async def run(
- self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
+ self,
+ input_data: Input,
+ *,
+ credentials: APIKeyCredentials,
+ execution_context: "ExecutionContext",
+ **kwargs,
) -> BlockOutput:
try:
_, text_output, stdout, stderr, sandbox_id, _ = await self.execute_code(
@@ -495,6 +519,9 @@ class InstantiateCodeSandboxBlock(Block, BaseE2BExecutorMixin):
template_id=input_data.template_id,
setup_commands=input_data.setup_commands,
timeout=input_data.timeout,
+ metadata=SandboxMetadata.for_block(
+ execution_context, "code", self.id, input_data.template_id
+ ).as_e2b(),
)
if sandbox_id:
yield "sandbox_id", sandbox_id
@@ -585,7 +612,12 @@ class ExecuteCodeStepBlock(Block, BaseE2BExecutorMixin):
)
async def run(
- self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
+ self,
+ input_data: Input,
+ *,
+ credentials: APIKeyCredentials,
+ execution_context: "ExecutionContext",
+ **kwargs,
) -> BlockOutput:
try:
results, text_output, stdout, stderr, _, _ = await self.execute_code(
@@ -594,6 +626,8 @@ class ExecuteCodeStepBlock(Block, BaseE2BExecutorMixin):
language=input_data.language,
sandbox_id=input_data.sandbox_id,
dispose_sandbox=input_data.dispose_sandbox,
+ # The ownership check on the supplied id needs the caller.
+ execution_context=execution_context,
)
# Determine result object shape & filter out empty formats
diff --git a/autogpt_platform/backend/backend/blocks/code_executor_test.py b/autogpt_platform/backend/backend/blocks/code_executor_test.py
index fe329e31a3..3d7ddf4413 100644
--- a/autogpt_platform/backend/backend/blocks/code_executor_test.py
+++ b/autogpt_platform/backend/backend/blocks/code_executor_test.py
@@ -10,7 +10,7 @@ injection -- analogous to parameterized SQL queries.
import base64
import json
import uuid
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -18,6 +18,7 @@ from backend.blocks.code_executor import (
TEST_CREDENTIALS,
TEST_CREDENTIALS_INPUT,
ExecuteCodeBlock,
+ ExecuteCodeStepBlock,
ProgrammingLanguage,
)
from backend.blocks.code_executor_helpers import (
@@ -263,3 +264,95 @@ class TestExecuteCodeBlockRun:
assert any(name == "error" for name, _ in outputs)
mock.assert_not_called()
+
+
+class TestConnectToExistingSandbox:
+ """A sandbox id is caller input; under the platform key any id connects,
+ so the box's stamped user decides whether it may be used."""
+
+ def _sandbox(self, stamped: dict) -> AsyncMock:
+ sandbox = AsyncMock()
+ sandbox.get_info = AsyncMock(return_value=AsyncMock(metadata=stamped))
+ return sandbox
+
+ @staticmethod
+ def _sdk(cls: MagicMock, box: AsyncMock) -> None:
+ """The stamp is read through the static ``get_info`` before any connect."""
+ cls.get_info = AsyncMock(return_value=box.get_info.return_value)
+ cls.connect = AsyncMock(return_value=box)
+
+ async def test_another_users_sandbox_is_refused_and_left_alone(self):
+ block = ExecuteCodeBlock()
+ context = ExecutionContext(user_id="user-b", graph_exec_id="gexec-1")
+ theirs = self._sandbox(
+ {
+ "service": "autogpt-platform",
+ "autogpt_owner": "user:user-a",
+ "autogpt_kind": "code",
+ "autogpt_source": "block",
+ "autogpt_env": "dev",
+ "autogpt_user": "user-a",
+ }
+ )
+ with patch("backend.blocks.code_executor.AsyncSandbox") as cls:
+ self._sdk(cls, theirs)
+ with pytest.raises(PermissionError, match="does not belong"):
+ await block.execute_code(
+ api_key="k",
+ code="print(1)",
+ language=ProgrammingLanguage.PYTHON,
+ sandbox_id="sb-theirs",
+ dispose_sandbox=True,
+ execution_context=context,
+ )
+ cls.get_info.assert_awaited_once_with("sb-theirs", api_key="k")
+ # Connecting would resume the other user's box on their bill.
+ cls.connect.assert_not_awaited()
+ theirs.run_code.assert_not_awaited()
+ # Not ours to kill either, even with dispose_sandbox set.
+ theirs.kill.assert_not_awaited()
+
+ async def test_step_block_hands_the_caller_to_the_ownership_check(self):
+ """The step block always reconnects by id, so without the caller's
+ context every run would be refused as not the user's box."""
+ block = ExecuteCodeStepBlock()
+ context = ExecutionContext(user_id="user-a", graph_exec_id="gexec-1")
+ with patch.object(
+ block, "execute_code", AsyncMock(return_value=([], "", "", "", "sb", []))
+ ) as execute:
+ async for _ in block.run(
+ ExecuteCodeStepBlock.Input(
+ credentials=TEST_CREDENTIALS_INPUT,
+ sandbox_id="sb-mine",
+ step_code="print(1)",
+ ),
+ credentials=TEST_CREDENTIALS,
+ execution_context=context,
+ ):
+ pass
+ assert execute.await_args.kwargs["execution_context"] is context
+ assert execute.await_args.kwargs["sandbox_id"] == "sb-mine"
+
+ async def test_without_a_user_no_existing_sandbox_can_be_used(self):
+ block = ExecuteCodeBlock()
+ box = self._sandbox(
+ {
+ "service": "autogpt-platform",
+ "autogpt_owner": "user:user-a",
+ "autogpt_kind": "code",
+ "autogpt_source": "block",
+ "autogpt_env": "dev",
+ "autogpt_user": "user-a",
+ }
+ )
+ with patch("backend.blocks.code_executor.AsyncSandbox") as cls:
+ self._sdk(cls, box)
+ with pytest.raises(PermissionError):
+ await block.execute_code(
+ api_key="k",
+ code="print(1)",
+ language=ProgrammingLanguage.PYTHON,
+ sandbox_id="sb-theirs",
+ execution_context=None,
+ )
+ cls.connect.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/blocks/desktop/_api.py b/autogpt_platform/backend/backend/blocks/desktop/_api.py
new file mode 100644
index 0000000000..d0b3b07fb4
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/desktop/_api.py
@@ -0,0 +1,349 @@
+"""Async E2B desktop client built on the base ``e2b`` SDK.
+
+The official ``e2b-desktop`` package is a thin synchronous wrapper around the
+``desktop`` template: xdotool for input, scrot for screenshots, and
+x11vnc + noVNC for the interactive stream. We replicate that surface on
+``AsyncSandbox`` directly because the backend is fully async and the sync
+package pins a conflicting pillow version.
+"""
+
+import asyncio
+import contextlib
+import secrets
+import shlex
+import string
+from typing import Literal, Mapping, Optional
+
+from e2b import AsyncSandbox, AsyncVolume, SandboxLifecycle
+from pydantic import BaseModel
+
+DESKTOP_TEMPLATE = "desktop"
+HOME_PATH = "/home/user"
+WORKSPACE_PATH = "/home/user/workspace"
+# Where an expert's own computer sees its owning user's shared workspace. The
+# expert's durable home is WORKSPACE_PATH, same as everyone else's.
+SHARED_PATH = "/home/user/shared"
+# Standard XFCE user dirs redirected into the volume so a person's natural
+# desktop activity (browser downloads, files saved to the desktop) persists.
+PERSISTENT_HOME_DIRS = ("Downloads", "Desktop", "Documents")
+DISPLAY = ":0"
+VNC_PORT = 5900
+STREAM_PORT = 6080
+# The stream password lives next to x11vnc's hashed one so a resume can hand
+# back the URL the user already holds instead of restarting the proxy.
+STREAM_PASSWORD_PATH = f"{HOME_PATH}/.vnc/agpt_stream_password"
+# Bound on the E2B volumes API (private beta) so a slow create cannot stall
+# sandbox creation; the by-name mount fallback is the normal path anyway.
+VOLUME_API_TIMEOUT_SECONDS = 10
+_KILL_TIMEOUT_SECONDS = 10
+
+_TYPE_CHUNK_SIZE = 25
+_TYPE_DELAY_MS = 75
+_READY_POLL_SECONDS = 0.5
+# XFCE routinely takes 10-15 s to bring up xfwm4 on a cold start.
+_READY_POLL_ATTEMPTS = 40
+
+
+class DesktopStream(BaseModel):
+ kind: Literal["desktop_stream"] = "desktop_stream"
+ url: str
+ provider: Literal["e2b"] = "e2b"
+ sandbox_id: str
+ requires_auth: bool = False
+
+
+class PersistenceInfo(BaseModel):
+ volume_mounted: bool = False
+ volume_name: Optional[str] = None
+ warning: Optional[str] = None
+ # Every mount path that was attached (WORKSPACE_PATH plus, for an expert's
+ # computer, SHARED_PATH). ``volume_mounted``/``volume_name`` describe the
+ # WORKSPACE_PATH mount specifically.
+ mounted_paths: list[str] = []
+
+
+class DesktopSession:
+ def __init__(self, sandbox: AsyncSandbox):
+ self.sandbox = sandbox
+
+ @property
+ def sandbox_id(self) -> str:
+ return self.sandbox.sandbox_id
+
+ @classmethod
+ async def create(
+ cls,
+ api_key: str,
+ timeout_seconds: int,
+ width: int,
+ height: int,
+ volume_mounts: Optional[Mapping[str, str]] = None,
+ template: str = DESKTOP_TEMPLATE,
+ metadata: Optional[Mapping[str, str]] = None,
+ ) -> tuple["DesktopSession", PersistenceInfo]:
+ """Create a desktop sandbox.
+
+ *volume_mounts* maps mount paths to durable volume names (see
+ ``workspace_volume_mounts``); *metadata* is stamped on the sandbox so
+ its owner can find it again through the E2B API if the cached id is
+ lost.
+ """
+ sandbox, persistence = await _create_sandbox_with_volumes(
+ volume_mounts, api_key, timeout_seconds, template, metadata
+ )
+ session = cls(sandbox)
+ try:
+ await session.ensure_display(width, height)
+ # WORKSPACE_PATH always exists (blocks default their cwd to it),
+ # mounted or not; mounted paths get their mkdir as well.
+ paths = dict.fromkeys([WORKSPACE_PATH, *persistence.mounted_paths])
+ await session.run_command(
+ "mkdir -p " + " ".join(shlex.quote(p) for p in paths)
+ )
+ if persistence.volume_mounted:
+ await session.ensure_persistent_home()
+ except BaseException:
+ # The box is on the meter but no caller has its id yet: kill it
+ # rather than leak a sandbox that would bill until timeout and
+ # then sit paused forever.
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(sandbox.kill(), timeout=_KILL_TIMEOUT_SECONDS)
+ raise
+ return session, persistence
+
+ @classmethod
+ async def connect(
+ cls, sandbox_id: str, api_key: str, timeout_seconds: Optional[int] = None
+ ) -> "DesktopSession":
+ """Reattach to a desktop; *timeout_seconds* re-arms its running-time
+ limit, otherwise the SDK's 300 s default would pause a resumed desktop
+ under the user long before a freshly created one."""
+ sandbox = await AsyncSandbox.connect(
+ sandbox_id, api_key=api_key, timeout=timeout_seconds
+ )
+ return cls(sandbox)
+
+ async def start_stream(self) -> DesktopStream:
+ """Return the live stream URL, starting the VNC stack only if needed.
+
+ A resume (or a second ``start_desktop`` call) must not restart x11vnc
+ and noVNC: that would sever the stream the user is watching and rotate
+ the password baked into the URL they already hold. When the proxy is
+ still serving — E2B's pause/resume restores processes — the saved
+ password is reused and the same URL comes back.
+ """
+ password = await self._running_stream_password()
+ if password is None:
+ password = "".join(
+ secrets.choice(string.ascii_letters + string.digits) for _ in range(16)
+ )
+ await self.run_command(
+ "pkill -f '[n]ovnc_proxy' || true; pkill -x x11vnc || true"
+ )
+ await self.run_command(
+ f"mkdir -p ~/.vnc && x11vnc -storepasswd {password} ~/.vnc/passwd"
+ )
+ await self.run_command(
+ f"x11vnc -bg -display {DISPLAY} -forever -wait 50 -shared "
+ f"-rfbport {VNC_PORT} -usepw >/tmp/x11vnc.log 2>/tmp/x11vnc_stderr.log"
+ )
+ await self.sandbox.commands.run(
+ f"cd /opt/noVNC/utils && ./novnc_proxy --vnc localhost:{VNC_PORT} "
+ f"--listen {STREAM_PORT} --web /opt/noVNC > /tmp/novnc.log 2>&1",
+ background=True,
+ )
+ await self._wait_for(f'netstat -tuln | grep ":{STREAM_PORT} "')
+ await self.run_command(
+ f"umask 077 && printf %s {shlex.quote(password)} "
+ f"> {shlex.quote(STREAM_PASSWORD_PATH)}"
+ )
+ host = self.sandbox.get_host(STREAM_PORT)
+ url = (
+ f"https://{host}/vnc.html"
+ f"?autoconnect=true&resize=scale&password={password}"
+ )
+ return DesktopStream(url=url, sandbox_id=self.sandbox_id)
+
+ async def _running_stream_password(self) -> Optional[str]:
+ """The saved stream password, only while noVNC is actually listening."""
+ if not await self._check(f'netstat -tuln | grep -q ":{STREAM_PORT} "'):
+ return None
+ try:
+ saved = await self.sandbox.files.read(STREAM_PASSWORD_PATH)
+ except Exception:
+ return None
+ saved = saved.strip() if isinstance(saved, str) else ""
+ return saved or None
+
+ async def run_command(
+ self, command: str, cwd: Optional[str] = None, timeout: int = 60
+ ):
+ return await self.sandbox.commands.run(
+ command, cwd=cwd, timeout=timeout, envs={"DISPLAY": DISPLAY}
+ )
+
+ async def is_workspace_mounted(self) -> bool:
+ return await self.is_mounted(WORKSPACE_PATH)
+
+ async def is_mounted(self, path: str) -> bool:
+ return await self._check(f"mountpoint -q {shlex.quote(path)}")
+
+ async def ensure_persistent_home(self) -> None:
+ """Redirect the standard user dirs into the mounted volume.
+
+ Downloads/Desktop/Documents become symlinks into ``WORKSPACE_PATH`` so
+ files a person creates through the live desktop persist on the volume
+ (surviving sandbox destroy) rather than living only in the ephemeral
+ rootfs. Idempotent: safe to run on every sandbox that mounts the
+ volume, migrating any pre-existing content in once.
+ """
+ dirs = " ".join(PERSISTENT_HOME_DIRS)
+ script = (
+ f"for d in {dirs}; do "
+ f'mkdir -p {WORKSPACE_PATH}/"$d"; '
+ f'if [ -d {HOME_PATH}/"$d" ] && [ ! -L {HOME_PATH}/"$d" ]; then '
+ f'cp -an {HOME_PATH}/"$d"/. {WORKSPACE_PATH}/"$d"/ 2>/dev/null || true; '
+ f'rm -rf {HOME_PATH}/"$d"; fi; '
+ f'ln -sfn {WORKSPACE_PATH}/"$d" {HOME_PATH}/"$d"; '
+ f"done"
+ )
+ await self.run_command(script)
+
+ async def pause(self) -> None:
+ await self.sandbox.pause()
+
+ async def kill(self) -> None:
+ await self.sandbox.kill()
+
+ async def ensure_display(self, width: int, height: int) -> None:
+ if await self._check("pgrep -x xfwm4"):
+ return
+ # Both daemons must write to files, never to the command's pipes: a
+ # background command's stdout/stderr stream to the SDK handle, and
+ # once that handle is dropped the pipe closes, so the first warning
+ # Xvfb logs afterwards (Chrome opening a second window is enough)
+ # kills it with SIGPIPE and every X client with it.
+ if not await self._check(f"xdpyinfo -display {DISPLAY}"):
+ await self.sandbox.commands.run(
+ f"Xvfb {DISPLAY} -ac -screen 0 {width}x{height}x24 -retro -dpi 96 "
+ "-nolisten tcp -nolisten unix > /tmp/xvfb.log 2>&1",
+ background=True,
+ )
+ await self._wait_for(f"xdpyinfo -display {DISPLAY}")
+ await self.sandbox.commands.run(
+ "startxfce4 > /tmp/xfce.log 2>&1",
+ background=True,
+ envs={"DISPLAY": DISPLAY},
+ )
+ # Without gating on the window manager, the first xdotool call blocks
+ # 10-15 s inside a half-started XFCE instead of failing fast here.
+ await self._wait_for("pgrep -x xfwm4")
+
+ async def _check(self, command: str) -> bool:
+ try:
+ await self.sandbox.commands.run(command)
+ return True
+ except Exception:
+ return False
+
+ async def _wait_for(self, command: str) -> None:
+ for _ in range(_READY_POLL_ATTEMPTS):
+ if await self._check(command):
+ return
+ await asyncio.sleep(_READY_POLL_SECONDS)
+ raise TimeoutError(f"Timed out waiting for: {command.split()[0]}")
+
+
+def _sandbox_create_kwargs(
+ api_key: str,
+ timeout_seconds: int,
+ template: str,
+ metadata: Optional[Mapping[str, str]] = None,
+) -> dict:
+ kwargs: dict = {
+ "template": template,
+ "api_key": api_key,
+ "timeout": timeout_seconds,
+ "lifecycle": SandboxLifecycle(on_timeout="pause", auto_resume=True),
+ }
+ if metadata:
+ kwargs["metadata"] = dict(metadata)
+ return kwargs
+
+
+async def resolve_volume(volume_name: str, api_key: str) -> "AsyncVolume | str":
+ """Create *volume_name* if it does not exist yet, else mount it by name.
+
+ E2B has no get-or-create, so the create is expected to fail on every run
+ after the first and the by-name fallback is the normal path. Bounded so a
+ slow volumes API cannot stall sandbox creation. Shared with the CoPilot
+ shell (``copilot/tools/e2b_sandbox``) so both surfaces resolve identically.
+ """
+ try:
+ return await asyncio.wait_for(
+ AsyncVolume.create(volume_name, api_key=api_key),
+ timeout=VOLUME_API_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return volume_name
+
+
+# Client-side deadline on each create call (E2B provisions in 5-15 s), and
+# the number of mounted attempts before the volumes are given up on. A
+# desktop keeps its volumes for life once created, so one transient failure
+# must not decide that for it.
+CREATE_TIMEOUT_SECONDS = 30
+MOUNTED_CREATE_ATTEMPTS = 2
+
+
+async def _create_sandbox_with_volumes(
+ volume_mounts: Optional[Mapping[str, str]],
+ api_key: str,
+ timeout_seconds: int,
+ template: str = DESKTOP_TEMPLATE,
+ metadata: Optional[Mapping[str, str]] = None,
+) -> tuple[AsyncSandbox, PersistenceInfo]:
+ kwargs = _sandbox_create_kwargs(api_key, timeout_seconds, template, metadata)
+ if not volume_mounts:
+ sandbox = await asyncio.wait_for(
+ AsyncSandbox.create(**kwargs), timeout=CREATE_TIMEOUT_SECONDS
+ )
+ return sandbox, PersistenceInfo()
+
+ paths = list(volume_mounts)
+ volumes = await asyncio.gather(
+ *(resolve_volume(volume_mounts[path], api_key) for path in paths)
+ )
+ mounts = dict(zip(paths, volumes))
+ mount_error: Exception | None = None
+ for attempt in range(1, MOUNTED_CREATE_ATTEMPTS + 1):
+ try:
+ sandbox = await asyncio.wait_for(
+ AsyncSandbox.create(**kwargs, volume_mounts=mounts),
+ timeout=CREATE_TIMEOUT_SECONDS,
+ )
+ except Exception as exc:
+ mount_error = exc
+ if attempt < MOUNTED_CREATE_ATTEMPTS:
+ await asyncio.sleep(attempt)
+ continue
+ return sandbox, PersistenceInfo(
+ volume_mounted=WORKSPACE_PATH in volume_mounts,
+ volume_name=volume_mounts.get(WORKSPACE_PATH),
+ mounted_paths=list(volume_mounts),
+ )
+
+ # The volumes really are unavailable: fall back to a volume-less box and
+ # say so in its stamp, so the Computer tab does not report mounts it
+ # does not have.
+ if kwargs.get("metadata"):
+ kwargs["metadata"] = {**kwargs["metadata"], "autogpt_mounts": "none"}
+ sandbox = await asyncio.wait_for(
+ AsyncSandbox.create(**kwargs), timeout=CREATE_TIMEOUT_SECONDS
+ )
+ return sandbox, PersistenceInfo(
+ warning=(
+ "Persistent volume unavailable (E2B volumes are in private beta); "
+ f"using suspend/resume persistence only: {mount_error}"
+ )
+ )
diff --git a/autogpt_platform/backend/backend/blocks/desktop/_common.py b/autogpt_platform/backend/backend/blocks/desktop/_common.py
new file mode 100644
index 0000000000..3dda4f6e48
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/desktop/_common.py
@@ -0,0 +1,81 @@
+from typing import Optional
+
+from pydantic import SecretStr
+
+from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.data.model import APIKeyCredentials
+
+__all__ = [
+ "CREDENTIALS_FIELD_DESCRIPTION",
+ "SHARED_PATH",
+ "TEST_CREDENTIALS",
+ "TEST_CREDENTIALS_INPUT",
+ "WORKSPACE_PATH",
+ "expert_volume_name",
+ "user_volume_name",
+ "workspace_volume_mounts",
+]
+
+TEST_CREDENTIALS = APIKeyCredentials(
+ id="01234567-89ab-cdef-0123-456789abcdef",
+ provider="e2b",
+ api_key=SecretStr("mock-e2b-api-key"),
+ title="Mock E2B API key",
+ expires_at=None,
+)
+
+TEST_CREDENTIALS_INPUT = {
+ "provider": TEST_CREDENTIALS.provider,
+ "id": TEST_CREDENTIALS.id,
+ "type": TEST_CREDENTIALS.type,
+ "title": TEST_CREDENTIALS.type,
+}
+
+CREDENTIALS_FIELD_DESCRIPTION = (
+ "Enter your API key for the E2B platform. "
+ "You can get it here - https://e2b.dev/dashboard"
+)
+
+
+def user_volume_name(user_id: str) -> str:
+ """Name of the per-user durable workspace volume.
+
+ Shared across every E2B surface for the user — desktop blocks, the CoPilot
+ on-demand desktop, and the CoPilot agent shell — so they all read/write the
+ same persistent ``/home/user/workspace``.
+ """
+ return f"autogpt-user-{user_id}"
+
+
+def expert_volume_name(expert_id: str) -> str:
+ """Name of a hired expert's own durable volume.
+
+ This is the expert's home, not the user's: it persists for the life of the
+ expert regardless of which chat, delegation or scheduled kickoff happens
+ to be running as them, and it outlives the expert's sandbox if that is
+ ever rebuilt.
+ """
+ return f"autogpt-expert-{expert_id}"
+
+
+def workspace_volume_mounts(
+ user_id: Optional[str], expert_id: Optional[str] = None
+) -> dict[str, str]:
+ """Mount map (``path -> volume name``) for a CoPilot shell or desktop.
+
+ Plain session: the user's volume at ``WORKSPACE_PATH``.
+
+ Expert session: the expert's own volume at ``WORKSPACE_PATH`` — the
+ persistent home it may customise freely — plus the owning user's volume
+ at ``SHARED_PATH``, so deliverables dropped there show up on the user's
+ desktop and in every other session on the account. Without a user there
+ is nothing to share, so only the expert's home is mounted.
+ """
+ mounts: dict[str, str] = {}
+ if expert_id:
+ mounts[WORKSPACE_PATH] = expert_volume_name(expert_id)
+ if user_id:
+ mounts[SHARED_PATH] = user_volume_name(user_id)
+ elif user_id:
+ mounts[WORKSPACE_PATH] = user_volume_name(user_id)
+ return mounts
diff --git a/autogpt_platform/backend/backend/blocks/desktop/_common_test.py b/autogpt_platform/backend/backend/blocks/desktop/_common_test.py
new file mode 100644
index 0000000000..f3a7d8e752
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/desktop/_common_test.py
@@ -0,0 +1,37 @@
+"""Volume naming and mount maps for the E2B desktop/shell surfaces."""
+
+from backend.blocks.desktop._common import (
+ SHARED_PATH,
+ WORKSPACE_PATH,
+ expert_volume_name,
+ user_volume_name,
+ workspace_volume_mounts,
+)
+
+
+class TestWorkspaceVolumeMounts:
+ def test_plain_session_mounts_user_volume_as_home(self):
+ assert workspace_volume_mounts("u1") == {WORKSPACE_PATH: user_volume_name("u1")}
+
+ def test_expert_gets_own_home_plus_users_shared_workspace(self):
+ mounts = workspace_volume_mounts("u1", "exp-1")
+ assert mounts == {
+ WORKSPACE_PATH: expert_volume_name("exp-1"),
+ SHARED_PATH: user_volume_name("u1"),
+ }
+ # The expert's home is its own, never the user's volume.
+ assert mounts[WORKSPACE_PATH] != user_volume_name("u1")
+
+ def test_expert_without_user_has_nothing_to_share(self):
+ assert workspace_volume_mounts(None, "exp-1") == {
+ WORKSPACE_PATH: expert_volume_name("exp-1")
+ }
+
+ def test_no_owner_means_no_mounts(self):
+ assert workspace_volume_mounts(None) == {}
+ assert workspace_volume_mounts("") == {}
+
+ def test_volume_names_are_scoped_by_kind(self):
+ assert expert_volume_name("abc") == "autogpt-expert-abc"
+ assert user_volume_name("abc") == "autogpt-user-abc"
+ assert expert_volume_name("abc") != user_volume_name("abc")
diff --git a/autogpt_platform/backend/backend/blocks/desktop/desktop_api_test.py b/autogpt_platform/backend/backend/blocks/desktop/desktop_api_test.py
new file mode 100644
index 0000000000..3d90db6fb0
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/desktop/desktop_api_test.py
@@ -0,0 +1,81 @@
+"""Desktop client: volumes are retried before they are given up on, every
+create has a client-side deadline, and a resumed desktop keeps its timeout."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.blocks.desktop._api import (
+ WORKSPACE_PATH,
+ DesktopSession,
+ _create_sandbox_with_volumes,
+)
+
+_M = "backend.blocks.desktop._api"
+_META = {"service": "autogpt-platform", "autogpt_mounts": "attached"}
+
+
[email protected](autouse=True)
+def _no_real_waits():
+ with patch(f"{_M}.asyncio.sleep", AsyncMock()):
+ yield
+
+
+async def test_mounted_create_is_retried_once_before_falling_back():
+ box = MagicMock()
+ with (
+ patch(f"{_M}.resolve_volume", AsyncMock(side_effect=lambda name, key: name)),
+ patch(f"{_M}.AsyncSandbox") as cls,
+ ):
+ cls.create = AsyncMock(side_effect=[RuntimeError("502"), box])
+ sandbox, info = await _create_sandbox_with_volumes(
+ {WORKSPACE_PATH: "vol-user"}, "k", 900, metadata=_META
+ )
+ assert sandbox is box and info.volume_mounted and info.warning is None
+ assert cls.create.await_count == 2
+ assert all("volume_mounts" in c.kwargs for c in cls.create.await_args_list)
+
+
+async def test_fallback_after_every_mounted_attempt_says_so_in_the_stamp():
+ box = MagicMock()
+ with (
+ patch(f"{_M}.resolve_volume", AsyncMock(side_effect=lambda name, key: name)),
+ patch(f"{_M}.AsyncSandbox") as cls,
+ ):
+ cls.create = AsyncMock(
+ side_effect=[RuntimeError("no volumes"), RuntimeError("no volumes"), box]
+ )
+ sandbox, info = await _create_sandbox_with_volumes(
+ {WORKSPACE_PATH: "vol-user"}, "k", 900, metadata=_META
+ )
+ assert sandbox is box and not info.volume_mounted and info.warning
+ fallback = cls.create.await_args_list[-1].kwargs
+ assert "volume_mounts" not in fallback
+ # The Computer tab reads this stamp; it must not claim mounts it lacks.
+ assert fallback["metadata"]["autogpt_mounts"] == "none"
+
+
+async def test_a_create_that_hangs_is_cut_off():
+ import asyncio
+
+ async def never(**_kwargs):
+ await asyncio.Event().wait()
+
+ with (
+ patch(f"{_M}.AsyncSandbox") as cls,
+ patch(f"{_M}.CREATE_TIMEOUT_SECONDS", 0.01),
+ ):
+ cls.create = AsyncMock(side_effect=never)
+ with pytest.raises(asyncio.TimeoutError):
+ await _create_sandbox_with_volumes(None, "k", 900)
+
+
+async def test_connect_rearms_the_running_time_limit():
+ with patch(f"{_M}.AsyncSandbox") as cls:
+ cls.connect = AsyncMock(return_value=MagicMock())
+ await DesktopSession.connect("sb-1", "k", timeout_seconds=900)
+ cls.connect.assert_awaited_once_with("sb-1", api_key="k", timeout=900)
diff --git a/autogpt_platform/backend/backend/blocks/desktop/desktop_display_test.py b/autogpt_platform/backend/backend/blocks/desktop/desktop_display_test.py
new file mode 100644
index 0000000000..ce7f3eed0e
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/desktop/desktop_display_test.py
@@ -0,0 +1,60 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from backend.blocks.desktop._api import DesktopSession
+
+
+def _session_with_fresh_box() -> tuple[DesktopSession, AsyncMock]:
+ """A sandbox where neither the window manager nor the X server is running yet."""
+ run = AsyncMock()
+
+ async def fake_run(command: str, **kwargs):
+ if command.startswith("pgrep -x xfwm4"):
+ # Not running until XFCE has been started.
+ if not any("startxfce4" in c.args[0] for c in run.await_args_list):
+ raise RuntimeError("no xfwm4")
+ return MagicMock()
+ if command.startswith("xdpyinfo"):
+ if not any("Xvfb" in c.args[0] for c in run.await_args_list):
+ raise RuntimeError("no X server")
+ return MagicMock()
+ return MagicMock()
+
+ run.side_effect = fake_run
+ sandbox = MagicMock()
+ sandbox.commands.run = run
+ return DesktopSession(sandbox), run
+
+
+class TestEnsureDisplay:
+ @pytest.mark.asyncio
+ async def test_display_daemons_never_write_to_the_dropped_command_pipes(self):
+ """Xvfb / XFCE run as background commands whose SDK handle is discarded.
+
+ Their stdout and stderr must go to files: once the handle is gone the
+ pipe is closed and the next warning Xvfb logs kills it with SIGPIPE,
+ taking every X client (the whole desktop) down with it.
+ """
+ session, run = _session_with_fresh_box()
+
+ await session.ensure_display(1280, 720)
+
+ background = [
+ c.args[0] for c in run.await_args_list if c.kwargs.get("background")
+ ]
+ xvfb = next(c for c in background if c.startswith("Xvfb"))
+ xfce = next(c for c in background if c.startswith("startxfce4"))
+ assert "1280x720x24" in xvfb
+ for command in (xvfb, xfce):
+ assert "> /tmp/" in command and "2>&1" in command, command
+
+ @pytest.mark.asyncio
+ async def test_running_window_manager_short_circuits(self):
+ session, run = _session_with_fresh_box()
+ run.side_effect = None # every check succeeds: xfwm4 is already up
+
+ await session.ensure_display(1280, 720)
+
+ assert run.await_count == 1
+ assert run.await_args.args[0].startswith("pgrep -x xfwm4")
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index 64945d8f47..9c9955d6e0 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -28,6 +28,7 @@ from openai.types import CompletionUsage
from openai.types.chat import ChatCompletionMessageParam, ChatCompletionToolParam
from opentelemetry import trace as otel_trace
+from backend.blocks.desktop._common import workspace_volume_mounts
from backend.copilot import engine_switch
from backend.copilot.anthropic_rate_card import (
compute_anthropic_cost_usd,
@@ -130,6 +131,11 @@ from backend.copilot.tools import (
get_available_tools,
kickoff_turn_disabled_tools,
)
+from backend.copilot.tools.e2b_sandbox import (
+ count_expert_turn,
+ get_or_create_sandbox,
+ pause_sandbox_direct,
+)
from backend.copilot.tools.session_context import build_session_context
from backend.copilot.tools.skills import (
build_skills_context,
@@ -175,6 +181,25 @@ logger = logging.getLogger(__name__)
# Set to hold background tasks to prevent garbage collection
_background_tasks: set[asyncio.Task[Any]] = set()
+
+def _pause_uncounted_box(
+ sandbox: Any, session_id: str, expert_id: str | None
+) -> asyncio.Task[Any] | None:
+ """Pause a box opened for a turn that ended before its turn was counted.
+
+ A session's box has nobody else on it, so it is paused straight away
+ (fire-and-forget, like the turn-end pause). An expert's box may be
+ carrying another turn and this one never counted itself, so it is left
+ for the lifecycle timeout rather than paused under someone else.
+ """
+ if expert_id:
+ return None
+ task = asyncio.create_task(pause_sandbox_direct(sandbox, session_id))
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+ return task
+
+
# Hint appended on the last tool round so the model wraps up with a summary
# instead of issuing another tool call that gets cut off cold. The shared
# ``tool_call_loop`` drops ``tools`` on the last iteration (see util/tool_call_loop.py),
@@ -1820,14 +1845,22 @@ async def stream_chat_completion_baseline(
e2b_api_key = config.active_e2b_api_key
if e2b_api_key:
try:
- from backend.copilot.tools.e2b_sandbox import get_or_create_sandbox
-
+ # An expert session runs on the expert's own persistent box;
+ # everything else gets a per-session sandbox.
e2b_sandbox = await get_or_create_sandbox(
session_id,
api_key=e2b_api_key,
template=config.e2b_sandbox_template,
timeout=config.e2b_sandbox_timeout,
on_timeout=config.e2b_sandbox_on_timeout,
+ volume_mounts=workspace_volume_mounts(user_id, session.expert_id),
+ expert_id=session.expert_id,
+ user_id=user_id,
+ # Counted just before the try/finally that releases it, below:
+ # everything between here and there can still fail or be
+ # stopped, and a count with no release keeps the expert's box
+ # unpaused at every later turn end.
+ count_turn=False,
)
except Exception:
logger.warning("[Baseline] E2B sandbox setup failed", exc_info=True)
@@ -2266,7 +2299,18 @@ async def stream_chat_completion_baseline(
list[ChatCompletionToolParam], _mark_tools_with_cache_control(tools)
)
- yield StreamStart(messageId=message_id, sessionId=session_id)
+ try:
+ yield StreamStart(messageId=message_id, sessionId=session_id)
+ except BaseException:
+ # Closed or cancelled while suspended on the first yield: the finally
+ # that pauses the box sits further down and would never run.
+ if e2b_sandbox is not None:
+ _pause_uncounted_box(e2b_sandbox, session_id, session.expert_id)
+ raise
+
+ if e2b_sandbox is not None:
+ # From here the finally below always runs, so the turn can be counted.
+ await count_expert_turn(session_id, session.expert_id)
# Propagate user/session context to Langfuse so all LLM calls within
# this request are grouped under a single trace with proper attribution.
@@ -2639,6 +2683,19 @@ async def stream_chat_completion_baseline(
session.clear_inflight_tool_calls()
state.tool_persistence.finish(state.session_messages)
+ # --- Pause E2B sandbox to stop billing between turns (parity with
+ # the SDK path). Fire-and-forget: best-effort and must not block the
+ # cleanup below. An expert's box is left running while another of
+ # its turns is still active.
+ if e2b_sandbox is not None:
+ pause_task = asyncio.create_task(
+ pause_sandbox_direct(
+ e2b_sandbox, session_id, expert_id=session.expert_id
+ )
+ )
+ _background_tasks.add(pause_task)
+ pause_task.add_done_callback(_background_tasks.discard)
+
# Pending messages are drained atomically at turn start and
# between tool rounds, so there's nothing to clear in finally.
# Any message pushed after the final drain window stays in the
diff --git a/autogpt_platform/backend/backend/copilot/baseline/uncounted_box_test.py b/autogpt_platform/backend/backend/copilot/baseline/uncounted_box_test.py
new file mode 100644
index 0000000000..96212f816d
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/baseline/uncounted_box_test.py
@@ -0,0 +1,27 @@
+"""A stream closed before the turn's try/finally is reached must not leave
+the session's box running; an expert's shared box is left to its timeout."""
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.copilot.baseline import service
+
+
+async def test_session_box_is_paused_when_the_stream_is_closed_early():
+ sandbox = MagicMock()
+ with patch.object(service, "pause_sandbox_direct", AsyncMock()) as pause:
+ task = service._pause_uncounted_box(sandbox, "sess-1", None)
+ assert task is not None
+ await task
+ pause.assert_awaited_once_with(sandbox, "sess-1")
+
+
+async def test_expert_box_is_left_running_because_its_turn_was_never_counted():
+ with patch.object(service, "pause_sandbox_direct", AsyncMock()) as pause:
+ assert service._pause_uncounted_box(MagicMock(), "sess-1", "exp-1") is None
+ await asyncio.sleep(0)
+ pause.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/computer.py b/autogpt_platform/backend/backend/copilot/computer.py
new file mode 100644
index 0000000000..a3f299fd83
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/computer.py
@@ -0,0 +1,319 @@
+"""An owner's computer on E2B: what exists, and how to get its desktop up.
+
+Both the expert page's Computer tab and the copilot side panel's Computer
+view read from here, so they agree on what "the box" is: the owner's shell
+sandbox and its on-demand desktop, found through the same E2B metadata the
+lifecycle module stamps on every sandbox (``copilot/tools/e2b_sandbox``).
+
+``describe_computer`` only lists — it never connects, so a paused box stays
+paused (connecting is what E2B's auto-resume reacts to). ``open_desktop`` is
+the one write: it creates or resumes the owner's desktop and hands back the
+live stream, exactly as the ``start_desktop`` tool does from inside a turn.
+"""
+
+import asyncio
+import contextlib
+import logging
+import uuid
+from datetime import datetime
+from typing import Any, Literal, Mapping, Optional
+
+from e2b import SandboxState
+from e2b.exceptions import NotFoundException
+from pydantic import BaseModel
+
+from backend.blocks.desktop._api import DesktopSession, DesktopStream
+from backend.blocks.desktop._common import (
+ SHARED_PATH,
+ WORKSPACE_PATH,
+ workspace_volume_mounts,
+)
+from backend.copilot.sdk.env import config as chat_config
+from backend.copilot.tools.e2b_sandbox import (
+ METADATA_MOUNTS,
+ SandboxKind,
+ SandboxNotOwnedError,
+ SandboxOwner,
+ connect_owned,
+ find_owned_sandbox_id,
+ list_owned_sandboxes,
+)
+from backend.data.redis_client import get_redis_async
+from backend.util.desktop_preview import create_preview_link
+
+logger = logging.getLogger(__name__)
+
+_DESKTOP_RESOLUTION = (1280, 720)
+_KILL_TIMEOUT_SECONDS = 10
+_RECONNECT_RETRY_DELAY_SECONDS = 1.0
+
+# Opening a desktop can be volume resolution, two create attempts, the
+# display coming up and the home setup: about four minutes at the very worst.
+# The open is cut off before the lock can lapse, so a second opener can
+# never slip in under a still-running first one, and it waits about as long.
+_DESKTOP_LOCK_TTL_SECONDS = 300
+_DESKTOP_OPEN_DEADLINE_SECONDS = _DESKTOP_LOCK_TTL_SECONDS - 15
+_DESKTOP_LOCK_WAIT_SECONDS = 300
+_DESKTOP_LOCK_POLL_SECONDS = 0.5
+_UNLOCK_SCRIPT = (
+ 'if redis.call("get", KEYS[1]) == ARGV[1] then '
+ 'return redis.call("del", KEYS[1]) else return 0 end'
+)
+
+
+class SandboxSummary(BaseModel):
+ """One of the owner's sandboxes as E2B reports it, without waking it."""
+
+ kind: SandboxKind
+ sandbox_id: str
+ state: Literal["running", "paused"]
+ started_at: datetime
+ cpu_count: int
+ memory_mb: int
+ template_id: str
+ mounts_attached: bool
+
+
+class ComputerInfo(BaseModel):
+ """The owner's computer: its boxes and the volumes they mount."""
+
+ owner_kind: Literal["session", "expert"]
+ owner_id: str
+ e2b_active: bool
+ shell: Optional[SandboxSummary] = None
+ desktop: Optional[SandboxSummary] = None
+ # path -> volume name; empty when the owner has no volumes (no user).
+ mounts: dict[str, str] = {}
+ workspace_path: str = WORKSPACE_PATH
+ shared_path: str = SHARED_PATH
+
+
+def computer_owner(session_id: str, expert_id: Optional[str]) -> SandboxOwner:
+ return SandboxOwner.for_session(session_id, expert_id)
+
+
+async def describe_computer(
+ owner: SandboxOwner, mounts: Mapping[str, str]
+) -> ComputerInfo:
+ """List the owner's shell and desktop boxes without resuming either."""
+ api_key = chat_config.active_e2b_api_key
+ info = ComputerInfo(
+ owner_kind=owner.kind,
+ owner_id=owner.id,
+ e2b_active=api_key is not None,
+ mounts=dict(mounts),
+ )
+ if not api_key:
+ return info
+ kinds: tuple[SandboxKind, ...] = ("shell", "desktop")
+ # Two independent E2B round-trips on an endpoint polled every 15 s.
+ listings = await asyncio.gather(
+ *(list_owned_sandboxes(owner, kind, api_key) for kind in kinds),
+ return_exceptions=True,
+ )
+ for kind, boxes in zip(kinds, listings):
+ if isinstance(boxes, BaseException):
+ # A listing is informational; show nothing rather than fail the page.
+ logger.warning("[E2B] describe_computer: %s", boxes)
+ continue
+ if not boxes:
+ continue
+ box = boxes[0]
+ summary = SandboxSummary(
+ kind=kind,
+ sandbox_id=box.sandbox_id,
+ state="running" if box.state == SandboxState.RUNNING else "paused",
+ started_at=box.started_at,
+ cpu_count=box.cpu_count,
+ memory_mb=box.memory_mb,
+ template_id=box.template_id,
+ mounts_attached=(box.metadata or {}).get(METADATA_MOUNTS) == "attached",
+ )
+ if kind == "shell":
+ info.shell = summary
+ else:
+ info.desktop = summary
+ return info
+
+
+async def open_desktop(
+ owner: SandboxOwner,
+ mounts: Mapping[str, str],
+ api_key: str,
+ *,
+ user_id: Optional[str],
+ session_id: Optional[str] = None,
+) -> tuple[DesktopStream, bool, bool]:
+ """Return ``(stream, created, shared)`` — resuming the owner's desktop if it exists.
+
+ Shared by the ``start_desktop`` tool and the HTTP endpoints, so a desktop
+ opened from the expert page is the same box the expert's next turn finds.
+ *user_id* is who the stream link is issued to (see ``_owner_bound``) and,
+ with *session_id*, provenance stamped on a newly created box. A cached
+ or recovered sandbox id is only reattached after E2B confirms the box is
+ the owner's.
+
+ One opener at a time per owner: the panel's Start and the model's
+ ``start_desktop`` (or two tabs) can both miss the cache, and without the
+ lock each would create a box, one of which nothing would ever find again.
+ A second opener waits for the first and then reattaches to its box.
+ """
+ if not user_id:
+ raise ValueError("A desktop needs an authenticated user to issue its link to")
+ redis = await get_redis_async()
+ key = owner.key("desktop")
+ lock_key = f"{key}:lock"
+ token = uuid.uuid4().hex
+ waited = 0.0
+ while not await redis.set(lock_key, token, nx=True, ex=_DESKTOP_LOCK_TTL_SECONDS):
+ if waited >= _DESKTOP_LOCK_WAIT_SECONDS:
+ raise RuntimeError(f"Another request is still opening {owner}'s desktop")
+ await asyncio.sleep(_DESKTOP_LOCK_POLL_SECONDS)
+ waited += _DESKTOP_LOCK_POLL_SECONDS
+ try:
+ return await asyncio.wait_for(
+ _open_desktop_locked(
+ owner,
+ mounts,
+ api_key,
+ redis,
+ key,
+ user_id=user_id,
+ session_id=session_id,
+ ),
+ timeout=_DESKTOP_OPEN_DEADLINE_SECONDS,
+ )
+ finally:
+ await redis.eval(_UNLOCK_SCRIPT, 1, lock_key, token)
+
+
+async def _open_desktop_locked(
+ owner: SandboxOwner,
+ mounts: Mapping[str, str],
+ api_key: str,
+ redis: Any,
+ key: str,
+ *,
+ user_id: str,
+ session_id: Optional[str],
+) -> tuple[DesktopStream, bool, bool]:
+ raw = await redis.get(key)
+ sandbox_id = raw.decode() if isinstance(raw, bytes) else raw
+ if not sandbox_id:
+ # An expert's desktop outlives the Redis cache; E2B metadata is the record.
+ sandbox_id = await find_owned_sandbox_id(owner, "desktop", api_key)
+ if sandbox_id:
+ desktop = await _reconnect_desktop(sandbox_id, owner, api_key, redis, key)
+ if desktop is not None:
+ # From here on a failure is a real error on a live box, not a
+ # reason to abandon it and create another. The reconnect woke
+ # the box, so a failed setup pauses it again rather than leaving
+ # it on the meter until the lifecycle timeout.
+ try:
+ await desktop.ensure_display(*_DESKTOP_RESOLUTION)
+ await redis.set(key, sandbox_id, ex=owner.ttl)
+ stream = _owner_bound(await desktop.start_stream(), user_id)
+ mounted = await desktop.is_workspace_mounted()
+ except Exception:
+ await _pause_quietly(desktop)
+ raise
+ return stream, False, mounted
+
+ desktop, persistence = await DesktopSession.create(
+ api_key=api_key,
+ timeout_seconds=chat_config.e2b_desktop_timeout,
+ width=_DESKTOP_RESOLUTION[0],
+ height=_DESKTOP_RESOLUTION[1],
+ volume_mounts=dict(mounts) or None,
+ template=chat_config.e2b_desktop_template,
+ metadata=owner.creation_metadata(
+ "desktop",
+ user_id=user_id,
+ session_id=session_id,
+ template=chat_config.e2b_desktop_template,
+ mounts="attached" if mounts else "none",
+ ),
+ )
+ try:
+ await redis.set(key, desktop.sandbox_id, ex=owner.ttl)
+ except Exception:
+ if not owner.is_expert:
+ # Nothing else can find a session desktop: no metadata recovery,
+ # no archive path. Kill it rather than bill it until timeout.
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(desktop.kill(), timeout=_KILL_TIMEOUT_SECONDS)
+ raise
+ try:
+ stream = _owner_bound(await desktop.start_stream(), user_id)
+ except Exception:
+ # The id is cached, so the next open resumes this box; pause it now
+ # rather than bill it until the lifecycle timeout.
+ await _pause_quietly(desktop)
+ raise
+ return stream, True, persistence.volume_mounted
+
+
+async def _pause_quietly(desktop: DesktopSession) -> None:
+ """Best-effort pause of a box a failed open would otherwise leave running."""
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(desktop.pause(), timeout=_KILL_TIMEOUT_SECONDS)
+
+
+async def _reconnect_desktop(
+ sandbox_id: str, owner: SandboxOwner, api_key: str, redis: Any, key: str
+) -> Optional[DesktopSession]:
+ """Reattach to a cached or recovered desktop, or ``None`` if it is gone.
+
+ Reattaches only once E2B confirms the box is the owner's. A box E2B no
+ longer has, or one stamped for someone else, is given up on and dropped
+ from the cache so the owner gets one of their own. A transient failure
+ is retried once and then raised: forking an expert's desktop over a
+ network blip is worse than asking the user to try again.
+ """
+ for attempt in (1, 2):
+ try:
+ return DesktopSession(
+ await connect_owned(
+ sandbox_id,
+ owner,
+ "desktop",
+ api_key,
+ timeout=chat_config.e2b_desktop_timeout,
+ )
+ )
+ except (NotFoundException, SandboxNotOwnedError) as exc:
+ logger.warning("[E2B] Desktop %.12s replaced: %s", sandbox_id, exc)
+ await redis.delete(key)
+ return None
+ except Exception as exc:
+ if attempt == 1:
+ logger.warning(
+ "[E2B] Desktop %.12s reconnect failed (%s); retrying once",
+ sandbox_id,
+ exc,
+ )
+ await asyncio.sleep(_RECONNECT_RETRY_DELAY_SECONDS)
+ continue
+ raise
+ raise AssertionError("unreachable")
+
+
+def _owner_bound(stream: DesktopStream, user_id: str) -> DesktopStream:
+ """The stream with its URL replaced by an owner-bound link.
+
+ The real URL carries the desktop's password. It stays on the backend;
+ what the tool result, the chat message and the API hand out is a link
+ to ``/api/desktop-preview`` that only *user_id* can redeem.
+ """
+ return stream.model_copy(
+ update={"url": create_preview_link(user_id, stream.url), "requires_auth": True}
+ )
+
+
+def mounts_for(user_id: Optional[str], expert_id: Optional[str]) -> dict[str, str]:
+ """The desktop mounts exactly what the owner's shell mounts.
+
+ Same rule as ``workspace_volume_mounts``: an expert always gets its own
+ home, the user's shared volume only when there is a user.
+ """
+ return workspace_volume_mounts(user_id, expert_id)
diff --git a/autogpt_platform/backend/backend/copilot/computer_test.py b/autogpt_platform/backend/backend/copilot/computer_test.py
new file mode 100644
index 0000000000..30f5ab876b
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/computer_test.py
@@ -0,0 +1,439 @@
+"""Tests for backend.copilot.computer: describe without waking, open by owner."""
+
+import asyncio
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from e2b import SandboxState
+
+from backend.blocks.desktop._api import DesktopStream, PersistenceInfo
+from backend.blocks.desktop._common import SHARED_PATH, WORKSPACE_PATH
+from backend.copilot.computer import (
+ ComputerInfo,
+ computer_owner,
+ describe_computer,
+ mounts_for,
+ open_desktop,
+)
+from backend.copilot.tools.e2b_sandbox import SandboxOwner
+from backend.util.sandbox_metadata import deployment_env
+
+_C = "backend.copilot.computer"
+_USER, _EXPERT, _SESSION = "user-1", "exp-1", "sess-1"
+
+
+def _info(kind: str, sandbox_id: str, state: SandboxState, mounts: str = "attached"):
+ return SimpleNamespace(
+ sandbox_id=sandbox_id,
+ state=state,
+ started_at=datetime(2026, 9, 5, 12, 0, tzinfo=timezone.utc),
+ cpu_count=2,
+ memory_mb=4096,
+ template_id="desktop" if kind == "desktop" else "base",
+ metadata={"autogpt_kind": kind, "autogpt_mounts": mounts},
+ )
+
+
+class TestComputerOwner:
+ def test_expert_session_is_the_experts_computer(self):
+ assert computer_owner(_SESSION, _EXPERT) == SandboxOwner(
+ kind="expert", id=_EXPERT
+ )
+ assert computer_owner(_SESSION, None) == SandboxOwner(
+ kind="session", id=_SESSION
+ )
+
+ def test_mounts_follow_the_shells_rule(self):
+ # An expert always has its home; the shared volume needs a user.
+ assert set(mounts_for(None, _EXPERT)) == {WORKSPACE_PATH}
+ assert set(mounts_for(_USER, _EXPERT)) == {WORKSPACE_PATH, SHARED_PATH}
+ assert set(mounts_for(_USER, None)) == {WORKSPACE_PATH}
+ assert mounts_for(None, None) == {}
+
+
+class TestDescribeComputer:
+ @pytest.mark.asyncio
+ async def test_reports_both_boxes_without_connecting(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ listed = {
+ "shell": [_info("shell", "sb-shell", SandboxState.PAUSED)],
+ "desktop": [
+ _info("desktop", "sb-desk", SandboxState.RUNNING, mounts="none")
+ ],
+ }
+ list_mock = AsyncMock(side_effect=lambda o, kind, key: listed[kind])
+ with (
+ patch(f"{_C}.chat_config") as cfg,
+ patch(f"{_C}.list_owned_sandboxes", list_mock),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ ):
+ cfg.active_e2b_api_key = "k"
+ info = await describe_computer(owner, mounts_for(_USER, _EXPERT))
+
+ assert isinstance(info, ComputerInfo)
+ assert info.owner_kind == "expert" and info.e2b_active
+ assert (
+ info.shell and info.shell.state == "paused" and info.shell.mounts_attached
+ )
+ assert info.desktop and info.desktop.state == "running"
+ assert info.desktop.mounts_attached is False
+ assert info.mounts[SHARED_PATH].startswith("autogpt-user-")
+ # Describing must never resume a paused box.
+ desktop_cls.connect.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_a_failed_listing_shows_nothing_rather_than_failing(self):
+ from backend.copilot.tools.e2b_sandbox import SandboxLookupError
+
+ with (
+ patch(f"{_C}.chat_config") as cfg,
+ patch(
+ f"{_C}.list_owned_sandboxes",
+ AsyncMock(side_effect=SandboxLookupError("e2b down")),
+ ),
+ ):
+ cfg.active_e2b_api_key = "k"
+ info = await describe_computer(SandboxOwner(kind="expert", id=_EXPERT), {})
+ assert info.e2b_active and info.shell is None and info.desktop is None
+
+ @pytest.mark.asyncio
+ async def test_without_e2b_it_says_so_and_lists_nothing(self):
+ owner = SandboxOwner(kind="session", id=_SESSION)
+ with (
+ patch(f"{_C}.chat_config") as cfg,
+ patch(f"{_C}.list_owned_sandboxes", AsyncMock()) as list_mock,
+ ):
+ cfg.active_e2b_api_key = None
+ info = await describe_computer(owner, {})
+ assert info.e2b_active is False and info.shell is None and info.desktop is None
+ list_mock.assert_not_awaited()
+
+
+def _redis(stored: str | None, lock_free: bool = True):
+ r = MagicMock()
+ r.get = AsyncMock(return_value=stored)
+ r.set = AsyncMock(return_value=lock_free)
+ r.delete = AsyncMock()
+ r.eval = AsyncMock(return_value=1)
+ return r
+
+
+def _desktop(sandbox_id="sb-desk"):
+ d = MagicMock()
+ d.sandbox_id = sandbox_id
+ d.ensure_display = AsyncMock()
+ d.start_stream = AsyncMock(
+ return_value=DesktopStream(
+ url="https://6080-x.e2b.app/vnc.html", sandbox_id=sandbox_id
+ )
+ )
+ d.is_workspace_mounted = AsyncMock(return_value=True)
+ return d
+
+
[email protected](autouse=True)
+def _owner_bound_links():
+ with patch(
+ f"{_C}.create_preview_link",
+ side_effect=lambda user_id, url: f"preview://{user_id}/{url}",
+ ):
+ yield
+
+
+class TestOpenDesktop:
+ @pytest.mark.asyncio
+ async def test_needs_a_user_to_issue_the_link_to(self):
+ with pytest.raises(ValueError, match="authenticated user"):
+ await open_desktop(
+ SandboxOwner(kind="expert", id=_EXPERT), {}, "k", user_id=None
+ )
+
+ @pytest.mark.asyncio
+ async def test_a_second_opener_waits_and_reattaches(self):
+ """Start from the panel and start_desktop from the model can both miss
+ the cache; only one may create, the other reattaches to its box."""
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis(None)
+ # Lock taken on the first attempt; free on the second, by which time
+ # the first opener has cached its box.
+ redis.set = AsyncMock(side_effect=[False, True, True])
+ redis.get = AsyncMock(side_effect=[b"sb-first"])
+ desktop = _desktop("sb-first")
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.asyncio.sleep", AsyncMock()) as sleep,
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned") as connect_owned,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ connect_owned.return_value = MagicMock()
+ desktop_cls.return_value = desktop
+ desktop_cls.create = AsyncMock()
+ stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)
+
+ assert not created and stream.sandbox_id == "sb-first"
+ sleep.assert_awaited_once()
+ desktop_cls.create.assert_not_awaited()
+ # The lock is released by token, never a bare delete of the key.
+ script, _, lock_key, token = redis.eval.await_args.args
+ assert lock_key == f"copilot:e2b:expert:{_EXPERT}:desktop:lock"
+ assert token == redis.set.await_args_list[1].args[1]
+
+ @pytest.mark.asyncio
+ async def test_the_open_is_cut_off_before_the_lock_can_lapse(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis(None)
+
+ async def never(**_kwargs):
+ await asyncio.Event().wait()
+
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.chat_config") as cfg,
+ patch(f"{_C}._DESKTOP_OPEN_DEADLINE_SECONDS", 0.01),
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ desktop_cls.create = AsyncMock(side_effect=never)
+ with pytest.raises(asyncio.TimeoutError):
+ await open_desktop(owner, {}, "k", user_id=_USER)
+ from backend.copilot import computer
+
+ assert (
+ computer._DESKTOP_OPEN_DEADLINE_SECONDS < computer._DESKTOP_LOCK_TTL_SECONDS
+ )
+ # The lock is still released on the way out.
+ redis.eval.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_a_gone_desktop_is_replaced(self):
+ from e2b.exceptions import NotFoundException
+
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis("sb-gone")
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned") as connect_owned,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ connect_owned.side_effect = NotFoundException("gone")
+ desktop_cls.create = AsyncMock(
+ return_value=(_desktop("sb-new"), PersistenceInfo())
+ )
+ stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)
+ assert created and stream.sandbox_id == "sb-new"
+ redis.delete.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_a_transient_reconnect_failure_is_retried_then_raised(self):
+ """A network blip must not fork an expert's desktop: one retry, then
+ the error surfaces and the cached id is kept for next time."""
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis("sb-live")
+ desktop = _desktop("sb-live")
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.asyncio.sleep", AsyncMock()),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned") as connect_owned,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ connect_owned.side_effect = [RuntimeError("502"), MagicMock()]
+ desktop_cls.return_value = desktop
+ desktop_cls.create = AsyncMock()
+ stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)
+ assert not created and stream.sandbox_id == "sb-live"
+ assert connect_owned.await_count == 2
+
+ connect_owned.side_effect = RuntimeError("502")
+ with pytest.raises(RuntimeError, match="502"):
+ await open_desktop(owner, {}, "k", user_id=_USER)
+ desktop_cls.create.assert_not_awaited()
+ redis.delete.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_a_failure_after_reconnect_is_an_error_not_a_new_box(self):
+ """Only a failed connect means the box is gone. A display or stream
+ failure on a live box must surface, not abandon the box and bill for
+ a second one."""
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis("sb-live")
+ desktop = _desktop("sb-live")
+ desktop.ensure_display = AsyncMock(side_effect=RuntimeError("no display"))
+ desktop.pause = AsyncMock()
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned", AsyncMock(return_value=MagicMock())),
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ desktop_cls.return_value = desktop
+ desktop_cls.create = AsyncMock()
+ with pytest.raises(RuntimeError, match="no display"):
+ await open_desktop(owner, {}, "k", user_id=_USER)
+ desktop_cls.create.assert_not_awaited()
+ redis.delete.assert_not_awaited()
+ # The reconnect woke the box; a failed setup does not leave it running.
+ desktop.pause.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_a_session_desktop_whose_id_cannot_be_saved_is_killed(self):
+ """Nothing can find a session desktop later; an unsaved one would bill
+ until timeout. An expert's is recoverable by metadata and is kept."""
+ session_owner = SandboxOwner(kind="session", id=_SESSION)
+ redis = _redis(None)
+ redis.set = AsyncMock(side_effect=[True, ConnectionError("redis down")])
+ desktop = _desktop("sb-new")
+ desktop.kill = AsyncMock()
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ desktop_cls.create = AsyncMock(return_value=(desktop, PersistenceInfo()))
+ with pytest.raises(ConnectionError):
+ await open_desktop(session_owner, {}, "k", user_id=_USER)
+ desktop.kill.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_a_new_desktop_whose_stream_fails_is_paused_not_lost(self):
+ """Its id is cached, so the next open resumes it; meanwhile it must
+ not sit running on the meter."""
+ owner = SandboxOwner(kind="session", id=_SESSION)
+ redis = _redis(None)
+ desktop = _desktop("sb-new")
+ desktop.start_stream = AsyncMock(side_effect=RuntimeError("no novnc"))
+ desktop.pause = AsyncMock()
+ desktop.kill = AsyncMock()
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ desktop_cls.create = AsyncMock(return_value=(desktop, PersistenceInfo()))
+ with pytest.raises(RuntimeError, match="no novnc"):
+ await open_desktop(owner, {}, "k", user_id=_USER)
+ desktop.pause.assert_awaited_once()
+ desktop.kill.assert_not_awaited()
+ assert any(c.args[1] == "sb-new" for c in redis.set.await_args_list)
+
+ @pytest.mark.asyncio
+ async def test_gives_up_when_the_lock_never_frees(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis(None, lock_free=False)
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.asyncio.sleep", AsyncMock()),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ ):
+ desktop_cls.create = AsyncMock()
+ with pytest.raises(RuntimeError, match="still opening"):
+ await open_desktop(owner, {}, "k", user_id=_USER)
+ desktop_cls.create.assert_not_awaited()
+ redis.eval.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_creates_under_the_owner_key_with_the_owner_mounts(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ mounts = mounts_for(_USER, _EXPERT)
+ redis = _redis(None)
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ desktop_cls.create = AsyncMock(
+ return_value=(_desktop(), PersistenceInfo(volume_mounted=True))
+ )
+ stream, created, shared = await open_desktop(
+ owner, mounts, "k", user_id=_USER
+ )
+
+ assert created and shared and stream.sandbox_id == "sb-desk"
+ # The password-bearing URL stays here; callers get an owner-bound link.
+ assert stream.url == f"preview://{_USER}/https://6080-x.e2b.app/vnc.html"
+ assert stream.requires_auth is True
+ kwargs = desktop_cls.create.await_args.kwargs
+ assert kwargs["volume_mounts"] == mounts
+ assert kwargs["metadata"] == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"expert:{_EXPERT}",
+ "autogpt_kind": "desktop",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_user": _USER,
+ "autogpt_expert": _EXPERT,
+ "autogpt_template": "desktop",
+ "autogpt_mounts": "attached",
+ }
+ assert redis.set.await_args.args[0] == f"copilot:e2b:expert:{_EXPERT}:desktop"
+
+ @pytest.mark.asyncio
+ async def test_resumes_the_recovered_box_and_reuses_its_stream(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis(None)
+ desktop = _desktop("sb-old")
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value="sb-old")),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned") as connect_owned,
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ connect_owned.return_value = box = MagicMock()
+ desktop_cls.return_value = desktop
+ desktop_cls.create = AsyncMock()
+ stream, created, shared = await open_desktop(owner, {}, "k", user_id=_USER)
+
+ assert not created and shared and stream.sandbox_id == "sb-old"
+ # Reattached only once E2B confirms the box is the owner's, and with
+ # the same running-time limit as a new one.
+ connect_owned.assert_awaited_once_with(
+ "sb-old", owner, "desktop", "k", timeout=900
+ )
+ desktop_cls.assert_called_once_with(box)
+ desktop_cls.create.assert_not_awaited()
+ desktop.start_stream.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_a_recovered_id_that_is_not_the_owners_box_is_replaced(self):
+ """The cached id said sb-old; E2B says sb-old is someone else's. The
+ stale cache entry goes and the owner gets a box of their own."""
+ from backend.copilot.tools.e2b_sandbox import SandboxNotOwnedError
+
+ owner = SandboxOwner(kind="expert", id=_EXPERT)
+ redis = _redis("sb-old")
+ with (
+ patch(f"{_C}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_C}.find_owned_sandbox_id", AsyncMock(return_value=None)),
+ patch(f"{_C}.DesktopSession") as desktop_cls,
+ patch(f"{_C}.connect_owned", side_effect=SandboxNotOwnedError("nope")),
+ patch(f"{_C}.chat_config") as cfg,
+ ):
+ cfg.e2b_desktop_timeout = 900
+ cfg.e2b_desktop_template = "desktop"
+ desktop_cls.create = AsyncMock(
+ return_value=(_desktop("sb-new"), PersistenceInfo())
+ )
+ stream, created, _ = await open_desktop(owner, {}, "k", user_id=_USER)
+
+ assert created and stream.sandbox_id == "sb-new"
+ redis.delete.assert_awaited_once()
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 86b9df995c..bf8ad4abae 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -713,6 +713,21 @@ class ChatConfig(BaseSettings):
default="pause",
description="E2B lifecycle action on timeout: 'pause' (default, free) or 'kill'.",
)
+ e2b_desktop_template: str = Field(
+ default="desktop",
+ description="E2B template for the on-demand start_desktop sandbox. E2B's "
+ "public 'desktop' is 8 vCPU / 8 GiB (~$0.53/h running); build a smaller "
+ "one on the team with `poetry run build-desktop-template` "
+ "(agpt-desktop-1x1: 1 vCPU / 1 GiB, ~$0.07/h) and set this to its alias.",
+ )
+ e2b_desktop_timeout: int = Field(
+ default=900,
+ description="Running-time timeout (seconds) for the on-demand desktop "
+ "sandbox. Unlike the bash sandbox it is NOT paused at turn end — the "
+ "user may still be watching the stream — so this is the only thing "
+ "stopping the meter; it auto-pauses (free) when the timeout lapses "
+ "and resumes on the next start_desktop call.",
+ )
@property
def openrouter_active(self) -> bool:
diff --git a/autogpt_platform/backend/backend/copilot/expert_context.py b/autogpt_platform/backend/backend/copilot/expert_context.py
index 77efba4cd4..2fe35d5c00 100644
--- a/autogpt_platform/backend/backend/copilot/expert_context.py
+++ b/autogpt_platform/backend/backend/copilot/expert_context.py
@@ -25,6 +25,8 @@ import asyncio
import logging
from backend.api.features.experts.models import PROTECTED_SOUL_RULES, Expert
+from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.copilot.config import ChatConfig
from backend.data.db_accessors import experts_db
from backend.util.exceptions import ExpertNotFoundError
from backend.util.feature_flag import Flag, is_feature_enabled
@@ -246,7 +248,7 @@ async def _expert_session_context(
# If the expert changes between those reads, omit only this optional block.
if expert is None or expert.is_archived:
return ""
- return render_expert_workflows_block(expert) + teammates
+ return render_expert_workflows_block(expert) + _expert_computer_block() + teammates
def render_expert_workflows_block(expert: Expert) -> str:
@@ -280,6 +282,38 @@ def render_expert_workflows_block(expert: Expert) -> str:
)
+def _expert_computer_block() -> str:
+ """Tell an expert about its own machine — only when E2B actually backs it.
+
+ Lives in the first user message with the other expert blocks so the
+ cacheable system-prompt prefix stays byte-identical.
+ """
+ try:
+ if not ChatConfig().e2b_active:
+ return ""
+ except Exception as e:
+ logger.warning(f"Failed to resolve E2B config for expert context: {e}")
+ return ""
+ return (
+ "<expert_computer>\n"
+ "You have your own persistent cloud computer. It is suspended, not "
+ "destroyed, when idle, so what you install stays.\n"
+ f"- {WORKSPACE_PATH}: your durable home. Keep your notes, configs, "
+ "scripts and tools here and customise it freely.\n"
+ f"- {SHARED_PATH}: the user's shared workspace, when mounted. Put "
+ "deliverables there so they show up on the user's desktop and in "
+ "their other sessions. Trust the tool output on whether it is "
+ "mounted: without the mount, say where the file really is instead "
+ "of calling it shared.\n"
+ "- Use start_desktop when a task needs a browser or GUI app. The "
+ "desktop is shared with the user, not private from either of you: "
+ "you can see everything on it, and so can they.\n"
+ "- Never ask the user to sign into personal accounts on this "
+ "desktop; use their connected integrations instead.\n"
+ "</expert_computer>\n\n"
+ )
+
+
async def _team_context(
user_id: str,
*,
diff --git a/autogpt_platform/backend/backend/copilot/expert_context_test.py b/autogpt_platform/backend/backend/copilot/expert_context_test.py
index 61c31d4e16..dc61f38b4b 100644
--- a/autogpt_platform/backend/backend/copilot/expert_context_test.py
+++ b/autogpt_platform/backend/backend/copilot/expert_context_test.py
@@ -1010,3 +1010,78 @@ class TestUntrustedContentEscaped:
assert "Maria</expert_identity>" not in result
assert "Maria&lt;/expert_identity&gt;" in result
assert result.count("</expert_identity>") == 1
+
+
+class TestExpertComputerBlock:
+ """An expert is told about its own machine only when E2B backs it."""
+
+ def _db(self):
+ mock_db = MagicMock()
+ mock_db.get_expert = AsyncMock(return_value=_expert())
+ mock_db.list_experts = AsyncMock(return_value=[_expert()])
+ return mock_db
+
+ @pytest.mark.asyncio
+ async def test_expert_learns_home_and_shared_paths_when_e2b_active(self):
+ from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+ from backend.copilot.expert_context import build_expert_context
+
+ config = MagicMock()
+ config.e2b_active = True
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ result = await build_expert_context("user-1", "exp-1")
+
+ assert "<expert_computer>" in result
+ assert WORKSPACE_PATH in result
+ assert SHARED_PATH in result
+ assert "start_desktop" in result
+ # Sits with the other first-message blocks, after the workflows.
+ assert result.index("</expert_workflows>") < result.index("<expert_computer>")
+
+ @pytest.mark.asyncio
+ async def test_no_computer_block_without_e2b(self):
+ from backend.copilot.expert_context import build_expert_context
+
+ config = MagicMock()
+ config.e2b_active = False
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ result = await build_expert_context("user-1", "exp-1")
+
+ assert "<expert_computer>" not in result
+ assert "<expert_workflows>" in result
+
+ @pytest.mark.asyncio
+ async def test_plain_session_gets_no_computer_block(self):
+ from backend.copilot.expert_context import build_expert_context
+
+ config = MagicMock()
+ config.e2b_active = True
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ result = await build_expert_context("user-1", None)
+
+ assert "<expert_computer>" not in result
+
+ @pytest.mark.asyncio
+ async def test_config_failure_degrades_to_no_block(self):
+ from backend.copilot.expert_context import build_expert_context
+
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(
+ f"{_EC}.ChatConfig",
+ side_effect=RuntimeError("bad env"),
+ ),
+ ):
+ result = await build_expert_context("user-1", "exp-1")
+
+ assert "<expert_computer>" not in result
+ assert "<expert_workflows>" in result
diff --git a/autogpt_platform/backend/backend/copilot/permissions.py b/autogpt_platform/backend/backend/copilot/permissions.py
index 010de6aa4c..1e7863ebee 100644
--- a/autogpt_platform/backend/backend/copilot/permissions.py
+++ b/autogpt_platform/backend/backend/copilot/permissions.py
@@ -73,6 +73,7 @@ ToolName = Literal[
"add_understanding",
"ask_question",
"bash_exec",
+ "start_desktop",
"browser_act",
"browser_navigate",
"browser_screenshot",
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 4eca3809a1..8226574f29 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -3,6 +3,7 @@
# isort: skip_file — double-dot relative imports must stay relative to avoid Pyright type collisions
import asyncio
+import contextlib
import base64
import functools
from copy import copy
@@ -42,6 +43,7 @@ from langsmith.integrations.claude_agent_sdk import configure_claude_agent_sdk
from opentelemetry import trace as otel_trace
from pydantic import BaseModel
+from backend.blocks.desktop._common import workspace_volume_mounts
from backend.copilot.model_router import (
ResolvedModel,
RoutingSource,
@@ -4776,8 +4778,13 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
# warm-context, and CLI session restore are all independent network
# calls. Running them concurrently saves ~500-1000ms vs sequential.
+ # Captured outside the closure: `session` is narrowed to ChatSession
+ # here, but that narrowing does not carry into nested functions.
+ owner_expert_id = session.expert_id
+
async def _setup_e2b():
"""Set up E2B sandbox if configured, return sandbox or None."""
+ nonlocal e2b_sandbox
if not (e2b_api_key := config.active_e2b_api_key):
if config.use_e2b_sandbox:
logger.warning(
@@ -4787,13 +4794,22 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
)
return None
try:
+ # An expert session runs on the expert's own persistent box;
+ # everything else gets a per-session sandbox.
sandbox = await get_or_create_sandbox(
session_id,
api_key=e2b_api_key,
template=config.e2b_sandbox_template,
timeout=config.e2b_sandbox_timeout,
on_timeout=config.e2b_sandbox_on_timeout,
+ volume_mounts=workspace_volume_mounts(user_id, owner_expert_id),
+ expert_id=owner_expert_id,
+ user_id=user_id,
)
+ # Publish the live box before the gather returns: if a sibling
+ # setup leg fails, the finally below still pauses it and
+ # releases the expert's turn slot instead of leaking both.
+ e2b_sandbox = sandbox
except Exception as e2b_err:
logger.error(
"[E2B] [%s] Setup failed: %s",
@@ -4805,27 +4821,35 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
return sandbox
- (
- e2b_sandbox,
- (base_system_prompt, understanding),
- (graphiti_enabled, warm_ctx),
- _restore,
- ) = await asyncio.gather(
- _setup_e2b(),
- _build_system_prompt(user_id if not has_history else None),
- _fetch_graphiti_context(user_id, session, message),
- # Restore CLI session — single GCS round-trip covers both
- # --resume and builder state. message_count watermark lives
- # in the companion .meta.json alongside the session file.
- _restore_cli_session_for_turn(
- user_id,
- session_id,
- session,
- sdk_cwd,
- transcript_builder,
- log_prefix,
- ),
- )
+ # The E2B leg runs as its own task: if a sibling leg fails first, the
+ # box it may already have opened (and the expert turn it counted)
+ # must still be published so the finally below pauses and releases it.
+ e2b_task = asyncio.create_task(_setup_e2b())
+ try:
+ (
+ (base_system_prompt, understanding),
+ (graphiti_enabled, warm_ctx),
+ _restore,
+ ) = await asyncio.gather(
+ _build_system_prompt(user_id if not has_history else None),
+ _fetch_graphiti_context(user_id, session, message),
+ # Restore CLI session — single GCS round-trip covers both
+ # --resume and builder state. message_count watermark lives
+ # in the companion .meta.json alongside the session file.
+ _restore_cli_session_for_turn(
+ user_id,
+ session_id,
+ session,
+ sdk_cwd,
+ transcript_builder,
+ log_prefix,
+ ),
+ )
+ except BaseException:
+ with contextlib.suppress(BaseException):
+ e2b_sandbox = await e2b_task
+ raise
+ e2b_sandbox = await e2b_task
use_e2b = e2b_sandbox is not None
# Append appropriate supplement (Claude gets tool schemas automatically)
@@ -6209,7 +6233,13 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
# Use pause_sandbox_direct to skip the Redis lookup and reconnect
# round-trip — e2b_sandbox is the live object from this turn.
if e2b_sandbox is not None:
- task = asyncio.create_task(pause_sandbox_direct(e2b_sandbox, session_id))
+ task = asyncio.create_task(
+ pause_sandbox_direct(
+ e2b_sandbox,
+ session_id,
+ expert_id=session.expert_id if session else None,
+ )
+ )
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
diff --git a/autogpt_platform/backend/backend/copilot/tools/__init__.py b/autogpt_platform/backend/backend/copilot/tools/__init__.py
index e60c8c1cc5..1887600cbe 100644
--- a/autogpt_platform/backend/backend/copilot/tools/__init__.py
+++ b/autogpt_platform/backend/backend/copilot/tools/__init__.py
@@ -78,6 +78,7 @@ from .schedule_followup import ScheduleFollowupTool
from .search_docs import SearchDocsTool
from .setup_agent_webhook_trigger import SetupAgentWebhookTriggerTool
from .skills import DeleteSkillTool, ListSkillsTool, ReadSkillTool, StoreSkillTool
+from .start_desktop import StartDesktopTool
from .todo_write import TodoWriteTool
from .update_expert import UpdateExpertTool
from .update_soul import ConfirmExpertSoulUpdateTool, UpdateExpertSoulTool
@@ -167,6 +168,7 @@ TOOL_REGISTRY: dict[str, BaseTool] = {
"browser_screenshot": BrowserScreenshotTool(),
# Sandboxed code execution (bubblewrap)
"bash_exec": BashExecTool(),
+ "start_desktop": StartDesktopTool(),
"connect_integration": ConnectIntegrationTool(),
# Persistent workspace tools (cloud storage, survives across sessions)
# Feature request tools
diff --git a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py b/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
index f122fca2aa..cd4b6e01ae 100644
--- a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
+++ b/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
@@ -74,7 +74,11 @@ class BashExecTool(BaseTool):
return (
"Execute a Bash command or script. Shares filesystem with SDK file tools. "
"Useful for scripts, data processing, and package installation. "
- "Killed after `timeout` seconds."
+ "Killed after `timeout` seconds. Anything that should persist across "
+ "sessions or show on the user's desktop belongs in ~/workspace (a "
+ "durable volume shared live with start_desktop); other paths are "
+ "scratch. Expert sessions: ~/workspace is the expert's own machine, "
+ "~/shared is the user's workspace."
)
@property
diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
index 6e8eb1cdb1..d7dfdb8a6e 100644
--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
@@ -6,6 +6,22 @@ directly on the sandbox via ``sandbox.commands.run()``. SDK file tools
``/home/user`` directory via E2B's HTTP-based filesystem API — all tools
share a single coherent filesystem with no local sync required.
+Ownership
+---------
+A sandbox belongs to a :class:`SandboxOwner`:
+
+* ``session`` — scratch for one chat; killed when the chat is deleted.
+* ``expert`` — the hired expert's own computer. Every session that runs as
+ that expert (chats, ``delegate_to_expert`` sub-sessions, scheduled
+ kickoffs) reconnects to the same box, so tools it installs, files it writes
+ and anything left signed in in its browser are still there next time. The
+ model has a root shell in that same VM, so nothing on the box, browser
+ sessions included, is private from it. Deleting a
+ chat never touches it; archiving the expert does
+ (``kill_expert_sandboxes``). The box mounts the expert's own durable volume
+ at ``~/workspace`` and the owning user's volume at ``~/shared`` (see
+ ``workspace_volume_mounts``).
+
Lifecycle
---------
1. **Turn start** – connect to the existing sandbox (sandbox_id in Redis) or
@@ -15,8 +31,10 @@ Lifecycle
sandbox's ``/home/user`` filesystem.
3. **Turn end** – the sandbox is paused via ``pause_sandbox()`` (fire-and-forget)
so idle time between turns costs nothing. Paused sandboxes have no compute
- cost.
-4. **Session delete** – ``kill_sandbox()`` fully terminates the sandbox.
+ cost. An expert's box is only paused once its *last* concurrent turn
+ ends — two sessions of the same expert may be mid-command on it at once,
+ and pausing under one of them would sever its command stream.
+4. **Session delete** – ``kill_sandbox()`` fully terminates a session sandbox.
Cost control
------------
@@ -38,27 +56,53 @@ E2B assigns each sandbox an absolute ``end_at`` timestamp at create time:
``connect()`` extends it (by ``timeout`` seconds from the moment of reconnect).
Active sessions therefore stay alive as long as turns arrive within the timeout
window. Orphaned sandboxes (e.g. leaked by a failed create retry) are paused
-(not killed) at ``end_at`` under the default ``on_timeout="pause"`` lifecycle;
-they persist until explicitly killed or until E2B's platform-level cleanup
-applies (30-day limit during beta).
+(not killed) at ``end_at`` under the default ``on_timeout="pause"`` lifecycle.
+
+Paused sandboxes have no time-to-live on E2B's side — no expiry, no storage
+billing, no concurrency cost — which is what makes an expert's box durable.
+Redis only *caches* its id: every sandbox is stamped with ``autogpt_owner`` /
+``autogpt_kind`` metadata, and ``find_owned_sandbox_id`` recovers an expert's
+box through the E2B API if the cache is ever lost.
"""
import asyncio
import contextlib
import logging
import math
-from typing import Any, Awaitable, Callable, Literal
-
-from e2b import AsyncSandbox, SandboxLifecycle
-
+from collections.abc import Awaitable, Callable, Mapping
+from typing import Any, Literal
+
+from e2b import (
+ AsyncSandbox,
+ AsyncVolume,
+ SandboxInfo,
+ SandboxLifecycle,
+ SandboxQuery,
+ SandboxState,
+)
+from pydantic import BaseModel, ConfigDict
+
+from backend.blocks.desktop._api import resolve_volume
from backend.data.redis_client import get_redis_async
from backend.util.e2b_template import ensure_template, forget_template
+from backend.util.sandbox_metadata import MountState, SandboxMetadata
logger = logging.getLogger(__name__)
_SANDBOX_KEY_PREFIX = "copilot:e2b:sandbox:"
+_DESKTOP_KEY_PREFIX = "copilot:e2b:desktop:"
+_EXPERT_KEY_PREFIX = "copilot:e2b:expert:"
_CREATING_SENTINEL = "creating"
+# E2B sandbox metadata that lets an owner find its box without Redis.
+METADATA_OWNER = "autogpt_owner"
+METADATA_KIND = "autogpt_kind"
+# "attached" when the workspace volumes were mounted, "none" when creation had
+# to fall back to a volume-less box — visible in the E2B dashboard and API.
+METADATA_MOUNTS = "autogpt_mounts"
+
+SandboxKind = Literal["shell", "desktop"]
+
# Per-attempt timeout for AsyncSandbox.create(). E2B normally provisions a
# sandbox in 5-15 s; 30 s gives generous headroom while ensuring a slow/hung
# E2B API call fails fast rather than blocking an executor goroutine for hours.
@@ -83,91 +127,429 @@ _WAIT_INTERVAL_SECONDS = 0.5
# sentinel expiry. Result: ceil(120 / 0.5 * 1.2) = 288 iterations ≈ 144 s.
_MAX_WAIT_ATTEMPTS = math.ceil(_CREATION_LOCK_TTL / _WAIT_INTERVAL_SECONDS * 1.2)
-# Timeout for E2B API calls (pause/kill) — short because these are control-plane
-# operations; if the sandbox is unreachable, fail fast and retry on the next turn.
+# Timeout for E2B API calls (pause/kill/list) — short because these are
+# control-plane operations; if the sandbox is unreachable, fail fast and retry
+# on the next turn.
_E2B_API_TIMEOUT_SECONDS = 10
-# Redis TTL for the sandbox key. Must be ≥ the E2B project "paused sandbox
-# lifetime" setting (recommended: set both to 48 h).
+# Redis TTL for a session sandbox key. Must be ≥ the E2B project "paused
+# sandbox lifetime" setting (recommended: set both to 48 h).
_SANDBOX_ID_TTL = 48 * 3600 # 48 hours
+# An expert's box is meant to outlive any one chat. The key is refreshed on
+# every use and E2B metadata recovers it after expiry, so this is only a cache
+# TTL — not a lifetime.
+_EXPERT_ID_TTL = 30 * 24 * 3600
+
+# Leak guard for the per-expert active-turn counter: a turn that dies without
+# releasing its slot stops blocking the pause after this long. A turn may
+# legitimately run for the 6 h executor limit (``consumer-timeout`` in
+# ``executor/utils.py``), so the guard sits just past that; every acquire
+# refreshes it.
+_ACTIVE_TURN_TTL = 6 * 3600 + 15 * 60
+
+# Count one turn and (re)arm the key's expiry in one script: an INCR that
+# lands without its EXPIRE would leave a count nothing ever releases, and the
+# box would never pause at turn end.
+_ACQUIRE_TURN_SCRIPT = (
+ 'local n = redis.call("incr", KEYS[1]) '
+ 'redis.call("expire", KEYS[1], ARGV[1]) '
+ "return n"
+)
+
+# Release one turn and report how many are left; the last one out deletes
+# the key. One script, so a turn that starts between the DECR and the DEL
+# can never have its count wiped.
+_RELEASE_TURN_SCRIPT = (
+ 'local n = redis.call("decr", KEYS[1]) '
+ 'if n <= 0 then redis.call("del", KEYS[1]) return 0 end '
+ "return n"
+)
+
+# One more reconnect attempt before a listed expert box is given up on: a
+# durable box that fails once on a transient error must not be forked.
+_RECONNECT_RETRY_DELAY_SECONDS = 1.0
+
+
+class SandboxOwner(BaseModel):
+ """Who a CoPilot sandbox belongs to — and therefore how long it lives.
+
+ ``session`` sandboxes are scratch for one chat. ``expert`` sandboxes are
+ a hired expert's own persistent computer, shared by every session that
+ runs as that expert — one box per expert, not one per account, so two
+ experts never see each other's files or browser sessions.
+ """
+
+ model_config = ConfigDict(frozen=True)
+
+ kind: Literal["session", "expert"]
+ id: str
+
+ @classmethod
+ def for_session(
+ cls, session_id: str, expert_id: str | None = None
+ ) -> "SandboxOwner":
+ """Expert sessions run on the expert's box; everything else per-session."""
+ if expert_id:
+ return cls(kind="expert", id=expert_id)
+ return cls(kind="session", id=session_id)
+
+ @property
+ def is_expert(self) -> bool:
+ return self.kind == "expert"
+
+ def key(self, sandbox_kind: SandboxKind = "shell") -> str:
+ """Redis key caching this owner's sandbox id (doubles as creation lock)."""
+ if self.is_expert:
+ return f"{_EXPERT_KEY_PREFIX}{self.id}:{sandbox_kind}"
+ prefix = _SANDBOX_KEY_PREFIX if sandbox_kind == "shell" else _DESKTOP_KEY_PREFIX
+ return f"{prefix}{self.id}"
+
+ @property
+ def ttl(self) -> int:
+ return _EXPERT_ID_TTL if self.is_expert else _SANDBOX_ID_TTL
+
+ def metadata(self, sandbox_kind: SandboxKind = "shell") -> dict[str, str]:
+ """The identity keys a lookup filters on; a subset of ``creation_metadata``."""
+ return {METADATA_OWNER: f"{self.kind}:{self.id}", METADATA_KIND: sandbox_kind}
+
+ def creation_metadata(
+ self,
+ sandbox_kind: SandboxKind = "shell",
+ *,
+ user_id: str | None = None,
+ session_id: str | None = None,
+ template: str | None = None,
+ mounts: MountState | None = None,
+ ) -> dict[str, str]:
+ """Identity plus provenance, stamped on the box when it is created."""
+ return SandboxMetadata.for_copilot(
+ f"{self.kind}:{self.id}",
+ sandbox_kind,
+ user_id=user_id,
+ session_id=session_id,
+ expert_id=self.id if self.is_expert else None,
+ template=template,
+ mounts=mounts,
+ ).as_e2b()
+
+ def __str__(self) -> str:
+ return f"{self.kind} {self.id[:12]}"
+
+
+class SandboxNotOwnedError(Exception):
+ """A sandbox id resolved to a box that is not the owner's."""
+
+
+async def connect_owned(
+ sandbox_id: str,
+ owner: SandboxOwner,
+ sandbox_kind: SandboxKind,
+ api_key: str,
+ *,
+ timeout: int | None = None,
+) -> AsyncSandbox:
+ """Connect to *sandbox_id* only if E2B says it belongs to *owner*.
+
+ Under the platform's E2B key any sandbox id connects, so the id alone
+ (from Redis, a message, or a caller) must never be trusted: the box's
+ stamped ``autogpt_owner`` / ``autogpt_kind`` metadata is the record of
+ who it belongs to. Raises :class:`SandboxNotOwnedError` otherwise.
+
+ The stamp is read *before* connecting: a connect is not passive, it
+ resumes a paused box and re-arms its running-time limit, so a foreign
+ id must be refused without ever waking someone else's box. *timeout*
+ is that limit for the owner's box (a resumed desktop would otherwise get
+ the SDK's default).
+ """
+ info = await AsyncSandbox.get_info(sandbox_id, api_key=api_key)
+ expected = owner.metadata(sandbox_kind)
+ stamped = info.metadata or {}
+ if any(stamped.get(key) != value for key, value in expected.items()):
+ raise SandboxNotOwnedError(
+ f"Sandbox {sandbox_id[:12]} is not {owner}'s {sandbox_kind} box"
+ )
+ return await AsyncSandbox.connect(sandbox_id, api_key=api_key, timeout=timeout)
+
+
+def _as_owner(owner: "SandboxOwner | str") -> SandboxOwner:
+ """Accept a bare session id where an owner is expected."""
+ if isinstance(owner, SandboxOwner):
+ return owner
+ return SandboxOwner(kind="session", id=owner)
+
def _sandbox_key(session_id: str) -> str:
- return f"{_SANDBOX_KEY_PREFIX}{session_id}"
+ return SandboxOwner(kind="session", id=session_id).key()
-async def _get_stored_sandbox_id(session_id: str) -> str | None:
+async def _get_stored_sandbox_id(
+ owner: SandboxOwner, sandbox_kind: SandboxKind = "shell"
+) -> str | None:
redis = await get_redis_async()
- raw = await redis.get(_sandbox_key(session_id))
+ raw = await redis.get(owner.key(sandbox_kind))
value = raw.decode() if isinstance(raw, bytes) else raw
return None if value == _CREATING_SENTINEL else value
-async def _set_stored_sandbox_id(session_id: str, sandbox_id: str) -> None:
+async def _set_stored_sandbox_id(
+ owner: SandboxOwner, sandbox_id: str, sandbox_kind: SandboxKind = "shell"
+) -> None:
redis = await get_redis_async()
- await redis.set(_sandbox_key(session_id), sandbox_id, ex=_SANDBOX_ID_TTL)
+ await redis.set(owner.key(sandbox_kind), sandbox_id, ex=owner.ttl)
-async def _clear_stored_sandbox_id(session_id: str) -> None:
+async def _clear_stored_sandbox_id(
+ owner: SandboxOwner, sandbox_kind: SandboxKind = "shell"
+) -> None:
redis = await get_redis_async()
- await redis.delete(_sandbox_key(session_id))
+ await redis.delete(owner.key(sandbox_kind))
+
+
+class SandboxLookupError(Exception):
+ """E2B could not tell us which boxes an owner has.
+
+ Distinct from "none": for an expert the box *is* the state, so a failed
+ lookup must not be read as "no box" and answered with a fresh one.
+ """
+
+
+async def list_owned_sandboxes(
+ owner: SandboxOwner, sandbox_kind: SandboxKind, api_key: str
+) -> list[SandboxInfo]:
+ """The owner's boxes of one kind as E2B lists them, newest first.
+
+ A running box sorts before a paused one. Listing never connects, so a
+ paused box stays paused — connecting is what auto-resume reacts to.
+ Raises :class:`SandboxLookupError` when the API call fails or times out.
+ """
+ try:
+ paginator = AsyncSandbox.list(
+ query=SandboxQuery(
+ metadata=owner.metadata(sandbox_kind),
+ state=[SandboxState.RUNNING, SandboxState.PAUSED],
+ ),
+ limit=10,
+ api_key=api_key,
+ )
+ infos = await asyncio.wait_for(
+ paginator.next_items(), timeout=_E2B_API_TIMEOUT_SECONDS
+ )
+ except Exception as exc:
+ raise SandboxLookupError(
+ f"E2B lookup of {owner}'s {sandbox_kind} boxes failed: {exc}"
+ ) from exc
+ infos = sorted(infos, key=lambda info: info.started_at, reverse=True)
+ running = [info for info in infos if info.state == SandboxState.RUNNING]
+ paused = [info for info in infos if info.state != SandboxState.RUNNING]
+ return running + paused
+
+
+async def find_owned_sandbox_id(
+ owner: SandboxOwner, sandbox_kind: SandboxKind, api_key: str
+) -> str | None:
+ """Recover an expert's box through the E2B API when Redis has forgotten it.
+
+ Session sandboxes are Redis-only: losing that key just means a fresh
+ scratch sandbox, which is not worth a control-plane round-trip. For an
+ expert the box *is* the state, so we query E2B by the owner metadata every
+ sandbox is stamped with, preferring a running box over a paused one and
+ the newest of several (a lost creation race can leave duplicates).
+ A failed lookup raises :class:`SandboxLookupError` rather than answering
+ ``None``: "unknown" must never turn into a second box.
+ """
+ if not owner.is_expert:
+ return None
+ infos = await list_owned_sandboxes(owner, sandbox_kind, api_key)
+ if not infos:
+ return None
+ chosen = infos[0]
+ if len(infos) > 1:
+ logger.warning(
+ "[E2B] %s has %d %s sandboxes; using %.12s",
+ owner,
+ len(infos),
+ sandbox_kind,
+ chosen.sandbox_id,
+ )
+ return chosen.sandbox_id
async def _try_reconnect(
- sandbox_id: str, session_id: str, api_key: str
+ sandbox_id: str, owner: "SandboxOwner | str", api_key: str
) -> "AsyncSandbox | None":
"""Try to reconnect to an existing sandbox. Returns None on failure."""
+ owner = _as_owner(owner)
try:
- sandbox = await AsyncSandbox.connect(sandbox_id, api_key=api_key)
+ sandbox = await connect_owned(sandbox_id, owner, "shell", api_key)
if await sandbox.is_running():
- # Refresh TTL so an active session cannot lose its sandbox_id at expiry.
- await _set_stored_sandbox_id(session_id, sandbox_id)
+ # Refresh TTL so an active owner cannot lose its sandbox_id at expiry.
+ await _set_stored_sandbox_id(owner, sandbox_id)
return sandbox
+ except SandboxNotOwnedError as exc:
+ logger.warning("[E2B] Refusing reconnect: %s", exc)
except Exception as exc:
logger.warning("[E2B] Reconnect to %.12s failed: %s", sandbox_id, exc)
# Stale — clear the sandbox_id from Redis so a new one can be created.
- await _clear_stored_sandbox_id(session_id)
+ await _clear_stored_sandbox_id(owner)
return None
+async def _resolve_volume_mounts(
+ volume_mounts: Mapping[str, str] | None, api_key: str
+) -> dict[str, "AsyncVolume | str"] | None:
+ """Build the ``volume_mounts`` mapping (path -> volume) for named volumes.
+
+ Creates each volume if it does not exist yet, otherwise mounts it by name
+ (``resolve_volume`` bounds each call). Volumes resolve concurrently so the
+ creation lock is held for one round-trip, not one per mount. Returns
+ ``None`` when no volumes are requested; never raises.
+ """
+ if not volume_mounts:
+ return None
+ paths = list(volume_mounts)
+ volumes = await asyncio.gather(
+ *(resolve_volume(volume_mounts[path], api_key) for path in paths)
+ )
+ return dict(zip(paths, volumes))
+
+
+# ---------------------------------------------------------------------------
+# Expert boxes: concurrent-turn accounting
+# ---------------------------------------------------------------------------
+
+
+def _active_turns_key(owner: SandboxOwner) -> str:
+ return f"{owner.key('shell')}:active"
+
+
+async def _acquire_turn(owner: SandboxOwner) -> None:
+ """Count this turn on an expert's box so another turn's end can't pause it.
+
+ Fails closed: if the count cannot be recorded the turn must not run on
+ the box, because its eventual release would decrement a count it never
+ added and could pause the box under a concurrent turn.
+ """
+ if not owner.is_expert:
+ return
+ redis = await get_redis_async()
+ key = _active_turns_key(owner)
+ await redis.eval(_ACQUIRE_TURN_SCRIPT, 1, key, _ACTIVE_TURN_TTL)
+
+
+async def count_expert_turn(session_id: str, expert_id: str | None) -> None:
+ """Count a turn on the expert's box (no-op for a plain session).
+
+ For callers that opened the box with ``count_turn=False`` because work
+ that could still fail sits between opening it and the ``try`` whose
+ ``finally`` releases the turn. Count only once that ``try`` is reached.
+ """
+ await _acquire_turn(SandboxOwner.for_session(session_id, expert_id))
+
+
+async def _release_turn(owner: SandboxOwner) -> bool:
+ """Return ``True`` when the box may be paused: no other turn is still on it."""
+ if not owner.is_expert:
+ return True
+ try:
+ redis = await get_redis_async()
+ key = _active_turns_key(owner)
+ remaining = int(await redis.eval(_RELEASE_TURN_SCRIPT, 1, key))
+ if remaining <= 0:
+ return True
+ logger.info(
+ "[E2B] %s still has %d active turn(s); leaving its box running",
+ owner,
+ remaining,
+ )
+ return False
+ except Exception as exc:
+ # Fail closed: without the counter we cannot know whether another turn
+ # is on the box, and pausing under one severs its command stream. The
+ # lifecycle timeout still pauses the box once it goes idle.
+ logger.warning(
+ "[E2B] Could not release active turn for %s (%s); leaving its box running",
+ owner,
+ exc,
+ )
+ return False
+
+
async def get_or_create_sandbox(
session_id: str,
api_key: str,
timeout: int,
template: str = "base",
on_timeout: Literal["kill", "pause"] = "pause",
+ volume_mounts: Mapping[str, str] | None = None,
+ *,
+ expert_id: str | None = None,
+ user_id: str | None = None,
+ count_turn: bool = True,
) -> AsyncSandbox:
- """Return the existing E2B sandbox for *session_id* or create a new one.
+ """Return the existing E2B sandbox for this turn's owner or create a new one.
- The sandbox key in Redis serves a dual purpose: it stores the sandbox_id
- and acts as a creation lock via a ``"creating"`` sentinel value. This
- removes the need for a separate lock key.
+ The owner is the session, or — when *expert_id* is set — the expert, whose
+ box every one of its sessions shares. The owner's key in Redis serves a
+ dual purpose: it stores the sandbox_id and acts as a creation lock via a
+ ``"creating"`` sentinel value. This removes the need for a separate lock
+ key.
*timeout* controls how long the e2b sandbox may run continuously before
the ``on_timeout`` lifecycle rule fires (default: 5 min).
*on_timeout* controls what happens on timeout: ``"pause"`` (default, free)
or ``"kill"``. When ``"pause"``, ``auto_resume`` is enabled so paused
sandboxes wake transparently on SDK activity.
+ *volume_mounts* maps mount paths to durable volume names (see
+ ``workspace_volume_mounts``) so the agent shell shares a persistent
+ ``~/workspace`` with the on-demand desktop and, for an expert, the owning
+ user's ``~/shared``. A mount failure degrades to a volume-less sandbox
+ rather than failing the session.
+ *count_turn* records this turn on an expert's box so another turn's end
+ cannot pause it; pass ``False`` and call ``count_expert_turn`` later when
+ the release is not yet guaranteed to run.
+
+ Raises :class:`SandboxLookupError` when E2B cannot say whether an expert
+ already has a box: a fresh box would fork the expert's durable state.
"""
+ owner = SandboxOwner.for_session(session_id, expert_id)
redis = await get_redis_async()
- key = _sandbox_key(session_id)
+ key = owner.key("shell")
+ # Boxes E2B still lists but we could not reconnect to twice (mid-teardown,
+ # an unresumable snapshot). Without this an expert owner would re-find the
+ # same id on every iteration and never fall through to creating a fresh one.
+ failed_ids: set[str] = set()
+ retried_ids: set[str] = set()
for _ in range(_MAX_WAIT_ATTEMPTS):
raw = await redis.get(key)
value = raw.decode() if isinstance(raw, bytes) else raw
+ if not value and owner.is_expert:
+ # Redis is only a cache for an expert's box; E2B is the record.
+ value = await find_owned_sandbox_id(owner, "shell", api_key)
+ if value in failed_ids:
+ value = None
+ elif value:
+ await _set_stored_sandbox_id(owner, value)
+
if value and value != _CREATING_SENTINEL:
# Existing sandbox ID — try to reconnect (auto-resumes if paused).
- sandbox = await _try_reconnect(value, session_id, api_key)
+ sandbox = await _try_reconnect(value, owner, api_key)
+ if sandbox is None and owner.is_expert and value not in retried_ids:
+ # A durable box gets one more chance before it is given up
+ # on; a single transient failure must not fork it.
+ retried_ids.add(value)
+ await asyncio.sleep(_RECONNECT_RETRY_DELAY_SECONDS)
+ sandbox = await _try_reconnect(value, owner, api_key)
if sandbox:
- logger.info(
- "[E2B] Reconnected to %.12s for session %.12s",
- value,
- session_id,
- )
+ logger.info("[E2B] Reconnected to %.12s for %s", value, owner)
+ if count_turn:
+ await _acquire_turn(owner)
return sandbox
# _try_reconnect cleared the key — loop to create a new sandbox.
+ failed_ids.add(value)
continue
if value == _CREATING_SENTINEL:
@@ -192,7 +574,7 @@ async def get_or_create_sandbox(
# We hold the slot — create the sandbox with per-attempt timeout and
# retry. The sentinel remains held throughout so concurrent callers
- # for the same session wait rather than racing to create duplicates.
+ # for the same owner wait rather than racing to create duplicates.
sandbox: AsyncSandbox | None = None
try:
lifecycle = SandboxLifecycle(
@@ -208,6 +590,7 @@ async def get_or_create_sandbox(
# killed) at end_at and persist until explicitly cleaned up.
# At most _SANDBOX_CREATE_MAX_RETRIES − 1 = 2 sandboxes can
# leak per incident.
+ mounts = await _resolve_volume_mounts(volume_mounts, api_key)
last_exc: Exception | None = None
for attempt in range(1, _SANDBOX_CREATE_MAX_RETRIES + 1):
try:
@@ -217,6 +600,14 @@ async def get_or_create_sandbox(
api_key=api_key,
timeout=timeout,
lifecycle=lifecycle,
+ volume_mounts=mounts,
+ metadata=owner.creation_metadata(
+ "shell",
+ user_id=user_id,
+ session_id=session_id,
+ template=template,
+ mounts="attached" if mounts else "none",
+ ),
),
timeout=_SANDBOX_CREATE_TIMEOUT_SECONDS,
)
@@ -225,12 +616,25 @@ async def get_or_create_sandbox(
except Exception as exc:
last_exc = exc
logger.warning(
- "[E2B] Sandbox creation attempt %d/%d failed for session %.12s: %s",
+ "[E2B] Sandbox creation attempt %d/%d failed for %s: %s",
attempt,
_SANDBOX_CREATE_MAX_RETRIES,
- session_id,
+ owner,
exc,
)
+ if (
+ mounts is not None
+ and attempt == _SANDBOX_CREATE_MAX_RETRIES - 1
+ ):
+ # A volume problem must not cost the user their shell,
+ # but a transient failure must not cost an expert its
+ # durable home either: keep the mounts through the
+ # retries and only make the last attempt volume-less.
+ logger.warning(
+ "[E2B] Final attempt for %s will run without workspace volumes",
+ owner,
+ )
+ mounts = None
if attempt < _SANDBOX_CREATE_MAX_RETRIES:
await asyncio.sleep(2 ** (attempt - 1)) # 1 s, 2 s
@@ -241,8 +645,13 @@ async def get_or_create_sandbox(
raise last_exc
assert sandbox is not None # guaranteed: last_exc is None iff break was hit
+ if mounts:
+ with contextlib.suppress(Exception):
+ await sandbox.commands.run(
+ "mkdir -p " + " ".join(f"'{path}'" for path in mounts)
+ )
try:
- await _set_stored_sandbox_id(session_id, sandbox.sandbox_id)
+ await _set_stored_sandbox_id(owner, sandbox.sandbox_id)
except Exception:
# Redis save failed — kill the sandbox to avoid leaking it.
with contextlib.suppress(Exception):
@@ -271,66 +680,79 @@ async def get_or_create_sandbox(
await redis.delete(key)
raise
- logger.info(
- "[E2B] Created sandbox %.12s for session %.12s",
- sandbox.sandbox_id,
- session_id,
- )
+ logger.info("[E2B] Created sandbox %.12s for %s", sandbox.sandbox_id, owner)
+ if count_turn:
+ await _acquire_turn(owner)
return sandbox
- raise RuntimeError(f"Could not acquire E2B sandbox for session {session_id[:12]}")
+ raise RuntimeError(f"Could not acquire E2B sandbox for {owner}")
async def _act_on_sandbox(
- session_id: str,
+ owner: SandboxOwner,
api_key: str,
action: str,
fn: Callable[[AsyncSandbox], Awaitable[Any]],
*,
+ sandbox_kind: SandboxKind = "shell",
+ sandbox_id: str | None = None,
clear_stored_id: bool = False,
) -> bool:
- """Connect to the sandbox for *session_id* and run *fn* on it.
+ """Connect to the owner's sandbox and run *fn* on it.
- Shared by ``pause_sandbox`` and ``kill_sandbox``. Returns ``True`` on
- success, ``False`` when no sandbox is found or the action fails.
- If *clear_stored_id* is ``True``, the sandbox_id is removed from Redis
- only after the action succeeds so a failed kill can be retried.
+ Shared by ``pause_sandbox``, ``kill_sandbox`` and
+ ``kill_expert_sandboxes``. Returns ``True`` on success, ``False`` when no
+ sandbox is found or the action fails. If *clear_stored_id* is ``True``,
+ the sandbox_id is removed from Redis only after the action succeeds so a
+ failed kill can be retried.
"""
- sandbox_id = await _get_stored_sandbox_id(session_id)
+ if sandbox_id is None:
+ sandbox_id = await _get_stored_sandbox_id(owner, sandbox_kind)
if not sandbox_id:
return False
async def _run() -> None:
- await fn(await AsyncSandbox.connect(sandbox_id, api_key=api_key))
+ await fn(await connect_owned(sandbox_id, owner, sandbox_kind, api_key))
try:
await asyncio.wait_for(_run(), timeout=_E2B_API_TIMEOUT_SECONDS)
if clear_stored_id:
- await _clear_stored_sandbox_id(session_id)
+ await _clear_stored_sandbox_id(owner, sandbox_kind)
logger.info(
- "[E2B] %s sandbox %.12s for session %.12s",
+ "[E2B] %s %s sandbox %.12s for %s",
action.capitalize(),
+ sandbox_kind,
sandbox_id,
- session_id,
+ owner,
)
return True
+ except SandboxNotOwnedError as exc:
+ # A cached id that is not the owner's box any more is never going to
+ # be: forget it, or every later pause and kill fails the same way.
+ logger.warning("[E2B] Refusing to %s: %s", action, exc)
+ await _clear_stored_sandbox_id(owner, sandbox_kind)
+ return False
except Exception as exc:
logger.warning(
- "[E2B] Failed to %s sandbox %.12s for session %.12s: %s",
+ "[E2B] Failed to %s %s sandbox %.12s for %s: %s",
action,
+ sandbox_kind,
sandbox_id,
- session_id,
+ owner,
exc,
)
return False
-async def pause_sandbox(session_id: str, api_key: str) -> bool:
- """Pause the E2B sandbox for *session_id* to stop billing between turns.
+async def pause_sandbox(
+ session_id: str, api_key: str, *, expert_id: str | None = None
+) -> bool:
+ """Pause the E2B sandbox for this turn's owner to stop billing between turns.
Paused sandboxes cost nothing and are resumed automatically by
``get_or_create_sandbox()`` on the next turn (via ``AsyncSandbox.connect()``).
- The sandbox_id is kept in Redis so reconnection works seamlessly.
+ The sandbox_id is kept in Redis so reconnection works seamlessly. An
+ expert's box is left running while another of its turns is still active.
Prefer ``pause_sandbox_direct()`` when the sandbox object is already in
scope — it skips the Redis lookup and reconnect round-trip.
@@ -338,31 +760,37 @@ async def pause_sandbox(session_id: str, api_key: str) -> bool:
Returns ``True`` if the sandbox was found and paused, ``False`` otherwise.
Safe to call even when no sandbox exists for the session.
"""
- return await _act_on_sandbox(session_id, api_key, "pause", lambda sb: sb.pause())
+ owner = SandboxOwner.for_session(session_id, expert_id)
+ if not await _release_turn(owner):
+ return False
+ return await _act_on_sandbox(owner, api_key, "pause", lambda sb: sb.pause())
-async def pause_sandbox_direct(sandbox: "AsyncSandbox", session_id: str) -> bool:
+async def pause_sandbox_direct(
+ sandbox: "AsyncSandbox", session_id: str, *, expert_id: str | None = None
+) -> bool:
"""Pause an already-connected sandbox without a reconnect round-trip.
Use this in callers that already hold the live sandbox object (e.g. turn
teardown in ``service.py``). Saves the Redis lookup and
- ``AsyncSandbox.connect()`` call that ``pause_sandbox()`` would make.
+ ``AsyncSandbox.connect()`` call that ``pause_sandbox()`` would make. An
+ expert's box is left running while another of its turns is still active.
- Returns ``True`` on success, ``False`` on failure or timeout.
+ Returns ``True`` on success, ``False`` on failure, timeout, or when the
+ box is deliberately left running.
"""
+ owner = SandboxOwner.for_session(session_id, expert_id)
+ if not await _release_turn(owner):
+ return False
try:
await asyncio.wait_for(sandbox.pause(), timeout=_E2B_API_TIMEOUT_SECONDS)
- logger.info(
- "[E2B] Paused sandbox %.12s for session %.12s",
- sandbox.sandbox_id,
- session_id,
- )
+ logger.info("[E2B] Paused sandbox %.12s for %s", sandbox.sandbox_id, owner)
return True
except Exception as exc:
logger.warning(
- "[E2B] Failed to pause sandbox %.12s for session %.12s: %s",
+ "[E2B] Failed to pause sandbox %.12s for %s: %s",
sandbox.sandbox_id,
- session_id,
+ owner,
exc,
)
return False
@@ -372,15 +800,64 @@ async def kill_sandbox(
session_id: str,
api_key: str,
) -> bool:
- """Kill the E2B sandbox for *session_id* and clear its Redis entry.
+ """Kill the session's E2B sandboxes (shell and on-demand desktop).
- Returns ``True`` if a sandbox was found and killed, ``False`` otherwise.
- Safe to call even when no sandbox exists for the session.
+ Only ever touches *session* sandboxes: an expert session has none under
+ its own id, so deleting an expert chat leaves the expert's computer alone
+ (see ``kill_expert_sandboxes`` for the archive path). The desktop is
+ included because nothing else ever stops it: it is deliberately not paused
+ at turn end, so an orphaned one would bill until its timeout and then sit
+ paused forever.
+
+ Returns ``True`` if at least one sandbox was found and killed, ``False``
+ otherwise. Safe to call even when no sandbox exists for the session.
"""
- return await _act_on_sandbox(
- session_id,
- api_key,
- "kill",
- lambda sb: sb.kill(),
- clear_stored_id=True,
- )
+ owner = SandboxOwner(kind="session", id=session_id)
+ killed = False
+ for sandbox_kind in ("shell", "desktop"):
+ if await _act_on_sandbox(
+ owner,
+ api_key,
+ "kill",
+ lambda sb: sb.kill(),
+ sandbox_kind=sandbox_kind,
+ clear_stored_id=True,
+ ):
+ killed = True
+ return killed
+
+
+async def kill_expert_sandboxes(expert_id: str, api_key: str) -> int:
+ """Kill an expert's shell and desktop boxes — its computer goes on archive.
+
+ The expert's volume is deliberately kept: files outlive the machine, and
+ destroying user data is not something a best-effort cleanup should do.
+ Falls back to the E2B metadata lookup for each box so a stale Redis cache
+ cannot leave a paused machine behind. Returns the number killed.
+ """
+ owner = SandboxOwner(kind="expert", id=expert_id)
+ killed = 0
+ for sandbox_kind in ("shell", "desktop"):
+ sandbox_id = await _get_stored_sandbox_id(owner, sandbox_kind)
+ if not sandbox_id:
+ try:
+ sandbox_id = await find_owned_sandbox_id(owner, sandbox_kind, api_key)
+ except SandboxLookupError as exc:
+ logger.warning("[E2B] Archive of %s: %s", owner, exc)
+ continue
+ if not sandbox_id:
+ continue
+ if await _act_on_sandbox(
+ owner,
+ api_key,
+ "kill",
+ lambda sb: sb.kill(),
+ sandbox_kind=sandbox_kind,
+ sandbox_id=sandbox_id,
+ clear_stored_id=True,
+ ):
+ killed += 1
+ with contextlib.suppress(Exception):
+ redis = await get_redis_async()
+ await redis.delete(_active_turns_key(owner))
+ return killed
diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
index 4adfe0094d..b8c2c74a7a 100644
--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
@@ -12,15 +12,34 @@ session-scoped event loop in conftest.py.
"""
import asyncio
+import contextlib
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from e2b import SandboxState
+
+from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.blocks.desktop._common import (
+ expert_volume_name,
+ user_volume_name,
+ workspace_volume_mounts,
+)
+from backend.util.sandbox_metadata import deployment_env
from .e2b_sandbox import (
_CREATING_SENTINEL,
_SANDBOX_CREATE_MAX_RETRIES,
+ SandboxLookupError,
+ SandboxNotOwnedError,
+ SandboxOwner,
_try_reconnect,
+ connect_owned,
+ count_expert_turn,
+ find_owned_sandbox_id,
get_or_create_sandbox,
+ kill_expert_sandboxes,
kill_sandbox,
pause_sandbox,
pause_sandbox_direct,
@@ -32,15 +51,61 @@ _SANDBOX_ID = "sb-abc"
_TIMEOUT = 300
-def _mock_sandbox(sandbox_id: str = _SANDBOX_ID, running: bool = True) -> MagicMock:
+def _mock_sandbox(
+ sandbox_id: str = _SANDBOX_ID,
+ running: bool = True,
+ *,
+ owner: SandboxOwner | None = None,
+ kind: str = "shell",
+) -> MagicMock:
+ """A connected box stamped as *owner*'s (the session's shell box by default).
+
+ ``connect_owned`` reads the stamp back through ``get_info``; a box with
+ the wrong stamp is refused, so every test that reconnects says whose box
+ it is.
+ """
sb = MagicMock()
sb.sandbox_id = sandbox_id
sb.is_running = AsyncMock(return_value=running)
sb.pause = AsyncMock()
sb.kill = AsyncMock()
+ stamped = (owner or SandboxOwner(kind="session", id=_SESSION_ID)).metadata(
+ kind # type: ignore[arg-type]
+ )
+ sb.get_info = AsyncMock(return_value=MagicMock(metadata=stamped))
+ _STAMPS[sandbox_id] = sb.get_info.return_value
return sb
+# sandbox_id -> the info the SDK reports for it, filled by ``_mock_sandbox``.
+_STAMPS: dict[str, MagicMock] = {}
+_SESSION_SHELL_STAMP = SandboxOwner(kind="session", id=_SESSION_ID).metadata("shell")
+
+
[email protected](autouse=True)
+def _fresh_stamps():
+ _STAMPS.clear()
+ yield
+
+
+def _patch_sdk():
+ """The E2B SDK, with ``get_info`` answering for the boxes a test built.
+
+ ``connect_owned`` reads a box's stamp through the *static* ``get_info``
+ before it connects, so the class mock has to know the stamp too: a box
+ from ``_mock_sandbox`` answers with its own, and an id no test built is
+ taken to be the session's shell box, which is what most tests reconnect.
+ """
+ with patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls:
+
+ async def _get_info(sandbox_id: str, **_):
+ return _STAMPS.get(sandbox_id) or MagicMock(metadata=_SESSION_SHELL_STAMP)
+
+ mock_cls.get_info = AsyncMock(side_effect=_get_info)
+ yield mock_cls
+
+
def _mock_redis(
set_nx_result: bool = True,
stored_sandbox_id: str | None = None,
@@ -55,7 +120,9 @@ def _mock_redis(
"""
r = AsyncMock()
raw = stored_sandbox_id.encode() if stored_sandbox_id else None
- r.get = AsyncMock(return_value=raw)
+ shell_key = f"copilot:e2b:sandbox:{_SESSION_ID}"
+ # Only the session's shell key holds the id; its desktop key is empty.
+ r.get = AsyncMock(side_effect=lambda key: raw if key == shell_key else None)
r.set = AsyncMock(return_value=set_nx_result)
r.delete = AsyncMock()
return r
@@ -75,12 +142,29 @@ def _patch_redis(redis: AsyncMock):
class TestTryReconnect:
+ def test_reconnect_refuses_a_box_stamped_for_someone_else(self):
+ """A cached id that resolves to another owner's box is dropped, not used."""
+ sb = _mock_sandbox(owner=SandboxOwner(kind="session", id="other-session"))
+ redis = _mock_redis()
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(return_value=sb)
+ result = asyncio.run(_try_reconnect(_SANDBOX_ID, _SESSION_ID, _API_KEY))
+
+ assert result is None
+ # Connecting would resume the other owner's box; it is never touched.
+ mock_cls.connect.assert_not_awaited()
+ sb.kill.assert_not_awaited()
+ redis.delete.assert_awaited_once()
+
def test_reconnect_success(self):
"""Returns the sandbox when it connects and is running; refreshes Redis TTL."""
sb = _mock_sandbox()
redis = _mock_redis()
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -96,7 +180,7 @@ class TestTryReconnect:
sb = _mock_sandbox(running=False)
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -109,7 +193,7 @@ class TestTryReconnect:
"""Clears sandbox_id in Redis when connect raises an exception."""
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(side_effect=ConnectionError("gone"))
@@ -130,7 +214,7 @@ class TestGetOrCreateSandbox:
sb = _mock_sandbox()
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -148,7 +232,7 @@ class TestGetOrCreateSandbox:
new_sb = _mock_sandbox("sb-new")
redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.create = AsyncMock(return_value=new_sb)
@@ -188,7 +272,7 @@ class TestGetOrCreateSandbox:
redis.set = AsyncMock(side_effect=fake_set)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
patch(
"backend.copilot.tools.e2b_sandbox.ensure_template",
side_effect=fake_ensure,
@@ -211,7 +295,7 @@ class TestGetOrCreateSandbox:
new_sb = _mock_sandbox("sb-new")
redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.create = AsyncMock(return_value=new_sb)
@@ -231,7 +315,7 @@ class TestGetOrCreateSandbox:
"""If sandbox creation fails, the Redis creation slot is deleted."""
redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.create = AsyncMock(side_effect=RuntimeError("quota"))
@@ -265,7 +349,7 @@ class TestGetOrCreateSandbox:
redis.set = AsyncMock(side_effect=_set_side_effect)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.create = AsyncMock(return_value=new_sb)
@@ -290,7 +374,7 @@ class TestGetOrCreateSandbox:
redis.delete = AsyncMock()
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
patch(
"backend.copilot.tools.e2b_sandbox.asyncio.sleep",
@@ -319,7 +403,7 @@ class TestGetOrCreateSandbox:
return new_sb
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
patch(
"backend.copilot.tools.e2b_sandbox.asyncio.sleep",
@@ -339,7 +423,7 @@ class TestGetOrCreateSandbox:
redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
patch(
"backend.copilot.tools.e2b_sandbox.asyncio.sleep",
@@ -371,7 +455,7 @@ class TestGetOrCreateSandbox:
return new_sb
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
patch(
"backend.copilot.tools.e2b_sandbox.asyncio.sleep",
@@ -394,7 +478,7 @@ class TestGetOrCreateSandbox:
raise asyncio.CancelledError
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
patch(
"backend.copilot.tools.e2b_sandbox.asyncio.sleep",
@@ -419,7 +503,7 @@ class TestGetOrCreateSandbox:
raise asyncio.CancelledError
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
patch(
"backend.copilot.tools.e2b_sandbox._set_stored_sandbox_id",
side_effect=_set_side_effect,
@@ -451,7 +535,7 @@ class TestGetOrCreateSandbox:
redis.delete = AsyncMock()
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=stale_sb)
@@ -476,7 +560,7 @@ class TestKillSandbox:
sb = _mock_sandbox()
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -487,6 +571,20 @@ class TestKillSandbox:
# Redis key cleared after successful kill
redis.delete.assert_awaited_once()
+ def test_kill_refuses_a_foreign_box_and_forgets_it(self):
+ """A cached id stamped for someone else is dropped so the next
+ pause or kill does not fail the same way; the box is never touched."""
+ foreign = _mock_sandbox(owner=SandboxOwner(kind="session", id="other"))
+ redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
+ with _patch_sdk() as mock_cls, _patch_redis(redis):
+ mock_cls.connect = AsyncMock(return_value=foreign)
+ result = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))
+
+ assert result is False
+ mock_cls.connect.assert_not_awaited()
+ foreign.kill.assert_not_awaited()
+ redis.delete.assert_awaited_once_with(f"copilot:e2b:sandbox:{_SESSION_ID}")
+
def test_kill_no_sandbox(self):
"""No-op when Redis has no sandbox_id."""
redis = _mock_redis(stored_sandbox_id=None)
@@ -502,7 +600,7 @@ class TestKillSandbox:
"""
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(side_effect=ConnectionError("gone"))
@@ -547,7 +645,7 @@ class TestPauseSandbox:
sb = _mock_sandbox()
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -570,7 +668,7 @@ class TestPauseSandbox:
"""Returns False if connect fails."""
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(side_effect=ConnectionError("gone"))
@@ -613,7 +711,7 @@ class TestPauseSandbox:
sb = _mock_sandbox(_SANDBOX_ID)
redis = _mock_redis(stored_sandbox_id=_SANDBOX_ID)
with (
- patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_sdk() as mock_cls,
_patch_redis(redis),
):
mock_cls.connect = AsyncMock(return_value=sb)
@@ -665,3 +763,651 @@ class TestPauseSandboxDirect:
result = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))
assert result is False
+
+
+# ---------------------------------------------------------------------------
+# Expert boxes: one persistent sandbox per hired expert
+# ---------------------------------------------------------------------------
+
+_EXPERT_ID = "exp-777"
+_USER_ID = "user-42"
+_EXPERT_SHELL_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:shell"
+_EXPERT_DESKTOP_KEY = f"copilot:e2b:expert:{_EXPERT_ID}:desktop"
+_EXPERT_ACTIVE_KEY = f"{_EXPERT_SHELL_KEY}:active"
+
+
+def _info(
+ sandbox_id: str, state: SandboxState, age_seconds: int = 0
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ sandbox_id=sandbox_id,
+ state=state,
+ started_at=datetime.now(timezone.utc) - timedelta(seconds=age_seconds),
+ )
+
+
+def _mock_list(infos: list) -> MagicMock:
+ """``AsyncSandbox.list`` is sync and returns an async paginator."""
+ paginator = MagicMock()
+ paginator.next_items = AsyncMock(return_value=infos)
+ return MagicMock(return_value=paginator)
+
+
+def _keyed_redis(values: dict[str, str | None], decr_result: int = 0) -> AsyncMock:
+ """Redis mock answering ``get`` per key (bytes, like the real client)."""
+ r = AsyncMock()
+ r.get = AsyncMock(side_effect=lambda key: (values.get(key) or "").encode() or None)
+ r.set = AsyncMock(return_value=True)
+ r.delete = AsyncMock()
+ # Two scripts: ``_acquire_turn`` (INCR + EXPIRE) and ``_release_turn``
+ # (DECR, and DEL when nothing is left).
+ r.eval = AsyncMock(
+ side_effect=lambda script, *_: 1 if "incr" in script else max(decr_result, 0)
+ )
+ return r
+
+
+def _turn_acquires(redis: AsyncMock) -> list[str]:
+ """Keys the acquire script ran against, in order."""
+ return [
+ call.args[2] for call in redis.eval.await_args_list if "incr" in call.args[0]
+ ]
+
+
+class TestConnectOwned:
+ def test_checks_the_stamp_then_connects(self):
+ owner = SandboxOwner(kind="expert", id=_EXPERT_ID)
+ sb = _mock_sandbox("sb-expert", owner=owner, kind="desktop")
+ with _patch_sdk() as mock_cls:
+ mock_cls.connect = AsyncMock(return_value=sb)
+ result = asyncio.run(connect_owned("sb-expert", owner, "desktop", _API_KEY))
+ assert result is sb
+ mock_cls.get_info.assert_awaited_once_with("sb-expert", api_key=_API_KEY)
+ mock_cls.connect.assert_awaited_once_with(
+ "sb-expert", api_key=_API_KEY, timeout=None
+ )
+
+ def test_timeout_reaches_the_connect(self):
+ """A resumed desktop gets its own running-time limit, not the SDK's."""
+ owner = SandboxOwner(kind="expert", id=_EXPERT_ID)
+ sb = _mock_sandbox("sb-expert", owner=owner, kind="desktop")
+ with _patch_sdk() as mock_cls:
+ mock_cls.connect = AsyncMock(return_value=sb)
+ asyncio.run(
+ connect_owned("sb-expert", owner, "desktop", _API_KEY, timeout=900)
+ )
+ mock_cls.connect.assert_awaited_once_with(
+ "sb-expert", api_key=_API_KEY, timeout=900
+ )
+
+ @pytest.mark.parametrize(
+ "stamp",
+ [
+ {"autogpt_owner": "expert:someone-else", "autogpt_kind": "shell"},
+ {"autogpt_owner": f"expert:{_EXPERT_ID}", "autogpt_kind": "desktop"},
+ {},
+ ],
+ ids=["another owner", "wrong kind", "unstamped"],
+ )
+ def test_refuses_a_box_that_is_not_the_owners_without_waking_it(self, stamp):
+ """Any id connects under the platform key; the stamp is the only record.
+
+ A connect resumes a paused box and extends its life, so the refusal
+ has to come from the stamp alone, before any connect.
+ """
+ owner = SandboxOwner(kind="expert", id=_EXPERT_ID)
+ with _patch_sdk() as mock_cls:
+ mock_cls.get_info = AsyncMock(return_value=MagicMock(metadata=stamp))
+ mock_cls.connect = AsyncMock()
+ with pytest.raises(SandboxNotOwnedError):
+ asyncio.run(connect_owned("sb-x", owner, "shell", _API_KEY))
+ mock_cls.connect.assert_not_awaited()
+
+
+class TestSandboxOwner:
+ def test_expert_session_is_owned_by_the_expert(self):
+ owner = SandboxOwner.for_session(_SESSION_ID, _EXPERT_ID)
+ assert owner == SandboxOwner(kind="expert", id=_EXPERT_ID)
+ assert owner.is_expert
+ assert owner.key() == _EXPERT_SHELL_KEY
+ assert owner.key("desktop") == _EXPERT_DESKTOP_KEY
+
+ def test_plain_session_keys_are_unchanged(self):
+ owner = SandboxOwner.for_session(_SESSION_ID, None)
+ assert not owner.is_expert
+ assert owner.key() == f"copilot:e2b:sandbox:{_SESSION_ID}"
+ assert owner.key("desktop") == f"copilot:e2b:desktop:{_SESSION_ID}"
+
+ def test_expert_cache_outlives_session_cache(self):
+ assert (
+ SandboxOwner(kind="expert", id=_EXPERT_ID).ttl
+ > SandboxOwner(kind="session", id=_SESSION_ID).ttl
+ )
+
+ def test_metadata_identifies_owner_and_kind(self):
+ assert SandboxOwner(kind="expert", id=_EXPERT_ID).metadata("desktop") == {
+ "autogpt_owner": f"expert:{_EXPERT_ID}",
+ "autogpt_kind": "desktop",
+ }
+
+
+class TestFindOwnedSandboxId:
+ def test_session_owner_never_hits_the_api(self):
+ with _patch_sdk() as mock_cls:
+ result = asyncio.run(
+ find_owned_sandbox_id(
+ SandboxOwner(kind="session", id=_SESSION_ID), "shell", _API_KEY
+ )
+ )
+ assert result is None
+ mock_cls.list.assert_not_called()
+
+ def test_prefers_running_box_over_newer_paused_one(self):
+ infos = [
+ _info("sb-paused-new", SandboxState.PAUSED, age_seconds=10),
+ _info("sb-running-old", SandboxState.RUNNING, age_seconds=500),
+ ]
+ with _patch_sdk() as mock_cls:
+ mock_cls.list = _mock_list(infos)
+ result = asyncio.run(
+ find_owned_sandbox_id(
+ SandboxOwner(kind="expert", id=_EXPERT_ID), "shell", _API_KEY
+ )
+ )
+ assert result == "sb-running-old"
+ query = mock_cls.list.call_args.kwargs["query"]
+ assert query.metadata == {
+ "autogpt_owner": f"expert:{_EXPERT_ID}",
+ "autogpt_kind": "shell",
+ }
+ assert set(query.state) == {SandboxState.RUNNING, SandboxState.PAUSED}
+
+ def test_newest_paused_box_when_none_running(self):
+ infos = [
+ _info("sb-old", SandboxState.PAUSED, age_seconds=900),
+ _info("sb-new", SandboxState.PAUSED, age_seconds=5),
+ ]
+ with _patch_sdk() as mock_cls:
+ mock_cls.list = _mock_list(infos)
+ result = asyncio.run(
+ find_owned_sandbox_id(
+ SandboxOwner(kind="expert", id=_EXPERT_ID), "desktop", _API_KEY
+ )
+ )
+ assert result == "sb-new"
+
+ def test_api_failure_is_an_error_not_a_miss(self):
+ """ "Unknown" must never be read as "none": the caller would create a
+ second box and fork the expert's durable state."""
+ with _patch_sdk() as mock_cls:
+ mock_cls.list = MagicMock(side_effect=RuntimeError("e2b down"))
+ with pytest.raises(SandboxLookupError, match="e2b down"):
+ asyncio.run(
+ find_owned_sandbox_id(
+ SandboxOwner(kind="expert", id=_EXPERT_ID), "shell", _API_KEY
+ )
+ )
+
+
+class TestExpertShellBox:
+ def test_recovers_expert_box_from_e2b_when_redis_is_empty(self):
+ """Redis is only a cache for an expert's box; E2B metadata is the record."""
+ sb = _mock_sandbox(
+ "sb-expert", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)
+ )
+ redis = _keyed_redis({})
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([_info("sb-expert", SandboxState.PAUSED)])
+ mock_cls.connect = AsyncMock(return_value=sb)
+ mock_cls.create = AsyncMock()
+ result = asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID, _API_KEY, timeout=_TIMEOUT, expert_id=_EXPERT_ID
+ )
+ )
+
+ assert result is sb
+ mock_cls.connect.assert_awaited_once_with(
+ "sb-expert", api_key=_API_KEY, timeout=None
+ )
+ mock_cls.create.assert_not_awaited()
+ # Re-cached under the expert key, never the session key.
+ keys = {call.args[0] for call in redis.set.await_args_list}
+ assert _EXPERT_SHELL_KEY in keys
+ assert f"copilot:e2b:sandbox:{_SESSION_ID}" not in keys
+ # This turn is counted so a concurrent turn's end cannot pause the box.
+ assert _turn_acquires(redis) == [_EXPERT_ACTIVE_KEY]
+
+ def test_lookup_failure_never_creates_a_second_expert_box(self):
+ redis = _keyed_redis({})
+ with (
+ _patch_sdk() as mock_cls,
+ patch("backend.copilot.tools.e2b_sandbox.ensure_template", AsyncMock()),
+ _patch_redis(redis),
+ ):
+ mock_cls.list = MagicMock(side_effect=RuntimeError("e2b down"))
+ mock_cls.create = AsyncMock()
+ with pytest.raises(SandboxLookupError):
+ asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID, _API_KEY, timeout=_TIMEOUT, expert_id=_EXPERT_ID
+ )
+ )
+ mock_cls.create.assert_not_awaited()
+
+ def test_listed_expert_box_gets_a_second_reconnect_before_being_forked(self):
+ """One transient connect failure must not leave the durable box behind."""
+ sb = _mock_sandbox(
+ "sb-expert", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)
+ )
+ redis = _keyed_redis({})
+ with (
+ _patch_sdk() as mock_cls,
+ patch("backend.copilot.tools.e2b_sandbox.asyncio.sleep", AsyncMock()),
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([_info("sb-expert", SandboxState.PAUSED)])
+ mock_cls.connect = AsyncMock(side_effect=[RuntimeError("502"), sb])
+ mock_cls.create = AsyncMock()
+ result = asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID, _API_KEY, timeout=_TIMEOUT, expert_id=_EXPERT_ID
+ )
+ )
+ assert result is sb
+ assert mock_cls.connect.await_count == 2
+ mock_cls.create.assert_not_awaited()
+
+ def test_count_turn_false_defers_the_turn_count(self):
+ sb = _mock_sandbox(
+ "sb-expert", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)
+ )
+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-expert"})
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(return_value=sb)
+ asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID,
+ _API_KEY,
+ timeout=_TIMEOUT,
+ expert_id=_EXPERT_ID,
+ count_turn=False,
+ )
+ )
+ assert _turn_acquires(redis) == []
+ asyncio.run(count_expert_turn(_SESSION_ID, _EXPERT_ID))
+ assert _turn_acquires(redis) == [_EXPERT_ACTIVE_KEY]
+
+ def test_counting_a_turn_arms_the_expiry_in_the_same_script(self):
+ """An INCR without its EXPIRE would be a count nothing releases."""
+ redis = _keyed_redis({})
+ with _patch_redis(redis):
+ asyncio.run(count_expert_turn(_SESSION_ID, _EXPERT_ID))
+ script, nkeys, key, ttl = redis.eval.await_args.args
+ assert nkeys == 1 and key == _EXPERT_ACTIVE_KEY
+ assert "incr" in script and "expire" in script and ttl > 0
+
+ def test_a_turn_that_cannot_be_counted_does_not_get_the_box(self):
+ """An uncounted turn's release would decrement someone else's count
+ and could pause the box under them, so the count failure surfaces."""
+ sb = _mock_sandbox(
+ "sb-expert", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)
+ )
+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-expert"})
+ redis.eval = AsyncMock(side_effect=ConnectionError("redis down"))
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(return_value=sb)
+ with pytest.raises(ConnectionError):
+ asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID, _API_KEY, timeout=_TIMEOUT, expert_id=_EXPERT_ID
+ )
+ )
+ # The failed acquire was the only script; no release ran for it.
+ assert [
+ c.args[0] for c in redis.eval.await_args_list if "decr" in c.args[0]
+ ] == []
+
+ def test_creates_expert_box_with_home_and_shared_volumes(self):
+ sb = _mock_sandbox("sb-expert-new")
+ sb.commands = MagicMock()
+ sb.commands.run = AsyncMock()
+ redis = _keyed_redis({})
+ mounts = workspace_volume_mounts(_USER_ID, _EXPERT_ID)
+ with (
+ _patch_sdk() as mock_cls,
+ # Volumes already exist -> mounted by name. Patched where the
+ # module looks it up, so no real E2B call can sneak through.
+ patch(
+ "backend.copilot.tools.e2b_sandbox.resolve_volume",
+ AsyncMock(side_effect=lambda name, key: name),
+ ),
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([])
+ mock_cls.create = AsyncMock(return_value=sb)
+ result = asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID,
+ _API_KEY,
+ timeout=_TIMEOUT,
+ volume_mounts=mounts,
+ expert_id=_EXPERT_ID,
+ )
+ )
+
+ assert result is sb
+ kwargs = mock_cls.create.call_args.kwargs
+ assert kwargs["volume_mounts"] == {
+ WORKSPACE_PATH: expert_volume_name(_EXPERT_ID),
+ SHARED_PATH: user_volume_name(_USER_ID),
+ }
+ assert kwargs["metadata"] == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"expert:{_EXPERT_ID}",
+ "autogpt_kind": "shell",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_session": _SESSION_ID,
+ "autogpt_expert": _EXPERT_ID,
+ "autogpt_template": "base",
+ "autogpt_mounts": "attached",
+ }
+ # Creation lock and cached id both live under the expert key.
+ lock_call = redis.set.await_args_list[0]
+ assert lock_call.args[0] == _EXPERT_SHELL_KEY
+ assert lock_call.args[1] == _CREATING_SENTINEL
+ # Both mount points exist before the first command runs.
+ mkdir = sb.commands.run.await_args.args[0]
+ assert WORKSPACE_PATH in mkdir and SHARED_PATH in mkdir
+
+ def test_plain_session_create_is_tagged_but_untouched_otherwise(self):
+ sb = _mock_sandbox("sb-plain")
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.create = AsyncMock(return_value=sb)
+ asyncio.run(get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT))
+
+ kwargs = mock_cls.create.call_args.kwargs
+ assert kwargs["metadata"] == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"session:{_SESSION_ID}",
+ "autogpt_kind": "shell",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_session": _SESSION_ID,
+ "autogpt_template": "base",
+ "autogpt_mounts": "none",
+ }
+ assert kwargs["volume_mounts"] is None
+ mock_cls.list.assert_not_called()
+ assert _turn_acquires(redis) == []
+
+
+class TestExpertPause:
+ def test_last_turn_pauses_the_box(self):
+ sb = _mock_sandbox()
+ redis = _keyed_redis({}, decr_result=0)
+ with _patch_redis(redis):
+ ok = asyncio.run(
+ pause_sandbox_direct(sb, _SESSION_ID, expert_id=_EXPERT_ID)
+ )
+ assert ok is True
+ sb.pause.assert_awaited_once()
+ # One script does the DECR and, at zero, the DEL: no window for a turn
+ # that starts in between to lose its count.
+ script, _, key = redis.eval.await_args.args
+ assert key == _EXPERT_ACTIVE_KEY and "decr" in script and "del" in script
+
+ def test_concurrent_turn_keeps_the_box_running(self):
+ """Pausing under another session of the same expert would sever its
+ command stream, so the box stays up until the last turn ends."""
+ sb = _mock_sandbox()
+ redis = _keyed_redis({}, decr_result=1)
+ with _patch_redis(redis):
+ ok = asyncio.run(
+ pause_sandbox_direct(sb, _SESSION_ID, expert_id=_EXPERT_ID)
+ )
+ assert ok is False
+ sb.pause.assert_not_awaited()
+
+ def test_lookup_pause_honours_the_counter_too(self):
+ redis = _keyed_redis({_EXPERT_SHELL_KEY: _SANDBOX_ID}, decr_result=2)
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock()
+ ok = asyncio.run(pause_sandbox(_SESSION_ID, _API_KEY, expert_id=_EXPERT_ID))
+ assert ok is False
+ mock_cls.connect.assert_not_awaited()
+
+ def test_session_pause_never_touches_the_counter(self):
+ sb = _mock_sandbox()
+ redis = _keyed_redis({})
+ with _patch_redis(redis):
+ ok = asyncio.run(pause_sandbox_direct(sb, _SESSION_ID))
+ assert ok is True
+ redis.eval.assert_not_awaited()
+
+
+class TestExpertKill:
+ def test_deleting_an_expert_chat_leaves_the_expert_box_alone(self):
+ """kill_sandbox only knows session keys; an expert box has none."""
+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-expert"})
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock()
+ ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))
+ assert ok is False
+ mock_cls.connect.assert_not_awaited()
+
+ def test_archive_kills_shell_and_desktop_and_clears_cache(self):
+ expert = SandboxOwner(kind="expert", id=_EXPERT_ID)
+ shell = _mock_sandbox("sb-shell", owner=expert)
+ desktop = _mock_sandbox("sb-desktop", owner=expert, kind="desktop")
+ redis = _keyed_redis(
+ {_EXPERT_SHELL_KEY: "sb-shell", _EXPERT_DESKTOP_KEY: "sb-desktop"}
+ )
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(
+ side_effect=lambda sid, api_key, timeout=None: {
+ "sb-shell": shell,
+ "sb-desktop": desktop,
+ }[sid]
+ )
+ killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))
+
+ assert killed == 2
+ shell.kill.assert_awaited_once()
+ desktop.kill.assert_awaited_once()
+ deleted = {call.args[0] for call in redis.delete.await_args_list}
+ assert deleted == {_EXPERT_SHELL_KEY, _EXPERT_DESKTOP_KEY, _EXPERT_ACTIVE_KEY}
+ mock_cls.list.assert_not_called()
+
+ def test_archive_falls_back_to_e2b_metadata_for_forgotten_boxes(self):
+ shell = _mock_sandbox(
+ "sb-shell", owner=SandboxOwner(kind="expert", id=_EXPERT_ID)
+ )
+ redis = _keyed_redis({})
+ lists = {
+ "shell": [_info("sb-shell", SandboxState.PAUSED)],
+ "desktop": [],
+ }
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+
+ def _list(query, **_):
+ paginator = MagicMock()
+ paginator.next_items = AsyncMock(
+ return_value=lists[query.metadata["autogpt_kind"]]
+ )
+ return paginator
+
+ mock_cls.list = MagicMock(side_effect=_list)
+ mock_cls.connect = AsyncMock(return_value=shell)
+ killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))
+
+ assert killed == 1
+ shell.kill.assert_awaited_once()
+ assert mock_cls.list.call_count == 2
+
+ def test_failed_kill_keeps_cache_for_retry(self):
+ _mock_sandbox("sb-shell", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))
+ redis = _keyed_redis({_EXPERT_SHELL_KEY: "sb-shell"})
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([])
+ mock_cls.connect = AsyncMock(side_effect=RuntimeError("boom"))
+ killed = asyncio.run(kill_expert_sandboxes(_EXPERT_ID, _API_KEY))
+ assert killed == 0
+ deleted = {call.args[0] for call in redis.delete.await_args_list}
+ assert _EXPERT_SHELL_KEY not in deleted
+
+
+class TestKillSandboxDesktop:
+ def test_deleting_a_chat_kills_its_desktop_too(self):
+ """start_desktop's box is never paused at turn end, so the session
+ delete is the only thing that ever stops it."""
+ shell, desktop = _mock_sandbox("sb-shell"), _mock_sandbox(
+ "sb-desk", kind="desktop"
+ )
+ redis = _keyed_redis(
+ {
+ f"copilot:e2b:sandbox:{_SESSION_ID}": "sb-shell",
+ f"copilot:e2b:desktop:{_SESSION_ID}": "sb-desk",
+ }
+ )
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(
+ side_effect=lambda sid, api_key, timeout=None: {
+ "sb-shell": shell,
+ "sb-desk": desktop,
+ }[sid]
+ )
+ ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))
+ assert ok is True
+ shell.kill.assert_awaited_once()
+ desktop.kill.assert_awaited_once()
+ deleted = {call.args[0] for call in redis.delete.await_args_list}
+ assert deleted == {
+ f"copilot:e2b:sandbox:{_SESSION_ID}",
+ f"copilot:e2b:desktop:{_SESSION_ID}",
+ }
+
+ def test_desktop_only_session_still_reports_a_kill(self):
+ desktop = _mock_sandbox("sb-desk", kind="desktop")
+ redis = _keyed_redis({f"copilot:e2b:desktop:{_SESSION_ID}": "sb-desk"})
+ with (
+ _patch_sdk() as mock_cls,
+ _patch_redis(redis),
+ ):
+ mock_cls.connect = AsyncMock(return_value=desktop)
+ ok = asyncio.run(kill_sandbox(_SESSION_ID, _API_KEY))
+ assert ok is True
+ desktop.kill.assert_awaited_once()
+
+
+class TestExpertBoxRecovery:
+ def test_unconnectable_listed_box_falls_through_to_create(self):
+ """E2B may keep listing a box we cannot reconnect to; the loop must
+ create a replacement instead of spinning on the same id."""
+ fresh = _mock_sandbox("sb-fresh")
+ # Stamped as the expert's, so the refusal is not what stops the reconnect.
+ _mock_sandbox("sb-dead", owner=SandboxOwner(kind="expert", id=_EXPERT_ID))
+ redis = _keyed_redis({})
+ with (
+ _patch_sdk() as mock_cls,
+ patch("backend.copilot.tools.e2b_sandbox.asyncio.sleep", AsyncMock()),
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([_info("sb-dead", SandboxState.PAUSED)])
+ mock_cls.connect = AsyncMock(side_effect=RuntimeError("unresumable"))
+ mock_cls.create = AsyncMock(return_value=fresh)
+ result = asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID, _API_KEY, timeout=_TIMEOUT, expert_id=_EXPERT_ID
+ )
+ )
+ assert result is fresh
+ # Exactly one retry of the listed box, then a replacement.
+ assert mock_cls.connect.await_count == 2
+ mock_cls.connect.assert_awaited_with("sb-dead", api_key=_API_KEY, timeout=None)
+ mock_cls.create.assert_awaited_once()
+ assert mock_cls.list.call_count <= 2
+
+ def test_mounts_survive_transient_create_failures(self):
+ """Only the final attempt goes volume-less; a slow first attempt must
+ not quietly cost an expert its durable home for 30 days."""
+ sb = _mock_sandbox("sb-late")
+ sb.commands = MagicMock()
+ sb.commands.run = AsyncMock()
+ redis = _keyed_redis({})
+ mounts = workspace_volume_mounts(_USER_ID, _EXPERT_ID)
+ with (
+ _patch_sdk() as mock_cls,
+ patch(
+ "backend.copilot.tools.e2b_sandbox.resolve_volume",
+ new=AsyncMock(side_effect=lambda name, key: name),
+ ),
+ patch("backend.copilot.tools.e2b_sandbox.asyncio.sleep", new=AsyncMock()),
+ _patch_redis(redis),
+ ):
+ mock_cls.list = _mock_list([])
+ mock_cls.create = AsyncMock(
+ side_effect=[asyncio.TimeoutError(), RuntimeError("busy"), sb]
+ )
+ result = asyncio.run(
+ get_or_create_sandbox(
+ _SESSION_ID,
+ _API_KEY,
+ timeout=_TIMEOUT,
+ volume_mounts=mounts,
+ expert_id=_EXPERT_ID,
+ )
+ )
+ assert result is sb
+ attempts = [c.kwargs for c in mock_cls.create.call_args_list]
+ assert len(attempts) == _SANDBOX_CREATE_MAX_RETRIES
+ assert attempts[0]["volume_mounts"] == mounts
+ assert attempts[1]["volume_mounts"] == mounts
+ assert attempts[2]["volume_mounts"] is None
+ assert attempts[0]["metadata"]["autogpt_mounts"] == "attached"
+ assert attempts[2]["metadata"]["autogpt_mounts"] == "none"
+
+ def test_release_fails_closed_when_redis_is_unavailable(self):
+ """Without the counter we cannot rule out a concurrent turn, so the
+ box is left running for the lifecycle timeout to pause."""
+ sb = _mock_sandbox()
+ redis = _keyed_redis({})
+ redis.eval = AsyncMock(side_effect=ConnectionError("redis down"))
+ with _patch_redis(redis):
+ ok = asyncio.run(
+ pause_sandbox_direct(sb, _SESSION_ID, expert_id=_EXPERT_ID)
+ )
+ assert ok is False
+ sb.pause.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/copilot/tools/models.py b/autogpt_platform/backend/backend/copilot/tools/models.py
index 9258bcea3b..8797d2d067 100644
--- a/autogpt_platform/backend/backend/copilot/tools/models.py
+++ b/autogpt_platform/backend/backend/copilot/tools/models.py
@@ -98,6 +98,7 @@ class ResponseType(str, Enum):
# Code execution
BASH_EXEC = "bash_exec"
+ DESKTOP_STREAM = "desktop_stream"
# Web
WEB_FETCH = "web_fetch"
@@ -965,6 +966,20 @@ class BashExecResponse(ToolResponseBase):
timed_out: bool = False
+class DesktopStreamToolResponse(ToolResponseBase):
+ """Response for start_desktop: an embeddable live desktop stream.
+
+ ``desktop_stream`` carries the same shape the desktop blocks emit
+ (kind/url/provider/sandbox_id/requires_auth). The copilot chat renders it
+ through DesktopStreamRenderer via ToolResult's ``start_desktop`` card, and
+ every surface that consults the output-renderer registry (block outputs,
+ attachments) embeds it the same way.
+ """
+
+ type: ResponseType = ResponseType.DESKTOP_STREAM
+ desktop_stream: dict
+
+
# Feature request models
class FeatureRequestInfo(BaseModel):
"""Information about a feature request issue."""
diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop.py
new file mode 100644
index 0000000000..5c4dc2805c
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop.py
@@ -0,0 +1,157 @@
+"""On-demand interactive desktop for CoPilot sessions (E2B).
+
+The session's bash sandbox stays on the cheap headless ``base`` template —
+E2B bills actual vCPU+RAM per running second, so putting every session on the
+GUI-capable ``desktop`` template would tax the majority of sessions that never
+need a screen. Instead this tool manages a SEPARATE desktop-template sandbox
+per owner: created on first use, kept running after the turn ends so the
+user can watch and control the live stream, auto-paused when idle (free), and
+resumed in about a second on the next call.
+
+The owner is the session — or, in an expert session, the expert itself, so
+the desktop is the expert's own persistent computer: the same box (browser
+profile, installed apps and all) comes back for every chat, delegation and
+scheduled run that happens as that expert. The model has a root shell in
+that same VM, so the desktop is shared with it, never private from it: any
+account a person signs into on it is readable by the model. The expert page's
+Computer tab and the copilot side panel open the very same box through
+``backend.copilot.computer``.
+
+Both the desktop and the owner's bash sandbox mount the SAME durable volumes
+(see ``workspace_volume_mounts``): ``/home/user/workspace`` is the owner's
+persistent home and, for an expert, ``/home/user/shared`` is the owning
+user's workspace — so files the agent writes appear on the desktop and vice
+versa, and both survive across sessions. The desktop additionally redirects
+Downloads/Desktop/Documents into the home volume, so a person's browser
+downloads and saved files persist too.
+"""
+
+import logging
+from typing import Any
+
+from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.copilot.model import ChatSession
+from backend.copilot.sdk.env import config as chat_config
+
+from .base import BaseTool
+from .models import DesktopStreamToolResponse, ErrorResponse, ToolResponseBase
+
+logger = logging.getLogger(__name__)
+
+
+class StartDesktopTool(BaseTool):
+ """Start (or resume) the owner's on-demand desktop and return its stream."""
+
+ @property
+ def name(self) -> str:
+ return "start_desktop"
+
+ @property
+ def description(self) -> str:
+ return (
+ "Start or resume an interactive cloud desktop and return a live "
+ "stream the user can watch and control. It shares your persistent "
+ "~/workspace with bash_exec (in an expert session it is the expert's "
+ "own persistent computer; ~/shared is the user's workspace). Stays "
+ "up after your turn, suspends when idle. Use when a task needs a "
+ "browser or GUI app."
+ )
+
+ @property
+ def parameters(self) -> dict[str, Any]:
+ return {"type": "object", "properties": {}, "required": []}
+
+ @property
+ def requires_auth(self) -> bool:
+ # A desktop is a billed box; only a signed-in user may start one.
+ return True
+
+ @property
+ def is_available(self) -> bool:
+ # Without E2B the tool can only fail; do not offer it to the model.
+ return bool(chat_config.active_e2b_api_key)
+
+ async def _execute(
+ self,
+ user_id: str | None,
+ session: ChatSession,
+ **kwargs: Any,
+ ) -> ToolResponseBase:
+ session_id = session.session_id if session else None
+ if not session_id:
+ return ErrorResponse(
+ message="No active session to attach a desktop to.",
+ error="session_unavailable",
+ )
+ api_key = chat_config.active_e2b_api_key
+ if not api_key:
+ return ErrorResponse(
+ message="E2B is not configured, so a desktop cannot be started.",
+ error="e2b_unconfigured",
+ session_id=session_id,
+ )
+
+ # ``execute()`` already refuses an unauthenticated caller (the tool
+ # requires auth); this guards a direct ``_execute`` call.
+ if not user_id:
+ return ErrorResponse(
+ message="A desktop needs a signed-in user to hand its link to.",
+ error="user_required",
+ session_id=session_id,
+ )
+
+ expert_id = session.expert_id if session else None
+ # Imported here: ``computer`` imports the sandbox module, whose package
+ # imports this tool, so a module-level import is a cycle.
+ from backend.copilot.computer import computer_owner, mounts_for, open_desktop
+
+ owner = computer_owner(session_id, expert_id)
+ try:
+ stream, created, shared = await open_desktop(
+ owner,
+ mounts_for(user_id, expert_id),
+ api_key,
+ user_id=user_id,
+ session_id=session_id,
+ )
+ except Exception as exc:
+ logger.error("[E2B] start_desktop failed: %s", exc, exc_info=True)
+ # The cause is in the server log; the model gets a stable message.
+ return ErrorResponse(
+ message="Failed to start the desktop. Try again in a moment.",
+ error="desktop_start_failed",
+ session_id=session_id,
+ )
+
+ return DesktopStreamToolResponse(
+ message=_build_message(created, shared, expert=owner.is_expert),
+ desktop_stream=stream.model_dump(),
+ session_id=session_id,
+ )
+
+
+def _build_message(created: bool, shared: bool, *, expert: bool = False) -> str:
+ state = "started" if created else "resumed"
+ if shared and expert:
+ files = (
+ f"This is your own persistent computer. {WORKSPACE_PATH} is your "
+ "durable home — customise it freely; installed tools survive "
+ f"between sessions — and {SHARED_PATH} is the user's shared "
+ "workspace, so put anything the user should see there. The desktop "
+ "is shared with the user, not private from either of you."
+ )
+ elif shared:
+ files = (
+ f"It shares your persistent {WORKSPACE_PATH} with bash_exec live, "
+ "so put anything the user should see (or that should persist) there."
+ )
+ else:
+ files = (
+ "No per-user volume is attached (no user context), so this desktop "
+ "is ephemeral and not shared with bash_exec."
+ )
+ return (
+ f"Desktop {state}. The user can watch and control it via the live "
+ f"stream. It stays up after this turn and suspends itself when idle; "
+ f"calling start_desktop again resumes it with state intact. {files}"
+ )
diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py
new file mode 100644
index 0000000000..02f3cb97a3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py
@@ -0,0 +1,329 @@
+"""Tests for StartDesktopTool — session-linked on-demand E2B desktop."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.blocks.desktop._api import (
+ SHARED_PATH,
+ WORKSPACE_PATH,
+ DesktopStream,
+ PersistenceInfo,
+)
+from backend.blocks.desktop._common import expert_volume_name, user_volume_name
+from backend.copilot.tools.e2b_sandbox import SandboxOwner
+from backend.util.sandbox_metadata import deployment_env
+
+from ._test_data import make_session
+from .models import DesktopStreamToolResponse, ErrorResponse
+from .start_desktop import StartDesktopTool
+
+_USER = "user-start-desktop-test"
+_STREAM = DesktopStream(
+ url="https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ sandbox_id="sbx-desktop-1",
+)
+
+
+def _preview_link(user_id: str, live_url: str) -> str:
+ return f"https://platform.example/api/proxy/api/desktop-preview?for={user_id}"
+
+
[email protected](autouse=True)
+def _owner_bound_links():
+ """The real link needs FRONTEND_BASE_URL and a Fernet key; stub the wrapper."""
+ with patch(
+ "backend.copilot.computer.create_preview_link", side_effect=_preview_link
+ ):
+ yield
+
+
+def _make_redis(stored: str | None = None) -> MagicMock:
+ redis = MagicMock()
+ redis.get = AsyncMock(return_value=stored)
+ redis.set = AsyncMock(return_value=True)
+ redis.delete = AsyncMock()
+ redis.eval = AsyncMock(return_value=1)
+ return redis
+
+
+def _make_desktop() -> MagicMock:
+ desktop = MagicMock()
+ desktop.sandbox_id = "sbx-desktop-1"
+ desktop.ensure_display = AsyncMock()
+ desktop.start_stream = AsyncMock(return_value=_STREAM)
+ desktop.is_workspace_mounted = AsyncMock(return_value=True)
+ return desktop
+
+
+class TestStartDesktop:
+ def test_is_only_offered_with_e2b_and_only_to_signed_in_users(self):
+ tool = StartDesktopTool()
+ assert tool.requires_auth is True
+ with patch("backend.copilot.tools.start_desktop.chat_config") as mock_config:
+ mock_config.active_e2b_api_key = None
+ assert tool.is_available is False
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ assert tool.is_available is True
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_unconfigured_e2b_returns_error(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER)
+ with patch("backend.copilot.tools.start_desktop.chat_config") as mock_config:
+ mock_config.active_e2b_api_key = None
+ result = await tool._execute(user_id=_USER, session=session)
+ assert isinstance(result, ErrorResponse)
+ assert result.error == "e2b_unconfigured"
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_creates_desktop_on_shared_user_volume(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER)
+ desktop = _make_desktop()
+ redis = _make_redis(stored=None)
+
+ with (
+ patch(
+ "backend.copilot.computer.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ),
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ mock_config.e2b_desktop_timeout = 900
+ mock_config.e2b_desktop_template = "desktop"
+ mock_session_cls.create = AsyncMock(
+ return_value=(desktop, PersistenceInfo(volume_mounted=True))
+ )
+
+ result = await tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, DesktopStreamToolResponse)
+ assert result.desktop_stream["kind"] == "desktop_stream"
+ # The tool result, and so the stored message, carries an owner-bound
+ # link, never the stream URL with the desktop's password in it.
+ assert result.desktop_stream["url"] == _preview_link(_USER, _STREAM.url)
+ assert result.desktop_stream["requires_auth"] is True
+ assert _STREAM.url not in result.model_dump_json()
+ # The desktop mounts the SAME per-user volume as the agent shell, and
+ # is tagged so its owner can be found through the E2B API.
+ create_kwargs = mock_session_cls.create.await_args.kwargs
+ assert create_kwargs["volume_mounts"] == {
+ WORKSPACE_PATH: user_volume_name(_USER)
+ }
+ assert create_kwargs["metadata"] == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"session:{session.session_id}",
+ "autogpt_kind": "desktop",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_user": _USER,
+ "autogpt_session": session.session_id,
+ "autogpt_template": "desktop",
+ "autogpt_mounts": "attached",
+ }
+ redis.set.assert_awaited()
+ assert redis.set.await_args.args[0] == (
+ f"copilot:e2b:desktop:{session.session_id}"
+ )
+ assert "started" in result.message
+ assert "workspace" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_resumes_existing_desktop_without_creating(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER)
+ desktop = _make_desktop()
+ redis = _make_redis(stored="sbx-desktop-1")
+
+ with (
+ patch(
+ "backend.copilot.computer.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ),
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.computer.connect_owned") as connect_owned,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ mock_config.e2b_desktop_timeout = 900
+ connect_owned.return_value = box = MagicMock()
+ mock_session_cls.return_value = desktop
+ mock_session_cls.create = AsyncMock()
+
+ result = await tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, DesktopStreamToolResponse)
+ # Reattached only after E2B confirmed the box is this session's desktop,
+ # with its running-time limit re-armed.
+ connect_owned.assert_awaited_once_with(
+ "sbx-desktop-1",
+ SandboxOwner(kind="session", id=session.session_id),
+ "desktop",
+ "e2b_test_key",
+ timeout=900,
+ )
+ mock_session_cls.assert_called_once_with(box)
+ mock_session_cls.create.assert_not_awaited()
+ desktop.ensure_display.assert_awaited_once()
+ assert "resumed" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_no_user_id_gets_no_desktop(self):
+ """The stream link is issued to a user; without one there is nobody to
+ issue it to, so no box is created."""
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER)
+
+ with (
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ mock_session_cls.create = AsyncMock()
+
+ result = await tool._execute(user_id=None, session=session)
+
+ assert isinstance(result, ErrorResponse)
+ assert result.error == "user_required"
+ mock_session_cls.create.assert_not_awaited()
+
+
+class TestExpertDesktop:
+ """An expert session's desktop is the expert's own persistent computer."""
+
+ _EXPERT = "exp-desktop-1"
+
+ def _patches(self, redis, mock_session_cls, mock_config, found: str | None):
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ mock_config.e2b_desktop_timeout = 900
+ mock_config.e2b_desktop_template = "desktop"
+ return (
+ patch(
+ "backend.copilot.computer.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ),
+ patch(
+ "backend.copilot.computer.find_owned_sandbox_id",
+ new=AsyncMock(return_value=found),
+ ),
+ )
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_expert_desktop_is_keyed_by_expert_with_home_and_shared(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER, expert_id=self._EXPERT)
+ desktop = _make_desktop()
+ redis = _make_redis(stored=None)
+
+ with (
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ redis_patch, find_patch = self._patches(
+ redis, mock_session_cls, mock_config, found=None
+ )
+ mock_session_cls.create = AsyncMock(
+ return_value=(
+ desktop,
+ PersistenceInfo(
+ volume_mounted=True,
+ mounted_paths=[WORKSPACE_PATH, SHARED_PATH],
+ ),
+ )
+ )
+ with redis_patch, find_patch:
+ result = await tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, DesktopStreamToolResponse)
+ create_kwargs = mock_session_cls.create.await_args.kwargs
+ # Own home first, then the user's shared workspace.
+ assert create_kwargs["volume_mounts"] == {
+ WORKSPACE_PATH: expert_volume_name(self._EXPERT),
+ SHARED_PATH: user_volume_name(_USER),
+ }
+ assert create_kwargs["metadata"] == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"expert:{self._EXPERT}",
+ "autogpt_kind": "desktop",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_user": _USER,
+ "autogpt_session": session.session_id,
+ "autogpt_expert": self._EXPERT,
+ "autogpt_template": "desktop",
+ "autogpt_mounts": "attached",
+ }
+ # Cached under the expert, not the session: every session of this
+ # expert must come back to the same desktop.
+ assert redis.set.await_args.args[0] == (
+ f"copilot:e2b:expert:{self._EXPERT}:desktop"
+ )
+ assert session.session_id not in redis.set.await_args.args[0]
+ assert "own persistent computer" in result.message
+ assert SHARED_PATH in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_expert_desktop_recovered_from_e2b_when_cache_is_empty(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER, expert_id=self._EXPERT)
+ desktop = _make_desktop()
+ redis = _make_redis(stored=None)
+
+ with (
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.computer.connect_owned") as connect_owned,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ redis_patch, find_patch = self._patches(
+ redis, mock_session_cls, mock_config, found="sbx-desktop-1"
+ )
+ connect_owned.return_value = MagicMock()
+ mock_session_cls.return_value = desktop
+ mock_session_cls.create = AsyncMock()
+ with redis_patch, find_patch:
+ result = await tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, DesktopStreamToolResponse)
+ connect_owned.assert_awaited_once_with(
+ "sbx-desktop-1",
+ SandboxOwner(kind="expert", id=self._EXPERT),
+ "desktop",
+ "e2b_test_key",
+ timeout=900,
+ )
+ mock_session_cls.create.assert_not_awaited()
+ # The recovered id is re-cached under the expert key.
+ assert redis.set.await_args.args[0] == (
+ f"copilot:e2b:expert:{self._EXPERT}:desktop"
+ )
+ assert "resumed" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_plain_session_never_asks_e2b_for_a_lost_desktop(self):
+ tool = StartDesktopTool()
+ session = make_session(user_id=_USER)
+ desktop = _make_desktop()
+ redis = _make_redis(stored=None)
+
+ with (
+ patch(
+ "backend.copilot.computer.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ),
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_sandbox,
+ patch("backend.copilot.computer.DesktopSession") as mock_session_cls,
+ patch("backend.copilot.tools.start_desktop.chat_config") as mock_config,
+ ):
+ mock_config.active_e2b_api_key = "e2b_test_key"
+ mock_config.e2b_desktop_timeout = 900
+ mock_config.e2b_desktop_template = "desktop"
+ mock_session_cls.create = AsyncMock(
+ return_value=(desktop, PersistenceInfo(volume_mounted=True))
+ )
+ result = await tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, DesktopStreamToolResponse)
+ mock_sandbox.list.assert_not_called()
diff --git a/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py b/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py
index 92d460e14a..b89baf3f9b 100644
--- a/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/tool_schema_test.py
@@ -131,6 +131,12 @@ from backend.copilot.tools import TOOL_REGISTRY
# #14244 agent-collab-architecture +0
# #14209 autopilot-auto-mode-v2 +0
# #14432 secrt-2593-publish +0
+# #14365 sandbox-e2b-desktop (start_desktop) +670 (branch measures 63,417)
+# Bumped 68_238 -> 68_997 on 2026-09-16 when #14365 merged dev (twice in one
+# morning: #14416's pause/resume_schedule landed between the two). Dev's
+# registry measures 68,503 with start_desktop removed, so start_desktop's
+# delta is +493 once merged (the +670 above was against an older dev).
+# Merged registry measures 68,996; the ceiling is that plus one.
# There is NO margin on top, deliberately. This limit is a brake: it exists to
# make every increase in what Otto pays per turn a decision someone took,
# so slack for growth nobody has measured is the one thing it must not carry.
@@ -153,7 +159,7 @@ from backend.copilot.tools import TOOL_REGISTRY
# counts against this ceiling. The tip reads 68,235 and `refs/pull/14416/merge`
# 68,237 — dev widened raise_expert by two characters after this line was first
# set, which reddened three interpreters on a branch that had added nothing.
-_CHAR_BUDGET = 68_238
+_CHAR_BUDGET = 68_997
@pytest.fixture(scope="module")
diff --git a/autogpt_platform/backend/backend/util/desktop_preview.py b/autogpt_platform/backend/backend/util/desktop_preview.py
new file mode 100644
index 0000000000..bfc0eb9c85
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/desktop_preview.py
@@ -0,0 +1,57 @@
+"""Owner-bound, encrypted links for E2B desktop streams.
+
+A desktop's live URL carries its VNC password, so anyone holding the URL
+controls the desktop and every login in it. The URL therefore never leaves
+the backend: callers get a link to ``/api/desktop-preview`` carrying an
+encrypted, owner-bound, expiring token, and the endpoint redirects the owner
+to the real URL with ``Cache-Control: no-store``. A chat message, a share
+link or a copied card holds the token, never the credential.
+"""
+
+from typing import Literal
+from urllib.parse import urlencode
+
+from cryptography.fernet import InvalidToken
+from pydantic import BaseModel, ValidationError
+
+from backend.util.encryption import JSONCryptor
+
+# The token is bound to the owner and only redeemable with the owner's
+# session, so its lifetime is about how long a stored chat card keeps
+# working, not about limiting exposure of the credential behind it.
+PREVIEW_LINK_TTL = 86400
+
+
+class DesktopPreview(BaseModel):
+ purpose: Literal["e2b-desktop-preview"]
+ user_id: str
+ url: str
+
+
+def create_preview_link(user_id: str, live_url: str) -> str:
+ """An owner-bound link to *live_url* that discloses nothing about it.
+
+ Origin-relative: the owner redeems it on whatever origin they are on, so
+ a preview deployment, a custom domain or a plain-http local stack still
+ sends the session cookie, and the link never leaks the platform's host.
+ """
+ if not user_id:
+ raise ValueError("Live view requires an authenticated user")
+ token = JSONCryptor().encrypt(
+ DesktopPreview(
+ purpose="e2b-desktop-preview", user_id=user_id, url=live_url
+ ).model_dump()
+ )
+ return f"/api/proxy/api/desktop-preview?{urlencode({'token': token})}"
+
+
+def resolve_preview_link(user_id: str, token: str) -> str | None:
+ """The live URL behind *token*, only for the user it was issued to."""
+ try:
+ data = JSONCryptor().fernet.decrypt(token.encode(), ttl=PREVIEW_LINK_TTL)
+ preview = DesktopPreview.model_validate_json(data)
+ except (InvalidToken, ValidationError):
+ return None
+ if not user_id or preview.user_id != user_id:
+ return None
+ return preview.url
diff --git a/autogpt_platform/backend/backend/util/desktop_preview_test.py b/autogpt_platform/backend/backend/util/desktop_preview_test.py
new file mode 100644
index 0000000000..35578d58f3
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/desktop_preview_test.py
@@ -0,0 +1,57 @@
+"""Preview links must not expose credentials or authorize other users."""
+
+from unittest.mock import patch
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+from cryptography.fernet import Fernet
+
+from backend.util.desktop_preview import create_preview_link, resolve_preview_link
+from backend.util.encryption import JSONCryptor
+
+
+def cryptor():
+ cryptor = JSONCryptor(Fernet.generate_key().decode())
+ with patch("backend.util.desktop_preview.JSONCryptor", return_value=cryptor):
+ yield cryptor
+
+
+def test_preview_link_hides_credentials_and_requires_owner(cryptor):
+ desktop_url = "https://6080-sandbox.e2b.app/vnc.html?password=private"
+ link = create_preview_link("owner", desktop_url)
+ assert "password" not in link
+ assert "private" not in link
+ # Origin-relative, so it works on any origin the owner is signed in to.
+ assert link.startswith("/api/proxy/api/desktop-preview?")
+ assert not link.startswith("//")
+ assert urlsplit(link).netloc == ""
+ token = parse_qs(urlsplit(link).query)["token"][0]
+ assert resolve_preview_link("owner", token) == desktop_url
+ assert resolve_preview_link("other-user", token) is None
+ assert resolve_preview_link("", token) is None
+ assert resolve_preview_link("owner", token[:-10]) is None
+
+
+def test_preview_link_requires_user(cryptor):
+ with pytest.raises(ValueError, match="authenticated user"):
+ create_preview_link("", "https://preview.example")
+
+
+ "data",
+ [
+ {"user_id": "owner", "url": "https://preview.example", "purpose": "other"},
+ {"user_id": "owner"},
+ ],
+)
+def test_invalid_payload_rejected(cryptor, data):
+ assert resolve_preview_link("owner", cryptor.encrypt(data)) is None
+
+
+def test_expired_preview_rejected(cryptor):
+ token = cryptor.fernet.encrypt_at_time(
+ b'{"purpose":"e2b-desktop-preview","user_id":"owner","url":"https://preview.example"}',
+ current_time=0,
+ ).decode()
+ assert resolve_preview_link("owner", token) is None
diff --git a/autogpt_platform/backend/backend/util/sandbox_metadata.py b/autogpt_platform/backend/backend/util/sandbox_metadata.py
new file mode 100644
index 0000000000..cc1aa92270
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/sandbox_metadata.py
@@ -0,0 +1,165 @@
+"""Metadata stamped on every E2B sandbox the platform creates.
+
+E2B sandbox metadata is the one record that outlives Redis keys and process
+restarts, and it is what ``SandboxQuery(metadata=...)`` filters on. Every
+creation site builds its metadata here, so a box seen in the E2B dashboard,
+or by a reaper, can be traced to the user, the session / expert / graph run,
+the block and the deployment that made it.
+
+Keys are prefixed ``autogpt_`` and values are always strings (E2B's
+``dict[str, str]`` contract). ``owner`` + ``kind`` are the identity CoPilot
+looks boxes up by; everything else is provenance.
+
+Several products share one E2B team, each with its own metadata schema. The
+one key they have in common is ``service``: a box is only parsed as ours when
+``service == "autogpt-platform"`` (see ``SandboxMetadata.parse``), so another
+product's keys are never misread and it can ignore ours the same way.
+"""
+
+from typing import TYPE_CHECKING, Literal, Mapping, Optional
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from backend.util.settings import Config
+
+if TYPE_CHECKING:
+ from backend.data.execution import ExecutionContext
+
+METADATA_PREFIX = "autogpt_"
+SERVICE_FIELD = "service"
+SERVICE = "autogpt-platform"
+
+SandboxSource = Literal["copilot", "block"]
+SandboxUse = Literal["shell", "desktop", "code", "claude_code"]
+MountState = Literal["attached", "none"]
+
+
+class SandboxMetadata(BaseModel):
+ """Typed view of the ``autogpt_*`` keys; ``as_e2b()`` flattens it for the SDK."""
+
+ model_config = ConfigDict(frozen=True)
+
+ owner: str
+ kind: SandboxUse
+ source: SandboxSource
+ env: str
+ user: Optional[str] = None
+ session: Optional[str] = None
+ expert: Optional[str] = None
+ graph: Optional[str] = None
+ graph_exec: Optional[str] = None
+ node_exec: Optional[str] = None
+ block: Optional[str] = None
+ template: Optional[str] = None
+ mounts: Optional[MountState] = None
+
+ @classmethod
+ def for_copilot(
+ cls,
+ owner: str,
+ kind: SandboxUse,
+ *,
+ user_id: Optional[str] = None,
+ session_id: Optional[str] = None,
+ expert_id: Optional[str] = None,
+ template: Optional[str] = None,
+ mounts: Optional[MountState] = None,
+ ) -> "SandboxMetadata":
+ """Metadata for a CoPilot box; *owner* is ``"<kind>:<id>"`` from ``SandboxOwner``."""
+ return cls(
+ owner=owner,
+ kind=kind,
+ source="copilot",
+ env=deployment_env(),
+ user=user_id,
+ session=session_id,
+ expert=expert_id,
+ template=template,
+ mounts=mounts,
+ )
+
+ @classmethod
+ def for_block(
+ cls,
+ context: "ExecutionContext | None",
+ kind: SandboxUse,
+ block_id: str,
+ template: Optional[str] = None,
+ ) -> "SandboxMetadata":
+ """Metadata for a box a graph block creates, traced to its node execution."""
+ if context is None:
+ return cls(
+ owner=f"block:{block_id}",
+ kind=kind,
+ source="block",
+ env=deployment_env(),
+ block=block_id,
+ template=template or None,
+ )
+ return cls(
+ owner=_block_owner(context, block_id),
+ kind=kind,
+ source="block",
+ env=deployment_env(),
+ user=context.user_id,
+ session=context.session_id,
+ expert=context.expert_id,
+ graph=context.graph_id,
+ graph_exec=context.graph_exec_id,
+ node_exec=context.node_exec_id,
+ block=block_id,
+ template=template or None,
+ )
+
+ @classmethod
+ def parse(cls, metadata: Mapping[str, str]) -> Optional["SandboxMetadata"]:
+ """Read a box's metadata back; ``None`` unless the box is this service's.
+
+ Unknown ``autogpt_*`` keys (from a newer or older build) are ignored;
+ a box missing the identity keys is treated as not ours.
+ """
+ if metadata.get(SERVICE_FIELD) != SERVICE:
+ return None
+ fields = {
+ key.removeprefix(METADATA_PREFIX): value
+ for key, value in metadata.items()
+ if key.startswith(METADATA_PREFIX)
+ and key.removeprefix(METADATA_PREFIX) in cls.model_fields
+ }
+ try:
+ return cls.model_validate(fields)
+ except ValidationError:
+ return None
+
+ def as_e2b(self) -> dict[str, str]:
+ """Flatten to E2B's ``dict[str, str]``: unset fields dropped, keys prefixed."""
+ return {SERVICE_FIELD: SERVICE} | {
+ f"{METADATA_PREFIX}{key}": value
+ for key, value in self.model_dump(exclude_none=True).items()
+ }
+
+
+def owned_by_user(metadata: Mapping[str, str], user_id: Optional[str]) -> bool:
+ """Whether a box's metadata says it was created for *user_id*.
+
+ Used before attaching to a sandbox id a caller supplied. Under the
+ platform's E2B key every sandbox id connects, so the stamped user is the
+ only thing that makes "reconnect" mean "reconnect to your own box". A
+ box with no parseable metadata is not ours, or predates stamping, and is
+ refused either way.
+ """
+ parsed = SandboxMetadata.parse(metadata)
+ return bool(user_id) and parsed is not None and parsed.user == user_id
+
+
+def deployment_env() -> str:
+ """The ``APP_ENV`` this process runs as (``local`` / ``dev`` / ``prod``)."""
+ return Config().app_env.value
+
+
+def _block_owner(context: "ExecutionContext", block_id: str) -> str:
+ if context.graph_exec_id:
+ return f"graph_exec:{context.graph_exec_id}"
+ if context.user_id:
+ return f"user:{context.user_id}"
+ return f"block:{block_id}"
diff --git a/autogpt_platform/backend/backend/util/sandbox_metadata_test.py b/autogpt_platform/backend/backend/util/sandbox_metadata_test.py
new file mode 100644
index 0000000000..aee5dc179e
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/sandbox_metadata_test.py
@@ -0,0 +1,171 @@
+from unittest.mock import patch
+
+from backend.data.execution import ExecutionContext
+from backend.util.sandbox_metadata import SandboxMetadata, deployment_env, owned_by_user
+
+_USER = "user-1111"
+_SESSION = "sess-2222"
+_EXPERT = "expert-3333"
+_GRAPH = "graph-4444"
+_GRAPH_EXEC = "gexec-5555"
+_NODE_EXEC = "nexec-6666"
+_BLOCK = "block-7777"
+
+
+def _context(**overrides) -> ExecutionContext:
+ fields = dict(
+ user_id=_USER,
+ session_id=_SESSION,
+ expert_id=_EXPERT,
+ graph_id=_GRAPH,
+ graph_exec_id=_GRAPH_EXEC,
+ node_exec_id=_NODE_EXEC,
+ )
+ fields.update(overrides)
+ return ExecutionContext(**fields)
+
+
+class TestAsE2B:
+ def test_prefixes_keys_and_drops_unset_fields(self):
+ meta = SandboxMetadata(
+ owner="session:s", kind="shell", source="copilot", env="x"
+ )
+ assert meta.as_e2b() == {
+ "service": "autogpt-platform",
+ "autogpt_owner": "session:s",
+ "autogpt_kind": "shell",
+ "autogpt_source": "copilot",
+ "autogpt_env": "x",
+ }
+
+ def test_values_are_all_strings(self):
+ meta = SandboxMetadata.for_block(_context(), "code", _BLOCK, "tpl")
+ assert all(isinstance(v, str) for v in meta.as_e2b().values())
+
+
+class TestParse:
+ def test_round_trips_our_own_metadata(self):
+ meta = SandboxMetadata.for_block(_context(), "code", _BLOCK, "tpl-1")
+ assert SandboxMetadata.parse(meta.as_e2b()) == meta
+
+ def test_another_services_metadata_is_not_ours(self):
+ reviewer = {"purpose": "review", "repo": "org/repo", "job_id": "j1"}
+ assert SandboxMetadata.parse(reviewer) is None
+ assert SandboxMetadata.parse({**reviewer, "service": "pr-reviewer"}) is None
+
+ def test_untagged_legacy_box_is_not_ours(self):
+ assert SandboxMetadata.parse({"autogpt_owner": "session:s"}) is None
+
+ def test_unknown_and_missing_keys(self):
+ stamped = {
+ "service": "autogpt-platform",
+ "autogpt_owner": "session:s",
+ "autogpt_kind": "shell",
+ "autogpt_source": "copilot",
+ "autogpt_env": "dev",
+ "autogpt_future_key": "ignored",
+ }
+ parsed = SandboxMetadata.parse(stamped)
+ assert parsed is not None and parsed.owner == "session:s"
+ del stamped["autogpt_kind"]
+ assert SandboxMetadata.parse(stamped) is None
+
+
+class TestForCopilot:
+ def test_stamps_env_from_settings(self):
+ with patch("backend.util.sandbox_metadata.deployment_env", return_value="prod"):
+ meta = SandboxMetadata.for_copilot("session:s", "shell")
+ assert meta.as_e2b()["autogpt_env"] == "prod"
+
+ def test_carries_provenance(self):
+ meta = SandboxMetadata.for_copilot(
+ f"expert:{_EXPERT}",
+ "desktop",
+ user_id=_USER,
+ session_id=_SESSION,
+ expert_id=_EXPERT,
+ template="desktop",
+ mounts="attached",
+ )
+ assert meta.as_e2b() == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"expert:{_EXPERT}",
+ "autogpt_kind": "desktop",
+ "autogpt_source": "copilot",
+ "autogpt_env": deployment_env(),
+ "autogpt_user": _USER,
+ "autogpt_session": _SESSION,
+ "autogpt_expert": _EXPERT,
+ "autogpt_template": "desktop",
+ "autogpt_mounts": "attached",
+ }
+
+
+class TestForBlock:
+ def test_traces_to_the_node_execution(self):
+ meta = SandboxMetadata.for_block(_context(), "code", _BLOCK, "tpl-1")
+ assert meta.as_e2b() == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"graph_exec:{_GRAPH_EXEC}",
+ "autogpt_kind": "code",
+ "autogpt_source": "block",
+ "autogpt_env": deployment_env(),
+ "autogpt_user": _USER,
+ "autogpt_session": _SESSION,
+ "autogpt_expert": _EXPERT,
+ "autogpt_graph": _GRAPH,
+ "autogpt_graph_exec": _GRAPH_EXEC,
+ "autogpt_node_exec": _NODE_EXEC,
+ "autogpt_block": _BLOCK,
+ "autogpt_template": "tpl-1",
+ }
+
+ def test_owner_falls_back_to_user_then_block(self):
+ by_user = SandboxMetadata.for_block(
+ _context(graph_exec_id=None), "desktop", _BLOCK
+ )
+ assert by_user.owner == f"user:{_USER}"
+ by_block = SandboxMetadata.for_block(
+ _context(graph_exec_id=None, user_id=None), "desktop", _BLOCK
+ )
+ assert by_block.owner == f"block:{_BLOCK}"
+
+ def test_empty_template_is_omitted(self):
+ meta = SandboxMetadata.for_block(_context(), "code", _BLOCK, "")
+ assert "autogpt_template" not in meta.as_e2b()
+
+ def test_without_context_still_identifies_the_block(self):
+ meta = SandboxMetadata.for_block(None, "claude_code", _BLOCK, "base")
+ assert meta.as_e2b() == {
+ "service": "autogpt-platform",
+ "autogpt_owner": f"block:{_BLOCK}",
+ "autogpt_kind": "claude_code",
+ "autogpt_source": "block",
+ "autogpt_env": deployment_env(),
+ "autogpt_block": _BLOCK,
+ "autogpt_template": "base",
+ }
+
+
+class TestOwnedByUser:
+ """A caller-supplied sandbox id is only usable if the box was stamped
+ for that user; anything else, including an unstamped box, is refused."""
+
+ def test_the_stamped_user_owns_it(self):
+ stamped = SandboxMetadata.for_block(_context(), "code", _BLOCK).as_e2b()
+ assert owned_by_user(stamped, _USER)
+
+ def test_another_user_does_not(self):
+ stamped = SandboxMetadata.for_block(_context(), "code", _BLOCK).as_e2b()
+ assert not owned_by_user(stamped, "user-9999")
+
+ def test_a_box_stamped_without_a_user_belongs_to_nobody(self):
+ stamped = SandboxMetadata.for_block(None, "code", _BLOCK).as_e2b()
+ assert not owned_by_user(stamped, _USER)
+ assert not owned_by_user(stamped, None)
+
+ def test_an_unstamped_or_foreign_box_is_refused(self):
+ assert not owned_by_user({}, _USER)
+ assert not owned_by_user(
+ {"service": "pr-reviewer", "autogpt_user": _USER}, _USER
+ )
diff --git a/autogpt_platform/backend/scripts/desktop_smoke_test.py b/autogpt_platform/backend/scripts/desktop_smoke_test.py
new file mode 100644
index 0000000000..bca49e5700
--- /dev/null
+++ b/autogpt_platform/backend/scripts/desktop_smoke_test.py
@@ -0,0 +1,110 @@
+"""Real-API smoke test for the E2B desktop sandbox blocks.
+
+Run manually (costs a few cents of E2B credits):
+ poetry run python scripts/desktop_smoke_test.py
+
+Reads E2B_API_KEY from backend/.env. Creates a desktop, verifies the live
+stream URL responds, exercises input + files + suspend/resume + the
+workspace volume, then destroys the sandbox. Prints timings.
+"""
+
+import asyncio
+import os
+import time
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import httpx
+from dotenv import load_dotenv
+
+from backend.blocks.desktop._api import WORKSPACE_PATH, DesktopSession
+
+VOLUME_NAME = "autogpt-smoke-test"
+TEST_FILE = f"{WORKSPACE_PATH}/persist-test.txt"
+
+
+def timed(label: str, started: float, timings: dict[str, float]) -> None:
+ timings[label] = round(time.monotonic() - started, 2)
+ print(f" {label}: {timings[label]}s")
+
+
+async def main() -> None:
+ load_dotenv(Path(__file__).parent.parent / ".env")
+ api_key = os.environ["E2B_API_KEY"]
+ timings: dict[str, float] = {}
+
+ print("1) create desktop sandbox (with volume attempt)")
+ t = time.monotonic()
+ session, persistence = await DesktopSession.create(
+ api_key=api_key,
+ timeout_seconds=300,
+ width=1280,
+ height=720,
+ volume_mounts={WORKSPACE_PATH: VOLUME_NAME},
+ )
+ timed("cold_start_ready", t, timings)
+ print(f" sandbox_id={session.sandbox_id}")
+ print(f" persistence={persistence.model_dump()}")
+
+ try:
+ print("2) start stream")
+ t = time.monotonic()
+ stream = await session.start_stream()
+ timed("stream_start", t, timings)
+ # The URL carries the VNC password; log where it points, not the secret.
+ print(
+ f" stream host={urlsplit(stream.url).netloc} sandbox={stream.sandbox_id}"
+ )
+ async with httpx.AsyncClient() as client:
+ resp = await client.get(stream.url, timeout=15)
+ resp.raise_for_status()
+ print(f" stream HTTP status: {resp.status_code}")
+
+ # Input actions (click / screenshot) live with the desktop blocks in
+ # #14381; this smoke test covers the box lifecycle and the stream.
+
+ print("4) workspace file write")
+ await session.sandbox.files.write(TEST_FILE, "persistence check")
+
+ print("5) suspend")
+ t = time.monotonic()
+ await session.pause()
+ timed("suspend", t, timings)
+
+ print("6) resume via connect")
+ t = time.monotonic()
+ resumed = await DesktopSession.connect(session.sandbox_id, api_key)
+ timed("resume", t, timings)
+ content = await resumed.sandbox.files.read(TEST_FILE)
+ assert (
+ content == "persistence check"
+ ), f"file changed across resume: {content!r}"
+ print(f" file after resume: {content!r}")
+ session = resumed
+
+ if persistence.volume_mounted:
+ print("7) cross-sandbox volume check")
+ other, _other_persistence = await DesktopSession.create(
+ api_key=api_key,
+ timeout_seconds=120,
+ width=1024,
+ height=768,
+ volume_mounts={WORKSPACE_PATH: VOLUME_NAME},
+ )
+ try:
+ cross = await other.sandbox.files.read(TEST_FILE)
+ assert cross == "persistence check", f"volume not shared: {cross!r}"
+ print(f" volume file in second sandbox: {cross!r}")
+ finally:
+ await other.kill()
+ else:
+ print(f"7) volume not mounted, skipping ({persistence.warning})")
+ finally:
+ print("8) destroy")
+ await session.kill()
+
+ print("\nTimings:", timings)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/autogpt_platform/backend/snapshots/desktop_preview_owner_redirect b/autogpt_platform/backend/snapshots/desktop_preview_owner_redirect
new file mode 100644
index 0000000000..00055be518
--- /dev/null
+++ b/autogpt_platform/backend/snapshots/desktop_preview_owner_redirect
@@ -0,0 +1,10 @@
+{
+ "body": "",
+ "headers": {
+ "cache-control": "no-store",
+ "content-length": "0",
+ "location": "https://6080-sandbox.e2b.app/vnc.html?password=private",
+ "referrer-policy": "no-referrer"
+ },
+ "status": 307
+}
\ No newline at end of file
diff --git a/autogpt_platform/backend/snapshots/desktop_preview_unauthenticated b/autogpt_platform/backend/snapshots/desktop_preview_unauthenticated
new file mode 100644
index 0000000000..a49e689559
--- /dev/null
+++ b/autogpt_platform/backend/snapshots/desktop_preview_unauthenticated
@@ -0,0 +1,8 @@
+{
+ "body": "{\"detail\":\"Authorization header is missing\"}",
+ "headers": {
+ "content-length": "44",
+ "content-type": "application/json"
+ },
+ "status": 401
+}
\ No newline at end of file
diff --git a/autogpt_platform/backend/snapshots/desktop_preview_unavailable b/autogpt_platform/backend/snapshots/desktop_preview_unavailable
new file mode 100644
index 0000000000..2b7f8f1121
--- /dev/null
+++ b/autogpt_platform/backend/snapshots/desktop_preview_unavailable
@@ -0,0 +1,8 @@
+{
+ "body": "{\"detail\":\"Preview unavailable\"}",
+ "headers": {
+ "content-length": "32",
+ "content-type": "application/json"
+ },
+ "status": 404
+}
\ No newline at end of file
diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/share/chat/[token]/__tests__/page.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/share/chat/[token]/__tests__/page.test.tsx
index 4bc13e5aa7..8585144d8e 100644
--- a/autogpt_platform/frontend/src/app/(no-navbar)/share/chat/[token]/__tests__/page.test.tsx
+++ b/autogpt_platform/frontend/src/app/(no-navbar)/share/chat/[token]/__tests__/page.test.tsx
@@ -39,6 +39,9 @@ function resetArtifactPanelStore() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
useCopilotUIStore.getState().resetAutoOpenState();
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/CopilotPage.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/CopilotPage.tsx
index 71968f8649..4fde37632f 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/CopilotPage.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/CopilotPage.tsx
@@ -179,7 +179,7 @@ function MainArea({
<ContextPanel sessionId={sessionId} />
)}
{!isMobile && isArtifactsEnabled && sessionId && (
- <ArtifactPanel hasExternalClose />
+ <ArtifactPanel hasExternalClose sessionId={sessionId} />
)}
</div>
);
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/store.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/store.test.ts
index 91b3859cee..f6a4e56d55 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/store.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/store.test.ts
@@ -30,6 +30,9 @@ function resetStore() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
// Clear the module-level auto-open flags so each test starts isolated —
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactCard/ArtifactCard.stories.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactCard/ArtifactCard.stories.tsx
index 6ec547750c..a28729b2f6 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactCard/ArtifactCard.stories.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactCard/ArtifactCard.stories.tsx
@@ -136,6 +136,9 @@ export const ActiveState: Story = {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
return <Story />;
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.stories.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.stories.tsx
index e18bbfdac2..a723f450a8 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.stories.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.stories.tsx
@@ -25,6 +25,9 @@ function openPanelWith(artifact: ArtifactRef) {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
}
@@ -186,6 +189,9 @@ export const Closed: Story = {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
return <Story />;
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.tsx
index 292480febc..9b1067fc77 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/ArtifactPanel.tsx
@@ -7,6 +7,7 @@ import { MIN_ARTIFACT_PANEL_WIDTH, PANEL_RESERVED_WIDTH } from "../../store";
import { PanelResizeHandle } from "../PanelResizeHandle";
import { ArtifactContent } from "./components/ArtifactContent";
import { ArtifactPanelHeader } from "./components/ArtifactPanelHeader";
+import { ComputerPanelContent } from "./components/ComputerPanelContent";
import { useArtifactPanel } from "./useArtifactPanel";
// Matched to the context panel so the two side rails move as one system.
@@ -15,13 +16,15 @@ const PANEL_DURATION = 0.3;
interface Props {
mobile?: boolean;
+ /** Enables the Computer face: the chat whose sandboxes the panel can show. */
+ sessionId?: string | null;
/** The desktop copilot chat renders its own sidebar-right close control;
* standalone hosts (share viewer, tour, mobile drawer) do not, so their
* header keeps its Close button. */
hasExternalClose?: boolean;
}
-export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
+export function ArtifactPanel({ mobile, hasExternalClose, sessionId }: Props) {
const {
activeArtifact,
history,
@@ -36,7 +39,11 @@ export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
handleDownload,
artifactPanelWidth,
setArtifactPanelWidth,
+ mode,
+ isComputerOpen,
+ setArtifactPanelMode,
} = useArtifactPanel();
+ const showComputer = !!sessionId && mode === "computer" && isComputerOpen;
// Hold the last live artifact so both the mobile drawer and the desktop
// panel can keep rendering its contents while the close animation plays —
@@ -59,7 +66,8 @@ export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
const [availableWidth, setAvailableWidth] = useState<number | null>(null);
const [isResizing, setIsResizing] = useState(false);
const shouldReduceMotion = useReducedMotion();
- const showDesktopPanel = !mobile && !!activeArtifact && !!classification;
+ const showDesktopPanel =
+ !mobile && ((!!activeArtifact && !!classification) || showComputer);
useEffect(() => {
if (!showDesktopPanel || typeof ResizeObserver === "undefined") return;
const parent = panelRef.current?.parentElement;
@@ -155,7 +163,7 @@ export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
return (
<AnimatePresence initial={false}>
- {showDesktopPanel && shown && (
+ {showDesktopPanel && (shown || showComputer) && (
<motion.div
ref={panelRef}
data-artifact-panel
@@ -182,12 +190,23 @@ export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
className="flex h-full min-h-0 flex-col overflow-hidden"
>
<ArtifactPanelHeader
- artifact={shown.artifact}
- classification={shown.classification}
+ artifact={
+ showComputer
+ ? (activeArtifact ?? null)
+ : (shown?.artifact ?? null)
+ }
+ classification={
+ showComputer
+ ? classification
+ : (shown?.classification ?? null)
+ }
+ mode={showComputer ? "computer" : "artifact"}
+ showModeSwitch={!!sessionId}
+ onModeChange={setArtifactPanelMode}
hasExternalClose={hasExternalClose}
canGoBack={history.length > 0}
isSourceView={isSourceView}
- hasSourceToggle={shown.classification.hasSourceToggle}
+ hasSourceToggle={shown?.classification.hasSourceToggle ?? false}
canCopy={canCopy}
onBack={goBackArtifact}
onClose={clearArtifactPreview}
@@ -196,11 +215,15 @@ export function ArtifactPanel({ mobile, hasExternalClose }: Props) {
onOpenFiles={showFilesTab}
onSourceToggle={setIsSourceView}
/>
- <ArtifactContent
- artifact={shown.artifact}
- isSourceView={isSourceView}
- classification={shown.classification}
- />
+ {showComputer && sessionId ? (
+ <ComputerPanelContent sessionId={sessionId} />
+ ) : shown ? (
+ <ArtifactContent
+ artifact={shown.artifact}
+ isSourceView={isSourceView}
+ classification={shown.classification}
+ />
+ ) : null}
</div>
</div>
</motion.div>
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx
new file mode 100644
index 0000000000..7a70f00871
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx
@@ -0,0 +1,140 @@
+import {
+ getGetV2GetSessionComputerMockHandler,
+ getPostV2StartSessionDesktopMockHandler,
+} from "@/app/api/__generated__/endpoints/chat/chat.msw";
+import { server } from "@/mocks/mock-server";
+import { render, screen, waitFor } from "@/tests/integrations/test-utils";
+import { cleanup } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { useCopilotUIStore } from "../../../store";
+import { ComputerPanelContent } from "../components/ComputerPanelContent";
+import { PanelModeSwitch } from "../components/PanelModeSwitch";
+
+const STREAM = {
+ url: "https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ sandbox_id: "sbx-1",
+ provider: "e2b" as const,
+};
+
+describe("copilot store: computer face", () => {
+ beforeEach(() => {
+ useCopilotUIStore.getState().resetArtifactPanel();
+ useCopilotUIStore.getState().closeArtifactPanel({ persist: false });
+ });
+
+ it("a started desktop opens the computer face when nothing else is showing", () => {
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.computer?.sandbox_id).toBe("sbx-1");
+ expect(panel.mode).toBe("computer");
+ expect(panel.isComputerOpen).toBe(true);
+ // A closed panel would hide the desktop that was just started.
+ expect(panel.isOpen).toBe(true);
+ });
+
+ it("does not steal the panel from an open artifact, but remembers the desktop", () => {
+ useCopilotUIStore.getState().openArtifact(
+ {
+ id: "a1",
+ title: "notes.md",
+ sourceUrl: "/x",
+ origin: "assistant",
+ } as never,
+ { persist: false },
+ );
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.mode).toBe("artifact");
+ expect(panel.computer?.sandbox_id).toBe("sbx-1");
+ useCopilotUIStore.getState().setArtifactPanelMode("computer");
+ expect(useCopilotUIStore.getState().artifactPanel.isComputerOpen).toBe(
+ true,
+ );
+ });
+
+ it("closing the panel and entering a new chat both drop the computer face", () => {
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ useCopilotUIStore.getState().closeArtifactPanel({ persist: false });
+ expect(useCopilotUIStore.getState().artifactPanel.isComputerOpen).toBe(
+ false,
+ );
+ useCopilotUIStore.getState().clearLastArtifact();
+ expect(useCopilotUIStore.getState().artifactPanel.computer).toBeNull();
+ });
+});
+
+describe("PanelModeSwitch", () => {
+ afterEach(() => cleanup());
+
+ it("marks the active face and disables Artifact when there is none", async () => {
+ const changes: string[] = [];
+ render(
+ <PanelModeSwitch
+ mode="computer"
+ hasArtifact={false}
+ onChange={(m) => changes.push(m)}
+ />,
+ );
+ const computer = screen.getByRole("button", { name: "Computer" });
+ const artifact = screen.getByRole("button", { name: "Artifact" });
+ expect(computer.getAttribute("aria-pressed")).toBe("true");
+ expect(artifact).toHaveProperty("disabled", true);
+ await userEvent.click(computer);
+ expect(changes).toEqual(["computer"]);
+ });
+});
+
+describe("ComputerPanelContent", () => {
+ beforeEach(() => {
+ useCopilotUIStore.getState().resetArtifactPanel();
+ });
+ afterEach(() => cleanup());
+
+ it("shows the chat's boxes and starts a desktop on demand", async () => {
+ server.use(
+ getGetV2GetSessionComputerMockHandler({
+ owner_kind: "expert",
+ owner_id: "exp-1",
+ e2b_active: true,
+ shell: {
+ kind: "shell",
+ sandbox_id: "sb",
+ state: "paused",
+ started_at: new Date("2026-09-05T12:00:00Z"),
+ cpu_count: 2,
+ memory_mb: 512,
+ template_id: "base",
+ mounts_attached: true,
+ },
+ desktop: null,
+ mounts: {},
+ workspace_path: "/home/user/workspace",
+ shared_path: "/home/user/shared",
+ }),
+ getPostV2StartSessionDesktopMockHandler({
+ kind: "desktop_stream",
+ requires_auth: false,
+ ...STREAM,
+ }),
+ );
+
+ render(<ComputerPanelContent sessionId="sess-1" />);
+
+ expect(await screen.findByText("Suspended")).toBeDefined();
+ expect(screen.getByText(/expert's own computer/)).toBeDefined();
+ await userEvent.click(
+ screen.getByRole("button", { name: "Start desktop" }),
+ );
+ await waitFor(
+ () =>
+ expect(
+ useCopilotUIStore.getState().artifactPanel.computer?.sandbox_id,
+ ).toBe("sbx-1"),
+ { timeout: 4000 },
+ );
+ await waitFor(() =>
+ expect(screen.getByTitle("Interactive desktop (sbx-1)")).toBeDefined(),
+ );
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ArtifactPanelHeader.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ArtifactPanelHeader.tsx
index 154f3f5094..e41a01d3e9 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ArtifactPanelHeader.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ArtifactPanelHeader.tsx
@@ -6,12 +6,14 @@ import {
TooltipTrigger,
} from "@/components/atoms/Tooltip/BaseTooltip";
import { cn } from "@/lib/utils";
-import type { ArtifactRef } from "../../../store";
+import type { ArtifactPanelMode, ArtifactRef } from "../../../store";
import type { ArtifactClassification } from "../helpers";
+import { PanelModeSwitch } from "./PanelModeSwitch";
import { SourceToggle } from "./SourceToggle";
import {
ArrowLeft02Icon,
Cancel01Icon,
+ ComputerIcon,
Copy01Icon,
Download01Icon,
Folder01Icon,
@@ -19,8 +21,14 @@ import {
import { Icon } from "@/components/atoms/Icon/Icon";
interface Props {
- artifact: ArtifactRef;
- classification: ArtifactClassification;
+ /** Null while the panel shows only its computer face. */
+ artifact: ArtifactRef | null;
+ classification: ArtifactClassification | null;
+ mode?: ArtifactPanelMode;
+ /** Render the Artifact/Computer switch; the chat passes a session, the
+ * share viewer and tour do not. */
+ showModeSwitch?: boolean;
+ onModeChange?: (mode: ArtifactPanelMode) => void;
canGoBack: boolean;
isSourceView: boolean;
hasSourceToggle: boolean;
@@ -77,47 +85,73 @@ export function ArtifactPanelHeader({
onOpenFiles,
onSourceToggle,
hasExternalClose = false,
+ mode = "artifact",
+ showModeSwitch = false,
+ onModeChange,
}: Props) {
+ const isComputer = mode === "computer" || artifact == null;
// An expert is not a workspace file: nothing to download, and "All files"
// would strand the user in the wrong tab.
- const isFile = !artifact.expert;
+ const isFile = !isComputer && !artifact?.expert;
// Height matches the chat thread header (36px avatar + py-2 + border) so
// the two top bars share one seam across the panel split.
return (
<div className="sticky top-0 z-10 flex h-[53px] items-center gap-2 border-b border-b-[#80808017] bg-sidebar px-3">
{/* Left section */}
<div className="flex min-w-0 flex-1 items-center gap-2">
- {canGoBack && (
+ {!isComputer && canGoBack && (
<HeaderButton onClick={onBack} title="Back">
<Icon icon={ArrowLeft02Icon} size={16} />
</HeaderButton>
)}
- <Icon
- icon={classification.icon}
- size={16}
- className="shrink-0 text-zinc-400"
- />
- <span className="truncate text-sm font-medium text-zinc-900">
- {artifact.title}
- </span>
- <span
- className={cn(
- "shrink-0 rounded-full px-2 py-0.5 text-xs font-medium",
- artifact.origin === "user-upload"
- ? "bg-blue-50 text-blue-600"
- : "bg-violet-50 text-violet-600",
- )}
- >
- {classification.label}
- </span>
+ {isComputer || !artifact || !classification ? (
+ <>
+ <Icon
+ icon={ComputerIcon}
+ size={16}
+ className="shrink-0 text-zinc-400"
+ />
+ <span className="truncate text-sm font-medium text-zinc-900">
+ Computer
+ </span>
+ </>
+ ) : (
+ <>
+ <Icon
+ icon={classification.icon}
+ size={16}
+ className="shrink-0 text-zinc-400"
+ />
+ <span className="truncate text-sm font-medium text-zinc-900">
+ {artifact.title}
+ </span>
+ <span
+ className={cn(
+ "shrink-0 rounded-full px-2 py-0.5 text-xs font-medium",
+ artifact.origin === "user-upload"
+ ? "bg-blue-50 text-blue-600"
+ : "bg-violet-50 text-violet-600",
+ )}
+ >
+ {classification.label}
+ </span>
+ </>
+ )}
</div>
{/* Right section */}
<div className="flex items-center gap-1">
- {hasSourceToggle && (
+ {showModeSwitch && onModeChange && (
+ <PanelModeSwitch
+ mode={isComputer ? "computer" : "artifact"}
+ hasArtifact={artifact != null}
+ onChange={onModeChange}
+ />
+ )}
+ {!isComputer && hasSourceToggle && (
<SourceToggle isSourceView={isSourceView} onToggle={onSourceToggle} />
)}
- {canCopy && (
+ {!isComputer && canCopy && (
<HeaderButton onClick={onCopy} title="Copy">
<Icon icon={Copy01Icon} size={16} />
</HeaderButton>
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx
new file mode 100644
index 0000000000..298b56a8e0
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/ComputerPanelContent.tsx
@@ -0,0 +1,132 @@
+"use client";
+
+import {
+ getGetV2GetSessionComputerQueryKey,
+ useGetV2GetSessionComputer,
+ usePostV2StartSessionDesktop,
+} from "@/app/api/__generated__/endpoints/chat/chat";
+import { okData } from "@/app/api/helpers";
+import { Button } from "@/components/atoms/Button/Button";
+import { Text } from "@/components/atoms/Text/Text";
+import { desktopStreamRenderer } from "@/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer";
+import { useToast } from "@/components/molecules/Toast/use-toast";
+import { cn } from "@/lib/utils";
+import { useQueryClient } from "@tanstack/react-query";
+import { useCopilotUIStore } from "../../../store";
+
+interface Props {
+ sessionId: string;
+}
+
+function StatePill({ state }: { state: "running" | "paused" | null }) {
+ return (
+ <span
+ className={cn(
+ "rounded-full px-2 py-0.5 text-xs font-medium",
+ state === "running"
+ ? "bg-emerald-50 text-emerald-700"
+ : state === "paused"
+ ? "bg-zinc-100 text-zinc-600"
+ : "bg-zinc-50 text-zinc-500",
+ )}
+ >
+ {state === "running"
+ ? "Running"
+ : state === "paused"
+ ? "Suspended"
+ : "None"}
+ </span>
+ );
+}
+
+export function ComputerPanelContent({ sessionId }: Props) {
+ const queryClient = useQueryClient();
+ const { toast } = useToast();
+ const stream = useCopilotUIStore((s) => s.artifactPanel.computer);
+ const registerComputerStream = useCopilotUIStore(
+ (s) => s.registerComputerStream,
+ );
+
+ const computerQuery = useGetV2GetSessionComputer(sessionId, {
+ query: {
+ select: (res) => okData(res) ?? null,
+ refetchInterval: 15_000,
+ },
+ });
+ const computer = computerQuery.data ?? null;
+
+ const { mutate: startDesktop, isPending } = usePostV2StartSessionDesktop({
+ mutation: {
+ onSuccess: (res) => {
+ const next = okData(res);
+ if (next) {
+ registerComputerStream({
+ url: next.url,
+ sandbox_id: next.sandbox_id,
+ provider: next.provider ?? "e2b",
+ });
+ }
+ queryClient.invalidateQueries({
+ queryKey: getGetV2GetSessionComputerQueryKey(sessionId),
+ });
+ },
+ onError: () => {
+ toast({
+ title: "Could not open the desktop",
+ description: "The sandbox did not come up. Try again in a moment.",
+ variant: "destructive",
+ });
+ },
+ },
+ });
+
+ const isExpert = computer?.owner_kind === "expert";
+ const hasDesktop = computer?.desktop != null;
+
+ return (
+ <div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto p-3">
+ <div className="flex items-center justify-between gap-3 rounded-lg border border-zinc-200 bg-white px-3 py-2">
+ <div className="flex items-center gap-3 text-sm">
+ <span className="flex items-center gap-1.5">
+ <span className="text-zinc-500">Shell</span>
+ <StatePill state={computer?.shell?.state ?? null} />
+ </span>
+ <span className="flex items-center gap-1.5">
+ <span className="text-zinc-500">Desktop</span>
+ <StatePill state={computer?.desktop?.state ?? null} />
+ </span>
+ </div>
+ <Button
+ variant="secondary"
+ size="small"
+ loading={isPending}
+ disabled={computer != null && !computer.e2b_active}
+ onClick={() => startDesktop({ sessionId })}
+ >
+ {stream ? "Refresh" : hasDesktop ? "Resume desktop" : "Start desktop"}
+ </Button>
+ </div>
+
+ {stream ? (
+ <div data-testid="session-desktop-stream">
+ {desktopStreamRenderer.render({ kind: "desktop_stream", ...stream })}
+ </div>
+ ) : (
+ <div className="flex flex-1 flex-col items-center justify-center gap-1 rounded-lg border border-dashed border-zinc-200 p-6 text-center">
+ <Text variant="small-medium">No desktop on screen yet</Text>
+ <Text variant="small" className="max-w-xs text-zinc-500">
+ {isExpert
+ ? "This chat runs on the expert's own computer. Start the desktop to watch it work, or ask for one in the chat."
+ : "Start the desktop to watch this chat work in a real browser, or ask for one in the chat."}
+ </Text>
+ </div>
+ )}
+
+ {computer && !computer.e2b_active ? (
+ <Text variant="small" className="text-amber-700">
+ Cloud sandboxes are not configured on this deployment.
+ </Text>
+ ) : null}
+ </div>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/PanelModeSwitch.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/PanelModeSwitch.tsx
new file mode 100644
index 0000000000..ff19dddb29
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/components/PanelModeSwitch.tsx
@@ -0,0 +1,57 @@
+"use client";
+
+import { Icon } from "@/components/atoms/Icon/Icon";
+import { cn } from "@/lib/utils";
+import { ComputerIcon, LicenseDraftIcon } from "@hugeicons/core-free-icons";
+import type { ArtifactPanelMode } from "../../../store";
+
+interface Props {
+ mode: ArtifactPanelMode;
+ hasArtifact: boolean;
+ onChange: (mode: ArtifactPanelMode) => void;
+}
+
+const OPTIONS: {
+ value: ArtifactPanelMode;
+ label: string;
+ icon: typeof ComputerIcon;
+}[] = [
+ { value: "artifact", label: "Artifact", icon: LicenseDraftIcon },
+ { value: "computer", label: "Computer", icon: ComputerIcon },
+];
+
+/** Two faces of one side panel: the document being worked on, or the
+ * machine doing the work. */
+export function PanelModeSwitch({ mode, hasArtifact, onChange }: Props) {
+ return (
+ <div
+ role="group"
+ aria-label="Panel view"
+ className="flex shrink-0 items-center rounded-md bg-zinc-100 p-0.5"
+ >
+ {OPTIONS.map((option) => {
+ const active = option.value === mode;
+ const disabled = option.value === "artifact" && !hasArtifact;
+ return (
+ <button
+ key={option.value}
+ type="button"
+ aria-pressed={active}
+ disabled={disabled}
+ onClick={() => onChange(option.value)}
+ className={cn(
+ "flex items-center gap-1 rounded px-2 py-1 text-xs font-medium transition-colors",
+ active
+ ? "bg-white text-zinc-900 shadow-sm"
+ : "text-zinc-500 hover:text-zinc-800",
+ disabled && "cursor-not-allowed opacity-40 hover:text-zinc-500",
+ )}
+ >
+ <Icon icon={option.icon} size={14} />
+ {option.label}
+ </button>
+ );
+ })}
+ </div>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/useArtifactPanel.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/useArtifactPanel.ts
index caea6951a4..af28a710ba 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/useArtifactPanel.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/useArtifactPanel.ts
@@ -16,6 +16,7 @@ export function useArtifactPanel() {
const setArtifactPanelWidth = useCopilotUIStore(
(s) => s.setArtifactPanelWidth,
);
+ const setArtifactPanelMode = useCopilotUIStore((s) => s.setArtifactPanelMode);
const [isSourceView, setIsSourceView] = useState(false);
@@ -92,5 +93,8 @@ export function useArtifactPanel() {
handleDownload,
artifactPanelWidth,
setArtifactPanelWidth,
+ mode: artifactPanel.mode,
+ isComputerOpen: artifactPanel.isComputerOpen,
+ setArtifactPanelMode,
};
}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/__tests__/ChatContainer.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/__tests__/ChatContainer.test.tsx
index f34dbc93a7..c7d5a3a85c 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/__tests__/ChatContainer.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/__tests__/ChatContainer.test.tsx
@@ -40,6 +40,9 @@ function resetCopilotStore() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
}
@@ -437,6 +440,9 @@ describe("ChatContainer", () => {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/useAutoOpenArtifacts.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/useAutoOpenArtifacts.test.ts
index f846f49d21..a2dc892129 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/useAutoOpenArtifacts.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatContainer/useAutoOpenArtifacts.test.ts
@@ -32,6 +32,9 @@ function resetStore() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
useCopilotUIStore.getState().resetAutoOpenState();
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
index 732d624c03..ff0e9a1f77 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
@@ -42,6 +42,13 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
const closeArtifactPanel = useCopilotUIStore((s) => s.closeArtifactPanel);
const lastArtifact = useCopilotUIStore((s) => s.artifactPanel.lastArtifact);
const openArtifact = useCopilotUIStore((s) => s.openArtifact);
+ const hasComputer = useCopilotUIStore(
+ (s) => s.artifactPanel.computer != null,
+ );
+ const isComputerOpen = useCopilotUIStore(
+ (s) => s.artifactPanel.isComputerOpen,
+ );
+ const openComputer = useCopilotUIStore((s) => s.openComputer);
const { generated } = useSessionFiles(sessionId);
const lastGenerated = getLastGeneratedFile(generated);
const isFilesCardOpen = useAreWorkspaceFileCardsOpen();
@@ -49,7 +56,7 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
// An open artifact preview and the artifacts tab are both faces of the
// right sidebar, so the toggle reads active for either and is the one
// control that closes them — the panel carries no close button.
- const isRightSidebarOpen = hasArtifact || isArtifactsOpen;
+ const isRightSidebarOpen = hasArtifact || isArtifactsOpen || isComputerOpen;
// The open activity card already lists the same file, so the labeled
// button floating above it is pure duplication — the card's rows are the
@@ -62,7 +69,7 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
const showFileName = lastGenerated != null && !isRightSidebarOpen;
function handleSidebarToggle() {
- if (hasArtifact) {
+ if (hasArtifact || isComputerOpen) {
closeArtifactPanel();
return;
}
@@ -79,6 +86,11 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
openArtifact(target);
return;
}
+ // No document yet but a desktop was started: the machine is the work.
+ if (hasComputer) {
+ openComputer();
+ return;
+ }
toggleContextPanelTab("artifacts");
}
@@ -91,10 +103,14 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
onClick={handleSidebarToggle}
aria-label={
isRightSidebarOpen
- ? "Hide artifacts"
+ ? isComputerOpen
+ ? "Hide computer"
+ : "Hide artifacts"
: lastGenerated
? `Open ${lastGenerated.item.name}`
- : "Open artifacts"
+ : hasComputer
+ ? "Open computer"
+ : "Open artifacts"
}
aria-pressed={isRightSidebarOpen}
className={cn(
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ArtifactsTab.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ArtifactsTab.test.tsx
index caed38a0b9..910a127f78 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ArtifactsTab.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ArtifactsTab.test.tsx
@@ -33,6 +33,9 @@ afterEach(() => {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
index 21d3bdcb63..0e730fafe3 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
@@ -46,6 +46,7 @@ function ReasoningStream({ text, live }: ReasoningStreamProps) {
interface Props {
row: ChainRow;
isLast: boolean;
+ readOnly?: boolean;
}
const SUB_SESSION_TOOLS = new Set([
@@ -71,7 +72,7 @@ function isLiveSubSessionRow(row: ChainRow): boolean {
);
}
-export function ChainRowView({ row, isLast }: Props) {
+export function ChainRowView({ row, isLast, readOnly = false }: Props) {
const [open, setOpen] = useState(row.requiresAction === true);
const isReasoning = row.category === "reasoning";
const artifactPanelOpen = useCopilotUIStore((s) => s.artifactPanel.isOpen);
@@ -203,7 +204,7 @@ export function ChainRowView({ row, isLast }: Props) {
live={liveReasoning}
/>
) : (
- <ToolResult row={row} />
+ <ToolResult row={row} readOnly={readOnly} />
)}
</div>
</div>
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx
new file mode 100644
index 0000000000..71510b9bff
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx
@@ -0,0 +1,45 @@
+"use client";
+
+import {
+ DesktopStreamPreview,
+ isDesktopStream,
+} from "@/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer";
+import { useEffect } from "react";
+import { useCopilotUIStore, type DesktopStreamRef } from "../../store";
+
+interface Props {
+ stream: unknown;
+ /** A shared transcript: the viewer is not the owner and has no panel. */
+ readOnly?: boolean;
+}
+
+function asStreamRef(value: unknown): DesktopStreamRef | null {
+ if (typeof value !== "object" || value === null) return null;
+ const v = value as Record<string, unknown>;
+ if (typeof v.url !== "string" || typeof v.sandbox_id !== "string")
+ return null;
+ return {
+ url: v.url,
+ sandbox_id: v.sandbox_id,
+ provider: typeof v.provider === "string" ? v.provider : "e2b",
+ };
+}
+
+/** The inline start_desktop card. Besides embedding the stream it tells the
+ * side panel a desktop exists, so the Computer face can show the same
+ * screen without the model being asked again. */
+export function DesktopStreamCard({ stream, readOnly = false }: Props) {
+ const registerComputerStream = useCopilotUIStore(
+ (s) => s.registerComputerStream,
+ );
+ const ref = asStreamRef(stream);
+ const sandboxId = ref?.sandbox_id;
+ const url = ref?.url;
+ useEffect(() => {
+ if (ref && !readOnly) registerComputerStream(ref);
+ // Re-register only when the stream itself changes, not on every render.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [sandboxId, url, readOnly]);
+ if (!isDesktopStream(stream)) return null;
+ return <DesktopStreamPreview value={stream} ownerView={!readOnly} />;
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolChain.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolChain.tsx
index d9f3079c5b..d9caf8de18 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolChain.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolChain.tsx
@@ -346,6 +346,7 @@ export function ToolChain({ parts, isStreaming, readOnly = false }: Props) {
<ChainRowView
row={row}
isLast={i === visible.length - 1 && !showDone}
+ readOnly={readOnly}
/>
</ChainActionsContext.Provider>
</m.div>
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
index f40db3b8b8..047a79fb18 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
@@ -6,6 +6,8 @@ import { PendingQuestionsContext } from "../QuestionDock/PendingQuestionsContext
import { QuestionsForm } from "../QuestionDock/QuestionDock";
import { SetupRequirementsCard } from "../SetupRequirementsCard/SetupRequirementsCard";
import { MCPSetupCard } from "../../tools/RunMCPTool/components/MCPSetupCard/MCPSetupCard";
+import { desktopStreamRenderer } from "@/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer";
+import { DesktopStreamCard } from "./DesktopStreamCard";
import {
AgentListCard,
AgentPreviewCard,
@@ -192,7 +194,11 @@ function setupRequirementsCard(row: ChainRow, output: Record<string, unknown>) {
);
}
-function toolCard(row: ChainRow, output: Record<string, unknown> | null) {
+function toolCard(
+ row: ChainRow,
+ output: Record<string, unknown> | null,
+ readOnly: boolean,
+) {
const input = asObject(row.input);
if (output) {
@@ -385,6 +391,16 @@ function toolCard(row: ChainRow, output: Record<string, unknown> | null) {
}
case "bash_exec":
return <Terminal row={row} />;
+ case "start_desktop": {
+ // The live desktop is the whole point of the tool: embed the stream
+ // instead of letting the payload fall through to a truncated key/value
+ // dump. The same renderer serves block outputs and attachments.
+ const stream = output ? output.desktop_stream : null;
+ if (stream && desktopStreamRenderer.canRender(stream)) {
+ return <DesktopStreamCard stream={stream} readOnly={readOnly} />;
+ }
+ return null;
+ }
case "TodoWrite":
return <TodoList row={row} />;
case "read_workspace_file":
@@ -399,9 +415,10 @@ function toolCard(row: ChainRow, output: Record<string, unknown> | null) {
interface Props {
row: ChainRow;
+ readOnly?: boolean;
}
-export function ToolResult({ row }: Props) {
+export function ToolResult({ row, readOnly = false }: Props) {
const output = asObject(row.output);
const pendingQuestions = useContext(PendingQuestionsContext);
@@ -430,7 +447,7 @@ export function ToolResult({ row }: Props) {
);
}
- const card = toolCard(row, output);
+ const card = toolCard(row, output, readOnly);
if (card) return card;
if (!output) return <KeyValueList value={row.output} />;
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
index 4de122aca8..8bf339e8cb 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
@@ -945,3 +945,69 @@ describe("ToolResult", () => {
});
});
});
+
+describe("ToolResult start_desktop", () => {
+ afterEach(() => cleanup());
+
+ it("embeds the live desktop stream instead of dumping the payload", () => {
+ render(
+ <ToolResult
+ row={row(
+ {
+ type: "desktop_stream",
+ message: "Desktop started.",
+ desktop_stream: {
+ kind: "desktop_stream",
+ url: "https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ provider: "e2b",
+ sandbox_id: "sbx-1",
+ },
+ },
+ "start_desktop",
+ )}
+ />,
+ );
+
+ const frame = screen.getByTitle("Interactive desktop (sbx-1)");
+ expect(frame.getAttribute("src")).toBe(
+ "https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ );
+ expect(screen.getByText("Open in new tab")).toBeDefined();
+ expect(screen.queryByText(/"kind"/)).toBeNull();
+ });
+
+ it("shows a shared-transcript viewer a notice, not the owner-only frame", () => {
+ render(
+ <ToolResult
+ readOnly
+ row={row(
+ {
+ type: "desktop_stream",
+ message: "Desktop started.",
+ desktop_stream: {
+ kind: "desktop_stream",
+ url: "/api/proxy/api/desktop-preview?token=abc",
+ provider: "e2b",
+ sandbox_id: "sbx-1",
+ requires_auth: true,
+ },
+ },
+ "start_desktop",
+ )}
+ />,
+ );
+
+ expect(screen.queryByTitle(/Interactive desktop/)).toBeNull();
+ expect(screen.getByText(/only visible to the owner/i)).toBeDefined();
+ });
+
+ it("falls back to the generic view when the stream is missing", () => {
+ render(
+ <ToolResult
+ row={row({ message: "Failed to start the desktop." }, "start_desktop")}
+ />,
+ );
+ expect(screen.getByText("Failed to start the desktop.")).toBeDefined();
+ expect(screen.queryByTitle(/Interactive desktop/)).toBeNull();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/toolCatalog.platform.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/toolCatalog.platform.ts
index 0fd4516468..788796e126 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/toolCatalog.platform.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/toolCatalog.platform.ts
@@ -47,6 +47,11 @@ export const PLATFORM_TOOL_CATALOG: Record<string, ToolMeta> = {
done: "Ran command",
subject: (input) => quoted(input, "command", 60),
},
+ start_desktop: {
+ category: "browser",
+ running: "Starting desktop",
+ done: "Started desktop",
+ },
list_workspace_files: {
category: "file-list",
running: "Listing workspace files",
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
index bd46cdca02..0a46b3f85b 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
@@ -86,6 +86,9 @@ function openFilesCard() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
}
@@ -98,6 +101,9 @@ function closeFilesCard() {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
}
@@ -156,6 +162,9 @@ afterEach(() => {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
useCopilotStreamStore.setState({ messageSnapshots: {} });
@@ -184,6 +193,9 @@ describe("WorkspaceFileCards", () => {
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
const { container } = render(<WorkspaceFileCards sessionId={SESSION} />);
@@ -198,6 +210,9 @@ describe("WorkspaceFileCards", () => {
history: [],
activeTab: "artifacts",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
});
const { container } = render(<WorkspaceFileCards sessionId={SESSION} />);
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
index 0e53acc93e..172c432939 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
@@ -58,6 +58,16 @@ export interface ArtifactRef {
expert?: ExpertArtifact;
}
+/** The live desktop behind a chat, as `start_desktop` reports it. */
+export interface DesktopStreamRef {
+ url: string;
+ sandbox_id: string;
+ provider: string;
+}
+
+/** Which face the side panel shows: the artifact preview or the computer. */
+export type ArtifactPanelMode = "artifact" | "computer";
+
interface ArtifactPanelState {
isOpen: boolean;
activeArtifact: ArtifactRef | null;
@@ -66,6 +76,11 @@ interface ArtifactPanelState {
/** Preview that was showing when the panel last closed — the sidebar
* toggle restores it on reopen instead of landing on the tabs view. */
lastArtifact: ArtifactRef | null;
+ mode: ArtifactPanelMode;
+ /** Stream of the desktop most recently started in this chat, if any. */
+ computer: DesktopStreamRef | null;
+ /** The computer face is open even with no artifact preview behind it. */
+ isComputerOpen: boolean;
}
export const DEFAULT_PANEL_WIDTH = 432; // context panel default (352 + 80)
@@ -214,6 +229,11 @@ interface CopilotUIState {
/** Forget the remembered preview — called on session entry so a new chat
* can never restore the previous chat's artifact. */
clearLastArtifact: () => void;
+ /** Remember the chat's desktop stream and show it if nothing else is open. */
+ registerComputerStream: (ref: DesktopStreamRef) => void;
+ /** Open the side panel on its computer face. */
+ openComputer: () => void;
+ setArtifactPanelMode: (mode: ArtifactPanelMode) => void;
openContextPanelForFiles: () => void;
showFilesTab: () => void;
@@ -338,6 +358,9 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
history: [],
activeTab: getPersistedTab(),
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
openArtifact: (ref, opts) =>
set((state) => {
@@ -360,6 +383,7 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
isOpen: true,
activeArtifact: ref,
history,
+ mode: "artifact",
},
};
}),
@@ -387,6 +411,7 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
// Remember what was previewing so the sidebar toggle can bring it
// back; closing from the tabs view remembers nothing.
lastArtifact: state.artifactPanel.activeArtifact,
+ isComputerOpen: false,
},
};
}),
@@ -396,6 +421,7 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
...state.artifactPanel,
activeArtifact: null,
history: [],
+ isComputerOpen: false,
},
})),
resetArtifactPanel: () =>
@@ -405,6 +431,45 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
activeArtifact: null,
history: [],
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
+ },
+ })),
+ registerComputerStream: (ref) =>
+ set((state) => {
+ const { activeArtifact, isComputerOpen } = state.artifactPanel;
+ // A desktop the model just started is the thing to look at: show it
+ // unless an artifact preview is already holding the panel.
+ const showNow = activeArtifact == null || isComputerOpen;
+ return {
+ artifactPanel: {
+ ...state.artifactPanel,
+ computer: ref,
+ // Selecting the face is not enough if the panel itself is closed.
+ isOpen: showNow ? true : state.artifactPanel.isOpen,
+ isComputerOpen: showNow || isComputerOpen,
+ mode: showNow ? "computer" : state.artifactPanel.mode,
+ },
+ };
+ }),
+ openComputer: () =>
+ set((state) => ({
+ artifactPanel: {
+ ...state.artifactPanel,
+ isOpen: true,
+ mode: "computer",
+ isComputerOpen: true,
+ },
+ })),
+ setArtifactPanelMode: (mode) =>
+ set((state) => ({
+ artifactPanel: {
+ ...state.artifactPanel,
+ mode,
+ // Switching to the artifact face closes the computer face, so a
+ // later registerComputerStream does not flip the panel back.
+ isComputerOpen: mode === "computer",
},
})),
goBackArtifact: () =>
@@ -437,12 +502,20 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
isOpen: nextOpen,
activeArtifact: null,
history: [],
+ isComputerOpen: false,
},
};
}),
clearLastArtifact: () =>
set((state) => ({
- artifactPanel: { ...state.artifactPanel, lastArtifact: null },
+ artifactPanel: {
+ ...state.artifactPanel,
+ lastArtifact: null,
+ // A new chat has its own computer; never show the previous chat's.
+ computer: null,
+ isComputerOpen: false,
+ mode: "artifact",
+ },
})),
toggleContextPanelTab: (tab) =>
set((state) => {
@@ -587,6 +660,9 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
history: [],
activeTab: "files",
lastArtifact: null,
+ mode: "artifact",
+ computer: null,
+ isComputerOpen: false,
},
copilotLlmModel: "standard",
copilotLlmAuth: null,
diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx
new file mode 100644
index 0000000000..acc5432118
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/ExpertComputerSection.tsx
@@ -0,0 +1,176 @@
+"use client";
+
+import type { SandboxSummary } from "@/app/api/__generated__/models/sandboxSummary";
+import { ACTION_BUTTON_CLASS } from "@/app/(platform)/team/helpers";
+import { Button } from "@/components/atoms/Button/Button";
+import { Icon } from "@/components/atoms/Icon/Icon";
+import { Skeleton } from "@/components/atoms/Skeleton/Skeleton";
+import { Text } from "@/components/atoms/Text/Text";
+import { desktopStreamRenderer } from "@/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer";
+import { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";
+import { cn } from "@/lib/utils";
+import { ComputerIcon, ComputerTerminalIcon } from "@hugeicons/core-free-icons";
+import type { IconSvgElement } from "@hugeicons/react";
+import {
+ describeMount,
+ desktopActionLabel,
+ formatResources,
+ formatSandboxState,
+} from "./helpers";
+import { useExpertComputerSection } from "./useExpertComputerSection";
+
+interface Props {
+ expertId: string;
+ expertName: string;
+ enabled: boolean;
+}
+
+interface SandboxCardProps {
+ title: string;
+ icon: IconSvgElement;
+ summary: SandboxSummary | null | undefined;
+ idleHint: string;
+}
+
+function SandboxCard({ title, icon, summary, idleHint }: SandboxCardProps) {
+ const running = summary?.state === "running";
+ const resources = formatResources(summary);
+ return (
+ <div className="rounded-lg border border-zinc-200 bg-white p-4">
+ <div className="flex items-center justify-between gap-2">
+ <div className="flex items-center gap-2">
+ <Icon icon={icon} size={16} className="text-zinc-500" />
+ <Text variant="small-medium">{title}</Text>
+ </div>
+ <span
+ className={cn(
+ "rounded-full px-2 py-0.5 text-xs font-medium",
+ running
+ ? "bg-emerald-50 text-emerald-700"
+ : summary
+ ? "bg-zinc-100 text-zinc-600"
+ : "bg-zinc-50 text-zinc-500",
+ )}
+ >
+ {running ? "Running" : summary ? "Suspended" : "None"}
+ </span>
+ </div>
+ <Text variant="small" className="mt-2 text-zinc-600">
+ {summary ? formatSandboxState(summary) : idleHint}
+ </Text>
+ {resources ? (
+ <Text variant="small" className="mt-1 text-zinc-500">
+ {resources}
+ {summary?.mounts_attached === false ? " · no volumes attached" : ""}
+ </Text>
+ ) : null}
+ </div>
+ );
+}
+
+export function ExpertComputerSection({
+ expertId,
+ expertName,
+ enabled,
+}: Props) {
+ const { computer, isLoading, refetch, stream, openDesktop, isOpening } =
+ useExpertComputerSection({ expertId, enabled });
+
+ if (isLoading) {
+ return (
+ <section className="space-y-3">
+ <Skeleton className="h-8 w-64 rounded-lg" />
+ <Skeleton className="h-28 w-full rounded-lg" />
+ </section>
+ );
+ }
+
+ // A failed background poll must not tear down a live desktop the user is
+ // watching; last-good data stays up and the next poll refreshes it.
+ if (!computer) {
+ return (
+ <ErrorCard
+ context="this expert's computer"
+ hint="We could not load the sandbox status."
+ onRetry={() => refetch()}
+ />
+ );
+ }
+
+ const mounts = Object.entries(computer.mounts ?? {});
+
+ return (
+ <section className="space-y-4">
+ <div className="flex flex-wrap items-start justify-between gap-3">
+ <div>
+ <Text variant="large-medium">{`${expertName}'s computer`}</Text>
+ <Text variant="small" className="mt-1 max-w-prose text-zinc-500">
+ A persistent cloud machine only {expertName} uses. It is suspended
+ when idle and costs nothing while suspended; installed tools and
+ files stay put between chats. {expertName} has full access to it, so
+ anything signed in on its desktop is visible to the AI. Do not sign
+ into personal accounts there.
+ </Text>
+ </div>
+ <Button
+ variant="primary"
+ size="small"
+ className={ACTION_BUTTON_CLASS}
+ loading={isOpening}
+ disabled={!computer.e2b_active}
+ onClick={openDesktop}
+ >
+ {desktopActionLabel(computer)}
+ </Button>
+ </div>
+
+ {!computer.e2b_active ? (
+ <div className="rounded-lg bg-amber-50 px-4 py-2.5 ring-1 ring-inset ring-amber-200">
+ <Text variant="small" className="text-amber-700">
+ Cloud sandboxes are not configured on this deployment.
+ </Text>
+ </div>
+ ) : null}
+
+ <div className="grid gap-3 sm:grid-cols-2">
+ <SandboxCard
+ title="Shell"
+ icon={ComputerTerminalIcon}
+ summary={computer.shell}
+ idleHint="Created on the first message that runs a command."
+ />
+ <SandboxCard
+ title="Desktop"
+ icon={ComputerIcon}
+ summary={computer.desktop}
+ idleHint="Created the first time a task needs a browser or GUI app."
+ />
+ </div>
+
+ {mounts.length > 0 ? (
+ <div className="rounded-lg border border-zinc-200 bg-white p-4">
+ <Text variant="small-medium">Volumes</Text>
+ <ul className="mt-2 space-y-2">
+ {mounts.map(([path, name]) => (
+ <li key={path} className="flex flex-col gap-0.5">
+ <code className="text-xs text-zinc-800">{path}</code>
+ <Text variant="small" className="text-zinc-500">
+ {describeMount(path, computer)}{" "}
+ <span className="font-mono text-xs text-zinc-400">
+ {name}
+ </span>
+ </Text>
+ </li>
+ ))}
+ </ul>
+ </div>
+ ) : null}
+
+ {stream ? (
+ <div data-testid="expert-desktop-stream">
+ {desktopStreamRenderer.render(stream)}
+ </div>
+ ) : null}
+ </section>
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx
new file mode 100644
index 0000000000..78d9dc4064
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/__tests__/ExpertComputerSection.test.tsx
@@ -0,0 +1,136 @@
+import {
+ getGetV2GetExpertComputerMockHandler,
+ getPostV2StartExpertDesktopMockHandler,
+} from "@/app/api/__generated__/endpoints/experts/experts.msw";
+import type { ComputerInfo } from "@/app/api/__generated__/models/computerInfo";
+import { server } from "@/mocks/mock-server";
+import { render, screen, waitFor } from "@/tests/integrations/test-utils";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it } from "vitest";
+import { ExpertComputerSection } from "../ExpertComputerSection";
+
+const computer: ComputerInfo = {
+ owner_kind: "expert",
+ owner_id: "expert-1",
+ e2b_active: true,
+ shell: {
+ kind: "shell",
+ sandbox_id: "sb-shell",
+ state: "paused",
+ started_at: new Date("2026-09-05T12:00:00Z"),
+ cpu_count: 2,
+ memory_mb: 512,
+ template_id: "base",
+ mounts_attached: true,
+ },
+ desktop: null,
+ mounts: {
+ "/home/user/workspace": "autogpt-expert-expert-1",
+ "/home/user/shared": "autogpt-user-u1",
+ },
+ workspace_path: "/home/user/workspace",
+ shared_path: "/home/user/shared",
+};
+
+describe("ExpertComputerSection", () => {
+ it("shows the expert's boxes and volumes without waking anything", async () => {
+ server.use(getGetV2GetExpertComputerMockHandler(computer));
+
+ render(
+ <ExpertComputerSection expertId="expert-1" expertName="Maria" enabled />,
+ );
+
+ expect(await screen.findByText("Maria's computer")).toBeDefined();
+ expect(screen.getByText("Suspended")).toBeDefined();
+ expect(screen.getByText("2 vCPU · 512 MiB")).toBeDefined();
+ expect(screen.getByText("/home/user/shared")).toBeDefined();
+ expect(screen.getByText("autogpt-expert-expert-1")).toBeDefined();
+ expect(screen.getByRole("button", { name: "Start desktop" })).toBeDefined();
+ });
+
+ it("starts the desktop and embeds its stream", async () => {
+ server.use(
+ getGetV2GetExpertComputerMockHandler(computer),
+ getPostV2StartExpertDesktopMockHandler({
+ kind: "desktop_stream",
+ url: "https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ provider: "e2b",
+ sandbox_id: "sbx-desktop",
+ requires_auth: false,
+ }),
+ );
+
+ render(
+ <ExpertComputerSection expertId="expert-1" expertName="Maria" enabled />,
+ );
+
+ await userEvent.click(
+ await screen.findByRole("button", { name: "Start desktop" }),
+ );
+
+ await waitFor(() =>
+ expect(
+ screen.getByTitle("Interactive desktop (sbx-desktop)"),
+ ).toBeDefined(),
+ );
+ });
+
+ it("does not show one expert's desktop under another expert", async () => {
+ server.use(
+ getGetV2GetExpertComputerMockHandler(computer),
+ getPostV2StartExpertDesktopMockHandler({
+ kind: "desktop_stream",
+ url: "https://6080-sbx.e2b.app/vnc.html?autoconnect=true",
+ provider: "e2b",
+ sandbox_id: "sbx-desktop",
+ requires_auth: false,
+ }),
+ );
+
+ const { rerender } = render(
+ <ExpertComputerSection expertId="expert-1" expertName="Maria" enabled />,
+ );
+ await userEvent.click(
+ await screen.findByRole("button", { name: "Start desktop" }),
+ );
+ await waitFor(() =>
+ expect(
+ screen.getByTitle("Interactive desktop (sbx-desktop)"),
+ ).toBeDefined(),
+ );
+
+ rerender(
+ <ExpertComputerSection expertId="expert-2" expertName="Ada" enabled />,
+ );
+
+ await waitFor(() =>
+ expect(
+ screen.queryByTitle("Interactive desktop (sbx-desktop)"),
+ ).toBeNull(),
+ );
+ });
+
+ it("explains when sandboxes are not configured", async () => {
+ server.use(
+ getGetV2GetExpertComputerMockHandler({
+ ...computer,
+ e2b_active: false,
+ shell: null,
+ mounts: {},
+ }),
+ );
+
+ render(
+ <ExpertComputerSection expertId="expert-1" expertName="Maria" enabled />,
+ );
+
+ expect(
+ await screen.findByText(
+ "Cloud sandboxes are not configured on this deployment.",
+ ),
+ ).toBeDefined();
+ expect(
+ screen.getByRole("button", { name: "Start desktop" }),
+ ).toHaveProperty("disabled", true);
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts
new file mode 100644
index 0000000000..c6cab33f2b
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/helpers.ts
@@ -0,0 +1,46 @@
+import type { ComputerInfo } from "@/app/api/__generated__/models/computerInfo";
+import type { SandboxSummary } from "@/app/api/__generated__/models/sandboxSummary";
+
+export function formatSandboxState(summary: SandboxSummary | null | undefined) {
+ if (!summary) return "Not created yet";
+ if (summary.state === "running") {
+ const since = new Date(summary.started_at);
+ return `Running since ${since.toLocaleTimeString([], {
+ hour: "numeric",
+ minute: "2-digit",
+ })}`;
+ }
+ return "Suspended. Resumes in about a second when needed.";
+}
+
+export function formatResources(summary: SandboxSummary | null | undefined) {
+ if (!summary) return null;
+ const gib = summary.memory_mb / 1024;
+ const memory =
+ gib >= 1
+ ? `${Number.isInteger(gib) ? gib : gib.toFixed(1)} GiB`
+ : `${summary.memory_mb} MiB`;
+ return `${summary.cpu_count} vCPU · ${memory}`;
+}
+
+export function describeMount(
+ path: string,
+ computer: Pick<ComputerInfo, "workspace_path" | "shared_path" | "owner_kind">,
+) {
+ if (path === computer.workspace_path) {
+ return computer.owner_kind === "expert"
+ ? "Its own home. Tools, notes and configs live here."
+ : "Your workspace.";
+ }
+ if (path === computer.shared_path) {
+ return "Your workspace, shared. Deliverables dropped here reach your desktop.";
+ }
+ return "Mounted volume.";
+}
+
+export function desktopActionLabel(computer: ComputerInfo | null | undefined) {
+ if (!computer?.desktop) return "Start desktop";
+ return computer.desktop.state === "running"
+ ? "Open desktop"
+ : "Resume desktop";
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/useExpertComputerSection.ts b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/useExpertComputerSection.ts
new file mode 100644
index 0000000000..5602c0def9
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/components/ExpertComputerSection/useExpertComputerSection.ts
@@ -0,0 +1,69 @@
+import {
+ getGetV2GetExpertComputerQueryKey,
+ useGetV2GetExpertComputer,
+ usePostV2StartExpertDesktop,
+} from "@/app/api/__generated__/endpoints/experts/experts";
+import type { DesktopStream } from "@/app/api/__generated__/models/desktopStream";
+import { okData } from "@/app/api/helpers";
+import { useToast } from "@/components/molecules/Toast/use-toast";
+import { useQueryClient } from "@tanstack/react-query";
+import { useState } from "react";
+
+interface Args {
+ expertId: string;
+ enabled: boolean;
+}
+
+export function useExpertComputerSection({ expertId, enabled }: Args) {
+ const queryClient = useQueryClient();
+ const { toast } = useToast();
+ // Kept with the expert it was opened for: the section can be re-rendered
+ // for another expert, whose page must not show this one's desktop.
+ const [opened, setOpened] = useState<{
+ expertId: string;
+ stream: DesktopStream;
+ } | null>(null);
+ const stream = opened?.expertId === expertId ? opened.stream : null;
+
+ const computerQuery = useGetV2GetExpertComputer(expertId, {
+ query: {
+ select: (res) => okData(res) ?? null,
+ enabled,
+ refetchInterval: 15_000,
+ },
+ });
+
+ const { mutate: startDesktop, isPending: isOpening } =
+ usePostV2StartExpertDesktop({
+ mutation: {
+ onSuccess: (res, variables) => {
+ const next = okData(res);
+ if (next) setOpened({ expertId: variables.expertId, stream: next });
+ queryClient.invalidateQueries({
+ queryKey: getGetV2GetExpertComputerQueryKey(expertId),
+ });
+ },
+ onError: () => {
+ toast({
+ title: "Could not open the desktop",
+ description: "The sandbox did not come up. Try again in a moment.",
+ variant: "destructive",
+ });
+ },
+ },
+ });
+
+ function openDesktop() {
+ startDesktop({ expertId });
+ }
+
+ return {
+ computer: computerQuery.data ?? null,
+ isLoading: computerQuery.isLoading,
+ isError: computerQuery.isError,
+ refetch: () => computerQuery.refetch(),
+ stream,
+ openDesktop,
+ isOpening,
+ };
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/page.tsx b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/page.tsx
index dce495658c..42034f05f5 100644
--- a/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/page.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/team/[expertId]/page.tsx
@@ -17,6 +17,7 @@ import { Flag, useFlagStatus } from "@/services/feature-flags/use-get-flag";
import {
Briefcase01Icon,
Calendar03Icon,
+ ComputerIcon,
PlugSocketIcon,
Settings01Icon,
SparklesIcon,
@@ -32,6 +33,7 @@ import { SoulDrawer } from "../components/SoulDrawer/SoulDrawer";
import { getLastRunLabel } from "../helpers";
import { ExpertAboutSection } from "./components/ExpertAboutSection";
import { ExpertBudgetSection } from "./components/ExpertBudgetSection";
+import { ExpertComputerSection } from "./components/ExpertComputerSection/ExpertComputerSection";
import { ExpertDetailHeader } from "./components/ExpertDetailHeader";
import { ExpertIntegrationsSection } from "./components/ExpertIntegrationsSection/ExpertIntegrationsSection";
import { ExpertNeedsYouSection } from "./components/ExpertNeedsYouSection/ExpertNeedsYouSection";
@@ -51,6 +53,7 @@ const TABS = [
{ value: "work", label: "Work", icon: Briefcase01Icon },
{ value: "schedules", label: "Schedules", icon: Calendar03Icon },
{ value: "workflows", label: "Workflows", icon: WorkflowSquare01Icon },
+ { value: "computer", label: "Computer", icon: ComputerIcon },
{ value: "integrations", label: "Integrations", icon: PlugSocketIcon },
{ value: "skills", label: "Skills", icon: SparklesIcon },
{ value: "settings", label: "Settings", icon: Settings01Icon },
@@ -229,6 +232,14 @@ export default function ExpertDetailPage() {
/>
</TabsLineContent>
+ <TabsLineContent value="computer">
+ <ExpertComputerSection
+ expertId={expert.id}
+ expertName={expert.name}
+ enabled={Boolean(enabled) && ready}
+ />
+ </TabsLineContent>
+
<TabsLineContent value="integrations">
<ExpertIntegrationsSection
expertId={expert.id}
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 4143e21ebf..26c56dfba2 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -4228,6 +4228,45 @@
}
}
},
+ "/api/chat/sessions/{session_id}/computer": {
+ "get": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "Get Session Computer",
+ "description": "The computer behind this chat.\n\nA plain chat has its own boxes; a chat that runs as a hired expert reports\nthe expert's persistent computer instead. Listing never wakes a paused box.",
+ "operationId": "getV2GetSessionComputer",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/ComputerInfo" }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Session not found or access denied" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/chat/sessions/{session_id}/connection": {
"put": {
"tags": ["v2", "chat", "chat"],
@@ -4281,6 +4320,47 @@
}
}
},
+ "/api/chat/sessions/{session_id}/desktop": {
+ "post": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "Start Session Desktop",
+ "description": "Start or resume the desktop behind this chat and return its live stream.\n\nSame box the ``start_desktop`` tool uses from inside a turn, so the side\npanel and the model always look at one screen.",
+ "operationId": "postV2StartSessionDesktop",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/DesktopStream" }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Session not found or access denied" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ },
+ "502": { "description": "The desktop could not be started" },
+ "503": { "description": "E2B is not configured" }
+ }
+ }
+ },
"/api/chat/sessions/{session_id}/messages/pending": {
"get": {
"tags": ["v2", "chat", "chat"],
@@ -5956,6 +6036,38 @@
}
}
},
+ "/api/desktop-preview": {
+ "get": {
+ "tags": ["v1"],
+ "summary": "Open Desktop Preview",
+ "description": "Redirect the desktop's owner to its live stream.\n\nThe stream URL carries the desktop's password, so it is only ever\ndisclosed here, to the user the token was issued to, and marked\n``no-store`` so no cache or referrer keeps a copy.",
+ "operationId": "getV1OpenDesktopPreview",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "token",
+ "in": "query",
+ "required": true,
+ "schema": { "type": "string", "maxLength": 8192, "title": "Token" }
+ }
+ ],
+ "responses": {
+ "307": { "description": "Successful Response" },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Preview unavailable" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/email/": {
"post": {
"tags": ["v1", "email"],
@@ -6849,6 +6961,86 @@
}
}
},
+ "/api/experts/{expert_id}/computer": {
+ "get": {
+ "tags": ["v2", "experts", "experts", "private"],
+ "summary": "Get Expert Computer",
+ "description": "The expert's own computer: its shell and desktop boxes as E2B lists them.\n\nListing never wakes a paused box, so the Computer tab can refresh freely.",
+ "operationId": "getV2GetExpertComputer",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "expert_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Expert Id" }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/ComputerInfo" }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Expert not found" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/experts/{expert_id}/computer/desktop": {
+ "post": {
+ "tags": ["v2", "experts", "experts", "private"],
+ "summary": "Start Expert Desktop",
+ "description": "Start or resume the expert's desktop and return its live stream.\n\nThis is the same box the expert's next ``start_desktop`` turn reconnects\nto, so what the user does here is what the expert sees.",
+ "operationId": "postV2StartExpertDesktop",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "expert_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Expert Id" }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/DesktopStream" }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Expert not found" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ },
+ "502": { "description": "The desktop could not be started" },
+ "503": { "description": "E2B is not configured" }
+ }
+ }
+ },
"/api/experts/{expert_id}/credentials": {
"get": {
"tags": ["v2", "experts", "experts", "private"],
@@ -18954,6 +19146,49 @@
"required": ["uuid"],
"title": "CommunitySummary"
},
+ "ComputerInfo": {
+ "properties": {
+ "owner_kind": {
+ "type": "string",
+ "enum": ["session", "expert"],
+ "title": "Owner Kind"
+ },
+ "owner_id": { "type": "string", "title": "Owner Id" },
+ "e2b_active": { "type": "boolean", "title": "E2B Active" },
+ "shell": {
+ "anyOf": [
+ { "$ref": "#/components/schemas/SandboxSummary" },
+ { "type": "null" }
+ ]
+ },
+ "desktop": {
+ "anyOf": [
+ { "$ref": "#/components/schemas/SandboxSummary" },
+ { "type": "null" }
+ ]
+ },
+ "mounts": {
+ "additionalProperties": { "type": "string" },
+ "type": "object",
+ "title": "Mounts",
+ "default": {}
+ },
+ "workspace_path": {
+ "type": "string",
+ "title": "Workspace Path",
+ "default": "/home/user/workspace"
+ },
+ "shared_path": {
+ "type": "string",
+ "title": "Shared Path",
+ "default": "/home/user/shared"
+ }
+ },
+ "type": "object",
+ "required": ["owner_kind", "owner_id", "e2b_active"],
+ "title": "ComputerInfo",
+ "description": "The owner's computer: its boxes and the volumes they mount."
+ },
"ConfirmLinkRequest": {
"properties": {
"telegram_auth": {
@@ -20132,6 +20367,32 @@
"title": "DemotionSummary",
"description": "One line per demoted RELATES_TO edge."
},
+ "DesktopStream": {
+ "properties": {
+ "kind": {
+ "type": "string",
+ "const": "desktop_stream",
+ "title": "Kind",
+ "default": "desktop_stream"
+ },
+ "url": { "type": "string", "title": "Url" },
+ "provider": {
+ "type": "string",
+ "const": "e2b",
+ "title": "Provider",
+ "default": "e2b"
+ },
+ "sandbox_id": { "type": "string", "title": "Sandbox Id" },
+ "requires_auth": {
+ "type": "boolean",
+ "title": "Requires Auth",
+ "default": false
+ }
+ },
+ "type": "object",
+ "required": ["url", "sandbox_id"],
+ "title": "DesktopStream"
+ },
"DeviceAuthInitiateResponse": {
"properties": {
"state_token": { "type": "string", "title": "State Token" },
@@ -27502,6 +27763,7 @@
"browser_act",
"browser_screenshot",
"bash_exec",
+ "desktop_stream",
"web_fetch",
"web_search",
"feature_request_search",
@@ -27696,6 +27958,43 @@
"title": "RunningExecutionsListResponse",
"description": "Response model for list of running executions"
},
+ "SandboxSummary": {
+ "properties": {
+ "kind": {
+ "type": "string",
+ "enum": ["shell", "desktop"],
+ "title": "Kind"
+ },
+ "sandbox_id": { "type": "string", "title": "Sandbox Id" },
+ "state": {
+ "type": "string",
+ "enum": ["running", "paused"],
+ "title": "State"
+ },
+ "started_at": {
+ "type": "string",
+ "format": "date-time",
+ "title": "Started At"
+ },
+ "cpu_count": { "type": "integer", "title": "Cpu Count" },
+ "memory_mb": { "type": "integer", "title": "Memory Mb" },
+ "template_id": { "type": "string", "title": "Template Id" },
+ "mounts_attached": { "type": "boolean", "title": "Mounts Attached" }
+ },
+ "type": "object",
+ "required": [
+ "kind",
+ "sandbox_id",
+ "state",
+ "started_at",
+ "cpu_count",
+ "memory_mb",
+ "template_id",
+ "mounts_attached"
+ ],
+ "title": "SandboxSummary",
+ "description": "One of the owner's sandboxes as E2B reports it, without waking it."
+ },
"ScheduleCleanupRequest": {
"properties": {
"schedule_ids": {
diff --git a/autogpt_platform/frontend/src/components/contextual/OutputRenderers/index.ts b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/index.ts
index 8b5948d007..406f686854 100644
--- a/autogpt_platform/frontend/src/components/contextual/OutputRenderers/index.ts
+++ b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/index.ts
@@ -10,8 +10,10 @@ import { jsonRenderer } from "./renderers/JSONRenderer";
import { markdownRenderer } from "./renderers/MarkdownRenderer";
import { workspaceFileRenderer } from "./renderers/WorkspaceFileRenderer";
import { linkRenderer } from "./renderers/LinkRenderer";
+import { desktopStreamRenderer } from "./renderers/DesktopStreamRenderer";
// Register all renderers in priority order
+globalRegistry.register(desktopStreamRenderer);
globalRegistry.register(workspaceFileRenderer);
globalRegistry.register(videoRenderer);
globalRegistry.register(audioRenderer);
diff --git a/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer.tsx b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer.tsx
new file mode 100644
index 0000000000..12aa7a694f
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer.tsx
@@ -0,0 +1,167 @@
+import React, { useRef } from "react";
+import {
+ ArrowExpandIcon,
+ ArrowUpRight01Icon,
+ ComputerIcon,
+} from "@hugeicons/core-free-icons";
+import { Icon } from "@/components/atoms/Icon/Icon";
+import {
+ OutputRenderer,
+ OutputMetadata,
+ DownloadContent,
+ CopyContent,
+} from "../types";
+
+export interface DesktopStreamValue {
+ kind: "desktop_stream";
+ url: string;
+ provider: string;
+ sandbox_id: string;
+ requires_auth?: boolean;
+}
+
+function isStreamUrl(url: unknown): url is string {
+ if (typeof url !== "string") return false;
+ // A root-relative path is our own proxy link; anything absolute must be
+ // https, or a javascript: URL would run in our origin under
+ // allow-scripts allow-same-origin.
+ if (url.startsWith("/") && !url.startsWith("//")) return true;
+ try {
+ return new URL(url).protocol === "https:";
+ } catch {
+ return false;
+ }
+}
+
+export function isDesktopStream(value: unknown): value is DesktopStreamValue {
+ if (typeof value !== "object" || value === null) return false;
+ const candidate = value as Record<string, unknown>;
+ // This renderer is in the global registry, so any output shaped like a
+ // stream reaches the iframe; only a URL that can resolve to our own origin
+ // or to an https host may.
+ return (
+ candidate.kind === "desktop_stream" &&
+ isStreamUrl(candidate.url) &&
+ typeof candidate.sandbox_id === "string"
+ );
+}
+
+interface PreviewProps {
+ value: DesktopStreamValue;
+ /** False on a shared transcript: the owner-bound link would only answer
+ * the viewer with a 404, so the frame is replaced by a notice. */
+ ownerView?: boolean;
+}
+
+export function DesktopStreamPreview({
+ value,
+ ownerView = true,
+}: PreviewProps) {
+ const frameRef = useRef<HTMLIFrameElement>(null);
+ const ownerOnly = value.requires_auth === true && !ownerView;
+
+ function handleFullscreen() {
+ frameRef.current?.requestFullscreen();
+ }
+
+ return (
+ <div className="overflow-hidden rounded-lg border border-zinc-200">
+ <div className="flex items-center justify-between border-b border-zinc-200 bg-zinc-50 px-3 py-2">
+ <div className="flex items-center gap-2 text-sm text-zinc-700">
+ <Icon icon={ComputerIcon} size={16} />
+ <span className="font-medium">Interactive Desktop</span>
+ <span className="rounded bg-zinc-200 px-1.5 py-0.5 text-xs uppercase text-zinc-600">
+ {value.provider}
+ </span>
+ </div>
+ {!ownerOnly && (
+ <div className="flex items-center gap-2">
+ <button
+ type="button"
+ onClick={handleFullscreen}
+ className="flex items-center gap-1 rounded px-2 py-1 text-xs text-zinc-600 hover:bg-zinc-200"
+ aria-label="Fullscreen"
+ >
+ <Icon icon={ArrowExpandIcon} size={14} />
+ Fullscreen
+ </button>
+ <a
+ href={value.url}
+ target="_blank"
+ rel="noopener noreferrer"
+ className="flex items-center gap-1 rounded px-2 py-1 text-xs text-zinc-600 hover:bg-zinc-200"
+ >
+ <Icon icon={ArrowUpRight01Icon} size={14} />
+ Open in new tab
+ </a>
+ </div>
+ )}
+ </div>
+ {ownerOnly ? (
+ <div className="flex aspect-video w-full items-center justify-center bg-zinc-900 px-6 text-center text-sm text-zinc-400">
+ The live desktop is only visible to the owner of this chat.
+ </div>
+ ) : (
+ <iframe
+ ref={frameRef}
+ src={value.url}
+ sandbox="allow-scripts allow-same-origin allow-popups"
+ allow="clipboard-read; clipboard-write; fullscreen"
+ className="aspect-video w-full bg-zinc-900"
+ title={`Interactive desktop (${value.sandbox_id})`}
+ />
+ )}
+ <p className="border-t border-zinc-200 bg-zinc-50 px-3 py-2 text-xs font-medium text-zinc-700">
+ {value.requires_auth
+ ? "Only the owner of this chat can view the live desktop. "
+ : ""}
+ The AI works on this desktop with full access, so anything signed in
+ here is visible to it. Do not sign into personal accounts.
+ </p>
+ </div>
+ );
+}
+
+function canRenderDesktopStream(
+ value: unknown,
+ _metadata?: OutputMetadata,
+): boolean {
+ return isDesktopStream(value);
+}
+
+function renderDesktopStream(
+ value: unknown,
+ _metadata?: OutputMetadata,
+): React.ReactNode {
+ if (!isDesktopStream(value)) return null;
+ return <DesktopStreamPreview value={value} />;
+}
+
+function getCopyContentDesktopStream(
+ value: unknown,
+ _metadata?: OutputMetadata,
+): CopyContent | null {
+ if (!isDesktopStream(value)) return null;
+ return {
+ mimeType: "text/plain",
+ data: value.url,
+ fallbackText: value.url,
+ };
+}
+
+function getDownloadContentDesktopStream(
+ _value: unknown,
+ _metadata?: OutputMetadata,
+): DownloadContent | null {
+ return null;
+}
+
+export const desktopStreamRenderer: OutputRenderer = {
+ name: "DesktopStreamRenderer",
+ priority: 80,
+ canRender: canRenderDesktopStream,
+ render: renderDesktopStream,
+ getCopyContent: getCopyContentDesktopStream,
+ getDownloadContent: getDownloadContentDesktopStream,
+ isConcatenable: () => false,
+};
diff --git a/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/__tests__/DesktopStreamRenderer.test.tsx b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/__tests__/DesktopStreamRenderer.test.tsx
new file mode 100644
index 0000000000..cd931426d9
--- /dev/null
+++ b/autogpt_platform/frontend/src/components/contextual/OutputRenderers/renderers/__tests__/DesktopStreamRenderer.test.tsx
@@ -0,0 +1,108 @@
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+ desktopStreamRenderer,
+ DesktopStreamPreview,
+} from "../DesktopStreamRenderer";
+
+const streamValue = {
+ kind: "desktop_stream" as const,
+ url: "https://6080-sandbox.e2b.app/vnc.html?autoconnect=true",
+ provider: "e2b",
+ sandbox_id: "sbx-123",
+ requires_auth: false,
+};
+
+describe("DesktopStreamRenderer", () => {
+ afterEach(() => {
+ cleanup();
+ });
+
+ it("canRender matches desktop_stream objects only", () => {
+ expect(desktopStreamRenderer.canRender(streamValue)).toBe(true);
+ expect(desktopStreamRenderer.canRender({ kind: "other" })).toBe(false);
+ expect(desktopStreamRenderer.canRender("https://example.com")).toBe(false);
+ expect(desktopStreamRenderer.canRender(null)).toBe(false);
+ expect(
+ desktopStreamRenderer.canRender({ kind: "desktop_stream", url: 42 }),
+ ).toBe(false);
+ });
+
+ it("refuses anything but an https URL or our own origin-relative link", () => {
+ for (const url of [
+ "javascript:alert(document.cookie)",
+ "http://6080-sandbox.e2b.app/vnc.html",
+ "data:text/html,<script>1</script>",
+ "not a url",
+ "//evil.example/vnc.html",
+ ]) {
+ expect(desktopStreamRenderer.canRender({ ...streamValue, url })).toBe(
+ false,
+ );
+ }
+ // The owner-bound preview link is root-relative so it works on any
+ // origin the owner is signed in to, plain-http local stacks included.
+ expect(
+ desktopStreamRenderer.canRender({
+ ...streamValue,
+ url: "/api/proxy/api/desktop-preview?token=abc",
+ requires_auth: true,
+ }),
+ ).toBe(true);
+ });
+
+ it("shows a notice instead of the frame to a viewer who is not the owner", () => {
+ const ownerBound = {
+ ...streamValue,
+ url: "/api/proxy/api/desktop-preview?token=abc",
+ requires_auth: true,
+ };
+ const { container } = render(
+ <DesktopStreamPreview value={ownerBound} ownerView={false} />,
+ );
+ expect(container.querySelector("iframe")).toBeNull();
+ expect(screen.getByText(/only visible to the owner/i)).toBeDefined();
+ expect(screen.queryByRole("link", { name: /open in new tab/i })).toBeNull();
+ });
+
+ it("renders an interactive iframe pointing at the stream URL", () => {
+ const { container } = render(
+ <>{desktopStreamRenderer.render(streamValue)}</>,
+ );
+ const iframe = container.querySelector("iframe");
+ expect(iframe).toBeTruthy();
+ expect(iframe?.getAttribute("src")).toBe(streamValue.url);
+ expect(iframe?.getAttribute("sandbox")).toContain("allow-scripts");
+ expect(screen.getByText("Interactive Desktop")).toBeDefined();
+ expect(screen.getByText("e2b")).toBeDefined();
+ });
+
+ it("provides an open-in-new-tab link", () => {
+ render(<>{desktopStreamRenderer.render(streamValue)}</>);
+ const link = screen.getByRole("link", { name: /open in new tab/i });
+ expect(link.getAttribute("href")).toBe(streamValue.url);
+ expect(link.getAttribute("target")).toBe("_blank");
+ });
+
+ it("tells the viewer the desktop is shared with the AI, and owner-only when auth is required", () => {
+ render(<>{desktopStreamRenderer.render(streamValue)}</>);
+ expect(screen.getByText(/visible to it/i)).toBeDefined();
+ expect(screen.queryByText(/only the owner/i)).toBeNull();
+ cleanup();
+ render(
+ <>
+ {desktopStreamRenderer.render({ ...streamValue, requires_auth: true })}
+ </>,
+ );
+ expect(screen.getByText(/only the owner/i)).toBeDefined();
+ });
+
+ it("copies the stream URL", () => {
+ const copy = desktopStreamRenderer.getCopyContent(streamValue);
+ expect(copy?.data).toBe(streamValue.url);
+ });
+
+ it("has no download content", () => {
+ expect(desktopStreamRenderer.getDownloadContent(streamValue)).toBeNull();
+ });
+});
diff --git a/docs/integrations/block-integrations/misc.md b/docs/integrations/block-integrations/misc.md
index 3cedff60d9..336d26fc3a 100644
--- a/docs/integrations/block-integrations/misc.md
+++ b/docs/integrations/block-integrations/misc.md
@@ -96,7 +96,7 @@ Tool and block identifiers provided in `tools` and `blocks` are validated at run
| transport | Run on platform credits, or on your connected ChatGPT subscription if supported by your plan. | "platform" \| "codex_app_server" | No |
| session_id | Session ID to continue an existing autopilot conversation. Leave empty to start a new session. Use the session_id output from a previous run to continue. | str | No |
| max_recursion_depth | Maximum nesting depth when the autopilot calls this block recursively (sub-agent pattern). Prevents infinite loops. | int | No |
-| tools | Tool names to filter. Works with tools_exclude to form an allow-list or deny-list. Leave empty to apply no tool filter. | List["add_understanding" \| "ask_question" \| "bash_exec" \| "browser_act" \| "browser_navigate" \| "browser_screenshot" \| "confirm_expert_change" \| "confirm_expert_soul_update" \| "connect_integration" \| "continue_run_block" \| "create_agent" \| "create_feature_request" \| "create_folder" \| "customize_agent" \| "decompose_goal" \| "delegate_to_expert" \| "delete_folder" \| "delete_preset" \| "delete_schedule" \| "delete_skill" \| "delete_workspace_file" \| "edit_agent" \| "enter_agent_building_mode" \| "expert_onboarding" \| "find_agent" \| "find_block" \| "find_library_agent" \| "fix_agent_graph" \| "get_agent_building_guide" \| "get_doc_page" \| "get_mcp_guide" \| "get_platform_info" \| "get_sub_session_result" \| "grant_expert_credential" \| "handoff_to_expert" \| "hire_expert" \| "install_expert_workflow" \| "list_agent_triggers" \| "list_chat_platform_channels" \| "list_expert_chats" \| "list_expert_credentials" \| "list_expert_workflows" \| "list_folders" \| "list_presets" \| "list_schedules" \| "list_skills" \| "list_team" \| "list_workspace_files" \| "memory_forget_confirm" \| "memory_forget_search" \| "memory_search" \| "memory_store" \| "move_agents_to_folder" \| "move_folder" \| "pause_schedule" \| "post_to_chat_platform" \| "raise_expert" \| "read_expert_chat" \| "read_skill" \| "read_workspace_file" \| "remove_expert_workflow" \| "request_credential_grant" \| "resume_schedule" \| "revoke_expert_credential" \| "run_agent" \| "run_block" \| "run_mcp_tool" \| "run_sub_session" \| "schedule_followup" \| "search_docs" \| "search_feature_requests" \| "setup_agent_webhook_trigger" \| "store_skill" \| "update_expert" \| "update_expert_soul" \| "update_folder" \| "update_preset" \| "validate_agent_graph" \| "view_agent_output" \| "web_fetch" \| "web_search" \| "write_workspace_file" \| "Agent" \| "Edit" \| "Glob" \| "Grep" \| "Read" \| "Task" \| "TodoWrite" \| "WebSearch" \| "Write"] | No |
+| tools | Tool names to filter. Works with tools_exclude to form an allow-list or deny-list. Leave empty to apply no tool filter. | List["add_understanding" \| "ask_question" \| "bash_exec" \| "start_desktop" \| "browser_act" \| "browser_navigate" \| "browser_screenshot" \| "confirm_expert_change" \| "confirm_expert_soul_update" \| "connect_integration" \| "continue_run_block" \| "create_agent" \| "create_feature_request" \| "create_folder" \| "customize_agent" \| "decompose_goal" \| "delegate_to_expert" \| "delete_folder" \| "delete_preset" \| "delete_schedule" \| "delete_skill" \| "delete_workspace_file" \| "edit_agent" \| "enter_agent_building_mode" \| "expert_onboarding" \| "find_agent" \| "find_block" \| "find_library_agent" \| "fix_agent_graph" \| "get_agent_building_guide" \| "get_doc_page" \| "get_mcp_guide" \| "get_platform_info" \| "get_sub_session_result" \| "grant_expert_credential" \| "handoff_to_expert" \| "hire_expert" \| "install_expert_workflow" \| "list_agent_triggers" \| "list_chat_platform_channels" \| "list_expert_chats" \| "list_expert_credentials" \| "list_expert_workflows" \| "list_folders" \| "list_presets" \| "list_schedules" \| "list_skills" \| "list_team" \| "list_workspace_files" \| "memory_forget_confirm" \| "memory_forget_search" \| "memory_search" \| "memory_store" \| "move_agents_to_folder" \| "move_folder" \| "pause_schedule" \| "post_to_chat_platform" \| "raise_expert" \| "read_expert_chat" \| "read_skill" \| "read_workspace_file" \| "remove_expert_workflow" \| "request_credential_grant" \| "resume_schedule" \| "revoke_expert_credential" \| "run_agent" \| "run_block" \| "run_mcp_tool" \| "run_sub_session" \| "schedule_followup" \| "search_docs" \| "search_feature_requests" \| "setup_agent_webhook_trigger" \| "store_skill" \| "update_expert" \| "update_expert_soul" \| "update_folder" \| "update_preset" \| "validate_agent_graph" \| "view_agent_output" \| "web_fetch" \| "web_search" \| "write_workspace_file" \| "Agent" \| "Edit" \| "Glob" \| "Grep" \| "Read" \| "Task" \| "TodoWrite" \| "WebSearch" \| "Write"] | No |
| tools_exclude | Controls how the 'tools' list is interpreted. True (default): 'tools' is a deny-list — listed tools are blocked, all others are allowed. An empty 'tools' list means allow everything. False: 'tools' is an allow-list — only listed tools are permitted. | bool | No |
| blocks | Block identifiers to filter when the copilot uses run_block. Each entry can be: a block name (e.g. 'HTTP Request'), a full block UUID, or the first 8 hex characters of the UUID (e.g. 'c069dc6b'). Works with blocks_exclude. Leave empty to apply no block filter. | List[str] | No |
| blocks_exclude | Controls how the 'blocks' list is interpreted. True (default): 'blocks' is a deny-list — listed blocks are blocked, all others are allowed. An empty 'blocks' list means allow everything. False: 'blocks' is an allow-list — only listed blocks are permitted. | bool | No |