Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): tell the main AutoPilot chat about its computer (#14701)

Krzysztof Czerwinski6262219728bf
diff --git a/autogpt_platform/backend/backend/copilot/eval/style/assembly.py b/autogpt_platform/backend/backend/copilot/eval/style/assembly.py
index 991f10f3d1..2ab7348146 100644
--- a/autogpt_platform/backend/backend/copilot/eval/style/assembly.py
+++ b/autogpt_platform/backend/backend/copilot/eval/style/assembly.py
@@ -148,7 +148,7 @@ def chat_system_prompt(expert: Expert | None) -> str:
suffix = render_expert_identity_suffix(expert) if expert else ""
return (
CACHEABLE_SYSTEM_PROMPT
- + get_sdk_supplement(use_e2b=True)
+ + get_sdk_supplement(use_e2b=True, expert_session=expert is not None)
+ (get_delegation_supplement() if DELEGATION_ENABLED else "")
+ get_expert_oversight_supplement(
experts_enabled=DELEGATION_ENABLED,
diff --git a/autogpt_platform/backend/backend/copilot/eval/style/assembly_test.py b/autogpt_platform/backend/backend/copilot/eval/style/assembly_test.py
index a05ab6418b..313bd279cf 100644
--- a/autogpt_platform/backend/backend/copilot/eval/style/assembly_test.py
+++ b/autogpt_platform/backend/backend/copilot/eval/style/assembly_test.py
@@ -124,7 +124,9 @@ def test_chat_prompt_is_base_plus_sdk_supplements_plus_suffix():
expert = roster_experts(["Frankie"])[0]
prompt = chat_system_prompt(expert)
assert prompt.startswith(CACHEABLE_SYSTEM_PROMPT)
- assert get_sdk_supplement(use_e2b=True) in prompt
+ assert get_sdk_supplement(use_e2b=True, expert_session=True) in prompt
+ assert "### Your computer" not in prompt
+ assert "### Your computer" in chat_system_prompt(None)
assert prompt.endswith("</expert_identity>")
assert "<expert_identity>" not in chat_system_prompt(None)
diff --git a/autogpt_platform/backend/backend/copilot/expert_context.py b/autogpt_platform/backend/backend/copilot/expert_context.py
index 8c9d0f1d87..d31e515bb1 100644
--- a/autogpt_platform/backend/backend/copilot/expert_context.py
+++ b/autogpt_platform/backend/backend/copilot/expert_context.py
@@ -27,7 +27,7 @@ import logging
from backend.api.features.experts.copy_policy import EXPERT_COPY_POLICY
from backend.api.features.experts.models import PROTECTED_SOUL_RULES, Expert
from backend.api.features.experts.models import ExpertRoutine as ExpertRoutineModel
-from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.blocks.desktop._api import DISPLAY, SHARED_PATH, WORKSPACE_PATH
from backend.copilot.config import ChatConfig
from backend.data.db_accessors import experts_db
from backend.util.exceptions import ExpertNotFoundError
@@ -228,7 +228,10 @@ async def build_expert_context(
own role, and a teammate's workflows are the easiest thing for the model
to borrow questions from. Plain sessions always get their roster.
- Returns ``""`` when there is nothing to inject or any lookup fails.
+ Returns ``""`` when there is nothing to inject or any lookup fails, except
+ that an expert session always keeps its computer block: the system prompt
+ leaves the plain chat's computer note out for every expert session, so
+ this is the only place the expert hears about its machine.
"""
if not user_id:
return ""
@@ -259,7 +262,7 @@ async def build_expert_context(
)
except Exception as e:
logger.warning(f"Failed to build expert context: {e}")
- return ""
+ return render_expert_computer_block() if expert_id else ""
async def _expert_session_context(
@@ -291,9 +294,9 @@ async def _expert_session_context(
_load_teammates(),
)
# Identity validation already failed closed before this context lookup.
- # If the expert changes between those reads, omit only this optional block.
+ # If the expert changes between those reads, omit only the optional blocks.
if expert is None or expert.is_archived:
- return ""
+ return render_expert_computer_block()
return (
render_expert_workflows_block(expert)
+ await _routines_block(user_id, expert_id)
@@ -458,6 +461,10 @@ def render_expert_computer_block() -> str:
"or GUI app; it is the same machine your commands run in. The desktop "
"is shared with the user, not private from either of you: you can see "
"everything on it, and so can they.\n"
+ "- The screen shows only what runs on this machine's display: launch "
+ "the app or browser with bash_exec, in the background with "
+ f"DISPLAY={DISPLAY}. browser_* tools run elsewhere and never appear on "
+ "it.\n"
"- Never ask the user to sign into personal accounts on this "
"desktop; use their connected integrations instead.\n"
"</expert_computer>\n\n"
diff --git a/autogpt_platform/backend/backend/copilot/expert_context_test.py b/autogpt_platform/backend/backend/copilot/expert_context_test.py
index 5a5cde67c2..f01b7ddb9c 100644
--- a/autogpt_platform/backend/backend/copilot/expert_context_test.py
+++ b/autogpt_platform/backend/backend/copilot/expert_context_test.py
@@ -530,7 +530,10 @@ class TestBuildExpertContextExpertSession:
mock_db = MagicMock()
mock_db.get_expert = AsyncMock(return_value=_expert(is_archived=True))
- with patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)):
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)),
+ patch(f"{_EC}.ChatConfig", return_value=MagicMock(e2b_active=False)),
+ ):
result = await build_expert_context("user-1", "exp-1")
assert result == ""
@@ -541,7 +544,10 @@ class TestBuildExpertContextExpertSession:
mock_db = MagicMock()
mock_db.get_expert = AsyncMock(return_value=None)
- with patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)):
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)),
+ patch(f"{_EC}.ChatConfig", return_value=MagicMock(e2b_active=False)),
+ ):
result = await build_expert_context("user-1", "exp-1")
assert result == ""
@@ -552,7 +558,10 @@ class TestBuildExpertContextExpertSession:
mock_db = MagicMock()
mock_db.get_expert = AsyncMock(side_effect=RuntimeError("db down"))
- with patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)):
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)),
+ patch(f"{_EC}.ChatConfig", return_value=MagicMock(e2b_active=False)),
+ ):
result = await build_expert_context("user-1", "exp-1")
assert result == ""
@@ -1206,6 +1215,76 @@ class TestExpertComputerBlock:
# Sits with the other first-message blocks, after the workflows.
assert result.index("</expert_workflows>") < result.index("<expert_computer>")
+ @pytest.mark.asyncio
+ async def test_expert_learns_the_screen_shows_only_its_own_machine(self):
+ from backend.blocks.desktop._api import DISPLAY
+ from backend.copilot.expert_context import build_expert_context
+
+ config = MagicMock()
+ config.e2b_active = True
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ result = await build_expert_context("user-1", "exp-1")
+
+ assert f"DISPLAY={DISPLAY}" in result
+ assert "browser_* tools run elsewhere" in result
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "get_expert",
+ [
+ AsyncMock(return_value=None),
+ AsyncMock(return_value=_expert(is_archived=True)),
+ AsyncMock(side_effect=RuntimeError("db down")),
+ ],
+ ids=["missing", "archived", "lookup-error"],
+ )
+ async def test_failed_expert_lookup_still_tells_the_expert_once(self, get_expert):
+ """The system prompt drops the plain chat's note for every expert
+ session, so a failed lookup must not cost the expert its block."""
+ from backend.copilot.expert_context import (
+ build_expert_context,
+ render_expert_computer_block,
+ )
+ from backend.copilot.prompting import get_sdk_supplement
+
+ mock_db = self._db()
+ mock_db.get_expert = get_expert
+ config = MagicMock()
+ config.e2b_active = True
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=mock_db)),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ context = await build_expert_context("user-1", "exp-1")
+ assert context == render_expert_computer_block()
+ turn = get_sdk_supplement(use_e2b=True, expert_session=True) + context
+
+ assert turn.count("### Your computer") + turn.count("<expert_computer>") == 1
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("expert_id", [None, "exp-1"])
+ async def test_a_session_is_told_about_its_computer_exactly_once(self, expert_id):
+ """The system prompt carries the plain chat's note and the first user
+ message carries the expert's block; a turn sees both strings."""
+ from backend.copilot.expert_context import build_expert_context
+ from backend.copilot.prompting import get_sdk_supplement
+
+ config = MagicMock()
+ config.e2b_active = True
+ with (
+ patch(f"{_EC}.experts_db", MagicMock(return_value=self._db())),
+ patch(f"{_EC}.ChatConfig", return_value=config),
+ ):
+ context = await build_expert_context("user-1", expert_id)
+ supplement = get_sdk_supplement(use_e2b=True, expert_session=bool(expert_id))
+ turn = supplement + context
+
+ assert turn.count("### Your computer") + turn.count("<expert_computer>") == 1
+ assert ("<expert_computer>" in turn) is bool(expert_id)
+
@pytest.mark.asyncio
async def test_rendered_context_is_hidden_from_chat_history(self):
from backend.copilot.expert_context import build_expert_context
diff --git a/autogpt_platform/backend/backend/copilot/prompting.py b/autogpt_platform/backend/backend/copilot/prompting.py
index f58d3b7f6f..71c58cff99 100644
--- a/autogpt_platform/backend/backend/copilot/prompting.py
+++ b/autogpt_platform/backend/backend/copilot/prompting.py
@@ -8,6 +8,8 @@ handling the distinction between:
from functools import cache
+from backend.blocks.desktop._api import DISPLAY
+
# Workflow rules appended to the system prompt on every copilot turn
# (baseline appends directly; SDK appends via the storage-supplement
# template). These are cross-tool rules (file sharing, @@agptfile: refs,
@@ -595,8 +597,27 @@ what happened so the user knows the turn is complete.
"""
+# Plain chats only: an expert session is told about its own machine by
+# ``expert_context.render_expert_computer_block``. bash_exec does not set DISPLAY.
+_COMPUTER_NOTE = f"""
+### Your computer
+The cloud sandbox is also a computer with a screen. `start_desktop` turns the
+screen on and streams it to the user; use it when a task needs a GUI app, or a
+browser the user should watch or take over.
+- The screen shows only what runs in the sandbox on its display: after
+ `start_desktop`, launch the app or browser with `bash_exec`, in the
+ background with `DISPLAY={DISPLAY}`. `browser_*` tools run elsewhere and never
+ appear on it.
+- It lives with this session: files and installed tools outside `~/workspace`
+ are lost when the session expires.
+- The desktop is shared with the user, not private from either of you. Never
+ ask them to sign into personal accounts on it; use their connected
+ integrations instead.
+"""
+
+
@cache
-def get_sdk_supplement(use_e2b: bool) -> str:
+def get_sdk_supplement(use_e2b: bool, expert_session: bool = False) -> str:
"""Get the supplement for SDK mode (Claude Agent SDK).
SDK mode does NOT include tool documentation because Claude automatically
@@ -613,16 +634,19 @@ def get_sdk_supplement(use_e2b: bool) -> str:
Args:
use_e2b: Whether E2B cloud sandbox is being used
+ expert_session: Whether the session belongs to an expert, whose own
+ ``<expert_computer>`` block replaces the computer note
Returns:
The supplement string to append to the system prompt
"""
- base = (
- _get_cloud_sandbox_supplement()
- if use_e2b
- else _get_local_storage_supplement("/tmp/copilot-<session-id>")
- )
- return base + _USER_FOLLOW_UP_NOTE
+ if not use_e2b:
+ return (
+ _get_local_storage_supplement("/tmp/copilot-<session-id>")
+ + _USER_FOLLOW_UP_NOTE
+ )
+ computer = "" if expert_session else _COMPUTER_NOTE
+ return _get_cloud_sandbox_supplement() + computer + _USER_FOLLOW_UP_NOTE
# The one reply a chat-platform bot does not deliver. A message on Discord,
diff --git a/autogpt_platform/backend/backend/copilot/prompting_test.py b/autogpt_platform/backend/backend/copilot/prompting_test.py
index d8a9cf8cda..4f3c115941 100644
--- a/autogpt_platform/backend/backend/copilot/prompting_test.py
+++ b/autogpt_platform/backend/backend/copilot/prompting_test.py
@@ -4,6 +4,7 @@ import importlib
import pytest
+from backend.blocks.desktop._api import DISPLAY
from backend.copilot import prompting
@@ -34,6 +35,49 @@ class TestGetSdkSupplementStaticPlaceholder:
assert "<session-id>" not in result
+class TestComputerNote:
+ """A plain chat on E2B is told it has a screen; an expert session is told
+ by its own ``<expert_computer>`` block instead, and a local session has no
+ computer at all."""
+
+ def test_plain_e2b_session_learns_about_the_screen(self):
+ result = prompting.get_sdk_supplement(use_e2b=True)
+ assert result.count("### Your computer") == 1
+ assert "`start_desktop`" in result
+ assert "lost when the session expires" in result
+ assert "sign into personal accounts" in result
+
+ def test_screen_shows_only_what_runs_in_the_sandbox(self):
+ """``browser_*`` drives a browser outside the sandbox, so the model
+ must not offer the user a takeover of something the screen never
+ shows."""
+ result = prompting.get_sdk_supplement(use_e2b=True)
+ assert f"DISPLAY={DISPLAY}" in result
+ assert "`browser_*` tools run elsewhere" in result
+
+ def test_no_computer_note_without_e2b(self):
+ result = prompting.get_sdk_supplement(use_e2b=False)
+ assert "### Your computer" not in result
+ assert "start_desktop" not in result
+ assert (
+ prompting.get_sdk_supplement(use_e2b=False, expert_session=True) == result
+ )
+
+ def test_expert_session_differs_only_by_the_computer_note(self):
+ plain = prompting.get_sdk_supplement(use_e2b=True)
+ expert = prompting.get_sdk_supplement(use_e2b=True, expert_session=True)
+ assert "### Your computer" not in expert
+ assert plain.replace(prompting._COMPUTER_NOTE, "") == expert
+
+ def test_note_sits_inside_the_tool_notes_before_the_follow_up_rules(self):
+ result = prompting.get_sdk_supplement(use_e2b=True)
+ assert (
+ result.index("## Tool notes")
+ < result.index("### Your computer")
+ < result.index("# `<user_follow_up>` blocks")
+ )
+
+
class TestCredentialsSurfacingGuardrails:
"""The system prompt must instruct the model to (a) surface sign-in cards
eagerly via tool calls and (b) never claim a card has appeared unless one
diff --git a/autogpt_platform/backend/backend/copilot/sdk/building_mode_restart_test.py b/autogpt_platform/backend/backend/copilot/sdk/building_mode_restart_test.py
index 8102fb6fee..17695e8deb 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/building_mode_restart_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/building_mode_restart_test.py
@@ -79,6 +79,8 @@ class TestApplyBuildingModeRestart:
delegation_supplement: str = "",
oversight_supplement: str = "",
team_building_supplement: str = "",
+ use_e2b: bool = False,
+ expert_id: str | None = None,
):
from backend.copilot.sdk.service import (
_BUILDING_MODE_CONTINUATION,
@@ -90,6 +92,7 @@ class TestApplyBuildingModeRestart:
new=mocker.AsyncMock(return_value=suffix),
)
session = _session(requested=True, guide_loaded=False)
+ session.expert_id = expert_id
state = self._state(
prior_emitted=prior_emitted, thinking_reprompted=thinking_reprompted
)
@@ -103,7 +106,7 @@ class TestApplyBuildingModeRestart:
oversight_supplement=oversight_supplement,
team_building_supplement=team_building_supplement,
graphiti_supplement="",
- use_e2b=False,
+ use_e2b=use_e2b,
session_id="sess-1",
message_id="msg-1",
log_prefix="[test]",
@@ -139,6 +142,26 @@ class TestApplyBuildingModeRestart:
text = prompt if isinstance(prompt, str) else prompt["append"]
assert marker in text
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "use_e2b, expert_id, expected",
+ [(True, None, 1), (True, "expert-1", 0), (False, None, 0)],
+ )
+ async def test_computer_note_only_for_a_plain_e2b_session(
+ self, mocker, use_e2b, expert_id, expected
+ ):
+ """An expert session's computer block rides its first user message, so
+ the system prompt must not add a second one."""
+ mocker.patch(
+ "backend.copilot.sdk.service.build_expert_identity_suffix",
+ new=mocker.AsyncMock(return_value=""),
+ )
+ _, state, _, _ = await self._run(mocker, use_e2b=use_e2b, expert_id=expert_id)
+
+ prompt = state.options.system_prompt
+ text = prompt if isinstance(prompt, str) else prompt["append"]
+ assert text.count("### Your computer") == expected
+
@pytest.mark.asyncio
async def test_empty_suffix_relaunches_without_the_confirmation(self, mocker):
"""An empty suffix means the guide is genuinely absent, so the model
diff --git a/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py b/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py
index 1af1e62b85..7232d1ae2a 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py
@@ -164,13 +164,21 @@ def _make_patches(*, hire_experts_enabled: bool):
return patches, mcp_server_mock, is_feature_enabled_mock
-async def _run_sdk_turn(*, user_id: str | None, hire_experts_enabled: bool):
+async def _run_sdk_turn(
+ *,
+ user_id: str | None,
+ hire_experts_enabled: bool,
+ expert_id: str | None = None,
+ extra_patches: list[tuple[str, dict]] | None = None,
+):
from backend.copilot.sdk.service import stream_chat_completion_sdk
session = _make_session()
+ session.expert_id = expert_id
patches, mcp_server_mock, is_feature_enabled_mock = _make_patches(
hire_experts_enabled=hire_experts_enabled
)
+ patches += extra_patches or []
events = []
with contextlib.ExitStack() as stack:
@@ -221,3 +229,32 @@ class TestSdkExpertsFlagGuard:
assert "update_expert_soul" in hidden
assert "hire_expert" not in hidden
assert "delegate_to_expert" not in hidden
+
+
+class TestSdkComputerNoteWiring:
+ """The plain chat's computer note rides ``get_sdk_supplement``; an expert
+ session must ask for the variant without it, because its own
+ ``<expert_computer>`` block arrives in the first user message."""
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("expert_id", [None, "expert-1"])
+ async def test_main_turn_tells_the_supplement_whether_it_is_an_expert_session(
+ self, expert_id: str | None
+ ) -> None:
+ supplement = MagicMock(return_value="")
+ await _run_sdk_turn(
+ user_id=None,
+ hire_experts_enabled=False,
+ expert_id=expert_id,
+ extra_patches=[
+ (f"{_SVC}.get_sdk_supplement", dict(new=supplement)),
+ (
+ f"{_SVC}.build_expert_identity_suffix",
+ dict(new_callable=AsyncMock, return_value=""),
+ ),
+ ],
+ )
+
+ supplement.assert_called_once_with(
+ use_e2b=False, expert_session=bool(expert_id)
+ )
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index c1d8a0edcb..03663be6a7 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -1756,7 +1756,7 @@ async def _apply_building_mode_restart(
# of the turn.
system_prompt = (
base_system_prompt
- + get_sdk_supplement(use_e2b=use_e2b)
+ + get_sdk_supplement(use_e2b=use_e2b, expert_session=bool(session.expert_id))
+ delegation_supplement
+ oversight_supplement
+ team_building_supplement
@@ -4918,7 +4918,9 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
session.guide_in_system_prompt = bool(builder_session_suffix)
system_prompt = (
base_system_prompt
- + get_sdk_supplement(use_e2b=use_e2b)
+ + get_sdk_supplement(
+ use_e2b=use_e2b, expert_session=bool(session.expert_id)
+ )
+ delegation_supplement
+ oversight_supplement
+ team_building_supplement
diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop.py
index 28415e2bb6..399ee7a87f 100644
--- a/autogpt_platform/backend/backend/copilot/tools/start_desktop.py
+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop.py
@@ -19,7 +19,7 @@ the screen comes back exactly as it was on the next resume.
import logging
from typing import Any
-from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH
+from backend.blocks.desktop._api import DISPLAY, SHARED_PATH, WORKSPACE_PATH
from backend.copilot.model import ChatSession
from backend.copilot.sdk.env import config as chat_config
@@ -144,6 +144,9 @@ def _build_message(first_time: bool, shared: bool, *, expert: bool = False) -> s
return (
f"{state}. This is the same machine bash_exec runs in, so files, "
"processes and installed tools are shared. The user can watch and "
- "control it via the live stream. The machine suspends between turns "
- f"and the screen comes back as you left it. {files}"
+ "control it via the live stream, which shows only what runs here on "
+ f"DISPLAY={DISPLAY}: start GUI apps or a browser with bash_exec in the "
+ "background; browser_* tools run elsewhere and never appear on it. "
+ "The machine suspends between turns and the screen comes back as you "
+ f"left it. {files}"
)
diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py
index 5b8e72d919..0abe5bf80b 100644
--- a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py
@@ -140,6 +140,9 @@ class TestStartDesktop:
assert "Screen is on" in result.message
assert "same machine" in result.message
assert WORKSPACE_PATH in result.message
+ # browser_* runs outside the box, so it is never on this screen.
+ assert "DISPLAY=:0" in result.message
+ assert "browser_* tools run elsewhere" in result.message
@pytest.mark.asyncio(loop_scope="session")
async def test_stored_result_carries_the_owner_link_not_the_password(self):