Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): put the desktop back in reach of the chat (#14696)

Krzysztof Czerwinski81778b759e1b
diff --git a/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py b/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py
index 1d93eceed0..5567b4822b 100644
--- a/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py
+++ b/autogpt_platform/backend/backend/copilot/capabilities/sources/static_tools.py
@@ -39,6 +39,10 @@ EAGER_CORE: frozenset[str] = frozenset(
"delegate_to_expert",
"handoff_to_expert",
"TodoWrite",
+ # The expert prompt tells the model to use ``start_desktop`` by name,
+ # and a deferred tool named directly is refused. Eager, the screen
+ # goes on in one call instead of a find/run round trip every turn.
+ "start_desktop",
# ``kickoff_turn_disabled_tools`` narrows a hire's first turn to this
# one tool. Deferred, that gate leaves the turn with no tools at all:
# the card it exists to open is unreachable, and so is the
diff --git a/autogpt_platform/backend/backend/copilot/tools/capability_tools_test.py b/autogpt_platform/backend/backend/copilot/tools/capability_tools_test.py
index 775321110a..ba278e1f64 100644
--- a/autogpt_platform/backend/backend/copilot/tools/capability_tools_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/capability_tools_test.py
@@ -67,6 +67,13 @@ def test_eager_and_deferred_split_the_registry():
assert len(get_available_tools(include_deferred=True)) > len(shown)
+def test_start_desktop_stays_eager_because_the_prompt_names_it():
+ """``expert_context`` tells the model "Use start_desktop"; a deferred
+ tool called by name is refused, so the instruction only works eager."""
+ assert "start_desktop" in EAGER_CORE
+ assert "start_desktop" not in DEFERRED_TOOL_NAMES
+
+
def test_prompt_names_only_registry_tools():
for legacy in (
"find_block",
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx
index 3185929fbb..e113a547e1 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ArtifactPanel/__tests__/computerFace.test.tsx
@@ -53,6 +53,78 @@ describe("copilot store: computer face", () => {
);
});
+ it("opening a document over the computer face turns the flag off with the mode", () => {
+ useCopilotUIStore.getState().openComputer();
+ useCopilotUIStore.getState().openArtifact(
+ {
+ id: "a1",
+ title: "notes.md",
+ sourceUrl: "/x",
+ origin: "assistant",
+ } as never,
+ { persist: false },
+ );
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.mode).toBe("artifact");
+ expect(panel.isComputerOpen).toBe(false);
+ // A desktop started now must not flip the panel over the document.
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ expect(useCopilotUIStore.getState().artifactPanel.mode).toBe("artifact");
+ });
+
+ it("the files tab takes the panel from the computer face instead of closing it", () => {
+ useCopilotUIStore.setState((s) => ({
+ artifactPanel: { ...s.artifactPanel, activeTab: "files" },
+ }));
+ useCopilotUIStore.getState().openComputer();
+ useCopilotUIStore.getState().toggleContextPanelTab("files");
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.isOpen).toBe(true);
+ expect(panel.activeTab).toBe("files");
+ expect(panel.isComputerOpen).toBe(false);
+ expect(panel.mode).toBe("artifact");
+ // The same click with the tab already showing closes, as before.
+ useCopilotUIStore.getState().toggleContextPanelTab("files");
+ expect(useCopilotUIStore.getState().artifactPanel.isOpen).toBe(false);
+ });
+
+ it("the panel header's files button leaves the computer face", () => {
+ useCopilotUIStore.getState().openComputer();
+ useCopilotUIStore.getState().showFilesTab();
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.activeTab).toBe("files");
+ expect(panel.isComputerOpen).toBe(false);
+ expect(panel.mode).toBe("artifact");
+ });
+
+ it("a card that remounts does not reopen a computer the user hid", () => {
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ useCopilotUIStore.getState().closeComputer();
+ // The streaming chain unmounts and remounts the desktop row.
+ useCopilotUIStore.getState().registerComputerStream(STREAM);
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.isComputerOpen).toBe(false);
+ expect(panel.isOpen).toBe(false);
+ // A different box is news again.
+ useCopilotUIStore
+ .getState()
+ .registerComputerStream({ ...STREAM, sandbox_id: "sbx-2" });
+ expect(useCopilotUIStore.getState().artifactPanel.isComputerOpen).toBe(
+ true,
+ );
+ });
+
+ it("remembers a desktop without opening anything when asked not to show it", () => {
+ useCopilotUIStore
+ .getState()
+ .registerComputerStream(STREAM, { show: false });
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.computer?.sandbox_id).toBe("sbx-1");
+ expect(panel.isComputerOpen).toBe(false);
+ expect(panel.isOpen).toBe(false);
+ expect(panel.mode).toBe("artifact");
+ });
+
it("closing the panel and entering a new chat both drop the computer face", () => {
useCopilotUIStore.getState().registerComputerStream(STREAM);
useCopilotUIStore.getState().closeArtifactPanel({ persist: false });
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
index ff0e9a1f77..c724afbf2d 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/ContextPanelToggle.tsx
@@ -9,8 +9,9 @@ import {
useSessionFiles,
type SessionFile,
} from "./components/FilesTab/useSessionFiles";
-import { LicenseDraftIcon } from "@hugeicons/core-free-icons";
+import { ComputerIcon, LicenseDraftIcon } from "@hugeicons/core-free-icons";
import { Icon } from "@/components/atoms/Icon/Icon";
+import { useIsMobile } from "../../useIsMobile";
interface Props {
sessionId?: string | null;
@@ -26,10 +27,14 @@ function getLastGeneratedFile(generated: SessionFile[]): SessionFile | null {
);
}
-/** The chat's one top-right control: the artifacts toggle. It wears the name
- * of the session's most recently generated file so the current working
+/** The chat's top-right controls: the Computer toggle and the artifacts
+ * toggle, one per face of the side panel. The artifacts toggle wears the
+ * name of the session's most recently generated file so the current working
* document stays visible, and clicking it opens that file directly in the
- * artifact panel. Workspace files open from the thread chip instead. */
+ * artifact panel. Workspace files open from the thread chip instead. The
+ * Computer toggle is always there for a chat with a session: the panel's
+ * own "Turn on screen" button lives on that face, so it must be reachable
+ * before any desktop exists and without an artifact to switch from. */
export function ContextPanelToggle({ sessionId = null }: Props) {
const isOpen = useCopilotUIStore((s) => s.artifactPanel.isOpen);
const hasArtifact = useCopilotUIStore(
@@ -42,43 +47,41 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
const closeArtifactPanel = useCopilotUIStore((s) => s.closeArtifactPanel);
const lastArtifact = useCopilotUIStore((s) => s.artifactPanel.lastArtifact);
const openArtifact = useCopilotUIStore((s) => s.openArtifact);
- const hasComputer = useCopilotUIStore(
- (s) => s.artifactPanel.computer != null,
- );
const isComputerOpen = useCopilotUIStore(
(s) => s.artifactPanel.isComputerOpen,
);
const openComputer = useCopilotUIStore((s) => s.openComputer);
+ const closeComputer = useCopilotUIStore((s) => s.closeComputer);
+ const setArtifactPanelMode = useCopilotUIStore((s) => s.setArtifactPanelMode);
+ const isMobile = useIsMobile();
const { generated } = useSessionFiles(sessionId);
const lastGenerated = getLastGeneratedFile(generated);
const isFilesCardOpen = useAreWorkspaceFileCardsOpen();
const isArtifactsOpen = isOpen && activeTab === "artifacts";
- // An open artifact preview and the artifacts tab are both faces of the
- // right sidebar, so the toggle reads active for either and is the one
- // control that closes them — the panel carries no close button.
- const isRightSidebarOpen = hasArtifact || isArtifactsOpen || isComputerOpen;
+ // An open artifact preview and the artifacts tab are both the document
+ // face of the right sidebar, so the toggle reads active for either and is
+ // the one control that closes them — the panel carries no close button.
+ // With the computer face on top, the artifact underneath is not showing.
+ const isDocumentOpen = !isComputerOpen && (hasArtifact || isArtifactsOpen);
+ const isRightSidebarOpen = isDocumentOpen || isComputerOpen;
+ // The mobile sheet has no computer face to open.
+ const showComputerToggle = !!sessionId && !isMobile;
// The open activity card already lists the same file, so the labeled
- // button floating above it is pure duplication — the card's rows are the
- // way in while it shows.
- if (isFilesCardOpen) return null;
+ // artifacts button floating above it is pure duplication — the card's rows
+ // are the way in while it shows. The Computer button is not about files
+ // and stays, or "Turn on screen" is out of reach until the card closes.
+ const showArtifactsToggle = !isFilesCardOpen;
+ if (!showArtifactsToggle && !showComputerToggle) return null;
// With the panel open its own header already names the document, so the
// button collapses to the bare icon; closed, the name is the reminder of
// what's being worked on.
const showFileName = lastGenerated != null && !isRightSidebarOpen;
- function handleSidebarToggle() {
- if (hasArtifact || isComputerOpen) {
- closeArtifactPanel();
- return;
- }
- if (isArtifactsOpen) {
- toggleContextPanelTab("artifacts");
- return;
- }
- // Straight to the document: the freshest generated file, then the
- // remembered preview, and only then the tabs view.
+ // Straight to the document: the freshest generated file, then the
+ // remembered preview, and only then the tabs view.
+ function openDocument() {
const target =
(lastGenerated ? fileItemToArtifactRef(lastGenerated.item) : null) ??
lastArtifact;
@@ -86,51 +89,93 @@ export function ContextPanelToggle({ sessionId = null }: Props) {
openArtifact(target);
return;
}
- // No document yet but a desktop was started: the machine is the work.
- if (hasComputer) {
- openComputer();
+ // Already on the library (the computer face was just turned off it):
+ // toggling would close it.
+ if (isArtifactsOpen) return;
+ toggleContextPanelTab("artifacts");
+ }
+
+ function handleSidebarToggle() {
+ if (isComputerOpen) {
+ // Turn the panel to its document face: the preview that was under
+ // the computer if there is one, else wherever a closed panel opens.
+ setArtifactPanelMode("artifact");
+ if (hasArtifact) return;
+ openDocument();
+ return;
+ }
+ if (hasArtifact) {
+ closeArtifactPanel();
return;
}
- toggleContextPanelTab("artifacts");
+ if (isArtifactsOpen) {
+ toggleContextPanelTab("artifacts");
+ return;
+ }
+ openDocument();
}
+ function handleComputerToggle() {
+ // Back to whatever the computer was covering: the preview, the tab, or
+ // nothing at all.
+ if (isComputerOpen) closeComputer();
+ else openComputer();
+ }
+
+ // Sized and stroked like the sidebar's nav icons so the chat's top-right
+ // controls read as the same family.
+ const toggleClass =
+ "shrink-0 rounded-md transition-[background-color,transform] duration-150 ease-out hover:bg-zinc-100 active:scale-[0.97] motion-reduce:transition-none";
+
return (
<div className="flex shrink-0 items-center gap-1 p-2">
- <Button
- type="button"
- variant="ghost"
- size={showFileName ? "sm" : "icon"}
- onClick={handleSidebarToggle}
- aria-label={
- isRightSidebarOpen
- ? isComputerOpen
- ? "Hide computer"
- : "Hide artifacts"
- : lastGenerated
- ? `Open ${lastGenerated.item.name}`
- : hasComputer
- ? "Open computer"
+ {showComputerToggle && (
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon"
+ onClick={handleComputerToggle}
+ aria-label={isComputerOpen ? "Hide computer" : "Open computer"}
+ aria-pressed={isComputerOpen}
+ className={cn(toggleClass, "size-8", isComputerOpen && "bg-zinc-100")}
+ >
+ <Icon
+ icon={ComputerIcon}
+ className="!size-4 text-sidebar-foreground/90"
+ />
+ </Button>
+ )}
+ {showArtifactsToggle && (
+ <Button
+ type="button"
+ variant="ghost"
+ size={showFileName ? "sm" : "icon"}
+ onClick={handleSidebarToggle}
+ aria-label={
+ isDocumentOpen
+ ? "Hide artifacts"
+ : lastGenerated
+ ? `Open ${lastGenerated.item.name}`
: "Open artifacts"
- }
- aria-pressed={isRightSidebarOpen}
- className={cn(
- // Sized and stroked like the sidebar's nav icons so the chat's
- // top-right control reads as the same family.
- "shrink-0 rounded-md transition-[background-color,transform] duration-150 ease-out hover:bg-zinc-100 active:scale-[0.97] motion-reduce:transition-none",
- showFileName ? "h-8 gap-1.5 px-2" : "size-8",
- isRightSidebarOpen && "bg-zinc-100",
- )}
- >
- <Icon
- icon={LicenseDraftIcon}
- className="!size-4 text-sidebar-foreground/90"
- />
- {showFileName && (
- <span className="max-w-[9rem] truncate text-xs font-medium text-sidebar-foreground/90">
- {lastGenerated.item.name}
- </span>
- )}
- </Button>
+ }
+ aria-pressed={isDocumentOpen}
+ className={cn(
+ toggleClass,
+ showFileName ? "h-8 gap-1.5 px-2" : "size-8",
+ isDocumentOpen && "bg-zinc-100",
+ )}
+ >
+ <Icon
+ icon={LicenseDraftIcon}
+ className="!size-4 text-sidebar-foreground/90"
+ />
+ {showFileName && (
+ <span className="max-w-[9rem] truncate text-xs font-medium text-sidebar-foreground/90">
+ {lastGenerated.item.name}
+ </span>
+ )}
+ </Button>
+ )}
</div>
);
}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanel.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanel.test.tsx
index 4ba685b5a2..cdc5cca26e 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanel.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanel.test.tsx
@@ -66,6 +66,19 @@ describe("ContextPanel", () => {
expect(container.querySelector("[data-context-panel]")).toBeNull();
});
+ test("hides itself while the computer face is showing, whatever tab is remembered", () => {
+ useCopilotUIStore.setState((s) => ({
+ artifactPanel: {
+ ...s.artifactPanel,
+ activeTab: "artifacts",
+ mode: "computer",
+ isComputerOpen: true,
+ },
+ }));
+ const { container } = render(<ContextPanel sessionId="session-1" />);
+ expect(container.querySelector("[data-context-panel]")).toBeNull();
+ });
+
test("renders nothing when closed", () => {
useCopilotUIStore.setState((s) => ({
artifactPanel: {
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanelToggle.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanelToggle.test.tsx
index c9db666542..e84ad6152b 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanelToggle.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/__tests__/ContextPanelToggle.test.tsx
@@ -1,8 +1,21 @@
import { fireEvent, render, screen } from "@/tests/integrations/test-utils";
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { Key, storage } from "@/services/storage/local-storage";
import { useCopilotUIStore } from "../../../store";
import { ContextPanelToggle } from "../ContextPanelToggle";
+vi.mock("../components/FilesTab/useSessionFiles", () => ({
+ useSessionFiles: () => ({ generated: [], uploaded: [] }),
+}));
+
+let mobile = false;
+function setMobile(value: boolean) {
+ mobile = value;
+}
+vi.mock("../../../useIsMobile", () => ({
+ useIsMobile: () => mobile,
+}));
+
vi.mock("@/services/feature-flags/use-get-flag", async (importOriginal) => {
const actual =
await importOriginal<
@@ -36,16 +49,23 @@ beforeEach(() => {
activeTab: "files",
lastArtifact: null,
history: [],
+ mode: "artifact",
+ isComputerOpen: false,
+ computer: null,
});
});
afterEach(() => {
+ setMobile(false);
setPanel({
isOpen: false,
activeArtifact: null,
activeTab: "files",
lastArtifact: null,
history: [],
+ mode: "artifact",
+ isComputerOpen: false,
+ computer: null,
});
});
@@ -91,4 +111,164 @@ describe("ContextPanelToggle", () => {
expect(screen.getByLabelText("Hide artifacts")).toBeDefined();
});
+
+ test("sidebar toggle turns the computer face back to the artifact under it", () => {
+ setPanel({
+ activeArtifact: ARTIFACT,
+ isOpen: true,
+ mode: "computer",
+ isComputerOpen: true,
+ });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open artifacts"));
+
+ expect(panelState().mode).toBe("artifact");
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().activeArtifact).toEqual(ARTIFACT);
+ expect(panelState().isOpen).toBe(true);
+ });
+
+ test("sidebar toggle turns the computer face back to the remembered preview", () => {
+ setPanel({
+ lastArtifact: ARTIFACT,
+ isOpen: true,
+ activeTab: "files",
+ mode: "computer",
+ isComputerOpen: true,
+ });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open artifacts"));
+
+ expect(panelState().activeArtifact).toEqual(ARTIFACT);
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().mode).toBe("artifact");
+ });
+
+ test("a panel the computer opened is stored as open once turned to the library", () => {
+ storage.set(Key.COPILOT_CONTEXT_PANEL_OPEN, "false");
+ setPanel({ isOpen: false, activeTab: "artifacts" });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open computer"));
+ fireEvent.click(screen.getByLabelText("Open artifacts"));
+
+ expect(panelState().isOpen).toBe(true);
+ expect(storage.get(Key.COPILOT_CONTEXT_PANEL_OPEN)).toBe("true");
+ });
+
+ test("sidebar toggle turns the computer face to the library when nothing is under it", () => {
+ setPanel({
+ isOpen: true,
+ activeTab: "artifacts",
+ mode: "computer",
+ isComputerOpen: true,
+ });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open artifacts"));
+
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().isOpen).toBe(true);
+ expect(panelState().activeTab).toBe("artifacts");
+ });
+});
+
+describe("ContextPanelToggle computer button", () => {
+ test("is there for a chat with a session and absent without one", () => {
+ const { unmount } = render(<ContextPanelToggle />);
+ expect(screen.queryByLabelText("Open computer")).toBeNull();
+ unmount();
+
+ render(<ContextPanelToggle sessionId="s1" />);
+ expect(screen.getByLabelText("Open computer")).toBeDefined();
+ });
+
+ test("opens the computer face with no artifact and no desktop started", () => {
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open computer"));
+
+ expect(panelState().isOpen).toBe(true);
+ expect(panelState().mode).toBe("computer");
+ expect(panelState().isComputerOpen).toBe(true);
+ });
+
+ test("opens the computer face over an artifact without dropping it", () => {
+ setPanel({ activeArtifact: ARTIFACT, isOpen: true });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open computer"));
+
+ expect(panelState().mode).toBe("computer");
+ expect(panelState().isComputerOpen).toBe(true);
+ expect(panelState().activeArtifact).toEqual(ARTIFACT);
+ expect(screen.getByLabelText("Hide computer")).toBeDefined();
+ expect(screen.getByLabelText("Open artifacts")).toBeDefined();
+ });
+
+ test("hiding the computer closes the panel it opened over, keeping the remembered preview", () => {
+ setPanel({ isOpen: false, lastArtifact: ARTIFACT });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open computer"));
+ fireEvent.click(screen.getByLabelText("Hide computer"));
+
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().isOpen).toBe(false);
+ expect(panelState().lastArtifact).toEqual(ARTIFACT);
+ });
+
+ test("hiding the computer returns to the tab that was open under it", () => {
+ setPanel({ isOpen: true, activeTab: "artifacts" });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Open computer"));
+ fireEvent.click(screen.getByLabelText("Hide computer"));
+
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().isOpen).toBe(true);
+ expect(panelState().activeTab).toBe("artifacts");
+ expect(screen.getByLabelText("Hide artifacts")).toBeDefined();
+ });
+
+ test("hiding the computer reveals the preview it was covering, history intact", () => {
+ const earlier = { ...ARTIFACT, id: "f0", title: "earlier.md" };
+ setPanel({
+ activeArtifact: ARTIFACT,
+ history: [earlier],
+ isOpen: true,
+ mode: "computer",
+ isComputerOpen: true,
+ });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ fireEvent.click(screen.getByLabelText("Hide computer"));
+
+ expect(panelState().isComputerOpen).toBe(false);
+ expect(panelState().mode).toBe("artifact");
+ expect(panelState().isOpen).toBe(true);
+ expect(panelState().activeArtifact).toEqual(ARTIFACT);
+ expect(panelState().history).toEqual([earlier]);
+ });
+
+ test("stays while the workspace files card is open, alone", () => {
+ setPanel({ isOpen: true, activeTab: "files" });
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ expect(screen.queryByLabelText("Open artifacts")).toBeNull();
+ fireEvent.click(screen.getByLabelText("Open computer"));
+
+ expect(panelState().isComputerOpen).toBe(true);
+ expect(panelState().mode).toBe("computer");
+ });
+
+ test("is hidden on mobile, whose sheet has no computer face", () => {
+ setMobile(true);
+ render(<ContextPanelToggle sessionId="s1" />);
+
+ expect(screen.queryByLabelText("Open computer")).toBeNull();
+ expect(screen.getByLabelText("Open artifacts")).toBeDefined();
+ });
});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/useContextPanel.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/useContextPanel.ts
index 866d838faa..0afd153e50 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/useContextPanel.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ContextPanel/useContextPanel.ts
@@ -8,10 +8,12 @@ export function useContextPanel() {
const contextPanelWidth = useCopilotUIStore((s) => s.contextPanelWidth);
const setContextPanelWidth = useCopilotUIStore((s) => s.setContextPanelWidth);
- // The artifact takes over the right region while it's open, so the Context
- // Panel is hidden then (you return to it by closing the artifact).
+ // The artifact or the computer takes over the right region while it's
+ // open, so the Context Panel is hidden then (you return to it by closing
+ // the artifact or turning the panel back to its document face).
const hasArtifact = artifactPanel.activeArtifact != null;
- const showExpanded = artifactPanel.isOpen && !hasArtifact;
+ const showExpanded =
+ artifactPanel.isOpen && !hasArtifact && !artifactPanel.isComputerOpen;
return {
isOpen: artifactPanel.isOpen,
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx
index 71510b9bff..c00c9f5001 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/DesktopStreamCard.tsx
@@ -6,6 +6,7 @@ import {
} from "@/components/contextual/OutputRenderers/renderers/DesktopStreamRenderer";
import { useEffect } from "react";
import { useCopilotUIStore, type DesktopStreamRef } from "../../store";
+import { useIsMobile } from "../../useIsMobile";
interface Props {
stream: unknown;
@@ -32,14 +33,16 @@ export function DesktopStreamCard({ stream, readOnly = false }: Props) {
const registerComputerStream = useCopilotUIStore(
(s) => s.registerComputerStream,
);
+ const isMobile = useIsMobile();
const ref = asStreamRef(stream);
const sandboxId = ref?.sandbox_id;
const url = ref?.url;
useEffect(() => {
- if (ref && !readOnly) registerComputerStream(ref);
+ // The mobile sheet has no computer face: remember the desktop, open nothing.
+ if (ref && !readOnly) registerComputerStream(ref, { show: !isMobile });
// Re-register only when the stream itself changes, not on every render.
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [sandboxId, url, readOnly]);
+ }, [sandboxId, url, readOnly, isMobile]);
if (!isDesktopStream(stream)) return null;
return <DesktopStreamPreview value={stream} ownerView={!readOnly} />;
}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
index bded7c59ee..a284c4bc95 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ToolResult.tsx
@@ -150,6 +150,16 @@ function chipStrings(value: unknown, key: string): string[] | null {
return labels.length > 0 ? labels : null;
}
+/** The live desktop is the whole point of start_desktop: embed the stream
+ * instead of letting the payload fall through to a truncated key/value
+ * dump. The same renderer serves block outputs and attachments, and the
+ * card is what tells the side panel a desktop exists. */
+function desktopCard(output: Record<string, unknown>, readOnly: boolean) {
+ const stream = output.desktop_stream;
+ if (!stream || !desktopStreamRenderer.canRender(stream)) return null;
+ return <DesktopStreamCard stream={stream} readOnly={readOnly} />;
+}
+
const CAPABILITY_RUN_TOOLS = new Set([
"run_capability",
"resume_capability",
@@ -340,6 +350,11 @@ function toolCard(
case "run_capability":
case "resume_capability": {
if (!output) return null;
+ // Transcripts recorded while start_desktop was deferred (#14569 until
+ // it went eager again) carry its result on a run_capability row: same
+ // payload, same card.
+ const desktop = desktopCard(output, readOnly);
+ if (desktop) return desktop;
const block = asObject(output.block);
if (block) return <BlockListCard blocks={[block]} />;
if (str(output, "block_name", "block_id"))
@@ -438,16 +453,8 @@ function toolCard(
}
case "bash_exec":
return <Terminal row={row} />;
- case "start_desktop": {
- // The live desktop is the whole point of the tool: embed the stream
- // instead of letting the payload fall through to a truncated key/value
- // dump. The same renderer serves block outputs and attachments.
- const stream = output ? output.desktop_stream : null;
- if (stream && desktopStreamRenderer.canRender(stream)) {
- return <DesktopStreamCard stream={stream} readOnly={readOnly} />;
- }
- return null;
- }
+ case "start_desktop":
+ return output ? desktopCard(output, readOnly) : null;
case "TodoWrite":
return <TodoList row={row} />;
case "read_workspace_file":
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/DesktopStreamCard.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/DesktopStreamCard.test.tsx
new file mode 100644
index 0000000000..3c7d732cc6
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/DesktopStreamCard.test.tsx
@@ -0,0 +1,48 @@
+import { render } from "@/tests/integrations/test-utils";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { useCopilotUIStore } from "../../../store";
+import { DesktopStreamCard } from "../DesktopStreamCard";
+
+let mobile = false;
+vi.mock("../../../useIsMobile", () => ({
+ useIsMobile: () => mobile,
+}));
+
+const STREAM = {
+ kind: "desktop_stream",
+ url: "/api/proxy/api/desktop-preview?token=abc",
+ provider: "e2b",
+ sandbox_id: "sbx-1",
+};
+
+beforeEach(() => {
+ mobile = false;
+ useCopilotUIStore.getState().resetArtifactPanel();
+ useCopilotUIStore.getState().closeArtifactPanel({ persist: false });
+});
+
+describe("DesktopStreamCard", () => {
+ it("opens the computer face for a desktop the model just started", () => {
+ render(<DesktopStreamCard stream={STREAM} />);
+
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.computer?.sandbox_id).toBe("sbx-1");
+ expect(panel.isComputerOpen).toBe(true);
+ });
+
+ it("on mobile remembers the desktop but opens no face the sheet cannot show", () => {
+ mobile = true;
+ render(<DesktopStreamCard stream={STREAM} />);
+
+ const panel = useCopilotUIStore.getState().artifactPanel;
+ expect(panel.computer?.sandbox_id).toBe("sbx-1");
+ expect(panel.isComputerOpen).toBe(false);
+ expect(panel.isOpen).toBe(false);
+ });
+
+ it("registers nothing for a shared transcript", () => {
+ render(<DesktopStreamCard stream={STREAM} readOnly />);
+
+ expect(useCopilotUIStore.getState().artifactPanel.computer).toBeNull();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
index 8bf339e8cb..4993924847 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/ToolResult.test.tsx
@@ -1011,3 +1011,73 @@ describe("ToolResult start_desktop", () => {
expect(screen.queryByTitle(/Interactive desktop/)).toBeNull();
});
});
+
+describe("ToolResult run_capability", () => {
+ afterEach(() => cleanup());
+
+ const desktopPayload = {
+ type: "desktop_stream",
+ message: "Desktop started.",
+ desktop_stream: {
+ kind: "desktop_stream",
+ url: "/api/proxy/api/desktop-preview?token=abc",
+ provider: "e2b",
+ sandbox_id: "sbx-2",
+ },
+ };
+
+ it("embeds the desktop when start_desktop ran as a capability", () => {
+ render(
+ <ToolResult
+ row={row(desktopPayload, "run_capability", {
+ id: "tool:start_desktop",
+ input: {},
+ })}
+ />,
+ );
+
+ const frame = screen.getByTitle("Interactive desktop (sbx-2)");
+ expect(frame.getAttribute("src")).toBe(
+ "/api/proxy/api/desktop-preview?token=abc",
+ );
+ expect(screen.queryByText(/"kind"/)).toBeNull();
+ expect(screen.queryByText(/desktop stream/i)).toBeNull();
+ });
+
+ it("keeps the shared-transcript notice on the capability path", () => {
+ render(
+ <ToolResult
+ readOnly
+ row={row(
+ {
+ ...desktopPayload,
+ desktop_stream: {
+ ...desktopPayload.desktop_stream,
+ requires_auth: true,
+ },
+ },
+ "resume_capability",
+ { id: "tool:start_desktop" },
+ )}
+ />,
+ );
+
+ expect(screen.queryByTitle(/Interactive desktop/)).toBeNull();
+ expect(screen.getByText(/only visible to the owner/i)).toBeDefined();
+ });
+
+ it("still renders a plain capability result as before", () => {
+ render(
+ <ToolResult
+ row={row(
+ { message: "Ran it.", result: { total: 3 } },
+ "run_capability",
+ { id: "tool:count" },
+ )}
+ />,
+ );
+
+ expect(screen.queryByTitle(/Interactive desktop/)).toBeNull();
+ expect(screen.getByText(/total/i)).toBeDefined();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
index 0a46b3f85b..1d355787cc 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/WorkspaceFileCards/__tests__/WorkspaceFileCards.test.tsx
@@ -219,6 +219,23 @@ describe("WorkspaceFileCards", () => {
expect(container.textContent).toBe("");
});
+ it("steps aside while the computer face covers the panel", () => {
+ useCopilotUIStore.setState({
+ artifactPanel: {
+ isOpen: true,
+ activeArtifact: null,
+ history: [],
+ activeTab: "files",
+ lastArtifact: null,
+ mode: "computer",
+ computer: null,
+ isComputerOpen: true,
+ },
+ });
+ const { container } = render(<WorkspaceFileCards sessionId={SESSION} />);
+ expect(container.textContent).toBe("");
+ });
+
it("requests the file list only while the card is open", async () => {
let listRequests = 0;
server.use(
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
index 172c432939..b612b6e12d 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/store.ts
@@ -229,10 +229,17 @@ interface CopilotUIState {
/** Forget the remembered preview — called on session entry so a new chat
* can never restore the previous chat's artifact. */
clearLastArtifact: () => void;
- /** Remember the chat's desktop stream and show it if nothing else is open. */
- registerComputerStream: (ref: DesktopStreamRef) => void;
+ /** Remember the chat's desktop stream and show it if nothing else is open.
+ * `show: false` only remembers it (mobile has no computer face to open). */
+ registerComputerStream: (
+ ref: DesktopStreamRef,
+ opts?: { show?: boolean },
+ ) => void;
/** Open the side panel on its computer face. */
openComputer: () => void;
+ /** Leave the computer face for whatever was under it: the preview, the
+ * tab, or a closed panel. Remembered previews survive either way. */
+ closeComputer: () => void;
setArtifactPanelMode: (mode: ArtifactPanelMode) => void;
openContextPanelForFiles: () => void;
showFilesTab: () => void;
@@ -270,6 +277,9 @@ interface CopilotUIState {
const _autoOpenKnownIds = new Set<string>();
let _autoOpenReady = false;
let _autoOpenUserClosed = false;
+// Whether the computer face was opened over a closed panel, so leaving it
+// closes the panel again instead of revealing a tab nobody had open.
+let _computerOverClosedPanel = false;
export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
initialPrompt: null,
@@ -384,6 +394,9 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
activeArtifact: ref,
history,
mode: "artifact",
+ // A document opened over the computer face takes the panel; the
+ // flag must follow, or the controls read the computer as showing.
+ isComputerOpen: false,
},
};
}),
@@ -436,12 +449,20 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
isComputerOpen: false,
},
})),
- registerComputerStream: (ref) =>
+ registerComputerStream: (ref, opts) =>
set((state) => {
- const { activeArtifact, isComputerOpen } = state.artifactPanel;
+ const { activeArtifact, isComputerOpen, isOpen, computer } =
+ state.artifactPanel;
// A desktop the model just started is the thing to look at: show it
- // unless an artifact preview is already holding the panel.
- const showNow = activeArtifact == null || isComputerOpen;
+ // unless an artifact preview is already holding the panel. A desktop
+ // already registered is not news: its card remounts whenever the
+ // streaming chain collapses and re-expands its rows, and that must not
+ // reopen a computer the user has hidden.
+ const isNews = computer?.sandbox_id !== ref.sandbox_id;
+ const showNow =
+ opts?.show !== false &&
+ (isComputerOpen || (isNews && activeArtifact == null));
+ if (showNow && !isComputerOpen) _computerOverClosedPanel = !isOpen;
return {
artifactPanel: {
...state.artifactPanel,
@@ -454,24 +475,54 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
};
}),
openComputer: () =>
- set((state) => ({
- artifactPanel: {
- ...state.artifactPanel,
- isOpen: true,
- mode: "computer",
- isComputerOpen: true,
- },
- })),
+ set((state) => {
+ if (!state.artifactPanel.isComputerOpen)
+ _computerOverClosedPanel = !state.artifactPanel.isOpen;
+ return {
+ artifactPanel: {
+ ...state.artifactPanel,
+ isOpen: true,
+ mode: "computer",
+ isComputerOpen: true,
+ },
+ };
+ }),
+ closeComputer: () =>
+ set((state) => {
+ const { activeArtifact } = state.artifactPanel;
+ // Under a preview or an open tab the document face is already there;
+ // over a panel that was closed, close it again. Neither path touches
+ // lastArtifact, unlike closeArtifactPanel.
+ const closeAgain = activeArtifact == null && _computerOverClosedPanel;
+ _computerOverClosedPanel = false;
+ if (closeAgain && isClient)
+ storage.set(Key.COPILOT_CONTEXT_PANEL_OPEN, "false");
+ return {
+ artifactPanel: {
+ ...state.artifactPanel,
+ isOpen: closeAgain ? false : state.artifactPanel.isOpen,
+ mode: "artifact",
+ isComputerOpen: false,
+ },
+ };
+ }),
setArtifactPanelMode: (mode) =>
- set((state) => ({
- artifactPanel: {
- ...state.artifactPanel,
- mode,
- // Switching to the artifact face closes the computer face, so a
- // later registerComputerStream does not flip the panel back.
- isComputerOpen: mode === "computer",
- },
- })),
+ set((state) => {
+ // openComputer persists nothing (a reload cannot restore the computer
+ // face), so a panel it opened is still stored as closed. Turned to its
+ // document face it is an ordinary open panel and must survive a reload.
+ if (isClient && mode === "artifact" && state.artifactPanel.isOpen)
+ storage.set(Key.COPILOT_CONTEXT_PANEL_OPEN, "true");
+ return {
+ artifactPanel: {
+ ...state.artifactPanel,
+ mode,
+ // Switching to the artifact face closes the computer face, so a
+ // later registerComputerStream does not flip the panel back.
+ isComputerOpen: mode === "computer",
+ },
+ };
+ }),
goBackArtifact: () =>
set((state) => {
const { history } = state.artifactPanel;
@@ -519,10 +570,17 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
})),
toggleContextPanelTab: (tab) =>
set((state) => {
- const { isOpen, activeTab, activeArtifact } = state.artifactPanel;
- // An open preview covers the panel, so a click there means "show me the
- // tab again" rather than "close" — only a visible matching tab closes.
- const nextOpen = !(isOpen && activeTab === tab && activeArtifact == null);
+ const { isOpen, activeTab, activeArtifact, isComputerOpen } =
+ state.artifactPanel;
+ // An open preview or the computer face covers the panel, so a click
+ // there means "show me the tab again" rather than "close" — only a
+ // visible matching tab closes.
+ const nextOpen = !(
+ isOpen &&
+ activeTab === tab &&
+ activeArtifact == null &&
+ !isComputerOpen
+ );
if (isClient) {
storage.set(Key.COPILOT_CONTEXT_PANEL_OPEN, String(nextOpen));
storage.set(Key.COPILOT_CONTEXT_PANEL_TAB, tab);
@@ -538,6 +596,9 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
// Closing from the tab view forgets the remembered preview, so the
// next sidebar click reopens the tab rather than an older artifact.
lastArtifact: nextOpen ? state.artifactPanel.lastArtifact : null,
+ // The tab takes the panel from the computer face.
+ mode: "artifact",
+ isComputerOpen: false,
},
};
}),
@@ -554,12 +615,15 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
activeTab: "files",
activeArtifact: null,
history: [],
+ mode: "artifact",
+ isComputerOpen: false,
},
}));
},
// Explicit user action (the artifact panel's files button): drops the open
- // preview and hands the region to the floating files card.
+ // preview or the computer face and hands the region to the floating files
+ // card.
showFilesTab: () => {
if (isClient) {
storage.set(Key.COPILOT_CONTEXT_PANEL_OPEN, "true");
@@ -572,6 +636,8 @@ export const useCopilotUIStore = create<CopilotUIState>((set, get) => ({
activeTab: "files",
activeArtifact: null,
history: [],
+ mode: "artifact",
+ isComputerOpen: false,
},
}));
},
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useAreWorkspaceFileCardsOpen.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useAreWorkspaceFileCardsOpen.ts
index 9423ea2f45..321a12979c 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useAreWorkspaceFileCardsOpen.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useAreWorkspaceFileCardsOpen.ts
@@ -7,11 +7,13 @@ import { useCopilotUIStore } from "./store";
// preview means the card is showing. The docked panel narrows the chat
// column on its own, so it must never also trigger the slide, or the two
// shifts stack and push the messages off screen under the app sidebar.
+// The computer face docks the same way, whatever tab is remembered under it.
export function useAreWorkspaceFileCardsOpen() {
return useCopilotUIStore(
(s) =>
s.artifactPanel.isOpen &&
s.artifactPanel.activeArtifact == null &&
+ !s.artifactPanel.isComputerOpen &&
s.artifactPanel.activeTab !== "artifacts",
);
}