AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(frontend/copilot): pin interactive tool cards outside reasoning collapse (#12346)
Ubbe8063391d0a94
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsxindex df253468ef..f7a72acc26 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx@@ -5,10 +5,17 @@ import {} from "@/components/ai-elements/conversation";import { Message, MessageContent } from "@/components/ai-elements/message";import { LoadingSpinner } from "@/components/atoms/LoadingSpinner/LoadingSpinner";-import { FileUIPart, ToolUIPart, UIDataTypes, UIMessage, UITools } from "ai";+import { FileUIPart, UIDataTypes, UIMessage, UITools } from "ai";import { TOOL_PART_PREFIX } from "../JobStatsBar/constants";import { TurnStatsBar } from "../JobStatsBar/TurnStatsBar";-import { parseSpecialMarkers } from "./helpers";+import {+ buildRenderSegments,+ getTurnMessages,+ type MessagePart,+ type RenderSegment,+ parseSpecialMarkers,+ splitReasoningAndResponse,+} from "./helpers";import { AssistantMessageActions } from "./components/AssistantMessageActions";import { CollapsedToolGroup } from "./components/CollapsedToolGroup";import { MessageAttachments } from "./components/MessageAttachments";@@ -16,8 +23,6 @@ import { MessagePartRenderer } from "./components/MessagePartRenderer";import { ReasoningCollapse } from "./components/ReasoningCollapse";import { ThinkingIndicator } from "./components/ThinkingIndicator";-type MessagePart = UIMessage<unknown, UIDataTypes, UITools>["parts"][number];-interface Props {messages: UIMessage<unknown, UIDataTypes, UITools>[];status: string;@@ -27,113 +32,6 @@ interface Props {sessionID?: string | null;}-function isCompletedToolPart(part: MessagePart): part is ToolUIPart {- return (- part.type.startsWith("tool-") &&- "state" in part &&- (part.state === "output-available" || part.state === "output-error")- );-}--type RenderSegment =- | { kind: "part"; part: MessagePart; index: number }- | { kind: "collapsed-group"; parts: ToolUIPart[] };--// Tool types that have custom renderers and should NOT be collapsed-const CUSTOM_TOOL_TYPES = new Set([- "tool-find_block",- "tool-find_agent",- "tool-find_library_agent",- "tool-search_docs",- "tool-get_doc_page",- "tool-run_block",- "tool-run_mcp_tool",- "tool-run_agent",- "tool-schedule_agent",- "tool-create_agent",- "tool-edit_agent",- "tool-view_agent_output",- "tool-search_feature_requests",- "tool-create_feature_request",-]);--/**- * Groups consecutive completed generic tool parts into collapsed segments.- * Non-generic tools (those with custom renderers) and active/streaming tools- * are left as individual parts.- */-function buildRenderSegments(- parts: MessagePart[],- baseIndex = 0,-): RenderSegment[] {- const segments: RenderSegment[] = [];- let pendingGroup: Array<{ part: ToolUIPart; index: number }> | null = null;-- function flushGroup() {- if (!pendingGroup) return;- if (pendingGroup.length >= 2) {- segments.push({- kind: "collapsed-group",- parts: pendingGroup.map((p) => p.part),- });- } else {- for (const p of pendingGroup) {- segments.push({ kind: "part", part: p.part, index: p.index });- }- }- pendingGroup = null;- }-- parts.forEach((part, i) => {- const absoluteIndex = baseIndex + i;- const isGenericCompletedTool =- isCompletedToolPart(part) && !CUSTOM_TOOL_TYPES.has(part.type);-- if (isGenericCompletedTool) {- if (!pendingGroup) pendingGroup = [];- pendingGroup.push({ part: part as ToolUIPart, index: absoluteIndex });- } else {- flushGroup();- segments.push({ kind: "part", part, index: absoluteIndex });- }- });-- flushGroup();- return segments;-}--/**- * For finalized assistant messages, split parts into "reasoning" (intermediate- * text + tools before the final response) and "response" (final text after the- * last tool). If there are no tools, everything is response.- */-function splitReasoningAndResponse(parts: MessagePart[]): {- reasoning: MessagePart[];- response: MessagePart[];-} {- const lastToolIndex = parts.findLastIndex((p) => p.type.startsWith("tool-"));-- // No tools → everything is response- if (lastToolIndex === -1) {- return { reasoning: [], response: parts };- }-- // Check if there's any text after the last tool- const hasResponseAfterTools = parts- .slice(lastToolIndex + 1)- .some((p) => p.type === "text");-- if (!hasResponseAfterTools) {- // No final text response → don't collapse anything- return { reasoning: [], response: parts };- }-- return {- reasoning: parts.slice(0, lastToolIndex + 1),- response: parts.slice(lastToolIndex + 1),- };-}-function renderSegments(segments: RenderSegment[],messageID: string,@@ -153,23 +51,6 @@ function renderSegments(});}-/** Collect all messages belonging to a turn: the user message + every- * assistant message up to (but not including) the next user message. */-function getTurnMessages(- messages: UIMessage<unknown, UIDataTypes, UITools>[],- lastAssistantIndex: number,-): UIMessage<unknown, UIDataTypes, UITools>[] {- const userIndex = messages.findLastIndex(- (m, i) => i < lastAssistantIndex && m.role === "user",- );- const nextUserIndex = messages.findIndex(- (m, i) => i > lastAssistantIndex && m.role === "user",- );- const start = userIndex >= 0 ? userIndex : lastAssistantIndex;- const end = nextUserIndex >= 0 ? nextUserIndex : messages.length;- return messages.slice(start, end);-}-export function ChatMessagesContainer({messages,status,@@ -258,6 +139,8 @@ export function ChatMessagesContainer({: { reasoning: [] as MessagePart[], response: message.parts };const hasReasoning = reasoning.length > 0;+ // Note: when interactive tools are pinned from reasoning into response,+ // this index approximates their position (used only for React keys).const responseStartIndex = message.parts.length - response.length;const responseSegments =message.role === "assistant"diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/helpers.tsindex 7ccf317e76..9a1950d65a 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/helpers.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/helpers.ts@@ -1,4 +1,170 @@import { getGetWorkspaceDownloadFileByIdUrl } from "@/app/api/__generated__/endpoints/workspace/workspace";+import { ResponseType } from "@/app/api/__generated__/models/responseType";+import { ToolUIPart, UIDataTypes, UIMessage, UITools } from "ai";++export type MessagePart = UIMessage<+ unknown,+ UIDataTypes,+ UITools+>["parts"][number];++export type RenderSegment =+ | { kind: "part"; part: MessagePart; index: number }+ | { kind: "collapsed-group"; parts: ToolUIPart[] };++const CUSTOM_TOOL_TYPES = new Set([+ "tool-find_block",+ "tool-find_agent",+ "tool-find_library_agent",+ "tool-search_docs",+ "tool-get_doc_page",+ "tool-run_block",+ "tool-run_mcp_tool",+ "tool-run_agent",+ "tool-schedule_agent",+ "tool-create_agent",+ "tool-edit_agent",+ "tool-view_agent_output",+ "tool-search_feature_requests",+ "tool-create_feature_request",+]);++const INTERACTIVE_RESPONSE_TYPES: ReadonlySet<string> = new Set([+ ResponseType.setup_requirements,+ ResponseType.agent_details,+ ResponseType.block_details,+ ResponseType.need_login,+ ResponseType.input_validation_error,+ ResponseType.clarification_needed,+ ResponseType.suggested_goal,+ ResponseType.agent_preview,+ ResponseType.agent_saved,+]);++export function isCompletedToolPart(part: MessagePart): part is ToolUIPart {+ return (+ part.type.startsWith("tool-") &&+ "state" in part &&+ (part.state === "output-available" || part.state === "output-error")+ );+}++export function isInteractiveToolPart(part: MessagePart): boolean {+ if (!part.type.startsWith("tool-")) return false;+ if (!("state" in part) || part.state !== "output-available") return false;++ let output = (part as ToolUIPart).output;+ if (!output) return false;++ if (typeof output === "string") {+ try {+ output = JSON.parse(output);+ } catch {+ return false;+ }+ }++ if (typeof output !== "object" || output === null) return false;++ const responseType = (output as Record<string, unknown>).type;+ return (+ typeof responseType === "string" &&+ INTERACTIVE_RESPONSE_TYPES.has(responseType)+ );+}++export function buildRenderSegments(+ parts: MessagePart[],+ baseIndex = 0,+): RenderSegment[] {+ const segments: RenderSegment[] = [];+ let pendingGroup: Array<{ part: ToolUIPart; index: number }> | null = null;++ function flushGroup() {+ if (!pendingGroup) return;+ if (pendingGroup.length >= 2) {+ segments.push({+ kind: "collapsed-group",+ parts: pendingGroup.map((p) => p.part),+ });+ } else {+ for (const p of pendingGroup) {+ segments.push({ kind: "part", part: p.part, index: p.index });+ }+ }+ pendingGroup = null;+ }++ parts.forEach((part, i) => {+ const absoluteIndex = baseIndex + i;+ const isGenericCompletedTool =+ isCompletedToolPart(part) && !CUSTOM_TOOL_TYPES.has(part.type);++ if (isGenericCompletedTool) {+ if (!pendingGroup) pendingGroup = [];+ pendingGroup.push({ part: part as ToolUIPart, index: absoluteIndex });+ } else {+ flushGroup();+ segments.push({ kind: "part", part, index: absoluteIndex });+ }+ });++ flushGroup();+ return segments;+}++export function splitReasoningAndResponse(parts: MessagePart[]): {+ reasoning: MessagePart[];+ response: MessagePart[];+} {+ const lastToolIndex = parts.findLastIndex((p) => p.type.startsWith("tool-"));++ if (lastToolIndex === -1) {+ return { reasoning: [], response: parts };+ }++ const hasResponseAfterTools = parts+ .slice(lastToolIndex + 1)+ .some((p) => p.type === "text");++ if (!hasResponseAfterTools) {+ return { reasoning: [], response: parts };+ }++ const rawReasoning = parts.slice(0, lastToolIndex + 1);+ const rawResponse = parts.slice(lastToolIndex + 1);++ const reasoning: MessagePart[] = [];+ const pinnedParts: MessagePart[] = [];++ for (const part of rawReasoning) {+ if (isInteractiveToolPart(part)) {+ pinnedParts.push(part);+ } else {+ reasoning.push(part);+ }+ }++ return {+ reasoning,+ response: [...pinnedParts, ...rawResponse],+ };+}++export function getTurnMessages(+ messages: UIMessage<unknown, UIDataTypes, UITools>[],+ lastAssistantIndex: number,+): UIMessage<unknown, UIDataTypes, UITools>[] {+ const userIndex = messages.findLastIndex(+ (m, i) => i < lastAssistantIndex && m.role === "user",+ );+ const nextUserIndex = messages.findIndex(+ (m, i) => i > lastAssistantIndex && m.role === "user",+ );+ const start = userIndex >= 0 ? userIndex : lastAssistantIndex;+ const end = nextUserIndex >= 0 ? nextUserIndex : messages.length;+ return messages.slice(start, end);+}// Special message prefixes for text-based markers (set by backend).// The hex suffix makes it virtually impossible for an LLM to accidentallydiff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/styleguide/page.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/styleguide/page.tsxindex ba582cea17..8a9f2739fd 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/styleguide/page.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/styleguide/page.tsx@@ -28,6 +28,8 @@ import { FindBlocksTool } from "../tools/FindBlocks/FindBlocks";import { RunAgentTool } from "../tools/RunAgent/RunAgent";import { RunBlockTool } from "../tools/RunBlock/RunBlock";import { SearchDocsTool } from "../tools/SearchDocs/SearchDocs";+import { ReasoningCollapse } from "../components/ChatMessagesContainer/components/ReasoningCollapse";+import { GenericTool } from "../tools/GenericTool/GenericTool";import { ViewAgentOutputTool } from "../tools/ViewAgentOutput/ViewAgentOutput";// ---------------------------------------------------------------------------@@ -57,6 +59,7 @@ const SECTIONS = ["Tool: Search Feature Requests","Tool: Create Feature Request","Full Conversation Example",+ "Reasoning Collapse: Interactive Tool Pinning",] as const;function Section({@@ -1833,6 +1836,258 @@ export default function StyleguidePage() {</ConversationContent></Conversation></Section>++ {/* ============================================================= */}+ {/* REASONING COLLAPSE: INTERACTIVE TOOL PINNING */}+ {/* ============================================================= */}++ <Section title="Reasoning Collapse: Interactive Tool Pinning">+ <p className="mb-4 text-sm text-neutral-600">+ When the stream finishes, intermediate tool calls are collapsed+ behind a "Show reasoning" button. However, tools whose+ output requires user interaction (credentials, inputs,+ clarification) are <strong>pinned</strong> and remain visible+ outside the collapse.+ </p>++ <SubSection label="Collapsed reasoning with pinned setup_requirements">+ <Conversation className="min-h-0 rounded-lg border bg-white">+ <ConversationContent className="gap-6 px-3 py-6">+ <Message from="assistant">+ <MessageContent className="text-[1rem] leading-relaxed group-[.is-assistant]:bg-transparent group-[.is-assistant]:text-slate-900">+ <ReasoningCollapse>+ <GenericTool+ part={{+ type: "tool-bash_exec",+ toolCallId: uid(),+ state: "output-available",+ input: { command: "echo hello" },+ output: {+ type: ResponseType.bash_exec,+ stdout: "hello",+ stderr: "",+ exit_code: 0,+ message: "Command completed",+ },+ }}+ />+ <FindBlocksTool+ part={{+ type: "tool-find_block",+ toolCallId: uid(),+ state: "output-available",+ input: { query: "weather" },+ output: {+ type: ResponseType.block_list,+ blocks: [+ {+ id: "block-1",+ name: "Get Weather",+ description: "Fetches weather data.",+ categories: [],+ },+ ],+ count: 1,+ },+ }}+ />+ </ReasoningCollapse>++ <RunBlockTool+ part={{+ type: "tool-run_block",+ toolCallId: uid(),+ state: "output-available",+ input: {+ block_id: "block-1",+ block_name: "Get Weather",+ },+ output: {+ type: ResponseType.setup_requirements,+ message:+ "Missing credentials for Get Weather block.",+ setup_info: {+ agent_id: "block-1",+ agent_name: "Get Weather",+ requirements: {+ credentials: [+ {+ id: "openweather-api",+ provider: "openweather",+ type: "api_key",+ title: "OpenWeather API Key",+ description: "Required for weather data.",+ },+ ],+ inputs: [],+ execution_modes: [],+ },+ user_readiness: {+ has_all_credentials: false,+ missing_credentials: {+ openweather: {+ id: "openweather-api",+ provider: "openweather",+ type: "api_key",+ title: "OpenWeather API Key",+ },+ },+ ready_to_run: false,+ },+ },+ },+ }}+ />++ <MessageResponse>+ The Get Weather block requires an OpenWeather API key.+ Please configure it in your credentials to proceed.+ </MessageResponse>+ </MessageContent>+ </Message>+ </ConversationContent>+ </Conversation>+ </SubSection>++ <SubSection label="Collapsed reasoning with pinned agent_details (inputs required)">+ <Conversation className="min-h-0 rounded-lg border bg-white">+ <ConversationContent className="gap-6 px-3 py-6">+ <Message from="assistant">+ <MessageContent className="text-[1rem] leading-relaxed group-[.is-assistant]:bg-transparent group-[.is-assistant]:text-slate-900">+ <ReasoningCollapse>+ <FindAgentsTool+ part={{+ type: "tool-find_library_agent",+ toolCallId: uid(),+ state: "output-available",+ input: { query: "email sender" },+ output: {+ type: ResponseType.agents_found,+ title: "Library Agents",+ agents: [+ {+ id: "agent-email",+ name: "Email Sender",+ description: "Sends emails via SMTP.",+ source: "library",+ in_library: true,+ },+ ],+ count: 1,+ },+ }}+ />+ </ReasoningCollapse>++ <RunAgentTool+ part={{+ type: "tool-run_agent",+ toolCallId: uid(),+ state: "output-available",+ input: { library_agent_id: "agent-email" },+ output: {+ type: ResponseType.agent_details,+ message:+ "Agent requires input values before it can run.",+ agent: {+ id: "agent-email",+ name: "Email Sender",+ description: "Sends emails via SMTP.",+ in_library: true,+ inputs: {+ properties: {+ to: {+ type: "string",+ description: "Recipient email",+ },+ subject: {+ type: "string",+ description: "Email subject",+ },+ body: {+ type: "string",+ description: "Email body",+ },+ },+ required: ["to", "subject", "body"],+ },+ credentials: [],+ },+ },+ }}+ />++ <MessageResponse>+ I found the Email Sender agent. It needs a few inputs+ before it can run. Please provide the recipient,+ subject, and body.+ </MessageResponse>+ </MessageContent>+ </Message>+ </ConversationContent>+ </Conversation>+ </SubSection>++ <SubSection label="Collapsed reasoning with pinned agent_saved">+ <Conversation className="min-h-0 rounded-lg border bg-white">+ <ConversationContent className="gap-6 px-3 py-6">+ <Message from="assistant">+ <MessageContent className="text-[1rem] leading-relaxed group-[.is-assistant]:bg-transparent group-[.is-assistant]:text-slate-900">+ <ReasoningCollapse>+ <FindBlocksTool+ part={{+ type: "tool-find_block",+ toolCallId: uid(),+ state: "output-available",+ input: { query: "HTTP request" },+ output: {+ type: ResponseType.block_list,+ blocks: [+ {+ id: "block-http",+ name: "HTTP Request",+ description: "Makes HTTP requests.",+ categories: [],+ },+ ],+ count: 1,+ },+ }}+ />+ </ReasoningCollapse>++ <CreateAgentTool+ part={{+ type: "tool-create_agent",+ toolCallId: uid(),+ state: "output-available",+ input: {+ description:+ "An agent that checks website uptime",+ },+ output: {+ type: ResponseType.agent_saved,+ message: "Agent saved to your library!",+ agent_id: "graph-123",+ agent_name: "Website Uptime Checker",+ library_agent_id: "lib-agent-456",+ library_agent_link:+ "/marketplace/agent/lib-agent-456",+ agent_page_link: "/build?agentId=graph-123",+ },+ }}+ />++ <MessageResponse>+ Your **Website Uptime Checker** agent has been created+ and saved to your library!+ </MessageResponse>+ </MessageContent>+ </Message>+ </ConversationContent>+ </Conversation>+ </SubSection>+ </Section></div></div></div>