🙌 OpenHands: AI-Driven Development
feat: add sub-agent task (TaskToolSet) visualizer to the chat UI (#14928)
Vasco Schiavobfd9af088ffb
diff --git a/frontend/__tests__/components/v1/get-event-content.test.tsx b/frontend/__tests__/components/v1/get-event-content.test.tsxindex 91851cd772..65de2e84ef 100644--- a/frontend/__tests__/components/v1/get-event-content.test.tsx+++ b/frontend/__tests__/components/v1/get-event-content.test.tsx@@ -58,6 +58,51 @@ const terminalObservationEvent: ObservationEvent = {},};+const taskActionEvent: ActionEvent = {+ id: "action-task-1",+ timestamp: new Date().toISOString(),+ source: "agent",+ thought: [{ type: "text", text: "I'll ask a sub-agent to count files." }],+ thinking_blocks: [],+ action: {+ kind: "TaskAction",+ description: "count repository files",+ prompt: "Count how many files are in the repo using git ls-files | wc -l.",+ subagent_type: "bash-runner",+ resume: null,+ },+ tool_name: "task",+ tool_call_id: "tool-task-1",+ tool_call: {+ id: "tool-task-1",+ type: "function",+ function: {+ name: "task",+ arguments: '{"prompt":"Count files","subagent_type":"bash-runner"}',+ },+ },+ llm_response_id: "response-task-1",+ security_risk: SecurityRisk.LOW,+ summary: "Ask sub-agent to count repository files",+};++const taskObservationEvent: ObservationEvent = {+ id: "obs-task-1",+ timestamp: new Date().toISOString(),+ source: "environment",+ tool_name: "task",+ tool_call_id: "tool-task-1",+ action_id: "action-task-1",+ observation: {+ kind: "TaskObservation",+ content: [{ type: "text", text: "tracked_count=1606" }],+ is_error: false,+ task_id: "task_00000002",+ subagent: "bash-runner",+ status: "completed",+ },+};+describe("getEventContent", () => {it("uses the action summary as the full action title", () => {const { title } = getEventContent(terminalActionEvent);@@ -145,4 +190,85 @@ describe("getEventContent", () => {expect(screen.getByText("Check repository status")).toBeInTheDocument();expect(screen.queryByText("$ git status")).not.toBeInTheDocument();});++ it("renders the sub-agent task card with subagent, task id, query and result", () => {+ const { details } = getEventContent(taskObservationEvent, taskActionEvent);++ render(<div>{details}</div>);++ // Labels render as raw i18n keys because i18n is not loaded in tests+ expect(+ screen.getByText("SUBAGENT_OBSERVATION$SUBAGENT:"),+ ).toBeInTheDocument();+ expect(screen.getByText("bash-runner")).toBeInTheDocument();+ expect(+ screen.getByText("SUBAGENT_OBSERVATION$TASK_ID:"),+ ).toBeInTheDocument();+ expect(screen.getByText("task_00000002")).toBeInTheDocument();+ expect(screen.getByText("SUBAGENT_OBSERVATION$QUERY")).toBeInTheDocument();+ expect(+ screen.getByText(+ "Count how many files are in the repo using git ls-files | wc -l.",+ ),+ ).toBeInTheDocument();+ expect(screen.getByText("SUBAGENT_OBSERVATION$RESULT")).toBeInTheDocument();+ expect(screen.getByText("tracked_count=1606")).toBeInTheDocument();+ });++ it("falls back to the task title key when the action summary is missing", () => {+ const actionWithoutSummary = { ...taskActionEvent, summary: undefined };+ const { title } = getEventContent(+ taskObservationEvent,+ actionWithoutSummary,+ );++ render(<span>{title}</span>);++ expect(screen.getByText("OBSERVATION_MESSAGE$TASK")).toBeInTheDocument();+ });++ it("shows the subagent and query in the in-progress task action card", () => {+ const { details } = getEventContent(taskActionEvent);++ expect(details).toContain("**Subagent:** `bash-runner`");+ expect(details).toContain(+ "Count how many files are in the repo using git ls-files | wc -l.",+ );+ });++ it("renders image content in the sub-agent result instead of dropping it", () => {+ const observationWithImage: ObservationEvent = {+ ...taskObservationEvent,+ observation: {+ kind: "TaskObservation",+ content: [+ { type: "text", text: "Here is a screenshot:" },+ { type: "image", image_urls: ["data:image/png;base64,abc123"] },+ ],+ is_error: false,+ task_id: "task_00000002",+ subagent: "bash-runner",+ status: "completed",+ },+ };++ const { details } = getEventContent(observationWithImage, taskActionEvent);++ const { container } = render(<div>{details}</div>);++ expect(screen.getByText("Here is a screenshot:")).toBeInTheDocument();+ const image = container.querySelector("img");+ expect(image).toHaveAttribute("src", "data:image/png;base64,abc123");+ });++ it("omits the query section when the task observation has no paired action", () => {+ const { details } = getEventContent(taskObservationEvent);++ render(<div>{details}</div>);++ expect(screen.getByText("tracked_count=1606")).toBeInTheDocument();+ expect(+ screen.queryByText("SUBAGENT_OBSERVATION$QUERY"),+ ).not.toBeInTheDocument();+ });});diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.tsindex c0f2bf9db8..217cc19da9 100644--- a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts+++ b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts@@ -23,6 +23,7 @@ import {BrowserCloseTabAction,GlobAction,GrepAction,+ TaskAction,} from "#/types/v1/core/base/action";const getRiskText = (risk: SecurityRisk) => {@@ -137,6 +138,16 @@ const getTaskTrackerActionContent = (action: TaskTrackerAction): string => {return content;};+// Sub-agent task action (TaskToolSet). Shown while the sub-agent is still+// running, before its TaskObservation arrives.+const getTaskActionContent = (action: TaskAction): string => {+ const parts: string[] = [`**Subagent:** \`${action.subagent_type}\``];+ if (action.prompt) {+ parts.push(`**Query**\n\n${action.prompt}`);+ }+ return parts.join("\n\n");+};+// Browser Actionstype BrowserAction =| BrowserNavigateAction@@ -236,6 +247,9 @@ export const getActionContent = (event: ActionEvent): string => {case "TaskTrackerAction":return getTaskTrackerActionContent(action);+ case "TaskAction":+ return getTaskActionContent(action);+case "BrowserNavigateAction":case "BrowserClickAction":case "BrowserTypeAction":diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsxindex 66a004ca99..165cb71c3e 100644--- a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx+++ b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx@@ -12,7 +12,11 @@ import { getActionContent } from "./get-action-content";import { getObservationContent } from "./get-observation-content";import { getACPToolCallContent } from "./get-acp-tool-call-content";import { TaskTrackingObservationContent } from "../task-tracking/task-tracking-observation-content";-import { TaskTrackerObservation } from "#/types/v1/core/base/observation";+import { SubagentObservationContent } from "../subagent/subagent-observation-content";+import {+ TaskTrackerObservation,+ TaskObservation,+} from "#/types/v1/core/base/observation";import { SkillReadyEvent, isSkillReadyEvent } from "./create-skill-ready-event";import i18n from "#/i18n";@@ -101,6 +105,9 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => {case "TaskTrackerAction":actionKey = "ACTION_MESSAGE$TASK_TRACKING";break;+ case "TaskAction":+ actionKey = "ACTION_MESSAGE$TASK";+ break;case "GrepAction":actionKey = "ACTION_MESSAGE$GREP";actionValues = {@@ -204,6 +211,9 @@ const getObservationEventTitle = (}break;}+ case "TaskObservation":+ observationKey = "OBSERVATION_MESSAGE$TASK";+ break;case "ThinkObservation":observationKey = "OBSERVATION_MESSAGE$THINK";break;@@ -265,6 +275,14 @@ export const getEventContent = (event={event as ObservationEvent<TaskTrackerObservation>}/>);+ } else if (event.observation.kind === "TaskObservation") {+ // The Query comes from the paired action's prompt.+ details = (+ <SubagentObservationContent+ event={event as ObservationEvent<TaskObservation>}+ correspondingAction={correspondingAction}+ />+ );} else {details = getObservationContent(event);}diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.tsindex 8bf3bda325..b893aff764 100644--- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts+++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts@@ -49,6 +49,7 @@ export const getObservationResult = (if (observation.error) return "error";return "success";case "MCPToolObservation":+ case "TaskObservation":if (observation.is_error) return "error";return "success";default:diff --git a/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsx b/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsxnew file mode 100644index 0000000000..fc92f49e76--- /dev/null+++ b/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsx@@ -0,0 +1,88 @@+import React from "react";+import { useTranslation } from "react-i18next";+import { I18nKey } from "#/i18n/declaration";+import { ActionEvent, ObservationEvent } from "#/types/v1/core";+import { TaskObservation } from "#/types/v1/core/base/observation";+import { MarkdownRenderer } from "#/components/features/markdown/markdown-renderer";++interface SubagentObservationContentProps {+ event: ObservationEvent<TaskObservation>;+ correspondingAction?: ActionEvent;+}++const getResultText = (observation: TaskObservation): string =>+ observation.content+ .filter((c) => c.type === "text")+ .map((c) => c.text)+ .join("\n");++const getResultImages = (observation: TaskObservation): string[] =>+ observation.content+ .filter((c) => c.type === "image")+ .flatMap((c) => c.image_urls);++const getQuery = (correspondingAction?: ActionEvent): string | undefined => {+ const action = correspondingAction?.action;+ return action?.kind === "TaskAction" ? action.prompt : undefined;+};++/**+ * Rich card body for a sub-agent task (TaskToolSet). Shows which sub-agent ran+ * the task, its task id, the query it was given (from the paired action) and+ * the result it returned. Mirrors the TaskTrackerObservation visualizer.+ */+export function SubagentObservationContent({+ event,+ correspondingAction,+}: SubagentObservationContentProps): React.ReactNode {+ const { t } = useTranslation();+ const { observation } = event;+ const query = getQuery(correspondingAction);+ const resultText = getResultText(observation);+ const resultImages = getResultImages(observation);++ return (+ <div className="flex flex-col gap-3 text-neutral-300">+ <div className="flex flex-col gap-1">+ <div className="flex gap-1">+ <span className="font-bold">+ {t(I18nKey.SUBAGENT_OBSERVATION$SUBAGENT)}:+ </span>+ <span>{observation.subagent}</span>+ </div>+ <div className="flex gap-1">+ <span className="font-bold">+ {t(I18nKey.SUBAGENT_OBSERVATION$TASK_ID)}:+ </span>+ <span>{observation.task_id}</span>+ </div>+ </div>++ {query && (+ <div className="flex flex-col gap-1">+ <span className="font-bold">+ {t(I18nKey.SUBAGENT_OBSERVATION$QUERY)}+ </span>+ <div className="rounded-md bg-neutral-700 p-2 whitespace-pre-wrap break-words">+ {query}+ </div>+ </div>+ )}++ <div className="flex flex-col gap-1">+ <span className="font-bold">+ {t(I18nKey.SUBAGENT_OBSERVATION$RESULT)}+ </span>+ {resultText && <MarkdownRenderer>{resultText}</MarkdownRenderer>}+ {resultImages.map((url) => (+ <img+ key={url}+ src={url}+ alt={t(I18nKey.SUBAGENT_OBSERVATION$RESULT)}+ className="max-w-full rounded-md"+ />+ ))}+ </div>+ </div>+ );+}diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.tsindex 8aa14eacff..2d6ffbdc6b 100644--- a/frontend/src/i18n/declaration.ts+++ b/frontend/src/i18n/declaration.ts@@ -53,6 +53,10 @@ export enum I18nKey {TASK_TRACKING_OBSERVATION$TASK_ID = "TASK_TRACKING_OBSERVATION$TASK_ID",TASK_TRACKING_OBSERVATION$TASK_NOTES = "TASK_TRACKING_OBSERVATION$TASK_NOTES",TASK_TRACKING_OBSERVATION$RESULT = "TASK_TRACKING_OBSERVATION$RESULT",+ SUBAGENT_OBSERVATION$SUBAGENT = "SUBAGENT_OBSERVATION$SUBAGENT",+ SUBAGENT_OBSERVATION$TASK_ID = "SUBAGENT_OBSERVATION$TASK_ID",+ SUBAGENT_OBSERVATION$QUERY = "SUBAGENT_OBSERVATION$QUERY",+ SUBAGENT_OBSERVATION$RESULT = "SUBAGENT_OBSERVATION$RESULT",OBSERVATION$ERROR_PREFIX = "OBSERVATION$ERROR_PREFIX",TASK$ADDRESSING_TASK = "TASK$ADDRESSING_TASK",SECRETS$SECRET_VALUE_REQUIRED = "SECRETS$SECRET_VALUE_REQUIRED",@@ -739,6 +743,7 @@ export enum I18nKey {ACTION_MESSAGE$TASK_TRACKING = "ACTION_MESSAGE$TASK_TRACKING",ACTION_MESSAGE$GREP = "ACTION_MESSAGE$GREP",ACTION_MESSAGE$GLOB = "ACTION_MESSAGE$GLOB",+ ACTION_MESSAGE$TASK = "ACTION_MESSAGE$TASK",OBSERVATION_MESSAGE$RUN = "OBSERVATION_MESSAGE$RUN",OBSERVATION_MESSAGE$RUN_IPYTHON = "OBSERVATION_MESSAGE$RUN_IPYTHON",OBSERVATION_MESSAGE$READ = "OBSERVATION_MESSAGE$READ",@@ -752,6 +757,7 @@ export enum I18nKey {OBSERVATION_MESSAGE$GREP = "OBSERVATION_MESSAGE$GREP",OBSERVATION_MESSAGE$TASK_TRACKING_PLAN = "OBSERVATION_MESSAGE$TASK_TRACKING_PLAN",OBSERVATION_MESSAGE$TASK_TRACKING_VIEW = "OBSERVATION_MESSAGE$TASK_TRACKING_VIEW",+ OBSERVATION_MESSAGE$TASK = "OBSERVATION_MESSAGE$TASK",EXPANDABLE_MESSAGE$SHOW_DETAILS = "EXPANDABLE_MESSAGE$SHOW_DETAILS",EXPANDABLE_MESSAGE$HIDE_DETAILS = "EXPANDABLE_MESSAGE$HIDE_DETAILS",AI_SETTINGS$TITLE = "AI_SETTINGS$TITLE",diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.jsonindex ae1c9912af..c26db5abb4 100644--- a/frontend/src/i18n/translation.json+++ b/frontend/src/i18n/translation.json@@ -900,6 +900,74 @@"uk": "Результат","ca": "Resultat"},+ "SUBAGENT_OBSERVATION$SUBAGENT": {+ "en": "Subagent",+ "zh-CN": "子代理",+ "zh-TW": "子代理",+ "ko-KR": "하위 에이전트",+ "ja": "サブエージェント",+ "no": "Underagent",+ "ar": "الوكيل الفرعي",+ "de": "Sub-Agent",+ "fr": "Sous-agent",+ "it": "Sub-agente",+ "pt": "Subagente",+ "es": "Subagente",+ "tr": "Alt aracı",+ "uk": "Субагент",+ "ca": "Subagent"+ },+ "SUBAGENT_OBSERVATION$TASK_ID": {+ "en": "Task ID",+ "zh-CN": "任务 ID",+ "zh-TW": "任務 ID",+ "ko-KR": "작업 ID",+ "ja": "タスク ID",+ "no": "Oppgave-ID",+ "ar": "معرف المهمة",+ "de": "Aufgaben-ID",+ "fr": "ID de tâche",+ "it": "ID attività",+ "pt": "ID da tarefa",+ "es": "ID de tarea",+ "tr": "Görev kimliği",+ "uk": "ID завдання",+ "ca": "ID de tasca"+ },+ "SUBAGENT_OBSERVATION$QUERY": {+ "en": "Query",+ "zh-CN": "查询",+ "zh-TW": "查詢",+ "ko-KR": "쿼리",+ "ja": "クエリ",+ "no": "Spørring",+ "ar": "الاستعلام",+ "de": "Anfrage",+ "fr": "Requête",+ "it": "Query",+ "pt": "Consulta",+ "es": "Consulta",+ "tr": "Sorgu",+ "uk": "Запит",+ "ca": "Consulta"+ },+ "SUBAGENT_OBSERVATION$RESULT": {+ "en": "Result",+ "zh-CN": "结果",+ "zh-TW": "結果",+ "ko-KR": "결과",+ "ja": "結果",+ "no": "Resultat",+ "ar": "النتيجة",+ "de": "Ergebnis",+ "fr": "Résultat",+ "it": "Risultato",+ "pt": "Resultado",+ "es": "Resultado",+ "tr": "Sonuç",+ "uk": "Результат",+ "ca": "Resultat"+ },"OBSERVATION$ERROR_PREFIX": {"en": "error:","ja": "エラー:",@@ -12562,6 +12630,23 @@"uk": "Пошук файлів: <cmd>{{pattern}}</cmd>","ca": "Cerca fitxers: <cmd>{{pattern}}</cmd>"},+ "ACTION_MESSAGE$TASK": {+ "en": "Delegating to sub-agent",+ "zh-CN": "委派给子代理",+ "zh-TW": "委派給子代理",+ "ko-KR": "하위 에이전트에 위임 중",+ "ja": "サブエージェントに委任中",+ "no": "Delegerer til underagent",+ "ar": "التفويض إلى وكيل فرعي",+ "de": "Delegiere an Sub-Agent",+ "fr": "Délégation au sous-agent",+ "it": "Delega al sub-agente",+ "pt": "Delegando para subagente",+ "es": "Delegando a subagente",+ "tr": "Alt aracıya devrediliyor",+ "uk": "Делегування субагенту",+ "ca": "Delegant a subagent"+ },"OBSERVATION_MESSAGE$RUN": {"en": "Ran <cmd>{{command}}</cmd>","zh-CN": "运行 <cmd>{{command}}</cmd>",@@ -12783,6 +12868,23 @@"uk": "Агент перевірив поточний план","ca": "L'agent ha consultat el pla actual"},+ "OBSERVATION_MESSAGE$TASK": {+ "en": "Sub-agent task",+ "zh-CN": "子代理任务",+ "zh-TW": "子代理任務",+ "ko-KR": "하위 에이전트 작업",+ "ja": "サブエージェントタスク",+ "no": "Underagentoppgave",+ "ar": "مهمة الوكيل الفرعي",+ "de": "Sub-Agent-Aufgabe",+ "fr": "Tâche du sous-agent",+ "it": "Attività del sub-agente",+ "pt": "Tarefa do subagente",+ "es": "Tarea del subagente",+ "tr": "Alt aracı görevi",+ "uk": "Завдання субагента",+ "ca": "Tasca del subagent"+ },"EXPANDABLE_MESSAGE$SHOW_DETAILS": {"en": "Show details","zh-CN": "显示详情",diff --git a/frontend/src/types/v1/core/base/action.ts b/frontend/src/types/v1/core/base/action.tsindex d2e0b9af88..2ba4f0175b 100644--- a/frontend/src/types/v1/core/base/action.ts+++ b/frontend/src/types/v1/core/base/action.ts@@ -270,6 +270,25 @@ export interface GrepAction extends ActionBase<"GrepAction"> {include: string | null;}+export interface TaskAction extends ActionBase<"TaskAction"> {+ /**+ * A short (3-5 word) description of the task.+ */+ description: string | null;+ /**+ * The task for the sub-agent to perform (shown as the "Query").+ */+ prompt: string;+ /**+ * The type of specialized agent to use for this task.+ */+ subagent_type: string;+ /**+ * Task ID of the task to resume from, if any.+ */+ resume: string | null;+}+export type Action =| MCPToolAction| FinishAction@@ -291,4 +310,5 @@ export type Action =| BrowserSwitchTabAction| BrowserCloseTabAction| GlobAction- | GrepAction;+ | GrepAction+ | TaskAction;diff --git a/frontend/src/types/v1/core/base/base.ts b/frontend/src/types/v1/core/base/base.tsindex a98c443ea2..9cb046cd17 100644--- a/frontend/src/types/v1/core/base/base.ts+++ b/frontend/src/types/v1/core/base/base.ts@@ -7,6 +7,7 @@ type EventType =| "FileEditor"| "StrReplaceEditor"| "TaskTracker"+ | "Task"| "PlanningFileEditor";type ActionOnlyType =diff --git a/frontend/src/types/v1/core/base/observation.ts b/frontend/src/types/v1/core/base/observation.tsindex 259a6f1643..5a1f027c2f 100644--- a/frontend/src/types/v1/core/base/observation.ts+++ b/frontend/src/types/v1/core/base/observation.ts@@ -298,6 +298,29 @@ export interface SwitchLLMObservation extends ObservationBase<"SwitchLLMObservatactive_model: string | null;}+export interface TaskObservation extends ObservationBase<"TaskObservation"> {+ /**+ * Result content returned by the sub-agent.+ */+ content: Array<TextContent | ImageContent>;+ /**+ * Whether the task resulted in an error.+ */+ is_error: boolean;+ /**+ * The unique identifier of the task.+ */+ task_id: string;+ /**+ * The sub-agent that executed the task.+ */+ subagent: string;+ /**+ * The status of the task.+ */+ status: string;+}+export type Observation =| MCPToolObservation| FinishObservation@@ -311,4 +334,5 @@ export type Observation =| PlanningFileEditorObservation| GlobObservation| GrepObservation- | SwitchLLMObservation;+ | SwitchLLMObservation+ | TaskObservation;