Skip to content

🙌 OpenHands: AI-Driven Development

feat: add sub-agent task (TaskToolSet) visualizer to the chat UI (#14928)

Vasco Schiavobfd9af088ffb
diff --git a/frontend/__tests__/components/v1/get-event-content.test.tsx b/frontend/__tests__/components/v1/get-event-content.test.tsx
index 91851cd772..65de2e84ef 100644
--- a/frontend/__tests__/components/v1/get-event-content.test.tsx
+++ b/frontend/__tests__/components/v1/get-event-content.test.tsx
@@ -58,6 +58,51 @@ const terminalObservationEvent: ObservationEvent = {
},
};
+const taskActionEvent: ActionEvent = {
+ id: "action-task-1",
+ timestamp: new Date().toISOString(),
+ source: "agent",
+ thought: [{ type: "text", text: "I'll ask a sub-agent to count files." }],
+ thinking_blocks: [],
+ action: {
+ kind: "TaskAction",
+ description: "count repository files",
+ prompt: "Count how many files are in the repo using git ls-files | wc -l.",
+ subagent_type: "bash-runner",
+ resume: null,
+ },
+ tool_name: "task",
+ tool_call_id: "tool-task-1",
+ tool_call: {
+ id: "tool-task-1",
+ type: "function",
+ function: {
+ name: "task",
+ arguments: '{"prompt":"Count files","subagent_type":"bash-runner"}',
+ },
+ },
+ llm_response_id: "response-task-1",
+ security_risk: SecurityRisk.LOW,
+ summary: "Ask sub-agent to count repository files",
+};
+
+const taskObservationEvent: ObservationEvent = {
+ id: "obs-task-1",
+ timestamp: new Date().toISOString(),
+ source: "environment",
+ tool_name: "task",
+ tool_call_id: "tool-task-1",
+ action_id: "action-task-1",
+ observation: {
+ kind: "TaskObservation",
+ content: [{ type: "text", text: "tracked_count=1606" }],
+ is_error: false,
+ task_id: "task_00000002",
+ subagent: "bash-runner",
+ status: "completed",
+ },
+};
+
describe("getEventContent", () => {
it("uses the action summary as the full action title", () => {
const { title } = getEventContent(terminalActionEvent);
@@ -145,4 +190,85 @@ describe("getEventContent", () => {
expect(screen.getByText("Check repository status")).toBeInTheDocument();
expect(screen.queryByText("$ git status")).not.toBeInTheDocument();
});
+
+ it("renders the sub-agent task card with subagent, task id, query and result", () => {
+ const { details } = getEventContent(taskObservationEvent, taskActionEvent);
+
+ render(<div>{details}</div>);
+
+ // Labels render as raw i18n keys because i18n is not loaded in tests
+ expect(
+ screen.getByText("SUBAGENT_OBSERVATION$SUBAGENT:"),
+ ).toBeInTheDocument();
+ expect(screen.getByText("bash-runner")).toBeInTheDocument();
+ expect(
+ screen.getByText("SUBAGENT_OBSERVATION$TASK_ID:"),
+ ).toBeInTheDocument();
+ expect(screen.getByText("task_00000002")).toBeInTheDocument();
+ expect(screen.getByText("SUBAGENT_OBSERVATION$QUERY")).toBeInTheDocument();
+ expect(
+ screen.getByText(
+ "Count how many files are in the repo using git ls-files | wc -l.",
+ ),
+ ).toBeInTheDocument();
+ expect(screen.getByText("SUBAGENT_OBSERVATION$RESULT")).toBeInTheDocument();
+ expect(screen.getByText("tracked_count=1606")).toBeInTheDocument();
+ });
+
+ it("falls back to the task title key when the action summary is missing", () => {
+ const actionWithoutSummary = { ...taskActionEvent, summary: undefined };
+ const { title } = getEventContent(
+ taskObservationEvent,
+ actionWithoutSummary,
+ );
+
+ render(<span>{title}</span>);
+
+ expect(screen.getByText("OBSERVATION_MESSAGE$TASK")).toBeInTheDocument();
+ });
+
+ it("shows the subagent and query in the in-progress task action card", () => {
+ const { details } = getEventContent(taskActionEvent);
+
+ expect(details).toContain("**Subagent:** `bash-runner`");
+ expect(details).toContain(
+ "Count how many files are in the repo using git ls-files | wc -l.",
+ );
+ });
+
+ it("renders image content in the sub-agent result instead of dropping it", () => {
+ const observationWithImage: ObservationEvent = {
+ ...taskObservationEvent,
+ observation: {
+ kind: "TaskObservation",
+ content: [
+ { type: "text", text: "Here is a screenshot:" },
+ { type: "image", image_urls: ["data:image/png;base64,abc123"] },
+ ],
+ is_error: false,
+ task_id: "task_00000002",
+ subagent: "bash-runner",
+ status: "completed",
+ },
+ };
+
+ const { details } = getEventContent(observationWithImage, taskActionEvent);
+
+ const { container } = render(<div>{details}</div>);
+
+ expect(screen.getByText("Here is a screenshot:")).toBeInTheDocument();
+ const image = container.querySelector("img");
+ expect(image).toHaveAttribute("src", "data:image/png;base64,abc123");
+ });
+
+ it("omits the query section when the task observation has no paired action", () => {
+ const { details } = getEventContent(taskObservationEvent);
+
+ render(<div>{details}</div>);
+
+ expect(screen.getByText("tracked_count=1606")).toBeInTheDocument();
+ expect(
+ screen.queryByText("SUBAGENT_OBSERVATION$QUERY"),
+ ).not.toBeInTheDocument();
+ });
});
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
index c0f2bf9db8..217cc19da9 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
@@ -23,6 +23,7 @@ import {
BrowserCloseTabAction,
GlobAction,
GrepAction,
+ TaskAction,
} from "#/types/v1/core/base/action";
const getRiskText = (risk: SecurityRisk) => {
@@ -137,6 +138,16 @@ const getTaskTrackerActionContent = (action: TaskTrackerAction): string => {
return content;
};
+// Sub-agent task action (TaskToolSet). Shown while the sub-agent is still
+// running, before its TaskObservation arrives.
+const getTaskActionContent = (action: TaskAction): string => {
+ const parts: string[] = [`**Subagent:** \`${action.subagent_type}\``];
+ if (action.prompt) {
+ parts.push(`**Query**\n\n${action.prompt}`);
+ }
+ return parts.join("\n\n");
+};
+
// Browser Actions
type BrowserAction =
| BrowserNavigateAction
@@ -236,6 +247,9 @@ export const getActionContent = (event: ActionEvent): string => {
case "TaskTrackerAction":
return getTaskTrackerActionContent(action);
+ case "TaskAction":
+ return getTaskActionContent(action);
+
case "BrowserNavigateAction":
case "BrowserClickAction":
case "BrowserTypeAction":
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
index 66a004ca99..165cb71c3e 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
@@ -12,7 +12,11 @@ import { getActionContent } from "./get-action-content";
import { getObservationContent } from "./get-observation-content";
import { getACPToolCallContent } from "./get-acp-tool-call-content";
import { TaskTrackingObservationContent } from "../task-tracking/task-tracking-observation-content";
-import { TaskTrackerObservation } from "#/types/v1/core/base/observation";
+import { SubagentObservationContent } from "../subagent/subagent-observation-content";
+import {
+ TaskTrackerObservation,
+ TaskObservation,
+} from "#/types/v1/core/base/observation";
import { SkillReadyEvent, isSkillReadyEvent } from "./create-skill-ready-event";
import i18n from "#/i18n";
@@ -101,6 +105,9 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => {
case "TaskTrackerAction":
actionKey = "ACTION_MESSAGE$TASK_TRACKING";
break;
+ case "TaskAction":
+ actionKey = "ACTION_MESSAGE$TASK";
+ break;
case "GrepAction":
actionKey = "ACTION_MESSAGE$GREP";
actionValues = {
@@ -204,6 +211,9 @@ const getObservationEventTitle = (
}
break;
}
+ case "TaskObservation":
+ observationKey = "OBSERVATION_MESSAGE$TASK";
+ break;
case "ThinkObservation":
observationKey = "OBSERVATION_MESSAGE$THINK";
break;
@@ -265,6 +275,14 @@ export const getEventContent = (
event={event as ObservationEvent<TaskTrackerObservation>}
/>
);
+ } else if (event.observation.kind === "TaskObservation") {
+ // The Query comes from the paired action's prompt.
+ details = (
+ <SubagentObservationContent
+ event={event as ObservationEvent<TaskObservation>}
+ correspondingAction={correspondingAction}
+ />
+ );
} else {
details = getObservationContent(event);
}
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
index 8bf3bda325..b893aff764 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
@@ -49,6 +49,7 @@ export const getObservationResult = (
if (observation.error) return "error";
return "success";
case "MCPToolObservation":
+ case "TaskObservation":
if (observation.is_error) return "error";
return "success";
default:
diff --git a/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsx b/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsx
new file mode 100644
index 0000000000..fc92f49e76
--- /dev/null
+++ b/frontend/src/components/v1/chat/subagent/subagent-observation-content.tsx
@@ -0,0 +1,88 @@
+import React from "react";
+import { useTranslation } from "react-i18next";
+import { I18nKey } from "#/i18n/declaration";
+import { ActionEvent, ObservationEvent } from "#/types/v1/core";
+import { TaskObservation } from "#/types/v1/core/base/observation";
+import { MarkdownRenderer } from "#/components/features/markdown/markdown-renderer";
+
+interface SubagentObservationContentProps {
+ event: ObservationEvent<TaskObservation>;
+ correspondingAction?: ActionEvent;
+}
+
+const getResultText = (observation: TaskObservation): string =>
+ observation.content
+ .filter((c) => c.type === "text")
+ .map((c) => c.text)
+ .join("\n");
+
+const getResultImages = (observation: TaskObservation): string[] =>
+ observation.content
+ .filter((c) => c.type === "image")
+ .flatMap((c) => c.image_urls);
+
+const getQuery = (correspondingAction?: ActionEvent): string | undefined => {
+ const action = correspondingAction?.action;
+ return action?.kind === "TaskAction" ? action.prompt : undefined;
+};
+
+/**
+ * Rich card body for a sub-agent task (TaskToolSet). Shows which sub-agent ran
+ * the task, its task id, the query it was given (from the paired action) and
+ * the result it returned. Mirrors the TaskTrackerObservation visualizer.
+ */
+export function SubagentObservationContent({
+ event,
+ correspondingAction,
+}: SubagentObservationContentProps): React.ReactNode {
+ const { t } = useTranslation();
+ const { observation } = event;
+ const query = getQuery(correspondingAction);
+ const resultText = getResultText(observation);
+ const resultImages = getResultImages(observation);
+
+ return (
+ <div className="flex flex-col gap-3 text-neutral-300">
+ <div className="flex flex-col gap-1">
+ <div className="flex gap-1">
+ <span className="font-bold">
+ {t(I18nKey.SUBAGENT_OBSERVATION$SUBAGENT)}:
+ </span>
+ <span>{observation.subagent}</span>
+ </div>
+ <div className="flex gap-1">
+ <span className="font-bold">
+ {t(I18nKey.SUBAGENT_OBSERVATION$TASK_ID)}:
+ </span>
+ <span>{observation.task_id}</span>
+ </div>
+ </div>
+
+ {query && (
+ <div className="flex flex-col gap-1">
+ <span className="font-bold">
+ {t(I18nKey.SUBAGENT_OBSERVATION$QUERY)}
+ </span>
+ <div className="rounded-md bg-neutral-700 p-2 whitespace-pre-wrap break-words">
+ {query}
+ </div>
+ </div>
+ )}
+
+ <div className="flex flex-col gap-1">
+ <span className="font-bold">
+ {t(I18nKey.SUBAGENT_OBSERVATION$RESULT)}
+ </span>
+ {resultText && <MarkdownRenderer>{resultText}</MarkdownRenderer>}
+ {resultImages.map((url) => (
+ <img
+ key={url}
+ src={url}
+ alt={t(I18nKey.SUBAGENT_OBSERVATION$RESULT)}
+ className="max-w-full rounded-md"
+ />
+ ))}
+ </div>
+ </div>
+ );
+}
diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts
index 8aa14eacff..2d6ffbdc6b 100644
--- a/frontend/src/i18n/declaration.ts
+++ b/frontend/src/i18n/declaration.ts
@@ -53,6 +53,10 @@ export enum I18nKey {
TASK_TRACKING_OBSERVATION$TASK_ID = "TASK_TRACKING_OBSERVATION$TASK_ID",
TASK_TRACKING_OBSERVATION$TASK_NOTES = "TASK_TRACKING_OBSERVATION$TASK_NOTES",
TASK_TRACKING_OBSERVATION$RESULT = "TASK_TRACKING_OBSERVATION$RESULT",
+ SUBAGENT_OBSERVATION$SUBAGENT = "SUBAGENT_OBSERVATION$SUBAGENT",
+ SUBAGENT_OBSERVATION$TASK_ID = "SUBAGENT_OBSERVATION$TASK_ID",
+ SUBAGENT_OBSERVATION$QUERY = "SUBAGENT_OBSERVATION$QUERY",
+ SUBAGENT_OBSERVATION$RESULT = "SUBAGENT_OBSERVATION$RESULT",
OBSERVATION$ERROR_PREFIX = "OBSERVATION$ERROR_PREFIX",
TASK$ADDRESSING_TASK = "TASK$ADDRESSING_TASK",
SECRETS$SECRET_VALUE_REQUIRED = "SECRETS$SECRET_VALUE_REQUIRED",
@@ -739,6 +743,7 @@ export enum I18nKey {
ACTION_MESSAGE$TASK_TRACKING = "ACTION_MESSAGE$TASK_TRACKING",
ACTION_MESSAGE$GREP = "ACTION_MESSAGE$GREP",
ACTION_MESSAGE$GLOB = "ACTION_MESSAGE$GLOB",
+ ACTION_MESSAGE$TASK = "ACTION_MESSAGE$TASK",
OBSERVATION_MESSAGE$RUN = "OBSERVATION_MESSAGE$RUN",
OBSERVATION_MESSAGE$RUN_IPYTHON = "OBSERVATION_MESSAGE$RUN_IPYTHON",
OBSERVATION_MESSAGE$READ = "OBSERVATION_MESSAGE$READ",
@@ -752,6 +757,7 @@ export enum I18nKey {
OBSERVATION_MESSAGE$GREP = "OBSERVATION_MESSAGE$GREP",
OBSERVATION_MESSAGE$TASK_TRACKING_PLAN = "OBSERVATION_MESSAGE$TASK_TRACKING_PLAN",
OBSERVATION_MESSAGE$TASK_TRACKING_VIEW = "OBSERVATION_MESSAGE$TASK_TRACKING_VIEW",
+ OBSERVATION_MESSAGE$TASK = "OBSERVATION_MESSAGE$TASK",
EXPANDABLE_MESSAGE$SHOW_DETAILS = "EXPANDABLE_MESSAGE$SHOW_DETAILS",
EXPANDABLE_MESSAGE$HIDE_DETAILS = "EXPANDABLE_MESSAGE$HIDE_DETAILS",
AI_SETTINGS$TITLE = "AI_SETTINGS$TITLE",
diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json
index ae1c9912af..c26db5abb4 100644
--- a/frontend/src/i18n/translation.json
+++ b/frontend/src/i18n/translation.json
@@ -900,6 +900,74 @@
"uk": "Результат",
"ca": "Resultat"
},
+ "SUBAGENT_OBSERVATION$SUBAGENT": {
+ "en": "Subagent",
+ "zh-CN": "子代理",
+ "zh-TW": "子代理",
+ "ko-KR": "하위 에이전트",
+ "ja": "サブエージェント",
+ "no": "Underagent",
+ "ar": "الوكيل الفرعي",
+ "de": "Sub-Agent",
+ "fr": "Sous-agent",
+ "it": "Sub-agente",
+ "pt": "Subagente",
+ "es": "Subagente",
+ "tr": "Alt aracı",
+ "uk": "Субагент",
+ "ca": "Subagent"
+ },
+ "SUBAGENT_OBSERVATION$TASK_ID": {
+ "en": "Task ID",
+ "zh-CN": "任务 ID",
+ "zh-TW": "任務 ID",
+ "ko-KR": "작업 ID",
+ "ja": "タスク ID",
+ "no": "Oppgave-ID",
+ "ar": "معرف المهمة",
+ "de": "Aufgaben-ID",
+ "fr": "ID de tâche",
+ "it": "ID attività",
+ "pt": "ID da tarefa",
+ "es": "ID de tarea",
+ "tr": "Görev kimliği",
+ "uk": "ID завдання",
+ "ca": "ID de tasca"
+ },
+ "SUBAGENT_OBSERVATION$QUERY": {
+ "en": "Query",
+ "zh-CN": "查询",
+ "zh-TW": "查詢",
+ "ko-KR": "쿼리",
+ "ja": "クエリ",
+ "no": "Spørring",
+ "ar": "الاستعلام",
+ "de": "Anfrage",
+ "fr": "Requête",
+ "it": "Query",
+ "pt": "Consulta",
+ "es": "Consulta",
+ "tr": "Sorgu",
+ "uk": "Запит",
+ "ca": "Consulta"
+ },
+ "SUBAGENT_OBSERVATION$RESULT": {
+ "en": "Result",
+ "zh-CN": "结果",
+ "zh-TW": "結果",
+ "ko-KR": "결과",
+ "ja": "結果",
+ "no": "Resultat",
+ "ar": "النتيجة",
+ "de": "Ergebnis",
+ "fr": "Résultat",
+ "it": "Risultato",
+ "pt": "Resultado",
+ "es": "Resultado",
+ "tr": "Sonuç",
+ "uk": "Результат",
+ "ca": "Resultat"
+ },
"OBSERVATION$ERROR_PREFIX": {
"en": "error:",
"ja": "エラー:",
@@ -12562,6 +12630,23 @@
"uk": "Пошук файлів: <cmd>{{pattern}}</cmd>",
"ca": "Cerca fitxers: <cmd>{{pattern}}</cmd>"
},
+ "ACTION_MESSAGE$TASK": {
+ "en": "Delegating to sub-agent",
+ "zh-CN": "委派给子代理",
+ "zh-TW": "委派給子代理",
+ "ko-KR": "하위 에이전트에 위임 중",
+ "ja": "サブエージェントに委任中",
+ "no": "Delegerer til underagent",
+ "ar": "التفويض إلى وكيل فرعي",
+ "de": "Delegiere an Sub-Agent",
+ "fr": "Délégation au sous-agent",
+ "it": "Delega al sub-agente",
+ "pt": "Delegando para subagente",
+ "es": "Delegando a subagente",
+ "tr": "Alt aracıya devrediliyor",
+ "uk": "Делегування субагенту",
+ "ca": "Delegant a subagent"
+ },
"OBSERVATION_MESSAGE$RUN": {
"en": "Ran <cmd>{{command}}</cmd>",
"zh-CN": "运行 <cmd>{{command}}</cmd>",
@@ -12783,6 +12868,23 @@
"uk": "Агент перевірив поточний план",
"ca": "L'agent ha consultat el pla actual"
},
+ "OBSERVATION_MESSAGE$TASK": {
+ "en": "Sub-agent task",
+ "zh-CN": "子代理任务",
+ "zh-TW": "子代理任務",
+ "ko-KR": "하위 에이전트 작업",
+ "ja": "サブエージェントタスク",
+ "no": "Underagentoppgave",
+ "ar": "مهمة الوكيل الفرعي",
+ "de": "Sub-Agent-Aufgabe",
+ "fr": "Tâche du sous-agent",
+ "it": "Attività del sub-agente",
+ "pt": "Tarefa do subagente",
+ "es": "Tarea del subagente",
+ "tr": "Alt aracı görevi",
+ "uk": "Завдання субагента",
+ "ca": "Tasca del subagent"
+ },
"EXPANDABLE_MESSAGE$SHOW_DETAILS": {
"en": "Show details",
"zh-CN": "显示详情",
diff --git a/frontend/src/types/v1/core/base/action.ts b/frontend/src/types/v1/core/base/action.ts
index d2e0b9af88..2ba4f0175b 100644
--- a/frontend/src/types/v1/core/base/action.ts
+++ b/frontend/src/types/v1/core/base/action.ts
@@ -270,6 +270,25 @@ export interface GrepAction extends ActionBase<"GrepAction"> {
include: string | null;
}
+export interface TaskAction extends ActionBase<"TaskAction"> {
+ /**
+ * A short (3-5 word) description of the task.
+ */
+ description: string | null;
+ /**
+ * The task for the sub-agent to perform (shown as the "Query").
+ */
+ prompt: string;
+ /**
+ * The type of specialized agent to use for this task.
+ */
+ subagent_type: string;
+ /**
+ * Task ID of the task to resume from, if any.
+ */
+ resume: string | null;
+}
+
export type Action =
| MCPToolAction
| FinishAction
@@ -291,4 +310,5 @@ export type Action =
| BrowserSwitchTabAction
| BrowserCloseTabAction
| GlobAction
- | GrepAction;
+ | GrepAction
+ | TaskAction;
diff --git a/frontend/src/types/v1/core/base/base.ts b/frontend/src/types/v1/core/base/base.ts
index a98c443ea2..9cb046cd17 100644
--- a/frontend/src/types/v1/core/base/base.ts
+++ b/frontend/src/types/v1/core/base/base.ts
@@ -7,6 +7,7 @@ type EventType =
| "FileEditor"
| "StrReplaceEditor"
| "TaskTracker"
+ | "Task"
| "PlanningFileEditor";
type ActionOnlyType =
diff --git a/frontend/src/types/v1/core/base/observation.ts b/frontend/src/types/v1/core/base/observation.ts
index 259a6f1643..5a1f027c2f 100644
--- a/frontend/src/types/v1/core/base/observation.ts
+++ b/frontend/src/types/v1/core/base/observation.ts
@@ -298,6 +298,29 @@ export interface SwitchLLMObservation extends ObservationBase<"SwitchLLMObservat
active_model: string | null;
}
+export interface TaskObservation extends ObservationBase<"TaskObservation"> {
+ /**
+ * Result content returned by the sub-agent.
+ */
+ content: Array<TextContent | ImageContent>;
+ /**
+ * Whether the task resulted in an error.
+ */
+ is_error: boolean;
+ /**
+ * The unique identifier of the task.
+ */
+ task_id: string;
+ /**
+ * The sub-agent that executed the task.
+ */
+ subagent: string;
+ /**
+ * The status of the task.
+ */
+ status: string;
+}
+
export type Observation =
| MCPToolObservation
| FinishObservation
@@ -311,4 +334,5 @@ export type Observation =
| PlanningFileEditorObservation
| GlobObservation
| GrepObservation
- | SwitchLLMObservation;
+ | SwitchLLMObservation
+ | TaskObservation;