Skip to content

πŸ™Œ OpenHands: AI-Driven Development

refactor(frontend): support TerminalObservation event (#11819)

Hiep Le5ef45cfec211
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
index fe6bf842c3..702c308733 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts
@@ -4,6 +4,7 @@ import i18n from "#/i18n";
import { SecurityRisk } from "#/types/v1/core/base/common";
import {
ExecuteBashAction,
+ TerminalAction,
FileEditorAction,
StrReplaceEditorAction,
MCPToolAction,
@@ -58,7 +59,7 @@ const getFileEditorActionContent = (
// Command Actions
const getExecuteBashActionContent = (
- event: ActionEvent<ExecuteBashAction>,
+ event: ActionEvent<ExecuteBashAction | TerminalAction>,
): string => {
let content = `Command:\n\`${event.action.command}\``;
@@ -164,8 +165,9 @@ export const getActionContent = (event: ActionEvent): string => {
return getFileEditorActionContent(action);
case "ExecuteBashAction":
+ case "TerminalAction":
return getExecuteBashActionContent(
- event as ActionEvent<ExecuteBashAction>,
+ event as ActionEvent<ExecuteBashAction | TerminalAction>,
);
case "MCPToolAction":
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
index 5c93b11be5..f6c67d9fcc 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx
@@ -50,6 +50,7 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => {
switch (actionType) {
case "ExecuteBashAction":
+ case "TerminalAction":
actionKey = "ACTION_MESSAGE$RUN";
actionValues = {
command: trimText(event.action.command, 80),
@@ -111,6 +112,7 @@ const getObservationEventTitle = (event: OpenHandsEvent): React.ReactNode => {
switch (observationType) {
case "ExecuteBashObservation":
+ case "TerminalObservation":
observationKey = "OBSERVATION_MESSAGE$RUN";
observationValues = {
command: event.observation.command
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts
index 6bfc3734e5..e043142e90 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts
@@ -8,6 +8,7 @@ import {
ThinkObservation,
BrowserObservation,
ExecuteBashObservation,
+ TerminalObservation,
FileEditorObservation,
StrReplaceEditorObservation,
TaskTrackerObservation,
@@ -42,8 +43,8 @@ const getFileEditorObservationContent = (
};
// Command Observations
-const getExecuteBashObservationContent = (
- event: ObservationEvent<ExecuteBashObservation>,
+const getTerminalObservationContent = (
+ event: ObservationEvent<ExecuteBashObservation | TerminalObservation>,
): string => {
const { observation } = event;
@@ -179,8 +180,9 @@ export const getObservationContent = (event: ObservationEvent): string => {
);
case "ExecuteBashObservation":
- return getExecuteBashObservationContent(
- event as ObservationEvent<ExecuteBashObservation>,
+ case "TerminalObservation":
+ return getTerminalObservationContent(
+ event as ObservationEvent<ExecuteBashObservation | TerminalObservation>,
);
case "BrowserObservation":
diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
index e5a52bfe95..790ecb00cf 100644
--- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
+++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts
@@ -17,6 +17,15 @@ export const getObservationResult = (
if (exitCode === 0 || metadata.exit_code === 0) return "success"; // Command executed successfully
return "error"; // Command failed
}
+ case "TerminalObservation": {
+ const exitCode =
+ observation.exit_code ?? observation.metadata.exit_code ?? null;
+
+ if (observation.timeout || exitCode === -1) return "timeout";
+ if (exitCode === 0) return "success";
+ if (observation.is_error) return "error";
+ return "success";
+ }
case "FileEditorObservation":
case "StrReplaceEditorObservation":
// Check if there's an error
diff --git a/frontend/src/types/v1/core/base/action.ts b/frontend/src/types/v1/core/base/action.ts
index ce08d5a1b9..77a76df848 100644
--- a/frontend/src/types/v1/core/base/action.ts
+++ b/frontend/src/types/v1/core/base/action.ts
@@ -41,6 +41,25 @@ export interface ExecuteBashAction extends ActionBase<"ExecuteBashAction"> {
reset: boolean;
}
+export interface TerminalAction extends ActionBase<"TerminalAction"> {
+ /**
+ * The terminal command to execute.
+ */
+ command: string;
+ /**
+ * If True, the command is an input to the running process. If False, the command is executed directly.
+ */
+ is_input: boolean;
+ /**
+ * Optional max time limit (seconds) for the command.
+ */
+ timeout: number | null;
+ /**
+ * If True, reset the terminal session before running the command.
+ */
+ reset: boolean;
+}
+
export interface FileEditorAction extends ActionBase<"FileEditorAction"> {
/**
* The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`.
@@ -206,6 +225,7 @@ export type Action =
| FinishAction
| ThinkAction
| ExecuteBashAction
+ | TerminalAction
| FileEditorAction
| StrReplaceEditorAction
| TaskTrackerAction
diff --git a/frontend/src/types/v1/core/base/base.ts b/frontend/src/types/v1/core/base/base.ts
index 5925e8599d..30c39cec89 100644
--- a/frontend/src/types/v1/core/base/base.ts
+++ b/frontend/src/types/v1/core/base/base.ts
@@ -3,6 +3,7 @@ type EventType =
| "Finish"
| "Think"
| "ExecuteBash"
+ | "Terminal"
| "FileEditor"
| "StrReplaceEditor"
| "TaskTracker";
@@ -24,7 +25,8 @@ type ObservationOnlyType = "Browser";
type ActionEventType = `${ActionOnlyType}Action` | `${EventType}Action`;
type ObservationEventType =
| `${ObservationOnlyType}Observation`
- | `${EventType}Observation`;
+ | `${EventType}Observation`
+ | "TerminalObservation";
export interface ActionBase<T extends ActionEventType = ActionEventType> {
kind: T;
diff --git a/frontend/src/types/v1/core/base/observation.ts b/frontend/src/types/v1/core/base/observation.ts
index e406e30593..0625c9d336 100644
--- a/frontend/src/types/v1/core/base/observation.ts
+++ b/frontend/src/types/v1/core/base/observation.ts
@@ -81,6 +81,34 @@ export interface ExecuteBashObservation
metadata: CmdOutputMetadata;
}
+export interface TerminalObservation
+ extends ObservationBase<"TerminalObservation"> {
+ /**
+ * Content returned from the terminal as a list of TextContent/ImageContent objects.
+ */
+ content: Array<TextContent | ImageContent>;
+ /**
+ * The bash command that was executed.
+ */
+ command: string | null;
+ /**
+ * The exit code of the command if it has finished.
+ */
+ exit_code: number | null;
+ /**
+ * Whether the command execution produced an error.
+ */
+ is_error: boolean;
+ /**
+ * Whether the command execution timed out.
+ */
+ timeout: boolean;
+ /**
+ * Additional metadata captured from the shell after command execution.
+ */
+ metadata: CmdOutputMetadata;
+}
+
export interface FileEditorObservation
extends ObservationBase<"FileEditorObservation"> {
/**
@@ -168,6 +196,7 @@ export type Observation =
| ThinkObservation
| BrowserObservation
| ExecuteBashObservation
+ | TerminalObservation
| FileEditorObservation
| StrReplaceEditorObservation
| TaskTrackerObservation;
diff --git a/frontend/src/types/v1/type-guards.ts b/frontend/src/types/v1/type-guards.ts
index b479e4697b..e88946e4bc 100644
--- a/frontend/src/types/v1/type-guards.ts
+++ b/frontend/src/types/v1/type-guards.ts
@@ -3,7 +3,9 @@ import {
ObservationEvent,
BaseEvent,
ExecuteBashAction,
+ TerminalAction,
ExecuteBashObservation,
+ TerminalObservation,
} from "./core";
import { AgentErrorEvent } from "./core/events/observation-event";
import { MessageEvent } from "./core/events/message-event";
@@ -98,17 +100,20 @@ export const isActionEvent = (event: OpenHandsEvent): event is ActionEvent =>
*/
export const isExecuteBashActionEvent = (
event: OpenHandsEvent,
-): event is ActionEvent<ExecuteBashAction> =>
- isActionEvent(event) && event.action.kind === "ExecuteBashAction";
+): event is ActionEvent<ExecuteBashAction | TerminalAction> =>
+ isActionEvent(event) &&
+ (event.action.kind === "ExecuteBashAction" ||
+ event.action.kind === "TerminalAction");
/**
- * Type guard function to check if an observation event is an ExecuteBashObservation
+ * Type guard function to check if an observation event contains terminal output
*/
export const isExecuteBashObservationEvent = (
event: OpenHandsEvent,
-): event is ObservationEvent<ExecuteBashObservation> =>
+): event is ObservationEvent<ExecuteBashObservation | TerminalObservation> =>
isObservationEvent(event) &&
- event.observation.kind === "ExecuteBashObservation";
+ (event.observation.kind === "ExecuteBashObservation" ||
+ event.observation.kind === "TerminalObservation");
/**
* Type guard function to check if an event is a system prompt event