Skip to content

🙌 OpenHands: AI-Driven Development

feat(automations): narrow needs review status (#17682)

Devinac127b6443c6
diff --git a/.pr/issue-17680-live-verification.html b/.pr/issue-17680-live-verification.html
new file mode 100644
index 0000000000..5f8f7e8400
--- /dev/null
+++ b/.pr/issue-17680-live-verification.html
@@ -0,0 +1,214 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <title>Issue #17680 — live verification</title>
+ <style>
+ :root {
+ color-scheme: dark;
+ font-family:
+ Inter,
+ ui-sans-serif,
+ system-ui,
+ -apple-system,
+ BlinkMacSystemFont,
+ "Segoe UI",
+ sans-serif;
+ background: #09090b;
+ color: #f4f4f5;
+ }
+ body {
+ margin: 0;
+ padding: 48px 24px;
+ }
+ main {
+ max-width: 960px;
+ margin: 0 auto;
+ }
+ h1,
+ h2 {
+ letter-spacing: -0.025em;
+ }
+ h1 {
+ margin: 0 0 8px;
+ font-size: 36px;
+ }
+ h2 {
+ margin: 0 0 12px;
+ font-size: 20px;
+ }
+ p,
+ li {
+ color: #d4d4d8;
+ line-height: 1.6;
+ }
+ .lede {
+ margin: 0 0 32px;
+ color: #a1a1aa;
+ }
+ .grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
+ gap: 16px;
+ margin: 20px 0 32px;
+ }
+ .card {
+ border: 1px solid #27272a;
+ border-radius: 16px;
+ padding: 20px;
+ background: #111113;
+ }
+ .badge {
+ display: inline-flex;
+ align-items: center;
+ border-radius: 999px;
+ padding: 5px 10px;
+ font-size: 12px;
+ font-weight: 700;
+ }
+ .success {
+ color: #86efac;
+ background: #052e16;
+ }
+ .review {
+ color: #fde68a;
+ background: #422006;
+ }
+ code,
+ pre {
+ font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
+ }
+ code {
+ color: #e4e4e7;
+ }
+ pre {
+ overflow-x: auto;
+ border: 1px solid #27272a;
+ border-radius: 12px;
+ padding: 16px;
+ background: #09090b;
+ color: #e4e4e7;
+ line-height: 1.5;
+ }
+ table {
+ width: 100%;
+ border-collapse: collapse;
+ margin: 16px 0 32px;
+ }
+ th,
+ td {
+ border-bottom: 1px solid #27272a;
+ padding: 12px;
+ text-align: left;
+ vertical-align: top;
+ }
+ th {
+ color: #a1a1aa;
+ font-size: 12px;
+ text-transform: uppercase;
+ letter-spacing: 0.08em;
+ }
+ .note {
+ border-left: 3px solid #f59e0b;
+ padding-left: 16px;
+ }
+ </style>
+ </head>
+ <body>
+ <main>
+ <h1>Narrow “Needs review” to genuine review cases</h1>
+ <p class="lede">
+ Reviewer evidence for OpenHands/OpenHands issue #17680. Verified on the
+ real local Agent Canvas, Agent Server, and automation service stack on
+ September 24, 2026.
+ </p>
+
+ <h2>Behavior change</h2>
+ <div class="grid">
+ <section class="card">
+ <span class="badge review">Before: Needs review</span>
+ <p>
+ Any completed run with a finish response but no recognized
+ <code>status</code> was treated as unknown, including a response
+ containing only <code>outcome_summary</code>.
+ </p>
+ </section>
+ <section class="card">
+ <span class="badge success">After: Successful</span>
+ <p>
+ A completed run now defaults to success when the response omits
+ <code>status</code>. The outcome summary is still displayed.
+ Explicit unrecognized status strings still require review.
+ </p>
+ </section>
+ </div>
+
+ <h2>Decision table</h2>
+ <table>
+ <thead>
+ <tr>
+ <th>Completed run response</th>
+ <th>Badge</th>
+ <th>Summary</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>{ "outcome_summary": "Report published" }</code></td>
+ <td><span class="badge success">Successful</span></td>
+ <td>Report published</td>
+ </tr>
+ <tr>
+ <td><code>{ "status": "unexpected", ... }</code></td>
+ <td><span class="badge review">Needs review</span></td>
+ <td>Preserved when provided</td>
+ </tr>
+ <tr>
+ <td>No finish response</td>
+ <td><span class="badge success">Successful</span></td>
+ <td>None</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <h2>Real-stack verification</h2>
+ <section class="card">
+ <p>
+ Created the isolated prompt automation
+ <code>PR 17680 live verification</code>
+ (<code>0d36598c-537d-4deb-92e5-ec2bcf680a33</code>) and manually
+ dispatched run <code>e1d6e30f-1556-439c-bbae-605377679acf</code>. The
+ prompt prohibited network, repository, secret, plugin, integration,
+ and file access. The automation was disabled after the run.
+ </p>
+ <p>
+ The live SDK (1.49.5) rejected a summary-only FinishTool call because
+ its schema requires <code>status</code>. The resulting fallback
+ metadata remained visibly marked
+ <span class="badge review">Needs review</span>, confirming that
+ explicit unknown/unrecognized outcomes retain the conservative state.
+ Existing real completed script runs with no finish response remained
+ <span class="badge success">Successful</span>.
+ </p>
+ <p class="note">
+ The live service cannot produce the exact backward-compatibility
+ payload from this issue because its current FinishTool schema requires
+ <code>status</code>. The summary-only and custom-metadata cases are
+ therefore exercised through the frontend's real display helper and
+ component tests, without changing the backend or its stored data.
+ </p>
+ </section>
+
+ <h2>Verification completed</h2>
+ <ul>
+ <li>
+ Focused display, modal, health-card, and manifest tests: 67 passed.
+ </li>
+ <li>Full test suite under UTC: 7,774 passed, 7 todo.</li>
+ <li>Application and library production builds passed.</li>
+ <li>Lint passed with pre-existing warnings and no errors.</li>
+ </ul>
+ </main>
+ </body>
+</html>
diff --git a/__tests__/components/automations/detail/run-logs-modal.test.tsx b/__tests__/components/automations/detail/run-logs-modal.test.tsx
index 761bbc49cc..d3e0898d9d 100644
--- a/__tests__/components/automations/detail/run-logs-modal.test.tsx
+++ b/__tests__/components/automations/detail/run-logs-modal.test.tsx
@@ -374,7 +374,34 @@ describe("RunLogsModal — run inspection summary", () => {
expect(screen.getByText(/Sandbox timed out/)).toBeInTheDocument();
});
- it("falls back to a readable JSON block for custom task metadata", () => {
+ it("shows a failed run's agent summary separately from its system error", () => {
+ useBashCommandLogsMock.mockReturnValue(makeHookResult());
+
+ render(
+ <RunLogsModal
+ isOpen
+ conversationId="conv-1"
+ bashCommandId="cmd-1"
+ onClose={() => {}}
+ run={makeRun({
+ status: AutomationRunStatus.FAILED,
+ error_detail: "Sandbox timed out.",
+ run_metadata: {
+ finish_tool_response: {
+ outcome_summary: "The report was generated before the timeout.",
+ },
+ },
+ })}
+ />,
+ );
+
+ expect(
+ screen.getByText("The report was generated before the timeout."),
+ ).toBeInTheDocument();
+ expect(screen.getByText(/Sandbox timed out/)).toBeInTheDocument();
+ });
+
+ it("shows a successful task with readable custom metadata when no status was reported", () => {
useBashCommandLogsMock.mockReturnValue(makeHookResult());
render(
@@ -396,7 +423,7 @@ describe("RunLogsModal — run inspection summary", () => {
);
expect(
- screen.getByText(I18nKey.AUTOMATIONS$DETAIL$NEEDS_REVIEW),
+ screen.getByText(I18nKey.AUTOMATIONS$DETAIL$SUCCESSFUL),
).toBeInTheDocument();
expect(
screen.getByText(I18nKey.AUTOMATIONS$DETAIL$CUSTOM_TASK_METADATA),
@@ -410,6 +437,33 @@ describe("RunLogsModal — run inspection summary", () => {
'"next_action": "Ask user to pick a contact."',
);
});
+
+ it("shows an outcome summary for a successful task with no reported status", () => {
+ useBashCommandLogsMock.mockReturnValue(makeHookResult());
+
+ render(
+ <RunLogsModal
+ isOpen
+ conversationId="conv-1"
+ bashCommandId="cmd-1"
+ onClose={() => {}}
+ run={makeRun({
+ run_metadata: {
+ finish_tool_response: {
+ outcome_summary: "Created and published the weekly report.",
+ },
+ },
+ })}
+ />,
+ );
+
+ expect(
+ screen.getByText(I18nKey.AUTOMATIONS$DETAIL$SUCCESSFUL),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByText("Created and published the weekly report."),
+ ).toBeInTheDocument();
+ });
});
describe("RunLogsModal — Debug with OpenHands button", () => {
diff --git a/__tests__/components/features/home/automation-run-health.test.ts b/__tests__/components/features/home/automation-run-health.test.ts
new file mode 100644
index 0000000000..dcf6fb09b9
--- /dev/null
+++ b/__tests__/components/features/home/automation-run-health.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import { deriveRunHealth } from "#/components/features/home/featured-automations/automation-run-health";
+import type { LatestAutomationRunState } from "#/hooks/query/use-latest-automation-runs";
+import {
+ AutomationRunStatus,
+ type AutomationFinishToolResponse,
+ type AutomationRun,
+} from "#/types/automation";
+
+function makeRun(
+ finishToolResponse: AutomationFinishToolResponse | string | null,
+): AutomationRun {
+ return {
+ id: "run-1",
+ status: AutomationRunStatus.COMPLETED,
+ conversation_id: "conv-1",
+ bash_command_id: "cmd-1",
+ error_detail: null,
+ started_at: "2026-01-01T10:00:00Z",
+ completed_at: "2026-01-01T10:02:00Z",
+ run_metadata: { finish_tool_response: finishToolResponse },
+ };
+}
+
+function makeState(run: AutomationRun): LatestAutomationRunState {
+ return {
+ latestRun: run,
+ recentRuns: [run],
+ isLoading: false,
+ isError: false,
+ };
+}
+
+describe("deriveRunHealth", () => {
+ it("treats a completed summary-only response as successful", () => {
+ const run = makeRun({
+ outcome_summary: "Created and published the weekly report.",
+ });
+
+ expect(deriveRunHealth(makeState(run))).toBe("success");
+ });
+
+ it.each(["blocked", "partial_success", "unexpected_status"])(
+ "treats an explicitly reported %s outcome as needing review",
+ (status) => {
+ expect(deriveRunHealth(makeState(makeRun({ status })))).toBe("warning");
+ },
+ );
+});
diff --git a/__tests__/manifests/automation-insights.test.ts b/__tests__/manifests/automation-insights.test.ts
index e3a8dead6b..79db494afe 100644
--- a/__tests__/manifests/automation-insights.test.ts
+++ b/__tests__/manifests/automation-insights.test.ts
@@ -122,7 +122,7 @@ describe("deriveAutomationHealth", () => {
"failing",
],
[
- "a completed latest task with custom metadata is unknown",
+ "a completed latest task with custom metadata is healthy",
createAutomation(),
settled({
latestRun: createRun({
@@ -131,7 +131,7 @@ describe("deriveAutomationHealth", () => {
},
}),
}),
- "unknown",
+ "healthy",
],
[
// Statuses the backend added after the dashboard's reference design:
diff --git a/__tests__/utils/automation-run-display.test.ts b/__tests__/utils/automation-run-display.test.ts
index 5b1ea48535..20bbb6042f 100644
--- a/__tests__/utils/automation-run-display.test.ts
+++ b/__tests__/utils/automation-run-display.test.ts
@@ -3,6 +3,7 @@ import {
getAutomationRunBadgeLabelKey,
getAutomationRunDisplay,
getAutomationRunTaskOutcome,
+ getAutomationRunTaskSummary,
} from "#/utils/automation-run-display";
import { AutomationRunStatus, type AutomationRun } from "#/types/automation";
import { I18nKey } from "#/i18n/declaration";
@@ -89,12 +90,54 @@ describe("automation run display", () => {
status: "unknown",
outcomeSummary: "Finished but could not classify the task.",
});
- expect(getAutomationRunDisplay(run).customTaskMetadataText).toContain(
+ const display = getAutomationRunDisplay(run);
+ expect(display.badgeStatus).toBe("unknown");
+ expect(display.summary).toBe("Finished but could not classify the task.");
+ expect(display.customTaskMetadataText).toContain(
'"status": "unexpected_status"',
);
});
- it("marks completed runs with arbitrary finish metadata as needs review", () => {
+ it("treats a completed run with an outcome summary but no status as successful", () => {
+ const run = makeRun({
+ run_metadata: {
+ finish_tool_response: {
+ outcome_summary: "Created and published the weekly report.",
+ },
+ },
+ });
+
+ expect(getAutomationRunTaskOutcome(run)).toBeNull();
+ expect(getAutomationRunDisplay(run)).toMatchObject({
+ badgeStatus: "success",
+ summary: "Created and published the weekly report.",
+ taskOutcome: null,
+ });
+ });
+
+ it("keeps agent summaries from failed runs without a task status", () => {
+ const run = makeRun({
+ status: AutomationRunStatus.FAILED,
+ error_detail: "Sandbox timed out.",
+ run_metadata: {
+ finish_tool_response: {
+ outcome_summary: "The report was generated before the timeout.",
+ },
+ },
+ });
+
+ expect(getAutomationRunTaskOutcome(run)).toBeNull();
+ expect(getAutomationRunTaskSummary(run)).toBe(
+ "The report was generated before the timeout.",
+ );
+ expect(getAutomationRunDisplay(run)).toMatchObject({
+ badgeStatus: AutomationRunStatus.FAILED,
+ summary: "Sandbox timed out.",
+ taskOutcome: null,
+ });
+ });
+
+ it("treats completed runs with arbitrary finish metadata as successful", () => {
const display = getAutomationRunDisplay(
makeRun({
run_metadata: {
@@ -107,7 +150,7 @@ describe("automation run display", () => {
}),
);
- expect(display.badgeStatus).toBe("unknown");
+ expect(display.badgeStatus).toBe("success");
expect(display.summary).toBeNull();
expect(display.taskOutcome).toBeNull();
expect(display.customTaskMetadataText).toContain(
@@ -118,6 +161,23 @@ describe("automation run display", () => {
);
});
+ it("treats a completed run with a non-object finish response as successful", () => {
+ const display = getAutomationRunDisplay(
+ makeRun({
+ run_metadata: {
+ finish_tool_response: "Report created and published.",
+ },
+ }),
+ );
+
+ expect(display.badgeStatus).toBe("success");
+ expect(display.summary).toBeNull();
+ expect(display.taskOutcome).toBeNull();
+ expect(display.customTaskMetadataText).toBe(
+ "Report created and published.",
+ );
+ });
+
it("keeps default task summary while exposing additional custom metadata", () => {
const display = getAutomationRunDisplay(
makeRun({
diff --git a/src/components/features/automations/detail/run-logs-modal.tsx b/src/components/features/automations/detail/run-logs-modal.tsx
index 89a0ad0987..eee06ca8f0 100644
--- a/src/components/features/automations/detail/run-logs-modal.tsx
+++ b/src/components/features/automations/detail/run-logs-modal.tsx
@@ -15,7 +15,10 @@ import {
type AutomationRun,
type AutomationRunStatusDetail,
} from "#/types/automation";
-import { getAutomationRunDisplay } from "#/utils/automation-run-display";
+import {
+ getAutomationRunDisplay,
+ getAutomationRunTaskSummary,
+} from "#/utils/automation-run-display";
import { DebugAutomationButton } from "./debug-automation-button";
import { RunStatusBadge } from "./run-status-badge";
@@ -102,7 +105,10 @@ function RunInspectionSummary({ run }: { run: AutomationRun | undefined }) {
if (!run) return null;
const display = getAutomationRunDisplay(run);
- const taskSummary = display.taskOutcome?.outcomeSummary ?? null;
+ const taskSummary =
+ run.status === AutomationRunStatus.COMPLETED
+ ? display.summary
+ : getAutomationRunTaskSummary(run);
const taskMetadataText = display.customTaskMetadataText;
const taskStatus =
run.status === AutomationRunStatus.COMPLETED || display.taskOutcome
diff --git a/src/utils/automation-run-display.ts b/src/utils/automation-run-display.ts
index 6c757e43cc..20548fc971 100644
--- a/src/utils/automation-run-display.ts
+++ b/src/utils/automation-run-display.ts
@@ -93,6 +93,11 @@ function getCustomFinishToolMetadata(response: unknown): unknown | null {
return null;
}
+export function getAutomationRunTaskSummary(run: AutomationRun): string | null {
+ const response = getAutomationRunFinishToolResponse(run);
+ return isRecord(response) ? trimString(response.outcome_summary) : null;
+}
+
export function getAutomationRunTaskOutcome(
run: AutomationRun,
): AutomationRunTaskOutcome | null {
@@ -101,7 +106,7 @@ export function getAutomationRunTaskOutcome(
const outcomeSummary = trimString(response.outcome_summary);
const hasStatus = typeof response.status === "string";
- if (!hasStatus && !outcomeSummary) return null;
+ if (!hasStatus) return null;
return {
status: isTaskOutcomeStatus(response.status) ? response.status : "unknown",
@@ -120,11 +125,15 @@ export function getAutomationRunDisplay(
const systemSummary = trimString(run.error_detail);
if (run.status === AutomationRunStatus.COMPLETED) {
+ const outcomeSummary =
+ taskOutcome?.outcomeSummary ??
+ (isRecord(finishToolResponse)
+ ? trimString(finishToolResponse.outcome_summary)
+ : null);
+
return {
- badgeStatus:
- taskOutcome?.status ??
- (finishToolResponse !== null ? "unknown" : "success"),
- summary: taskOutcome?.outcomeSummary ?? null,
+ badgeStatus: taskOutcome?.status ?? "success",
+ summary: outcomeSummary,
taskOutcome,
customTaskMetadata,
customTaskMetadataText,