🙌 OpenHands: AI-Driven Development
feat(automations): narrow needs review status (#17682)
Devinac127b6443c6
diff --git a/.pr/issue-17680-live-verification.html b/.pr/issue-17680-live-verification.htmlnew file mode 100644index 0000000000..5f8f7e8400--- /dev/null+++ b/.pr/issue-17680-live-verification.html@@ -0,0 +1,214 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="utf-8" />+ <meta name="viewport" content="width=device-width, initial-scale=1" />+ <title>Issue #17680 — live verification</title>+ <style>+ :root {+ color-scheme: dark;+ font-family:+ Inter,+ ui-sans-serif,+ system-ui,+ -apple-system,+ BlinkMacSystemFont,+ "Segoe UI",+ sans-serif;+ background: #09090b;+ color: #f4f4f5;+ }+ body {+ margin: 0;+ padding: 48px 24px;+ }+ main {+ max-width: 960px;+ margin: 0 auto;+ }+ h1,+ h2 {+ letter-spacing: -0.025em;+ }+ h1 {+ margin: 0 0 8px;+ font-size: 36px;+ }+ h2 {+ margin: 0 0 12px;+ font-size: 20px;+ }+ p,+ li {+ color: #d4d4d8;+ line-height: 1.6;+ }+ .lede {+ margin: 0 0 32px;+ color: #a1a1aa;+ }+ .grid {+ display: grid;+ grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));+ gap: 16px;+ margin: 20px 0 32px;+ }+ .card {+ border: 1px solid #27272a;+ border-radius: 16px;+ padding: 20px;+ background: #111113;+ }+ .badge {+ display: inline-flex;+ align-items: center;+ border-radius: 999px;+ padding: 5px 10px;+ font-size: 12px;+ font-weight: 700;+ }+ .success {+ color: #86efac;+ background: #052e16;+ }+ .review {+ color: #fde68a;+ background: #422006;+ }+ code,+ pre {+ font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace;+ }+ code {+ color: #e4e4e7;+ }+ pre {+ overflow-x: auto;+ border: 1px solid #27272a;+ border-radius: 12px;+ padding: 16px;+ background: #09090b;+ color: #e4e4e7;+ line-height: 1.5;+ }+ table {+ width: 100%;+ border-collapse: collapse;+ margin: 16px 0 32px;+ }+ th,+ td {+ border-bottom: 1px solid #27272a;+ padding: 12px;+ text-align: left;+ vertical-align: top;+ }+ th {+ color: #a1a1aa;+ font-size: 12px;+ text-transform: uppercase;+ letter-spacing: 0.08em;+ }+ .note {+ border-left: 3px solid #f59e0b;+ padding-left: 16px;+ }+ </style>+ </head>+ <body>+ <main>+ <h1>Narrow “Needs review” to genuine review cases</h1>+ <p class="lede">+ Reviewer evidence for OpenHands/OpenHands issue #17680. Verified on the+ real local Agent Canvas, Agent Server, and automation service stack on+ September 24, 2026.+ </p>++ <h2>Behavior change</h2>+ <div class="grid">+ <section class="card">+ <span class="badge review">Before: Needs review</span>+ <p>+ Any completed run with a finish response but no recognized+ <code>status</code> was treated as unknown, including a response+ containing only <code>outcome_summary</code>.+ </p>+ </section>+ <section class="card">+ <span class="badge success">After: Successful</span>+ <p>+ A completed run now defaults to success when the response omits+ <code>status</code>. The outcome summary is still displayed.+ Explicit unrecognized status strings still require review.+ </p>+ </section>+ </div>++ <h2>Decision table</h2>+ <table>+ <thead>+ <tr>+ <th>Completed run response</th>+ <th>Badge</th>+ <th>Summary</th>+ </tr>+ </thead>+ <tbody>+ <tr>+ <td><code>{ "outcome_summary": "Report published" }</code></td>+ <td><span class="badge success">Successful</span></td>+ <td>Report published</td>+ </tr>+ <tr>+ <td><code>{ "status": "unexpected", ... }</code></td>+ <td><span class="badge review">Needs review</span></td>+ <td>Preserved when provided</td>+ </tr>+ <tr>+ <td>No finish response</td>+ <td><span class="badge success">Successful</span></td>+ <td>None</td>+ </tr>+ </tbody>+ </table>++ <h2>Real-stack verification</h2>+ <section class="card">+ <p>+ Created the isolated prompt automation+ <code>PR 17680 live verification</code>+ (<code>0d36598c-537d-4deb-92e5-ec2bcf680a33</code>) and manually+ dispatched run <code>e1d6e30f-1556-439c-bbae-605377679acf</code>. The+ prompt prohibited network, repository, secret, plugin, integration,+ and file access. The automation was disabled after the run.+ </p>+ <p>+ The live SDK (1.49.5) rejected a summary-only FinishTool call because+ its schema requires <code>status</code>. The resulting fallback+ metadata remained visibly marked+ <span class="badge review">Needs review</span>, confirming that+ explicit unknown/unrecognized outcomes retain the conservative state.+ Existing real completed script runs with no finish response remained+ <span class="badge success">Successful</span>.+ </p>+ <p class="note">+ The live service cannot produce the exact backward-compatibility+ payload from this issue because its current FinishTool schema requires+ <code>status</code>. The summary-only and custom-metadata cases are+ therefore exercised through the frontend's real display helper and+ component tests, without changing the backend or its stored data.+ </p>+ </section>++ <h2>Verification completed</h2>+ <ul>+ <li>+ Focused display, modal, health-card, and manifest tests: 67 passed.+ </li>+ <li>Full test suite under UTC: 7,774 passed, 7 todo.</li>+ <li>Application and library production builds passed.</li>+ <li>Lint passed with pre-existing warnings and no errors.</li>+ </ul>+ </main>+ </body>+</html>diff --git a/__tests__/components/automations/detail/run-logs-modal.test.tsx b/__tests__/components/automations/detail/run-logs-modal.test.tsxindex 761bbc49cc..d3e0898d9d 100644--- a/__tests__/components/automations/detail/run-logs-modal.test.tsx+++ b/__tests__/components/automations/detail/run-logs-modal.test.tsx@@ -374,7 +374,34 @@ describe("RunLogsModal — run inspection summary", () => {expect(screen.getByText(/Sandbox timed out/)).toBeInTheDocument();});- it("falls back to a readable JSON block for custom task metadata", () => {+ it("shows a failed run's agent summary separately from its system error", () => {+ useBashCommandLogsMock.mockReturnValue(makeHookResult());++ render(+ <RunLogsModal+ isOpen+ conversationId="conv-1"+ bashCommandId="cmd-1"+ onClose={() => {}}+ run={makeRun({+ status: AutomationRunStatus.FAILED,+ error_detail: "Sandbox timed out.",+ run_metadata: {+ finish_tool_response: {+ outcome_summary: "The report was generated before the timeout.",+ },+ },+ })}+ />,+ );++ expect(+ screen.getByText("The report was generated before the timeout."),+ ).toBeInTheDocument();+ expect(screen.getByText(/Sandbox timed out/)).toBeInTheDocument();+ });++ it("shows a successful task with readable custom metadata when no status was reported", () => {useBashCommandLogsMock.mockReturnValue(makeHookResult());render(@@ -396,7 +423,7 @@ describe("RunLogsModal — run inspection summary", () => {);expect(- screen.getByText(I18nKey.AUTOMATIONS$DETAIL$NEEDS_REVIEW),+ screen.getByText(I18nKey.AUTOMATIONS$DETAIL$SUCCESSFUL),).toBeInTheDocument();expect(screen.getByText(I18nKey.AUTOMATIONS$DETAIL$CUSTOM_TASK_METADATA),@@ -410,6 +437,33 @@ describe("RunLogsModal — run inspection summary", () => {'"next_action": "Ask user to pick a contact."',);});++ it("shows an outcome summary for a successful task with no reported status", () => {+ useBashCommandLogsMock.mockReturnValue(makeHookResult());++ render(+ <RunLogsModal+ isOpen+ conversationId="conv-1"+ bashCommandId="cmd-1"+ onClose={() => {}}+ run={makeRun({+ run_metadata: {+ finish_tool_response: {+ outcome_summary: "Created and published the weekly report.",+ },+ },+ })}+ />,+ );++ expect(+ screen.getByText(I18nKey.AUTOMATIONS$DETAIL$SUCCESSFUL),+ ).toBeInTheDocument();+ expect(+ screen.getByText("Created and published the weekly report."),+ ).toBeInTheDocument();+ });});describe("RunLogsModal — Debug with OpenHands button", () => {diff --git a/__tests__/components/features/home/automation-run-health.test.ts b/__tests__/components/features/home/automation-run-health.test.tsnew file mode 100644index 0000000000..dcf6fb09b9--- /dev/null+++ b/__tests__/components/features/home/automation-run-health.test.ts@@ -0,0 +1,49 @@+import { describe, expect, it } from "vitest";+import { deriveRunHealth } from "#/components/features/home/featured-automations/automation-run-health";+import type { LatestAutomationRunState } from "#/hooks/query/use-latest-automation-runs";+import {+ AutomationRunStatus,+ type AutomationFinishToolResponse,+ type AutomationRun,+} from "#/types/automation";++function makeRun(+ finishToolResponse: AutomationFinishToolResponse | string | null,+): AutomationRun {+ return {+ id: "run-1",+ status: AutomationRunStatus.COMPLETED,+ conversation_id: "conv-1",+ bash_command_id: "cmd-1",+ error_detail: null,+ started_at: "2026-01-01T10:00:00Z",+ completed_at: "2026-01-01T10:02:00Z",+ run_metadata: { finish_tool_response: finishToolResponse },+ };+}++function makeState(run: AutomationRun): LatestAutomationRunState {+ return {+ latestRun: run,+ recentRuns: [run],+ isLoading: false,+ isError: false,+ };+}++describe("deriveRunHealth", () => {+ it("treats a completed summary-only response as successful", () => {+ const run = makeRun({+ outcome_summary: "Created and published the weekly report.",+ });++ expect(deriveRunHealth(makeState(run))).toBe("success");+ });++ it.each(["blocked", "partial_success", "unexpected_status"])(+ "treats an explicitly reported %s outcome as needing review",+ (status) => {+ expect(deriveRunHealth(makeState(makeRun({ status })))).toBe("warning");+ },+ );+});diff --git a/__tests__/manifests/automation-insights.test.ts b/__tests__/manifests/automation-insights.test.tsindex e3a8dead6b..79db494afe 100644--- a/__tests__/manifests/automation-insights.test.ts+++ b/__tests__/manifests/automation-insights.test.ts@@ -122,7 +122,7 @@ describe("deriveAutomationHealth", () => {"failing",],[- "a completed latest task with custom metadata is unknown",+ "a completed latest task with custom metadata is healthy",createAutomation(),settled({latestRun: createRun({@@ -131,7 +131,7 @@ describe("deriveAutomationHealth", () => {},}),}),- "unknown",+ "healthy",],[// Statuses the backend added after the dashboard's reference design:diff --git a/__tests__/utils/automation-run-display.test.ts b/__tests__/utils/automation-run-display.test.tsindex 5b1ea48535..20bbb6042f 100644--- a/__tests__/utils/automation-run-display.test.ts+++ b/__tests__/utils/automation-run-display.test.ts@@ -3,6 +3,7 @@ import {getAutomationRunBadgeLabelKey,getAutomationRunDisplay,getAutomationRunTaskOutcome,+ getAutomationRunTaskSummary,} from "#/utils/automation-run-display";import { AutomationRunStatus, type AutomationRun } from "#/types/automation";import { I18nKey } from "#/i18n/declaration";@@ -89,12 +90,54 @@ describe("automation run display", () => {status: "unknown",outcomeSummary: "Finished but could not classify the task.",});- expect(getAutomationRunDisplay(run).customTaskMetadataText).toContain(+ const display = getAutomationRunDisplay(run);+ expect(display.badgeStatus).toBe("unknown");+ expect(display.summary).toBe("Finished but could not classify the task.");+ expect(display.customTaskMetadataText).toContain('"status": "unexpected_status"',);});- it("marks completed runs with arbitrary finish metadata as needs review", () => {+ it("treats a completed run with an outcome summary but no status as successful", () => {+ const run = makeRun({+ run_metadata: {+ finish_tool_response: {+ outcome_summary: "Created and published the weekly report.",+ },+ },+ });++ expect(getAutomationRunTaskOutcome(run)).toBeNull();+ expect(getAutomationRunDisplay(run)).toMatchObject({+ badgeStatus: "success",+ summary: "Created and published the weekly report.",+ taskOutcome: null,+ });+ });++ it("keeps agent summaries from failed runs without a task status", () => {+ const run = makeRun({+ status: AutomationRunStatus.FAILED,+ error_detail: "Sandbox timed out.",+ run_metadata: {+ finish_tool_response: {+ outcome_summary: "The report was generated before the timeout.",+ },+ },+ });++ expect(getAutomationRunTaskOutcome(run)).toBeNull();+ expect(getAutomationRunTaskSummary(run)).toBe(+ "The report was generated before the timeout.",+ );+ expect(getAutomationRunDisplay(run)).toMatchObject({+ badgeStatus: AutomationRunStatus.FAILED,+ summary: "Sandbox timed out.",+ taskOutcome: null,+ });+ });++ it("treats completed runs with arbitrary finish metadata as successful", () => {const display = getAutomationRunDisplay(makeRun({run_metadata: {@@ -107,7 +150,7 @@ describe("automation run display", () => {}),);- expect(display.badgeStatus).toBe("unknown");+ expect(display.badgeStatus).toBe("success");expect(display.summary).toBeNull();expect(display.taskOutcome).toBeNull();expect(display.customTaskMetadataText).toContain(@@ -118,6 +161,23 @@ describe("automation run display", () => {);});+ it("treats a completed run with a non-object finish response as successful", () => {+ const display = getAutomationRunDisplay(+ makeRun({+ run_metadata: {+ finish_tool_response: "Report created and published.",+ },+ }),+ );++ expect(display.badgeStatus).toBe("success");+ expect(display.summary).toBeNull();+ expect(display.taskOutcome).toBeNull();+ expect(display.customTaskMetadataText).toBe(+ "Report created and published.",+ );+ });+it("keeps default task summary while exposing additional custom metadata", () => {const display = getAutomationRunDisplay(makeRun({diff --git a/src/components/features/automations/detail/run-logs-modal.tsx b/src/components/features/automations/detail/run-logs-modal.tsxindex 89a0ad0987..eee06ca8f0 100644--- a/src/components/features/automations/detail/run-logs-modal.tsx+++ b/src/components/features/automations/detail/run-logs-modal.tsx@@ -15,7 +15,10 @@ import {type AutomationRun,type AutomationRunStatusDetail,} from "#/types/automation";-import { getAutomationRunDisplay } from "#/utils/automation-run-display";+import {+ getAutomationRunDisplay,+ getAutomationRunTaskSummary,+} from "#/utils/automation-run-display";import { DebugAutomationButton } from "./debug-automation-button";import { RunStatusBadge } from "./run-status-badge";@@ -102,7 +105,10 @@ function RunInspectionSummary({ run }: { run: AutomationRun | undefined }) {if (!run) return null;const display = getAutomationRunDisplay(run);- const taskSummary = display.taskOutcome?.outcomeSummary ?? null;+ const taskSummary =+ run.status === AutomationRunStatus.COMPLETED+ ? display.summary+ : getAutomationRunTaskSummary(run);const taskMetadataText = display.customTaskMetadataText;const taskStatus =run.status === AutomationRunStatus.COMPLETED || display.taskOutcomediff --git a/src/utils/automation-run-display.ts b/src/utils/automation-run-display.tsindex 6c757e43cc..20548fc971 100644--- a/src/utils/automation-run-display.ts+++ b/src/utils/automation-run-display.ts@@ -93,6 +93,11 @@ function getCustomFinishToolMetadata(response: unknown): unknown | null {return null;}+export function getAutomationRunTaskSummary(run: AutomationRun): string | null {+ const response = getAutomationRunFinishToolResponse(run);+ return isRecord(response) ? trimString(response.outcome_summary) : null;+}+export function getAutomationRunTaskOutcome(run: AutomationRun,): AutomationRunTaskOutcome | null {@@ -101,7 +106,7 @@ export function getAutomationRunTaskOutcome(const outcomeSummary = trimString(response.outcome_summary);const hasStatus = typeof response.status === "string";- if (!hasStatus && !outcomeSummary) return null;+ if (!hasStatus) return null;return {status: isTaskOutcomeStatus(response.status) ? response.status : "unknown",@@ -120,11 +125,15 @@ export function getAutomationRunDisplay(const systemSummary = trimString(run.error_detail);if (run.status === AutomationRunStatus.COMPLETED) {+ const outcomeSummary =+ taskOutcome?.outcomeSummary ??+ (isRecord(finishToolResponse)+ ? trimString(finishToolResponse.outcome_summary)+ : null);+return {- badgeStatus:- taskOutcome?.status ??- (finishToolResponse !== null ? "unknown" : "success"),- summary: taskOutcome?.outcomeSummary ?? null,+ badgeStatus: taskOutcome?.status ?? "success",+ summary: outcomeSummary,taskOutcome,customTaskMetadata,customTaskMetadataText,