Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(automations): show saved draft errors before historical test status (#18617)

🐾 smolpawsd140bcedbf1f
diff --git a/.pr/evidence/draft-status/after.gif b/.pr/evidence/draft-status/after.gif
new file mode 100644
index 0000000000..50efebd83b
Binary files /dev/null and b/.pr/evidence/draft-status/after.gif differ
diff --git a/.pr/evidence/draft-status/after.mp4 b/.pr/evidence/draft-status/after.mp4
new file mode 100644
index 0000000000..8a07c6331c
Binary files /dev/null and b/.pr/evidence/draft-status/after.mp4 differ
diff --git a/.pr/evidence/draft-status/after.png b/.pr/evidence/draft-status/after.png
new file mode 100644
index 0000000000..fc07e7f39d
Binary files /dev/null and b/.pr/evidence/draft-status/after.png differ
diff --git a/.pr/evidence/draft-status/before.gif b/.pr/evidence/draft-status/before.gif
new file mode 100644
index 0000000000..990885f6d7
Binary files /dev/null and b/.pr/evidence/draft-status/before.gif differ
diff --git a/.pr/evidence/draft-status/before.mp4 b/.pr/evidence/draft-status/before.mp4
new file mode 100644
index 0000000000..118fd4ee28
Binary files /dev/null and b/.pr/evidence/draft-status/before.mp4 differ
diff --git a/.pr/evidence/draft-status/before.png b/.pr/evidence/draft-status/before.png
new file mode 100644
index 0000000000..cd87684142
Binary files /dev/null and b/.pr/evidence/draft-status/before.png differ
diff --git a/.pr/evidence/draft-status/desktop-proof.json b/.pr/evidence/draft-status/desktop-proof.json
new file mode 100644
index 0000000000..adc4730d7a
--- /dev/null
+++ b/.pr/evidence/draft-status/desktop-proof.json
@@ -0,0 +1,20 @@
+{
+ "ok": true,
+ "id": "F28.draft-validation",
+ "verdict": "verified",
+ "blocks": [
+ {
+ "name": "stale-test-success",
+ "sha": "60152038d683d659",
+ "verdict": "verified",
+ "reasons": [],
+ "baseline": {
+ "xfail": 3
+ },
+ "fix": {
+ "xpass": 2
+ },
+ "ignored": 0
+ }
+ ]
+}
diff --git a/.pr/evidence/draft-status/phone-proof.json b/.pr/evidence/draft-status/phone-proof.json
new file mode 100644
index 0000000000..171a0499d8
--- /dev/null
+++ b/.pr/evidence/draft-status/phone-proof.json
@@ -0,0 +1,20 @@
+{
+ "ok": true,
+ "id": "F28.draft-validation",
+ "verdict": "verified",
+ "blocks": [
+ {
+ "name": "stale-test-success",
+ "sha": "0e785f89f9ca5eb6",
+ "verdict": "verified",
+ "reasons": [],
+ "baseline": {
+ "xfail": 2
+ },
+ "fix": {
+ "xpass": 2
+ },
+ "ignored": 0
+ }
+ ]
+}
diff --git a/.pr/evidence/draft-status/phone-recipe.md b/.pr/evidence/draft-status/phone-recipe.md
new file mode 100644
index 0000000000..2f816e910f
--- /dev/null
+++ b/.pr/evidence/draft-status/phone-recipe.md
@@ -0,0 +1,36 @@
+I am an AI agent helping Engel Nyst (@enyst) with project work.
+
+Phone viewport variant of the unchanged validation assertion from the read-only maintenance map; only the viewport differs.
+
+```sh known-bug F28.draft-validation stale-test-success
+control-openhands browser viewport phone
+control-openhands browser goto /automations/setup
+control-openhands browser fill 'testid=automation-setup-name' QA_Stale_Test_Success
+control-openhands browser click 'testid=automation-setup-kind-custom'
+control-openhands browser click 'testid=automation-setup-frequency-custom'
+control-openhands browser fill 'testid=automation-setup-custom-schedule' '0 0 1 1 *'
+control-openhands browser click 'testid=automation-setup-save-draft'
+control-openhands browser hover 'testid=automation-setup-name'
+control-openhands browser wait-url 'draftId='
+QA_DRAFT_ID=$(control-openhands browser url | jq -r '.url|capture("draftId=(?<id>[a-f0-9-]+)").id')
+trap 'control-openhands api DELETE "/api/automation/v1/drafts/$QA_DRAFT_ID" --write >/dev/null; control-openhands browser goto /automations >/dev/null' EXIT
+control-openhands browser click 'testid=automation-setup-test'
+control-openhands browser wait 'testid=automation-setup-draft-run'
+QA_MATERIALIZED_ID=$(control-openhands api GET "/api/automation/v1/drafts/$QA_DRAFT_ID" | jq -r .body.materialized_automation_id)
+QA_RUN_STATE=''
+for QA_ATTEMPT in $(seq 1 20); do
+ QA_RUN_STATE=$(control-openhands api GET "/api/automation/v1/$QA_MATERIALIZED_ID/runs?limit=10" | jq -r '.body.runs[0].status // ""')
+ [ "$QA_RUN_STATE" = COMPLETED ] && break
+ sleep 3
+done
+[ "$QA_RUN_STATE" = COMPLETED ]
+control-openhands browser reload
+control-openhands browser text 'testid=automation-setup-draft-validity'
+control-openhands browser fill 'testid=automation-setup-custom-schedule' 'every day'
+control-openhands browser click 'testid=automation-setup-save-draft'
+control-openhands browser hover 'testid=automation-setup-name'
+control-openhands browser reload
+control-openhands assert --field body.dispatchable --equals false -- api GET "/api/automation/v1/drafts/$QA_DRAFT_ID"
+control-openhands browser screenshot --feature F28.draft-validation --name stale-test-success
+control-openhands assert --field text --equals 'Value error, Invalid cron expression: every day' --bug-shows 'Latest test passed' -- browser text 'testid=automation-setup-draft-validity' # bug
+```
diff --git a/.pr/evidence/draft-status/summary.json b/.pr/evidence/draft-status/summary.json
new file mode 100644
index 0000000000..8d9f3b796c
--- /dev/null
+++ b/.pr/evidence/draft-status/summary.json
@@ -0,0 +1,16 @@
+{
+ "attribution": "I am an AI agent helping Engel Nyst (@enyst) with project work.",
+ "issue": 124,
+ "base": "61a3bd93e642a0ff4def73b29496a73245628209",
+ "tested_fix": "891a3f05156b8eb73dfbb6ad13b8d28a21445696",
+ "backend": {
+ "agent_server": "1.55.1",
+ "automation": "1.20.1"
+ },
+ "model_calls": "None; genuine generated Custom Python callback automation tests. Test dispatch materializes a disabled automation; no enabled recurring job was created.",
+ "desktop": "3 baseline xfail including recorded transition, 2 fix xpass",
+ "phone": "2 baseline xfail, 2 fix xpass",
+ "tests": "3 new regression cases fail on base; all 12 focused tests pass on fix; typecheck and focused lint pass.",
+ "verification_skill": "Read-only; no map/harness changes in this product PR. Retirement is separate maintenance after the fix lands.",
+ "cleanup": "Fixed owned run stopped with private purge; ports closed; evidence retained. Shared baseline remains owned by this four-fix task until its other comparisons finish."
+}
diff --git a/.pr/evidence/draft-status/task-cleanup.json b/.pr/evidence/draft-status/task-cleanup.json
new file mode 100644
index 0000000000..8ee80853dd
--- /dev/null
+++ b/.pr/evidence/draft-status/task-cleanup.json
@@ -0,0 +1,102 @@
+{
+ "checked_at": "2026-10-17T23:40:38.477040+00:00",
+ "owned_runs": {
+ "base": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 46
+ },
+ "fix-124": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 12
+ },
+ "fix-125": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 15
+ },
+ "fix-126-preliminary": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 8
+ },
+ "fix-126-final": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 8
+ },
+ "fix-127": {
+ "ports": {
+ "ingress": "closed",
+ "agentServer": "closed",
+ "automation": "closed",
+ "frontend": "closed",
+ "vscode": "closed"
+ },
+ "private_purged": true,
+ "evidence_files_retained": 14
+ }
+ },
+ "external_hosts": {
+ "processes": [
+ {
+ "variant": "base",
+ "pid": 310633,
+ "port": 21580,
+ "state": "owned-group-sigterm"
+ },
+ {
+ "variant": "fix",
+ "pid": 310635,
+ "port": 21680,
+ "state": "zombie-not-a-service"
+ },
+ {
+ "variant": "fix",
+ "pid": 316536,
+ "port": 21680,
+ "state": "owned-group-sigterm"
+ }
+ ],
+ "ports_closed": [
+ 21580,
+ 21680
+ ]
+ },
+ "unique_port_count": 27,
+ "evidence_files": 103,
+ "scope": "Only this four-fix task resources were stopped; shared images/dependencies and original weekly evidence were preserved.",
+ "all_owned_ports_closed": true,
+ "count_definition": "Unique regular files beneath the six owned run evidence directories; current symlink aliases are excluded."
+}
diff --git a/__tests__/components/automations/automation-setup-draft-service.test.ts b/__tests__/components/automations/automation-setup-draft-service.test.ts
index fd000d8e82..368281953e 100644
--- a/__tests__/components/automations/automation-setup-draft-service.test.ts
+++ b/__tests__/components/automations/automation-setup-draft-service.test.ts
@@ -2,10 +2,13 @@ import { describe, expect, it } from "vitest";
import { HttpError } from "@openhands/typescript-client";
import {
extractDraftDispatchErrors,
+ getDraftExecutionStatusText,
formFromServerDraft,
setupDraftFromServerDraft,
} from "#/components/features/automations/setup/automation-setup-draft-service";
import type { AutomationSetupFormValues } from "#/api/automation-setup-types";
+import { AutomationRunStatus } from "#/types/automation";
+import { I18nKey } from "#/i18n/declaration";
import type { AutomationDraftApiResponse } from "#/manifests/types";
const detailEnvelope = {
@@ -60,6 +63,54 @@ const baseSavedDraft: AutomationDraftApiResponse = {
updatedAt: "2026-01-01T00:00:00.000Z",
};
+describe("getDraftExecutionStatusText", () => {
+ const run = {
+ id: "run-1",
+ status: AutomationRunStatus.COMPLETED,
+ conversation_id: null,
+ bash_command_id: null,
+ error_detail: null,
+ started_at: "2026-01-01T00:00:00Z",
+ completed_at: "2026-01-01T00:01:00Z",
+ };
+ const translate = (key: I18nKey) => String(key);
+
+ it.each([
+ AutomationRunStatus.COMPLETED,
+ AutomationRunStatus.FAILED,
+ AutomationRunStatus.RUNNING,
+ ])(
+ "shows the current draft error ahead of a historical %s test",
+ (status) => {
+ const invalidDraft = {
+ ...baseSavedDraft,
+ dispatchable: false,
+ validationErrors: [
+ {
+ field: "trigger",
+ code: "invalid",
+ message: "Invalid cron expression: every day",
+ },
+ ],
+ };
+
+ const text = getDraftExecutionStatusText(
+ invalidDraft,
+ [{ ...run, status }],
+ translate,
+ );
+
+ expect(text).toBe("Invalid cron expression: every day");
+ },
+ );
+
+ it("keeps the successful test status for a currently valid draft", () => {
+ expect(getDraftExecutionStatusText(baseSavedDraft, [run], translate)).toBe(
+ I18nKey.AUTOMATION_SETUP$LATEST_TEST_PASSED,
+ );
+ });
+});
+
const baseForm: AutomationSetupFormValues = {
name: "",
prompt: "",
diff --git a/src/components/features/automations/setup/automation-setup-draft-service.ts b/src/components/features/automations/setup/automation-setup-draft-service.ts
index 3c5d3e4ba4..4ac83df5cd 100644
--- a/src/components/features/automations/setup/automation-setup-draft-service.ts
+++ b/src/components/features/automations/setup/automation-setup-draft-service.ts
@@ -301,6 +301,9 @@ export function getDraftExecutionStatusText(
runs: AutomationRun[],
t: (key: I18nKey) => string,
): string {
+ const validationMessage = draft.validationErrors?.[0]?.message;
+ if (validationMessage) return validationMessage;
+
const latestRun = runs[0];
if (latestRun) {
if (draftRunFinishedSuccessfully(latestRun)) {
@@ -311,10 +314,7 @@ export function getDraftExecutionStatusText(
}
return t(I18nKey.AUTOMATION_SETUP$LATEST_TEST_RUNNING);
}
- return (
- draft.validationErrors?.[0]?.message ??
- t(I18nKey.AUTOMATION_SETUP$READY_TO_TEST)
- );
+ return t(I18nKey.AUTOMATION_SETUP$READY_TO_TEST);
}
export function draftRunPageErrors(