The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(agents): report actionable lifecycle failure steps (#16418)
Dotta250429942e27
diff --git a/maps/agent-access-map/agents.md b/maps/agent-access-map/agents.mdindex 5988d4eace..26e162cf03 100644--- a/maps/agent-access-map/agents.md+++ b/maps/agent-access-map/agents.md@@ -132,6 +132,8 @@ API: `present` — representative operation IDs:- `POST /api/agents/{id}/pause`.- `POST /api/agents/{id}/resume`.- `POST /api/agents/{id}/terminate`.+- `POST /api/agents/{id}/lifecycle/retry`.+- `GET /api/agents/{id}` returns the safe failed phase/step and action in `lifecycleError`.Agent access: `partial` — REST / native `call_api` resume under agent_config:updatechange-grant authorization; pause and termination remain board-only.diff --git a/maps/feature-map/agents.md b/maps/feature-map/agents.mdindex 621e271ed1..4443306897 100644--- a/maps/feature-map/agents.md+++ b/maps/feature-map/agents.md@@ -80,6 +80,12 @@ Automated: [instruction service](../../server/src/__tests__/agent-instructions-sManual: Save a harmless instruction edit, reload, and verify its revision and effect in the next eligible run. Pause and resume the disposable agent; confirm pause prevents new work and does not erase previous runs.+For failed setup, use an unavailable model or invalid command on a disposable agent.+Confirm the profile names the failed verification step and gives a relevant action.+Check that it explains automatic retry and offers **Retry now**. Fix the configuration,+retry, and reload; the failure must disappear only after readiness commits.+Provider output and credentials must not appear in the lifecycle message.+An explicit runner change is a configuration change, with normal revisions andsession invalidation. Incompatible settings must produce a field-specific error;selecting another runner must not silently discard them or change an active run'sdiff --git a/server/src/__tests__/agent-harness-verification.test.ts b/server/src/__tests__/agent-harness-verification.test.tsindex 4728cdabd1..3d43bdd0a8 100644--- a/server/src/__tests__/agent-harness-verification.test.ts+++ b/server/src/__tests__/agent-harness-verification.test.ts@@ -8,6 +8,22 @@ describe("saved harness verification", () => {adapterType: "test", status, testedAt: new Date().toISOString(),checks: codes.map(code => ({ code, level: "info", message: code })),});+ it.each([+ ["opencode_hello_probe_auth_required", "harness_auth_failed"],+ ["opencode_hello_probe_model_unavailable", "harness_model_unavailable"],+ ["opencode_command_unresolvable", "harness_command_unavailable"],+ ["opencode_cwd_invalid", "harness_cwd_invalid"],+ ["opencode_hello_probe_timed_out", "harness_probe_timed_out"],+ ["opencode_hello_probe_unexpected_output", "harness_probe_unexpected_output"],+ ["private provider error", "harness_test_failed"],+ ])("classifies %s without keeping provider output", (checkCode, code) => {+ try {+ assertHarnessTestPassed(result("fail", [checkCode]));+ expect.fail("Expected verification to fail");+ } catch (error) {+ expect(error).toMatchObject({ code, message: "The harness test failed" });+ }+ });it.each([["fail", []], ["warn", ["claude_hello_probe_auth_required"]],["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]],diff --git a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.tsindex 06b7e246d4..39c7895c30 100644--- a/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts+++ b/server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts@@ -5,6 +5,7 @@ import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../applicaimport { startAgentLifecycle } from "../index.js";import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";import { logger } from "../../../middleware/logger.js";+import { lifecycleFailureMessage } from "../application/failure.js";const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3,lifecycleRequiredPluginIds: null, lifecycleError: null,@@ -66,6 +67,30 @@ describe("lifecycle readiness handoff", () => {});describe("lifecycle failure diagnostics", () => {+ it.each([+ ["host", "harness_model_unavailable", "Execution configuration check failed. The selected model is unavailable."],+ ["plugin", "required_plugin_unavailable", "Execution configuration check: required plugin step failed. A required plugin is unavailable."],+ ["select_plugins", "unknown private code", "Execution configuration check: finding required plugins failed. Paperclip could not complete this step."],+ ["transition", "40P01", "Execution configuration check: saving progress failed. Paperclip could not complete this step."],+ ] as const)("persists safe actionable errors for %s", async (stage, code, expected) => {+ const { store, driver, worker } = fixture();+ const error = Object.assign(new Error("secret provider output"), { code });+ if (stage === "host") driver.runHost.mockRejectedValue(error);+ if (stage === "plugin") driver.runPlugin.mockRejectedValue(error);+ if (stage === "select_plugins") driver.requiredPluginIds.mockRejectedValue(error);+ if (stage === "transition") store.change.mockRejectedValue(error);+ await worker.process(agent.id);+ expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date));+ expect(JSON.stringify(store.defer.mock.calls)).not.toContain("secret provider output");+ expect(JSON.stringify(store.defer.mock.calls)).not.toContain("unknown private code");+ await worker.stop();+ });++ it("does not treat prototype properties as allowed failure codes", () => {+ expect(lifecycleFailureMessage({ stage: "host", phase: "verifying" }, { code: "toString" }))+ .toContain("Paperclip could not complete this step");+ });+it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => {const { store, driver, report, worker } = fixture();const error = new Error("private provider detail");@@ -100,6 +125,7 @@ describe("lifecycle failure diagnostics", () => {it.each([[{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"],[{ code: "harness_test_failed" }, "harness_test_failed"],+ [{ code: "harness_auth_failed" }, "harness_auth_failed"],[{ cause: { code: "40P01", message: "private query" } }, "40P01"],[{ code: "private credential value" }, "unclassified"],])("logs known codes without provider error contents", (details, failureCode) => {diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.tsindex fc84aa7921..725a4adf1c 100644--- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts+++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts@@ -140,7 +140,7 @@ const support = await getEmbeddedPostgresTestSupport();const work = worker(createLifecycleDriver(db, {} as never).runHost);await work.process(agent.id);expect(probe).toHaveBeenCalled();- expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." });+ expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: expect.stringContaining("Execution configuration check failed. Authentication failed.") });} finally { probe.mockRestore(); }});diff --git a/server/src/modules/agent-lifecycle/application/failure.ts b/server/src/modules/agent-lifecycle/application/failure.tsnew file mode 100644index 0000000000..0c5da80e34--- /dev/null+++ b/server/src/modules/agent-lifecycle/application/failure.ts@@ -0,0 +1,40 @@+import type { LifecycleFailure } from "./ports.js";++// Only fixed text may leave this boundary. Provider errors can contain secrets.+const reasons = {+ harness_test_failed: "The execution test did not pass. Open Configuration and run the agent test to inspect the failed checks.",+ harness_auth_failed: "Authentication failed. Check the agent's AI connection or sign in to its execution tool, then retry.",+ harness_model_unavailable: "The selected model is unavailable. Choose a model supported by the agent's AI connection in Configuration, then retry.",+ harness_command_unavailable: "The execution command was not found. Check the command and its installation in the agent's environment, then retry.",+ harness_cwd_invalid: "The working directory is unavailable. Check the working directory in Configuration, then retry.",+ harness_probe_timed_out: "The execution test timed out. Paperclip will retry automatically. If this continues, run the agent test in Configuration.",+ harness_probe_unexpected_output: "The execution test did not return the expected response. Run the agent test in Configuration to inspect the failed checks.",+ test_environment_unavailable: "The test environment is unavailable. Check the agent's environment in Configuration. Paperclip will retry automatically.",+ required_plugin_unavailable: "A required plugin is unavailable. Check the plugin's status in Settings. Paperclip will retry automatically.",+ invalid_lifecycle_result: "A required plugin returned an invalid result. Check the plugin's status in Settings. Paperclip will retry automatically.",+} as const;++const transientCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]);++export function lifecycleFailureCode(error: unknown): string | undefined {+ const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null;+ return [candidate?.code, candidate?.cause?.code].find((code): code is string =>+ typeof code === "string" && (Object.hasOwn(reasons, code) || transientCodes.has(code)));+}++export function lifecycleFailureMessage(context: LifecycleFailure, error: unknown): string {+ const phases = {+ preparing: "Agent preparation", verifying: "Execution configuration check", pausing: "Stopping agent execution",+ resuming: "Starting agent resources", terminating: "Stopping agent execution", cleaning_up: "Agent resource cleanup",+ };+ const phase = context.phase && Object.hasOwn(phases, context.phase)+ ? phases[context.phase as keyof typeof phases] : "Agent setup";+ const step = context.stage === "plugin" ? `${phase}: required plugin step`+ : context.stage === "select_plugins" ? `${phase}: finding required plugins`+ : ["save_plugins", "complete_host", "complete_plugin", "transition"].includes(context.stage)+ ? `${phase}: saving progress` : phase;+ const code = lifecycleFailureCode(error);+ const reason = code && Object.hasOwn(reasons, code) ? reasons[code as keyof typeof reasons]+ : "Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator.";+ return `${step} failed. ${reason}`;+}diff --git a/server/src/modules/agent-lifecycle/application/worker.ts b/server/src/modules/agent-lifecycle/application/worker.tsindex 3b90b22767..3c362dcbb1 100644--- a/server/src/modules/agent-lifecycle/application/worker.ts+++ b/server/src/modules/agent-lifecycle/application/worker.ts@@ -1,6 +1,7 @@import { randomUUID } from "node:crypto";import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js";import { isLifecycleWorkPending } from "../domain/policy.js";+import { lifecycleFailureMessage } from "./failure.js";/** Database leases protect each attempt across server processes. */export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) {@@ -20,10 +21,13 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDrif (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return;const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState,operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion };- const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) =>- attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work);+ let failureContext: LifecycleFailure = { ...context, stage: "host" };+ const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) => {+ const stepContext = { ...context, stage, ...(pluginId ? { pluginId } : {}) };+ return attempt(stepContext, work).catch(error => { failureContext = stepContext; throw error; });+ };const renewal = setInterval(() => {- void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {});+ void attempt({ ...context, stage: "renew" }, () => store.renew(agent, owner, new Date())).catch(() => {});}, 30_000);renewal.unref?.();try {@@ -65,8 +69,9 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDrasync () => { await driver.onReady?.(changed); }).catch(() => {});}if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return;- } catch {- if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue;+ } catch (error) {+ const message = lifecycleFailureMessage(failureContext, error);+ if (!await runStep("defer", () => store.defer(agent, owner, message, new Date()))) continue;return;} finally {clearInterval(renewal);diff --git a/server/src/modules/agent-lifecycle/index.ts b/server/src/modules/agent-lifecycle/index.tsindex ba8abce8c3..99fc03bb11 100644--- a/server/src/modules/agent-lifecycle/index.ts+++ b/server/src/modules/agent-lifecycle/index.ts@@ -142,3 +142,5 @@ export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: L},};}++export { lifecycleFailureCode } from "./application/failure.js";diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.tsindex 92e08a92ce..295397a445 100644--- a/server/src/services/agent-harness-verification.ts+++ b/server/src/services/agent-harness-verification.ts@@ -19,7 +19,17 @@ export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) {const authenticationFailed = result.checks.some(check =>check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code));const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe")));- if (result.status === "fail" || authenticationFailed || incomplete) throw Object.assign(new Error("The harness test failed"), { code: "harness_test_failed" });+ if (result.status === "fail" || authenticationFailed || incomplete) {+ const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix));+ const code = authenticationFailed ? "harness_auth_failed"+ : has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable"+ : has("_command_unresolvable") ? "harness_command_unavailable"+ : has("_cwd_invalid") ? "harness_cwd_invalid"+ : has("_hello_probe_timed_out") ? "harness_probe_timed_out"+ : has("_hello_probe_unexpected_output") ? "harness_probe_unexpected_output"+ : "harness_test_failed";+ throw Object.assign(new Error("The harness test failed"), { code });+ }}export function agentHarnessVerificationService(db: Db, manager: PluginWorkerManager) {diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.tsindex 25ef5e5873..a69ae1d912 100644--- a/server/src/services/agent-lifecycle.ts+++ b/server/src/services/agent-lifecycle.ts@@ -16,11 +16,7 @@ export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycleexport function createAgentLifecycleEffects(hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects {return {reportFailure(context, error) {- // Only known codes are safe; provider errors can contain credentials or private data.- const allowedCodes = new Set(["required_plugin_unavailable", "invalid_lifecycle_result", "harness_test_failed",- "test_environment_unavailable", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]);- const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null;- const code = [candidate?.code, candidate?.cause?.code].find(value => typeof value === "string" && allowedCodes.has(value));+ const code = lifecycle.lifecycleFailureCode(error);logger.warn({ ...context, failureCode: code ?? "unclassified" }, "Agent lifecycle operation failed");},transaction: withAccountingTransaction,diff --git a/ui/src/components/AgentLifecycleStatus.test.tsx b/ui/src/components/AgentLifecycleStatus.test.tsxindex 8e4467f2f4..e2fe7baf49 100644--- a/ui/src/components/AgentLifecycleStatus.test.tsx+++ b/ui/src/components/AgentLifecycleStatus.test.tsx@@ -34,7 +34,9 @@ it("retries a failed step once while pending and removes the failure after recovtry {await act(async () => root.render(<AgentLifecycleStatus agent={agent} onRetry={retry} retryPending={false} />));expect(container.textContent).toContain(agent.lifecycleError);- expect(container.textContent).toContain("Termination is not complete");+ expect(container.textContent).toContain("Agent cleanup failed");+ expect(container.textContent).toContain("Paperclip retries automatically");+ expect(container.textContent).not.toContain("Required resources are being removed");await act(async () => container.querySelector("button")!.click());expect(retry).toHaveBeenCalledOnce();await act(async () => root.render(<AgentLifecycleStatus agent={agent} onRetry={retry} retryPending />));diff --git a/ui/src/components/AgentLifecycleStatus.tsx b/ui/src/components/AgentLifecycleStatus.tsxindex 29a6984da7..689a54b831 100644--- a/ui/src/components/AgentLifecycleStatus.tsx+++ b/ui/src/components/AgentLifecycleStatus.tsx@@ -15,6 +15,12 @@ const phases = {cleaning_up: ["Cleaning up agent", "Required resources are being removed. Termination is not complete."],} satisfies Partial<Record<AgentLifecycleState, [string, string]>>;+const failedPhases = {+ preparing: "Agent preparation failed", verifying: "Agent verification failed",+ pausing: "Agent pause failed", resuming: "Agent resume failed",+ terminating: "Agent termination failed", cleaning_up: "Agent cleanup failed",+};+function pendingPhase(state?: AgentLifecycleState) {return state && state in phases ? phases[state as keyof typeof phases] : undefined;}@@ -48,15 +54,16 @@ export function AgentLifecycleStatus({ agent, onRetry, retryPending, refreshErro{phase && <div className="flex flex-wrap items-center gap-2"><Badge variant={agent.lifecycleError ? "destructive" : "secondary"}>{!agent.lifecycleError && <Loader2 aria-hidden="true" className="mr-1 size-3 animate-spin motion-reduce:animate-none" />}- {phase[0]}+ {agent.lifecycleError ? failedPhases[agent.lifecycleState as keyof typeof failedPhases] : phase[0]}</Badge>- <span className="text-muted-foreground">{phase[1]}</span>+ {!agent.lifecycleError && <span className="text-muted-foreground">{phase[1]}</span>}</div>}{phase && agent.lifecycleError && <div className="flex flex-wrap items-center gap-2 text-destructive"><span>{agent.lifecycleError}</span><Button variant="outline" size="sm" disabled={retryPending} onClick={onRetry}>- {retryPending ? "Retrying…" : "Retry"}+ {retryPending ? "Retrying…" : "Retry now"}</Button></div>}+ {phase && agent.lifecycleError && <p className="text-muted-foreground">Paperclip retries automatically; Retry now checks again immediately.</p>}</div>;}