The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(recovery): expose settled holds and their preserved decisions (#16169)
Tao Xie976cdc77ed56
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex 06d1e73dba..d008805a39 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -1274,6 +1274,13 @@ Examples:The recovery action stays source-scoped by default. Stranded-task escalation is board-owned and records the cause, evidence, next action, source and return owner, `routingPolicy: board_escalation_no_takeover_v1`, and wake or monitor policy in the source thread/detail surface.+The recovery-actions read API also lists settled records whose automatic recovery+receipt still blocks replay. A resolved recovery record does not by itself prove+that work resumed. The task's stopped-run notice lets authorized board users+inspect the matching decision under Recovery details, even when there is no+active recovery action. Existing run inspection, verified continuation, new+messages, and required-file repair remain the supported recovery paths.+The board owns the recovery decision, not the source deliverable. Automatic recovery must preserve both source assignee fields. Only an explicit operator decision or applicable serious-failure policy may transfer the deliverable.An upgrade may encounter an already-active agent-owned recovery action. Paperclip keeps that record readable and resolvable for compatibility, but periodic reconciliation does not enqueue another takeover wake from it.diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.mdindex 686273869f..e6fc1098fe 100644--- a/maps/agent-access-map/recovery.md+++ b/maps/agent-access-map/recovery.md@@ -75,6 +75,13 @@ revised live agent authorization and callback delivery remain `not-run`.## `source-action`+The recovery-actions read API includes settled execution reconciliation holds+with an automatic `replay: blocked` receipt, as well as retained workspace holds.+The existing company and task-read checks remain in force. The board notice+reads this history to explain its current hold; this adds no agent mutation+authority. Route and notice regressions cover the source contract, without+claiming live agent or provider qualification.+Review: `baseline`.API: `present` — representative operation IDs:diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.mdindex c5f4065663..251c79327b 100644--- a/maps/feature-map/recovery.md+++ b/maps/feature-map/recovery.md@@ -38,6 +38,13 @@ Open the affected task from Tasks, Inbox/Blocked, search, or a direct task link.Inspect the recovery notice/card in its history, expand details if needed, anduse the currently offered action. Follow any linked repair task and return owner.+An authorized board user can expand Recovery details on a stopped-run notice to+read the decision that still blocks replay, including a settled recovery record.+The notice matches the current action and run; it does not show another run's+historical decision or authorize a retry. Source-reviewed behavior; the+[notice regressions](../../ui/src/components/ExecutionBlockerNotice.test.tsx)+cover empty active recovery, a mismatched source, and failed history reads.+### `run-detail`Open the agent's run (`/agents/:agentId/runs/:runId`) from activity or the task'sdiff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.tsindex 05e96216f6..0cd24ecd5f 100644--- a/server/src/__tests__/issue-recovery-actions.test.ts+++ b/server/src/__tests__/issue-recovery-actions.test.ts@@ -1753,6 +1753,36 @@ describeEmbeddedPostgres("issue recovery actions", () => {expect(list.body.actions).toHaveLength(1);});+ it("lists a settled reconciliation hold that still blocks replay", async () => {+ const { companyId, coderId, sourceIssueId } = await seedCompany();+ const runId = randomUUID();+ const [action] = await db.insert(issueRecoveryActions).values({+ companyId,+ sourceIssueId,+ kind: "active_run_watchdog",+ status: "resolved",+ outcome: "blocked",+ ownerType: "board",+ returnOwnerAgentId: coderId,+ cause: "legacy_execution_requires_reconciliation",+ fingerprint: runId,+ nextAction: "Preserve recorded work without replay.",+ evidence: { runId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } },+ }).returning();++ const list = await request(createApp()).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);++ expect(list.body.active).toBeNull();+ expect(list.body.actions).toEqual([+ expect.objectContaining({+ id: action!.id,+ status: "resolved",+ cause: "legacy_execution_requires_reconciliation",+ evidence: { runId, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } },+ }),+ ]);+ });+it("projects recovery action metadata into the structured wake payload", async () => {const { companyId, managerId, coderId, sourceIssueId } = await seedCompany();const action = await issueRecoveryActionService(db).upsertSourceScoped({diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.tsindex fa2abf0c52..0265eb18db 100644--- a/server/src/routes/issues.ts+++ b/server/src/routes/issues.ts@@ -23,7 +23,7 @@ import { queuedInteractionId, readQueuedInteractionResponse, hasQueuedInteractioimport { deliverConversationComments, isConversation } from "../services/agent-conversations.js";import { issueRecoveryActionReadModel } from "../services/issue-recovery-actions.js";import { getExecutionBlocker } from "../services/execution-blocker.js";-import { requiresExecutionReconciliation } from "@paperclipai/shared";+import { EXECUTION_RECONCILIATION_CAUSES, requiresExecutionReconciliation } from "@paperclipai/shared";import {validateExecutionReconciliation,markExecutionReconciliation,@@ -39,7 +39,7 @@ import { createHash, randomUUID } from "node:crypto";import { Router, type Request, type Response } from "express";import multer from "multer";import { z } from "zod";-import { and, asc, desc, eq, inArray, isNull, notInArray, sql, type SQL } from "drizzle-orm";+import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";import type { Db } from "@paperclipai/db";import type { ChatChannelService } from "../services/chat-channels.js";import {@@ -9503,8 +9503,11 @@ export function issueRoutes(});const retainedHolds = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id),- sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`,+ or(+ sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`,+ inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]),+ ),)).orderBy(desc(issueRecoveryActions.createdAt));res.json({active,diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.tsindex ae713e1119..40881632c3 100644--- a/ui/src/api/issues.ts+++ b/ui/src/api/issues.ts@@ -287,6 +287,8 @@ export const issuesApi = {api.post<{ runId: string; resultId: string; status: "queued" }>(`/issues/${id}/recovery-actions/resume-workspace-finalization`, data),retryWorkspaceExport: (id: string, data: { actionId: string; runId: string; repairNote: string }) =>api.post<{ runId: string; resultId: string; leaseId: string; status: "queued" }>(`/issues/${id}/recovery-actions/retry-workspace-export`, data),+ recoveryActions: (id: string) =>+ api.get<{ active: IssueRecoveryAction | null; actions: IssueRecoveryAction[] }>(`/issues/${id}/recovery-actions`),resolveRecoveryAction: (id: string,data: {diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsxindex 3fd0f6a9b8..1ff43ab65e 100644--- a/ui/src/components/ExecutionBlockerNotice.test.tsx+++ b/ui/src/components/ExecutionBlockerNotice.test.tsx@@ -6,11 +6,13 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query";import { ExecutionBlockerNotice } from "./ExecutionBlockerNotice";import { agentsApi } from "../api/agents";import { activityApi } from "../api/activity";+import { issuesApi } from "../api/issues";vi.mock("../lib/router", () => ({Link: ({ to, ...props }: AnchorHTMLAttributes<HTMLAnchorElement> & { to: string }) => <a href={to} {...props} />,}));vi.mock("../api/agents", () => ({ agentsApi: { retryFailedRun: vi.fn() } }));vi.mock("../api/activity", () => ({ activityApi: { runsForIssue: vi.fn() } }));+vi.mock("../api/issues", () => ({ issuesApi: { recoveryActions: vi.fn() } }));describe("stopped task recovery notice", () => {let root: Root;@@ -19,6 +21,7 @@ describe("stopped task recovery notice", () => {const onRetried = vi.fn();beforeEach(async () => {vi.clearAllMocks();+ vi.mocked(issuesApi.recoveryActions).mockResolvedValue({ active: null, actions: [] });container = document.createElement("div"); document.body.append(container);root = createRoot(container);client = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } });@@ -169,4 +172,75 @@ describe("stopped task recovery notice", () => {expect(container.querySelector<HTMLButtonElement>("button")!.disabled).toBe(false);expect(onRetried).not.toHaveBeenCalled();});+ it("shows a settled hold's decision even when there is no active recovery action", async () => {+ vi.mocked(issuesApi.recoveryActions).mockResolvedValue({ active: null, actions: [{+ id: "recovery", status: "resolved", resolutionNote: "The last tool outcome could not be verified.",+ evidence: { runId: "cancelled-run", automaticRecovery: { replay: "blocked" } },+ }] as never });+ await act(async () => root.render(<QueryClientProvider client={client}>+ <ExecutionBlockerNotice companyId="company" issueId="task" issueStatus="blocked" canManage onRetried={onRetried} blocker={{+ recoveryActionId: "recovery", runId: "cancelled-run", agentId: "agent", runStatus: "cancelled",+ cause: "legacy_execution_requires_reconciliation", nextAction: "Send a new message after inspecting the run.",+ }} />+ </QueryClientProvider>));+ await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)); });+ expect(issuesApi.recoveryActions).toHaveBeenCalledWith("task");+ expect(container.querySelector("details")?.textContent).toContain("The last tool outcome could not be verified.");+ expect(container.querySelector("details")?.textContent).toContain("stopped automatic retries");+ expect(container.querySelector("button")).toBeNull();+ expect(agentsApi.retryFailedRun).not.toHaveBeenCalled();+ });+ it("does not show a different run's historical recovery decision", async () => {+ vi.mocked(issuesApi.recoveryActions).mockResolvedValue({ active: null, actions: [{+ id: "recovery", status: "resolved", resolutionNote: "Old decision",+ evidence: { runId: "another-run", automaticRecovery: { replay: "blocked" } },+ }] as never });+ await act(async () => root.render(<QueryClientProvider client={client}>+ <ExecutionBlockerNotice companyId="company" issueId="task" issueStatus="blocked" canManage onRetried={onRetried} blocker={{+ recoveryActionId: "recovery", runId: "cancelled-run", agentId: "agent",+ cause: "legacy_execution_requires_reconciliation", nextAction: "Inspect the current run.",+ }} />+ </QueryClientProvider>));+ await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)); });+ expect(container.querySelector("details")).toBeNull();+ expect(container.textContent).not.toContain("Old decision");+ });+ it("reports a recovery history read failure without hiding the current guidance", async () => {+ vi.mocked(issuesApi.recoveryActions).mockRejectedValue(new Error("History unavailable"));+ await act(async () => root.render(<QueryClientProvider client={client}>+ <ExecutionBlockerNotice companyId="company" issueId="task" issueStatus="blocked" canManage onRetried={onRetried} blocker={{+ recoveryActionId: "recovery", runId: "cancelled-run", agentId: "agent",+ cause: "legacy_execution_requires_reconciliation", nextAction: "Inspect the current run.",+ }} />+ </QueryClientProvider>));+ await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)); });+ expect(container.querySelector('[role="alert"]')?.textContent).toContain("History unavailable");+ expect(container.textContent).toContain("Inspect the current run.");+ });+ it("refreshes a decision while the same stopped run remains open", async () => {+ vi.useFakeTimers();+ try {+ const action = { id: "recovery", status: "active", resolutionNote: "Inspection pending", evidence: { runId: "cancelled-run" } };+ vi.mocked(issuesApi.recoveryActions).mockResolvedValue({ active: null, actions: [action] } as never);+ await act(async () => root.render(<QueryClientProvider client={client}>+ <ExecutionBlockerNotice companyId="company" issueId="task" issueStatus="blocked" canManage onRetried={onRetried} blocker={{+ recoveryActionId: "recovery", runId: "cancelled-run", agentId: "agent",+ cause: "legacy_execution_requires_reconciliation", nextAction: "Inspect the current run.",+ }} />+ </QueryClientProvider>));+ await act(async () => { await vi.advanceTimersByTimeAsync(1); });+ expect(container.textContent).toContain("Inspection pending");+ vi.mocked(issuesApi.recoveryActions).mockResolvedValue({ active: null, actions: [{ ...action,+ status: "resolved", resolutionNote: "Preserved work; automatic replay stopped", evidence: {+ runId: "cancelled-run", automaticRecovery: { replay: "blocked" },+ },+ }] } as never);+ await act(async () => { await vi.advanceTimersByTimeAsync(5_001); });+ expect(issuesApi.recoveryActions).toHaveBeenCalledTimes(2);+ expect(container.textContent).toContain("Preserved work; automatic replay stopped");+ expect(container.textContent).not.toContain("Inspection pending");+ } finally {+ vi.useRealTimers();+ }+ });});diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsxindex e56dbf1659..f6f82de177 100644--- a/ui/src/components/ExecutionBlockerNotice.tsx+++ b/ui/src/components/ExecutionBlockerNotice.tsx@@ -3,6 +3,7 @@ import type { ExecutionBlocker, Issue } from "@paperclipai/shared";import { WorkspaceFileRecovery } from "./WorkspaceFileRecovery";import { agentsApi } from "../api/agents";import { activityApi } from "../api/activity";+import { issuesApi } from "../api/issues";import { queryKeys } from "../lib/queryKeys";import { Button } from "./ui/button";import { cn } from "../lib/utils";@@ -18,6 +19,15 @@ export function ExecutionBlockerNotice({ companyId, issueId, issueStatus, canManonRetried: () => void;}) {const queryClient = useQueryClient();+ const { data: recovery, error: recoveryError } = useQuery({+ queryKey: queryKeys.issues.recoveryActions(issueId, blocker.recoveryActionId ?? null, blocker.runId ?? null),+ queryFn: () => issuesApi.recoveryActions(issueId),+ enabled: canManage && Boolean(blocker.recoveryActionId) && !["done", "cancelled"].includes(issueStatus ?? ""),+ refetchInterval: 5_000,+ });+ const hold = recovery?.actions.find(action => action.id === blocker.recoveryActionId &&+ (action.evidence.runId ?? action.evidence.sourceRunId) === blocker.runId);+ const replayBlocked = (hold?.evidence.automaticRecovery as { replay?: unknown } | undefined)?.replay === "blocked";const { data: runs, error: runsError } = useQuery({queryKey: queryKeys.issues.runs(issueId),queryFn: () => activityApi.runsForIssue(issueId),@@ -75,6 +85,13 @@ export function ExecutionBlockerNotice({ companyId, issueId, issueStatus, canMan<p role="alert" className="w-full text-destructive">{retry.error.message}</p>)}{runsError && <p role="alert" className="w-full text-destructive">{runsError.message}</p>}+ {canManage && hold && <details className="w-full">+ <summary className="cursor-pointer">Recovery details</summary>+ {replayBlocked &&+ <p>Paperclip stopped automatic retries to avoid repeating work with an unknown outcome. Inspect the run before requesting continuation.</p>}+ {hold.resolutionNote && <p>{hold.resolutionNote}</p>}+ </details>}+ {recoveryError && <p role="alert" className="w-full text-destructive">Unable to load recovery details: {recoveryError.message}</p>}{issueStatus && <WorkspaceFileRecovery key={`${blocker.recoveryActionId}:${blocker.runId}`} issueId={issueId}issueStatus={issueStatus} blocker={blocker} canManage={canManage} onRecorded={onRetried} />}</div>diff --git a/ui/src/lib/queryKeys.ts b/ui/src/lib/queryKeys.tsindex ea25bee263..92ac838341 100644--- a/ui/src/lib/queryKeys.ts+++ b/ui/src/lib/queryKeys.ts@@ -296,6 +296,8 @@ export const queryKeys = {dryRun: (id: string) => ["status-cards", "detail", id, "dry-run"] as const,},issues: {+ recoveryActions: (id: string, actionId: string | null, runId: string | null) =>+ ["issues", id, "recovery-actions", actionId, runId] as const,list: (companyId: string) => ["issues", companyId] as const,mentionPool: (companyId: string) =>["issues", companyId, "mention-pool"] as const,