The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(recovery): reject stale productive continuation wakes (#13173)
Dottad10cbde81523
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex b8d05bbb32..973d35c0e7 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -552,6 +552,8 @@ Recovery rule:This is an active-work continuity recovery.+After a productive successful run, recovery checks that the issue is still `in_progress` and assigned to the same agent under the enqueue transaction's issue lock. The sweep's earlier snapshot cannot authorize a continuation after completion, cancellation, reassignment, or a move to another status. A mismatch records a skipped wake receipt without creating a run. An empty queued continuation cancelled because the issue became terminal is omitted from task chat; its cancellation remains in the run log. Runs that actually started still show their stop state.+The same bounded rule applies when the previous heartbeat reported waiting on a local/background watcher and that watcher was killed, disappeared, or was never represented by a durable Paperclip primitive. Paperclip queues at most one continuation for the same recovery fingerprint. If the continuation also leaves only local watcher evidence, Paperclip must surface a real blocker or explicit recovery action instead of repeating continuation recovery. A new monitor, scheduled wake, healthy delegated blocker issue, or other durable source mutation resolves that recovery fingerprint normally.#### Deliberate wait is not a lost rundiff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.tsindex ba9fad4ecf..faf7f719c9 100644--- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts+++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts@@ -25,6 +25,7 @@ import {heartbeatService,} from "../services/heartbeat.ts";import { runningProcesses } from "../adapters/index.ts";+import { recoveryService } from "../services/recovery/service.ts";const mockAdapterExecute = vi.hoisted(() =>vi.fn(async () => ({@@ -395,6 +396,93 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {]);});+ it.each([+ { runtimeMode: "native", status: "done", reassigned: false },+ { runtimeMode: "legacy", status: "done", reassigned: false },+ { runtimeMode: "native", status: "cancelled", reassigned: false },+ { runtimeMode: "native", status: "backlog", reassigned: false },+ { runtimeMode: "native", status: "in_review", reassigned: false },+ { runtimeMode: "native", status: "blocked", reassigned: false },+ { runtimeMode: "native", status: "in_progress", reassigned: true },+ ] as const)("skips stale $runtimeMode productive recovery after status=$status reassigned=$reassigned commits under the enqueue lock", async ({ runtimeMode, status, reassigned }) => {+ const { companyId, agentId } = await seedCompanyAndAgent();+ const issueId = randomUUID();+ const runId = randomUUID();+ await db.insert(issues).values({+ id: issueId,+ companyId,+ title: "Completion racing with productive recovery",+ status: "in_progress",+ assigneeAgentId: agentId,+ });+ await db.insert(heartbeatRuns).values({+ id: runId,+ companyId,+ agentId,+ invocationSource: "assignment",+ runtimeMode,+ status: "succeeded",+ livenessState: "completed",+ contextSnapshot: { issueId, wakeReason: "issue_assigned" },+ startedAt: new Date(),+ finishedAt: new Date(),+ });++ // Let the real sweep select its in-progress snapshot, then hold the issue+ // lock until the real enqueue transaction is waiting on the newer state.+ const enqueueWakeup = vi.fn(async (...[targetAgentId, options]: Parameters<typeof heartbeat.wakeup>) => {+ let pendingWake!: ReturnType<typeof heartbeat.wakeup>;+ await db.transaction(async (tx) => {+ await tx.update(issues).set({+ status,+ ...(reassigned ? { assigneeAgentId: null, assigneeUserId: "responsible-user" } : {}),+ }).where(eq(issues.id, issueId));+ const [{ pid }] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`);+ pendingWake = heartbeat.wakeup(targetAgentId, options);+ try {+ expect(await waitForCondition(async () => {+ const [{ waiting }] = await db.execute<{ waiting: boolean }>(sql`+ select exists (+ select 1 from pg_stat_activity+ where ${pid} = any(pg_blocking_pids(pid))+ ) as waiting+ `);+ return waiting;+ })).toBe(true);+ } catch (error) {+ // Observe a pending rejection even if the lock assertion fails.+ void pendingWake.catch(() => {});+ throw error;+ }+ });+ return pendingWake;+ });+ const recovery = recoveryService(db, { enqueueWakeup });+ const result = await recovery.reconcileStrandedAssignedIssues();++ expect(enqueueWakeup).toHaveBeenCalledOnce();+ expect(result).toMatchObject({ continuationRequeued: 0, escalated: 0, skipped: 1, issueIds: [] });+ expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)).toEqual([{ id: runId }]);+ expect(await db.select().from(issueComments)).toHaveLength(0);+ expect(mockAdapterExecute).not.toHaveBeenCalled();+ const [wakeup] = await db.select().from(agentWakeupRequests);+ expect(wakeup).toMatchObject({+ status: "skipped",+ reason: "issue_state_guard_mismatch",+ runId: null,+ payload: {+ heartbeatSkip: {+ expectedStatuses: ["in_progress"],+ actualStatus: status,+ expectedAssigneeAgentId: agentId,+ actualAssigneeAgentId: reassigned ? null : agentId,+ },+ },+ });+ const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));+ expect(issue).toMatchObject({ status, assigneeAgentId: reassigned ? null : agentId });+ });+it("cancels a resolved connection-intent wake parked before queued-run claim", async () => {const { companyId, agentId } = await seedCompanyAndAgent();const issueId = randomUUID();diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex be14888d3b..0361621ec1 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -182,6 +182,10 @@ type RecoveryWakeupOptions = {requestedByActorType?: "user" | "agent" | "system";requestedByActorId?: string | null;contextSnapshot?: Record<string, unknown>;+ issueStateGuard?: {+ statuses: string[];+ assigneeAgentId: string;+ };};type RecoveryWakeup = (@@ -1921,6 +1925,17 @@ export function recoveryService(source: "automation",triggerDetail: "system",reason: input.reason,+ // The sweep can combine an old in-progress issue snapshot with a newer+ // successful run. Validate eligibility under the enqueue issue lock so+ // completion or reassignment cannot create a redundant continuation.+ ...(input.source === "issue.productive_terminal_continuation_recovery"+ ? {+ issueStateGuard: {+ statuses: ["in_progress"],+ assigneeAgentId: input.agentId,+ },+ }+ : {}),payload: withRecoveryContext({issueId: input.issueId,diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsxindex 533ef704d1..a5199d77af 100644--- a/ui/src/components/TaskChatThread.test.tsx+++ b/ui/src/components/TaskChatThread.test.tsx@@ -1229,31 +1229,72 @@ describe("TaskChatThread runtime transcript selection", () => {},);- it("does not show a completed-response notice for a redundant cancelled continuation", () => {- render(- <TaskChatThread- comments={[]}- onAdd={async () => {}}- linkedRuns={[- {- runId: "connection-continuation-skipped",- status: "cancelled",- errorCode: "issue_not_in_progress",- startedAt: null,- agentId: "agent-1",- agentName: "Runner",- adapterType: "paperclip_runner",- createdAt: "2026-09-07T18:00:00.000Z",- finishedAt: "2026-09-07T18:00:01.000Z",- },- ]}- />,- );- expect(container.textContent).not.toContain(- "The runner returned no user-facing response.",- );- expect(container.textContent).not.toContain("Run completed");- });+ it.each([+ ["legacy", "issue_not_in_progress"],+ ["native", "issue_not_in_progress"],+ ["legacy", "issue_terminal_status"],+ ["native", "issue_terminal_status"],+ ] as const)(+ "hides a redundant cancelled continuation (%s, %s)",+ (runtimeMode, errorCode) => {+ render(+ <TaskChatThread+ comments={[]}+ onAdd={async () => {}}+ linkedRuns={[+ {+ runId: "connection-continuation-skipped",+ runtimeMode,+ status: "cancelled",+ errorCode,+ startedAt: null,+ agentId: "agent-1",+ agentName: "Runner",+ adapterType: "paperclip_runner",+ createdAt: "2026-09-07T18:00:00.000Z",+ finishedAt: "2026-09-07T18:00:01.000Z",+ },+ ]}+ />,+ );+ expect(container.textContent).not.toContain(+ "The runner returned no user-facing response.",+ );+ expect(container.textContent).not.toContain("Run completed");+ expect(container.textContent).not.toContain("Couldn't start");+ expect(container.textContent).not.toContain("Run cancelled");+ expect(container.textContent).not.toContain("before returning an answer");+ },+ );++ it.each(["legacy", "native"] as const)(+ "keeps a cancellation visible when the %s run had already started",+ (runtimeMode) => {+ render(+ <TaskChatThread+ comments={[]}+ onAdd={async () => {}}+ linkedRuns={[+ {+ runId: "started-cancellation",+ runtimeMode,+ status: "cancelled",+ errorCode: "issue_terminal_status",+ agentId: "agent-1",+ agentName: "Runner",+ adapterType: "paperclip_runner",+ createdAt: "2026-09-07T18:00:00.000Z",+ startedAt: "2026-09-07T18:00:00.500Z",+ finishedAt: "2026-09-07T18:00:01.000Z",+ },+ ]}+ />,+ );+ expect(container.textContent).toContain(+ runtimeMode === "native" ? "Run cancelled" : "Stopped",+ );+ },+ );it("does not treat a progress comment as the final response of a failed native run", () => {nativeTranscriptState.transcriptByRun.set("native-progress-failed", [diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsxindex 5335b102ab..1fff76d2db 100644--- a/ui/src/components/TaskChatThread.tsx+++ b/ui/src/components/TaskChatThread.tsx@@ -1384,6 +1384,18 @@ export function TaskChatThread(props: TaskChatThreadProps) {if (liveRun && source.id === liveRun.id) continue;const entries = transcriptByRun.get(source.id) ?? [];const meta = linkedRunMetaById.get(source.id);+ // A queued continuation can become unnecessary while another turn finishes+ // the task. Keep that cancellation in the run log, not the conversation.+ // Apply this before native stop markers are assembled as well.+ if (+ source.status === "cancelled" &&+ entries.length === 0 &&+ (meta?.errorCode === "issue_not_in_progress" ||+ (meta?.errorCode === "issue_terminal_status" && !meta.startedAt))+ ) {+ settledRunIds.add(source.id);+ continue;+ }const acceptedSummary = acceptedSemanticResultSummary(meta?.resultJson);const parsedSource = transcriptToTaskChatItems(entries, {runId: source.id,@@ -1516,16 +1528,6 @@ export function TaskChatThread(props: TaskChatThreadProps) {});}if (entries.length === 0) {- // A queued continuation cancelled after the task was completed or parked- // never produced a provider turn. Keep its record in the run log without- // presenting it as a completed chat response.- if (- source.status === "cancelled" &&- meta?.errorCode === "issue_not_in_progress"- ) {- settledRunIds.add(source.id);- continue;- }if (sourceIsPaperclipRunner && sourceYielded) {settledRunIds.add(source.id);continue;