The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(recovery): preserve hand-back wake liveness (#10562)
Dottacd7f84965c5b
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex a155634c6c..2daf6fb37f 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -519,6 +519,11 @@ Recovery rule:This is a dispatch recovery, not a continuation recovery.+Recovery hand-back is covered by the same liveness guarantee:++- an `issue_recovery_action_restored` wake requested while the resolving recovery run is still active is persisted as a follow-up and dispatched only after that run exits, so it cannot be coalesced into the run that requested it+- if that follow-up is nevertheless lost, the stranded-work backstop treats an assigned `todo` issue with a resolved `handed_back` recovery action from during or after its latest successful run as stranded and queues the bounded assignment recovery wake; the successful resolving run is not, by itself, evidence that the handed-back source work is live+### 9.2 Stranded assigned `in_progress`Example:diff --git a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts b/server/src/__tests__/heartbeat-comment-wake-batching.test.tsindex 3e6d30fe8c..4a07e57c82 100644--- a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts+++ b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts@@ -295,6 +295,125 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {expect(runs[0]?.id).toBe(runId);});+ it("defers recovery hand-back wakes until the resolving run exits", async () => {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const issueId = randomUUID();+ const runId = randomUUID();+ const recoveryActionId = randomUUID();+ const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;+ const heartbeat = heartbeatService(db);++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix,+ requireBoardApprovalForNewAgents: false,+ defaultResponsibleUserId: "responsible-user",+ });++ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "Recovery owner",+ role: "engineer",+ status: "running",+ adapterType: "process",+ adapterConfig: {},+ runtimeConfig: {},+ permissions: {},+ });++ await db.insert(heartbeatRuns).values({+ id: runId,+ companyId,+ agentId,+ invocationSource: "assignment",+ triggerDetail: "system",+ status: "running",+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "source_scoped_recovery_action",+ },+ });+ runningProcesses.set(runId, {+ child: {} as never,+ graceSec: 0,+ processGroupId: null,+ });++ await db.insert(issues).values({+ id: issueId,+ companyId,+ title: "Resume handed-back work",+ status: "todo",+ priority: "medium",+ responsibleUserId: "responsible-user",+ assigneeAgentId: agentId,+ executionRunId: runId,+ executionAgentNameKey: "recovery-owner",+ executionLockedAt: new Date(),+ issueNumber: 1,+ identifier: `${issuePrefix}-1`,+ });++ const followupRun = await heartbeat.wakeup(agentId, {+ source: "automation",+ triggerDetail: "system",+ reason: "issue_recovery_action_restored",+ payload: {+ issueId,+ recoveryActionId,+ mutation: "recovery_action_resolution",+ },+ contextSnapshot: {+ issueId,+ taskId: issueId,+ recoveryActionId,+ wakeReason: "issue_recovery_action_restored",+ source: "issue.recovery_action_resolution",+ },+ requestedByActorType: "agent",+ requestedByActorId: agentId,+ });++ expect(followupRun).toBeNull();++ const deferred = await db+ .select()+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, companyId),+ eq(agentWakeupRequests.agentId, agentId),+ eq(agentWakeupRequests.status, "deferred_issue_execution"),+ ),+ )+ .then((rows) => rows[0] ?? null);++ expect(deferred).toMatchObject({+ reason: "issue_execution_deferred",+ runId: null,+ payload: expect.objectContaining({+ issueId,+ recoveryActionId,+ mutation: "recovery_action_resolution",+ }),+ });+ expect((deferred?.payload as Record<string, unknown>)._paperclipWakeContext).toMatchObject({+ issueId,+ taskId: issueId,+ recoveryActionId,+ wakeReason: "issue_recovery_action_restored",+ source: "issue.recovery_action_resolution",+ });++ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId));+ expect(runs).toHaveLength(1);+ expect(runs[0]?.id).toBe(runId);+ });+it("batches deferred comment wakes and forwards the ordered batch to the next run", async () => {const gateway = await createControlledGatewayServer();const companyId = randomUUID();diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 1ce6c93858..fd23f53a49 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -3751,6 +3751,60 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {}});+ it("re-enqueues handed-back todo work when its resolving run succeeded but the wake was lost", async () => {+ const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({+ status: "todo",+ runStatus: "succeeded",+ });+ const resolvedAt = new Date("2026-03-19T00:04:00.000Z");+ await db.insert(issueRecoveryActions).values({+ companyId,+ sourceIssueId: issueId,+ kind: "stranded_assigned_issue",+ status: "resolved",+ ownerType: "agent",+ ownerAgentId: agentId,+ previousOwnerAgentId: agentId,+ returnOwnerAgentId: agentId,+ cause: "stranded_assigned_issue",+ fingerprint: `handed-back:${issueId}`,+ nextAction: "Resume source work",+ outcome: "handed_back",+ resolutionNote: "Returned source work to the original owner",+ resolvedAt,+ createdAt: new Date("2026-03-19T00:01:00.000Z"),+ updatedAt: resolvedAt,+ });+ const heartbeat = heartbeatService(db);++ const result = await heartbeat.reconcileStrandedAssignedIssues();+ expect(result.assignmentDispatched).toBe(0);+ expect(result.dispatchRequeued).toBe(1);+ expect(result.continuationRequeued).toBe(0);+ expect(result.escalated).toBe(0);+ expect(result.issueIds).toEqual([issueId]);++ const runs = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.agentId, agentId));+ expect(runs).toHaveLength(2);++ const retryRun = runs.find((row) => row.id !== runId);+ expect(retryRun?.contextSnapshot).toMatchObject({+ issueId,+ taskId: issueId,+ wakeReason: "issue_assignment_recovery",+ retryReason: "assignment_recovery",+ source: "issue.assignment_recovery",+ retryOfRunId: runId,+ });+ expect(retryRun?.contextSnapshot as Record<string, unknown>).not.toHaveProperty("modelProfile");+ if (retryRun) {+ await waitForRunToSettle(heartbeat, retryRun.id);+ }+ });+it("re-enqueues an already stranded execution-review participant during reconciliation", async () => {const { agentId, issueId, runId, wakeupRequestId, stageId } = await seedInReviewParticipantRunFixture();const finishedAt = new Date("2026-03-19T00:05:00.000Z");diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 8f94007c56..bcc54e70c3 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -544,6 +544,7 @@ function mergeAdapterRecoveryMetadata(input: {const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set(["approval_approved",ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,+ "issue_recovery_action_restored",]);const ISSUE_RESPONSIBLE_USER_WAKE_REASONS = new Set(["issue_assigned",diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex d3af94c6f3..0610362e3a 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -140,7 +140,15 @@ type ResolvedDependencyWakeBackstopOptions = {type LatestIssueRun = Pick<typeof heartbeatRuns.$inferSelect,- "id" | "agentId" | "status" | "error" | "errorCode" | "contextSnapshot" | "livenessState"+ | "id"+ | "agentId"+ | "status"+ | "error"+ | "errorCode"+ | "contextSnapshot"+ | "livenessState"+ | "startedAt"+ | "createdAt"> & {resultJson?: unknown;} | null;@@ -784,6 +792,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })contextSnapshot: heartbeatRuns.contextSnapshot,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,+ startedAt: heartbeatRuns.startedAt,+ createdAt: heartbeatRuns.createdAt,}).from(heartbeatRuns).where(@@ -812,6 +822,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })contextSnapshot: heartbeatRuns.contextSnapshot,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,+ startedAt: heartbeatRuns.startedAt,+ createdAt: heartbeatRuns.createdAt,}).from(heartbeatRuns).where(@@ -948,6 +960,29 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }).then((rows) => Boolean(rows[0]));}+ async function wasTodoHandedBackDuringOrAfterLatestRun(+ issue: typeof issues.$inferSelect,+ latestRun: LatestIssueRun,+ ) {+ if (issue.status !== "todo" || latestRun?.status !== "succeeded") return false;+ const runBeganAt = latestRun.startedAt ?? latestRun.createdAt;++ return db+ .select({ id: issueRecoveryActions.id })+ .from(issueRecoveryActions)+ .where(+ and(+ eq(issueRecoveryActions.companyId, issue.companyId),+ eq(issueRecoveryActions.sourceIssueId, issue.id),+ eq(issueRecoveryActions.status, "resolved"),+ eq(issueRecoveryActions.outcome, "handed_back"),+ gte(issueRecoveryActions.resolvedAt, runBeganAt),+ ),+ )+ .limit(1)+ .then((rows) => Boolean(rows[0]));+ }+async function hasQueuedIssueWake(companyId: string, issueId: string, agentId?: string | null) {return db.select({ id: agentWakeupRequests.id })@@ -1026,6 +1061,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })contextSnapshot: heartbeatRuns.contextSnapshot,livenessState: heartbeatRuns.livenessState,resultJson: heartbeatRuns.resultJson,+ startedAt: heartbeatRuns.startedAt,+ createdAt: heartbeatRuns.createdAt,}).from(heartbeatRuns).where(@@ -4001,7 +4038,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })continue;}- if (latestRun.status === "succeeded") {+ if (+ latestRun.status === "succeeded" &&+ !(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun))+ ) {result.skipped += 1;continue;}