Skip to content

The open-source app everyone uses to manage agents at work

fix(recovery): preserve hand-back wake liveness (#10562)

Dottacd7f84965c5b
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md
index a155634c6c..2daf6fb37f 100644
--- a/doc/execution-semantics.md
+++ b/doc/execution-semantics.md
@@ -519,6 +519,11 @@ Recovery rule:
This is a dispatch recovery, not a continuation recovery.
+Recovery hand-back is covered by the same liveness guarantee:
+
+- an `issue_recovery_action_restored` wake requested while the resolving recovery run is still active is persisted as a follow-up and dispatched only after that run exits, so it cannot be coalesced into the run that requested it
+- if that follow-up is nevertheless lost, the stranded-work backstop treats an assigned `todo` issue with a resolved `handed_back` recovery action from during or after its latest successful run as stranded and queues the bounded assignment recovery wake; the successful resolving run is not, by itself, evidence that the handed-back source work is live
+
### 9.2 Stranded assigned `in_progress`
Example:
diff --git a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
index 3e6d30fe8c..4a07e57c82 100644
--- a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
+++ b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts
@@ -295,6 +295,125 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
expect(runs[0]?.id).toBe(runId);
});
+ it("defers recovery hand-back wakes until the resolving run exits", async () => {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const issueId = randomUUID();
+ const runId = randomUUID();
+ const recoveryActionId = randomUUID();
+ const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+ const heartbeat = heartbeatService(db);
+
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix,
+ requireBoardApprovalForNewAgents: false,
+ defaultResponsibleUserId: "responsible-user",
+ });
+
+ await db.insert(agents).values({
+ id: agentId,
+ companyId,
+ name: "Recovery owner",
+ role: "engineer",
+ status: "running",
+ adapterType: "process",
+ adapterConfig: {},
+ runtimeConfig: {},
+ permissions: {},
+ });
+
+ await db.insert(heartbeatRuns).values({
+ id: runId,
+ companyId,
+ agentId,
+ invocationSource: "assignment",
+ triggerDetail: "system",
+ status: "running",
+ contextSnapshot: {
+ issueId,
+ taskId: issueId,
+ wakeReason: "source_scoped_recovery_action",
+ },
+ });
+ runningProcesses.set(runId, {
+ child: {} as never,
+ graceSec: 0,
+ processGroupId: null,
+ });
+
+ await db.insert(issues).values({
+ id: issueId,
+ companyId,
+ title: "Resume handed-back work",
+ status: "todo",
+ priority: "medium",
+ responsibleUserId: "responsible-user",
+ assigneeAgentId: agentId,
+ executionRunId: runId,
+ executionAgentNameKey: "recovery-owner",
+ executionLockedAt: new Date(),
+ issueNumber: 1,
+ identifier: `${issuePrefix}-1`,
+ });
+
+ const followupRun = await heartbeat.wakeup(agentId, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: "issue_recovery_action_restored",
+ payload: {
+ issueId,
+ recoveryActionId,
+ mutation: "recovery_action_resolution",
+ },
+ contextSnapshot: {
+ issueId,
+ taskId: issueId,
+ recoveryActionId,
+ wakeReason: "issue_recovery_action_restored",
+ source: "issue.recovery_action_resolution",
+ },
+ requestedByActorType: "agent",
+ requestedByActorId: agentId,
+ });
+
+ expect(followupRun).toBeNull();
+
+ const deferred = await db
+ .select()
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+
+ expect(deferred).toMatchObject({
+ reason: "issue_execution_deferred",
+ runId: null,
+ payload: expect.objectContaining({
+ issueId,
+ recoveryActionId,
+ mutation: "recovery_action_resolution",
+ }),
+ });
+ expect((deferred?.payload as Record<string, unknown>)._paperclipWakeContext).toMatchObject({
+ issueId,
+ taskId: issueId,
+ recoveryActionId,
+ wakeReason: "issue_recovery_action_restored",
+ source: "issue.recovery_action_resolution",
+ });
+
+ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId));
+ expect(runs).toHaveLength(1);
+ expect(runs[0]?.id).toBe(runId);
+ });
+
it("batches deferred comment wakes and forwards the ordered batch to the next run", async () => {
const gateway = await createControlledGatewayServer();
const companyId = randomUUID();
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 1ce6c93858..fd23f53a49 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -3751,6 +3751,60 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
});
+ it("re-enqueues handed-back todo work when its resolving run succeeded but the wake was lost", async () => {
+ const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
+ status: "todo",
+ runStatus: "succeeded",
+ });
+ const resolvedAt = new Date("2026-03-19T00:04:00.000Z");
+ await db.insert(issueRecoveryActions).values({
+ companyId,
+ sourceIssueId: issueId,
+ kind: "stranded_assigned_issue",
+ status: "resolved",
+ ownerType: "agent",
+ ownerAgentId: agentId,
+ previousOwnerAgentId: agentId,
+ returnOwnerAgentId: agentId,
+ cause: "stranded_assigned_issue",
+ fingerprint: `handed-back:${issueId}`,
+ nextAction: "Resume source work",
+ outcome: "handed_back",
+ resolutionNote: "Returned source work to the original owner",
+ resolvedAt,
+ createdAt: new Date("2026-03-19T00:01:00.000Z"),
+ updatedAt: resolvedAt,
+ });
+ const heartbeat = heartbeatService(db);
+
+ const result = await heartbeat.reconcileStrandedAssignedIssues();
+ expect(result.assignmentDispatched).toBe(0);
+ expect(result.dispatchRequeued).toBe(1);
+ expect(result.continuationRequeued).toBe(0);
+ expect(result.escalated).toBe(0);
+ expect(result.issueIds).toEqual([issueId]);
+
+ const runs = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.agentId, agentId));
+ expect(runs).toHaveLength(2);
+
+ const retryRun = runs.find((row) => row.id !== runId);
+ expect(retryRun?.contextSnapshot).toMatchObject({
+ issueId,
+ taskId: issueId,
+ wakeReason: "issue_assignment_recovery",
+ retryReason: "assignment_recovery",
+ source: "issue.assignment_recovery",
+ retryOfRunId: runId,
+ });
+ expect(retryRun?.contextSnapshot as Record<string, unknown>).not.toHaveProperty("modelProfile");
+ if (retryRun) {
+ await waitForRunToSettle(heartbeat, retryRun.id);
+ }
+ });
+
it("re-enqueues an already stranded execution-review participant during reconciliation", async () => {
const { agentId, issueId, runId, wakeupRequestId, stageId } = await seedInReviewParticipantRunFixture();
const finishedAt = new Date("2026-03-19T00:05:00.000Z");
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 8f94007c56..bcc54e70c3 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -544,6 +544,7 @@ function mergeAdapterRecoveryMetadata(input: {
const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([
"approval_approved",
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
+ "issue_recovery_action_restored",
]);
const ISSUE_RESPONSIBLE_USER_WAKE_REASONS = new Set([
"issue_assigned",
diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts
index d3af94c6f3..0610362e3a 100644
--- a/server/src/services/recovery/service.ts
+++ b/server/src/services/recovery/service.ts
@@ -140,7 +140,15 @@ type ResolvedDependencyWakeBackstopOptions = {
type LatestIssueRun = Pick<
typeof heartbeatRuns.$inferSelect,
- "id" | "agentId" | "status" | "error" | "errorCode" | "contextSnapshot" | "livenessState"
+ | "id"
+ | "agentId"
+ | "status"
+ | "error"
+ | "errorCode"
+ | "contextSnapshot"
+ | "livenessState"
+ | "startedAt"
+ | "createdAt"
> & {
resultJson?: unknown;
} | null;
@@ -784,6 +792,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
contextSnapshot: heartbeatRuns.contextSnapshot,
livenessState: heartbeatRuns.livenessState,
resultJson: heartbeatRuns.resultJson,
+ startedAt: heartbeatRuns.startedAt,
+ createdAt: heartbeatRuns.createdAt,
})
.from(heartbeatRuns)
.where(
@@ -812,6 +822,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
contextSnapshot: heartbeatRuns.contextSnapshot,
livenessState: heartbeatRuns.livenessState,
resultJson: heartbeatRuns.resultJson,
+ startedAt: heartbeatRuns.startedAt,
+ createdAt: heartbeatRuns.createdAt,
})
.from(heartbeatRuns)
.where(
@@ -948,6 +960,29 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
.then((rows) => Boolean(rows[0]));
}
+ async function wasTodoHandedBackDuringOrAfterLatestRun(
+ issue: typeof issues.$inferSelect,
+ latestRun: LatestIssueRun,
+ ) {
+ if (issue.status !== "todo" || latestRun?.status !== "succeeded") return false;
+ const runBeganAt = latestRun.startedAt ?? latestRun.createdAt;
+
+ return db
+ .select({ id: issueRecoveryActions.id })
+ .from(issueRecoveryActions)
+ .where(
+ and(
+ eq(issueRecoveryActions.companyId, issue.companyId),
+ eq(issueRecoveryActions.sourceIssueId, issue.id),
+ eq(issueRecoveryActions.status, "resolved"),
+ eq(issueRecoveryActions.outcome, "handed_back"),
+ gte(issueRecoveryActions.resolvedAt, runBeganAt),
+ ),
+ )
+ .limit(1)
+ .then((rows) => Boolean(rows[0]));
+ }
+
async function hasQueuedIssueWake(companyId: string, issueId: string, agentId?: string | null) {
return db
.select({ id: agentWakeupRequests.id })
@@ -1026,6 +1061,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
contextSnapshot: heartbeatRuns.contextSnapshot,
livenessState: heartbeatRuns.livenessState,
resultJson: heartbeatRuns.resultJson,
+ startedAt: heartbeatRuns.startedAt,
+ createdAt: heartbeatRuns.createdAt,
})
.from(heartbeatRuns)
.where(
@@ -4001,7 +4038,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
continue;
}
- if (latestRun.status === "succeeded") {
+ if (
+ latestRun.status === "succeeded" &&
+ !(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun))
+ ) {
result.skipped += 1;
continue;
}