The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(recovery): escalate an issue whose interrupted run has spent its retry budget (#14046)
Devin Foley6bc830b62bc9
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex ccf848eb18..4443e9805e 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -3745,21 +3745,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {}).where(eq(heartbeatRuns.id, runId));await heartbeatService(db).drainRunningRunsForShutdown("SIGTERM");- await heartbeatService(db).reconcileStrandedAssignedIssues();+ const reconciled = await heartbeatService(db).reconcileStrandedAssignedIssues();+ // The budget stays spent across the restart: no successor run is+ // queued, by the process-loss retry or by the sweeper.expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)),).toHaveLength(1);+ // A spent budget is no longer left to sit: the sweeper escalates the+ // issue to a board-owned recovery action instead of skipping it on+ // every tick with no live path. The action spawns no run of its own.+ expect(reconciled.escalated).toBe(1);+ const recoveryActions = await db+ .select()+ .from(issueRecoveryActions)+ .where(eq(issueRecoveryActions.sourceIssueId, issueId));+ expect(recoveryActions).toHaveLength(1);+ expect(recoveryActions[0]).toMatchObject({ status: "active", ownerType: "board", ownerAgentId: null });+ const escalatedIssue = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(escalatedIssue?.status).toBe("blocked");+ // And the escalation is idempotent across further sweeps.+ const again = await heartbeatService(db).reconcileStrandedAssignedIssues();+ expect(again.escalated).toBe(0);expect(await db.select()- .from(issueRecoveryActions)- .where(eq(issueRecoveryActions.sourceIssueId, issueId)),- ).toEqual([]);- const run = await heartbeatService(db).getRun(runId);- expect(await getExecutionBlocker(db, run!.companyId, issueId)).toBeNull();+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.agentId, agentId)),+ ).toHaveLength(1);});it("releases active environment leases when an orphaned run is reaped", async () => {@@ -6006,6 +6025,176 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(sourceIssue?.status).toBe("blocked");});+ it("escalates an in_progress issue whose interrupted run has spent its transient retry budget", async () => {+ // Three deploy restarts in a row interrupted a run and both of its+ // bounded transient retries. The retry scheduler then reports the+ // budget exhausted and queues nothing, and the sweeper used to skip+ // the issue on every tick: in_progress, no run, no path, no notice.+ const { companyId, agentId, runId, issueId } =+ await seedStrandedIssueFixture({+ status: "in_progress",+ runStatus: "failed",+ runErrorCode: "server_shutdown_interrupted",+ runError: "Interrupted by graceful server shutdown (SIGTERM)",+ // A conversation adapter's interrupted run keeps its session for+ // continuation, so legacy reconciliation does not terminalize it;+ // the sweeper reaches the retry lane with a spent budget.+ resultJson: {+ stopReason: "interrupted",+ conversationContinuation: "continue_conversation_v1",+ },+ });+ const originalRunId = randomUUID();+ await db+ .update(heartbeatRuns)+ .set({+ status: "interrupted",+ scheduledRetryAttempt: 2,+ scheduledRetryReason: "transient_failure",+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "transient_failure_retry",+ retryOfRunId: originalRunId,+ },+ })+ .where(eq(heartbeatRuns.id, runId));++ const heartbeat = heartbeatService(db);+ const result = await heartbeat.reconcileStrandedAssignedIssues();+ expect(result.escalated).toBe(1);+ expect(result.continuationRequeued).toBe(0);+ expect(result.skipped).toBe(0);+ expect(result.issueIds).toEqual([issueId]);+ const successors = await db+ .select({ id: heartbeatRuns.id })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.retryOfRunId, runId));+ expect(successors).toHaveLength(0);++ const sourceIssue = await db+ .select()+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(sourceIssue?.status).toBe("blocked");+ expect(sourceIssue?.assigneeAgentId).toBe(agentId);+ const recoveryActions = await db+ .select()+ .from(issueRecoveryActions)+ .where(eq(issueRecoveryActions.sourceIssueId, issueId));+ expect(recoveryActions).toHaveLength(1);+ expect(recoveryActions[0]).toMatchObject({+ companyId,+ status: "active",+ previousOwnerAgentId: agentId,+ });+ const comments = await db+ .select()+ .from(issueComments)+ .where(eq(issueComments.issueId, issueId));+ expect(comments).toHaveLength(1);+ expect(comments[0]?.body).toContain("bounded retry budget");+ expect(comments[0]?.body).toContain("no live execution path");+ expect(comments[0]?.authorType).toBe("system");+ expect(comments[0]?.presentation).toMatchObject({+ kind: "system_notice",+ tone: "danger",+ });++ // The escalated issue is board-owned now: a second sweep leaves it alone.+ const again = await heartbeat.reconcileStrandedAssignedIssues();+ expect(again.escalated).toBe(0);+ expect(again.continuationRequeued).toBe(0);+ });++ it("keeps retrying an interrupted run while its transient retry budget remains", async () => {+ const { agentId, runId, issueId } = await seedStrandedIssueFixture({+ status: "in_progress",+ runStatus: "failed",+ runErrorCode: "server_shutdown_interrupted",+ runError: "Interrupted by graceful server shutdown (SIGTERM)",+ });+ await db+ .update(heartbeatRuns)+ .set({+ status: "interrupted",+ scheduledRetryAttempt: 1,+ scheduledRetryReason: "transient_failure",+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "transient_failure_retry",+ },+ })+ .where(eq(heartbeatRuns.id, runId));++ const heartbeat = heartbeatService(db);+ const result = await heartbeat.reconcileStrandedAssignedIssues();+ expect(result.continuationRequeued).toBe(1);+ expect(result.escalated).toBe(0);+ const successor = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.retryOfRunId, runId))+ .then((rows) => rows[0] ?? null);+ expect(successor).toMatchObject({+ agentId,+ status: "scheduled_retry",+ scheduledRetryAttempt: 2,+ scheduledRetryReason: "transient_failure",+ });+ const sourceIssue = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(sourceIssue?.status).toBe("in_progress");+ });++ it("escalates an assigned todo issue whose lost dispatch has spent its transient retry budget", async () => {+ const { agentId, runId, issueId } = await seedStrandedIssueFixture({+ status: "todo",+ runStatus: "failed",+ runErrorCode: "server_shutdown_interrupted",+ runError: "Interrupted by graceful server shutdown (SIGTERM)",+ resultJson: {+ stopReason: "interrupted",+ conversationContinuation: "continue_conversation_v1",+ },+ });+ await db+ .update(heartbeatRuns)+ .set({+ status: "interrupted",+ scheduledRetryAttempt: 2,+ scheduledRetryReason: "transient_failure",+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "transient_failure_retry",+ },+ })+ .where(eq(heartbeatRuns.id, runId));++ const heartbeat = heartbeatService(db);+ const result = await heartbeat.reconcileStrandedAssignedIssues();+ expect(result.escalated).toBe(1);+ expect(result.dispatchRequeued).toBe(0);+ const sourceIssue = await db+ .select()+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(sourceIssue?.status).toBe("blocked");+ expect(sourceIssue?.assigneeAgentId).toBe(agentId);+ const comments = await db+ .select()+ .from(issueComments)+ .where(eq(issueComments.issueId, issueId));+ expect(comments[0]?.body).toContain("bounded retry budget");+ });+it("escalates an exhausted successful handoff run that still leaves no disposition", async () => {const { companyId, agentId, runId, issueId } =await seedStrandedIssueFixture({@@ -8809,6 +8998,148 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {).not.toHaveProperty("modelProfile");});+ it("escalates an execution-review participant whose interrupted run has spent its transient retry budget", async () => {+ const { companyId, agentId, issueId, runId, wakeupRequestId } =+ await seedInReviewParticipantRunFixture();+ const finishedAt = new Date("2026-03-19T00:05:00.000Z");+ await db+ .update(heartbeatRuns)+ .set({+ status: "interrupted",+ errorCode: "server_shutdown_interrupted",+ error: "Interrupted by graceful server shutdown (SIGTERM)",+ scheduledRetryAttempt: 2,+ scheduledRetryReason: "transient_failure",+ resultJson: {+ stopReason: "interrupted",+ conversationContinuation: "continue_conversation_v1",+ },+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "transient_failure_retry",+ },+ startedAt: new Date("2026-03-19T00:00:00.000Z"),+ finishedAt,+ updatedAt: finishedAt,+ })+ .where(eq(heartbeatRuns.id, runId));+ await db+ .update(agentWakeupRequests)+ .set({ status: "cancelled", finishedAt, updatedAt: finishedAt })+ .where(eq(agentWakeupRequests.id, wakeupRequestId));++ const heartbeat = heartbeatService(db);+ const result = await heartbeat.reconcileStrandedAssignedIssues();+ expect(result.escalated).toBe(1);+ expect(result.reviewParticipantRequeued).toBe(0);+ expect(result.issueIds).toEqual([issueId]);+ const successors = await db+ .select({ id: heartbeatRuns.id })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.retryOfRunId, runId));+ expect(successors).toHaveLength(0);+ const sourceIssue = await db+ .select()+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(sourceIssue?.status).toBe("blocked");+ const recoveryActions = await db+ .select()+ .from(issueRecoveryActions)+ .where(eq(issueRecoveryActions.sourceIssueId, issueId));+ expect(recoveryActions).toHaveLength(1);+ expect(recoveryActions[0]).toMatchObject({+ companyId,+ status: "active",+ cause: "execution_review_participant_recovery",+ previousOwnerAgentId: agentId,+ });+ const again = await heartbeat.reconcileStrandedAssignedIssues();+ expect(again.escalated).toBe(0);+ });++ it("does not block an issue whose review participant spent its retry budget while another agent is still working it", async () => {+ // `blocked` is issue-wide. The exhausted participant is a dead end for+ // the review stage, but another agent with a live run on the same issue+ // must not be blocked under it: the escalation re-reads the live path+ // for ANY agent and stands down.+ const { companyId, issueId, runId, wakeupRequestId } =+ await seedInReviewParticipantRunFixture();+ const finishedAt = new Date("2026-03-19T00:05:00.000Z");+ await db+ .update(heartbeatRuns)+ .set({+ status: "interrupted",+ errorCode: "server_shutdown_interrupted",+ error: "Interrupted by graceful server shutdown (SIGTERM)",+ scheduledRetryAttempt: 2,+ scheduledRetryReason: "transient_failure",+ resultJson: {+ stopReason: "interrupted",+ conversationContinuation: "continue_conversation_v1",+ },+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "transient_failure_retry",+ },+ startedAt: new Date("2026-03-19T00:00:00.000Z"),+ finishedAt,+ updatedAt: finishedAt,+ })+ .where(eq(heartbeatRuns.id, runId));+ await db+ .update(agentWakeupRequests)+ .set({ status: "cancelled", finishedAt, updatedAt: finishedAt })+ .where(eq(agentWakeupRequests.id, wakeupRequestId));+ const otherAgentId = randomUUID();+ await db.insert(agents).values({+ id: otherAgentId,+ companyId,+ name: "CodexImplementor",+ role: "engineer",+ status: "running",+ adapterType: "codex_local",+ adapterConfig: {},+ runtimeConfig: {},+ permissions: {},+ });+ await db.insert(heartbeatRuns).values({+ id: randomUUID(),+ companyId,+ agentId: otherAgentId,+ invocationSource: "automation",+ triggerDetail: "system",+ status: "running",+ contextSnapshot: {+ issueId,+ taskId: issueId,+ wakeReason: "issue_commented",+ },+ startedAt: new Date("2026-03-19T00:10:00.000Z"),+ createdAt: new Date(Date.now() + 1_000),+ updatedAt: new Date("2026-03-19T00:10:00.000Z"),+ });++ const result = await heartbeatService(db).reconcileStrandedAssignedIssues();+ expect(result.escalated).toBe(0);+ expect(result.reviewParticipantRequeued).toBe(0);+ const sourceIssue = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(sourceIssue?.status).toBe("in_review");+ expect(+ await db+ .select()+ .from(issueRecoveryActions)+ .where(eq(issueRecoveryActions.sourceIssueId, issueId)),+ ).toEqual([]);+ });+it("re-enqueues a stranded execution-review participant when another agent has the latest issue run", async () => {const { companyId, agentId, issueId, runId, wakeupRequestId, stageId } =await seedInReviewParticipantRunFixture();diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 3acfcd71d4..c543541ad8 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -9499,6 +9499,12 @@ export function heartbeatService(const result = await scheduleBoundedRetryForRun(run, agent);return result.outcome === "scheduled" ? result.run : null;},+ // Mirrors scheduleBoundedRetryForRun's transient budget check: a failed+ // or interrupted run that has already consumed every bounded transient+ // attempt cannot be retried again through this lane.+ transientRetryBudgetSpent: (run) =>+ executionFailureRetryCount(run) >=+ BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,});const runDispatch = createRunDispatch(db);diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex 98be0347ae..148b8199bf 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -901,6 +901,16 @@ export function recoveryService(scheduleRecoveryRetry?: (runId: string,) => Promise<typeof heartbeatRuns.$inferSelect | null>;+ /**+ * Whether a failed or interrupted run has consumed every bounded+ * transient retry, so `scheduleRecoveryRetry` can no longer produce a+ * successor for it. Lets the sweeper tell "no retry because the budget+ * is spent" (escalate) from "no retry because something else owns the+ * run" (leave alone).+ */+ transientRetryBudgetSpent?: (+ run: typeof heartbeatRuns.$inferSelect,+ ) => boolean;liveRunExecutions?: Readonly<{ has(id: string): boolean }>;beforeOrphanedRunTerminalWrite?: (runId: string) => Promise<void>;},@@ -1875,6 +1885,13 @@ export function recoveryService(source: string;retryOfRunId?: string | null;extraContext?: Record<string, unknown>;+ /**+ * Out-parameter: set `retryExhausted` when no successor was queued+ * because the failed predecessor has spent its bounded transient retry+ * budget. A null return without it means another authority owns the+ * run (native runtime, reconciliation) and the caller must leave it.+ */+ outcome?: { retryExhausted?: boolean };}) {if (input.retryOfRunId) {const [predecessor] = await db@@ -1903,8 +1920,19 @@ export function recoveryService(});return null;}- if (deps.scheduleRecoveryRetry)- return deps.scheduleRecoveryRetry(predecessor.id);+ if (deps.scheduleRecoveryRetry) {+ const retry = await deps.scheduleRecoveryRetry(predecessor.id);+ if (retry) return retry;+ // A spent budget is the one "no retry" the sweeper must not wait+ // out: nothing else will ever queue a successor for this run, so+ // it reports the exhaustion instead of leaving the issue with no+ // live path (2026-09-25: three deploy restarts in a row spent the+ // budget and the issue sat in_progress with no run, unescalated).+ if (input.outcome && deps.transientRetryBudgetSpent?.(predecessor)) {+ input.outcome.retryExhausted = true;+ }+ return null;+ }return null;}}@@ -4855,6 +4883,7 @@ export function recoveryService(continue;}+ const reviewOutcome: { retryExhausted?: boolean } = {};const queued = await enqueueStrandedIssueRecovery({issueId: issue.id,agentId: participantAgentId,@@ -4868,10 +4897,35 @@ export function recoveryService(reviewRecoveryInstruction:"The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",},+ outcome: reviewOutcome,});if (queued) {result.reviewParticipantRequeued += 1;result.issueIds.push(issue.id);+ } else if (+ reviewOutcome.retryExhausted &&+ !(await hasActiveExecutionPath(issue.companyId, issue.id, null))+ ) {+ // Same exhaustion as the other lanes: the reviewer run's bounded+ // retries are spent, so escalate as the review-recovery failure it+ // is instead of skipping on every sweep with no live path. The+ // live-path re-read guards the write against a run or wake that+ // started after the loop's check — for ANY agent, not only the+ // participant: `blocked` is issue-wide, so another agent still+ // working the issue must never be blocked under.+ const updated = await escalateStrandedAssignedIssue({+ issue,+ previousStatus: "in_review",+ latestRun: participantLatestRun,+ notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),+ recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,+ });+ if (updated) {+ result.escalated += 1;+ result.issueIds.push(issue.id);+ } else {+ result.skipped += 1;+ }} else {result.skipped += 1;}@@ -4950,6 +5004,7 @@ export function recoveryService(continue;}+ const dispatchOutcome: { retryExhausted?: boolean } = {};const queued = await enqueueStrandedIssueRecovery({issueId: issue.id,agentId,@@ -4957,10 +5012,39 @@ export function recoveryService(retryReason: "assignment_recovery",source: "issue.assignment_recovery",retryOfRunId: latestRun.id,+ outcome: dispatchOutcome,});if (queued) {result.dispatchRequeued += 1;result.issueIds.push(issue.id);+ } else if (+ dispatchOutcome.retryExhausted &&+ !(await hasActiveExecutionPath(issue.companyId, issue.id, null))+ ) {+ // Same exhaustion as the in_progress lane: the lost dispatch's+ // bounded retries are spent, so escalate instead of skipping on+ // every sweep with no live path. The live-path re-read guards the+ // `blocked` write against a run or wake that started after the+ // loop's check.+ const updated = await escalateStrandedAssignedIssue({+ issue,+ previousStatus: "todo",+ latestRun,+ notice: {+ body:+ "Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " ++ "but the bounded retry budget is spent and it still has no live execution path. " ++ "Moving it to `blocked` so it is visible for intervention.",+ title: "No live execution path",+ tone: "danger",+ },+ });+ if (updated) {+ result.escalated += 1;+ result.issueIds.push(issue.id);+ } else {+ result.skipped += 1;+ }} else {result.skipped += 1;}@@ -5217,6 +5301,7 @@ export function recoveryService(continue;}+ const recoveryOutcome: { retryExhausted?: boolean } = {};const queued = await enqueueStrandedIssueRecovery({issueId: issue.id,agentId,@@ -5224,10 +5309,40 @@ export function recoveryService(retryReason: "issue_continuation_needed",source: "issue.continuation_recovery",retryOfRunId: latestRun?.id ?? issue.checkoutRunId ?? null,+ outcome: recoveryOutcome,});if (queued) {result.continuationRequeued += 1;result.issueIds.push(issue.id);+ } else if (+ recoveryOutcome.retryExhausted &&+ !(await hasActiveExecutionPath(issue.companyId, issue.id, null))+ ) {+ // The failed run's bounded transient retries are all spent, so no+ // successor will ever be queued for it. Escalate rather than skip+ // on every sweep forever: the issue would otherwise stay+ // `in_progress` with no run and no path until a person noticed.+ // The live-path re-read guards the `blocked` write against a run or+ // wake that started after the loop's check.+ const updated = await escalateStrandedAssignedIssue({+ issue,+ previousStatus: "in_progress",+ latestRun,+ notice: {+ body:+ "Paperclip retried this issue's run after it ended without finishing, but the bounded retry budget " ++ "is spent and it still has no live execution path. " ++ "Moving it to `blocked` so it is visible for intervention.",+ title: "No live execution path",+ tone: "danger",+ },+ });+ if (updated) {+ result.escalated += 1;+ result.issueIds.push(issue.id);+ } else {+ result.skipped += 1;+ }} else {result.skipped += 1;}