The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
import {
WORKSPACE_VALIDATION_FAILURE_CODE,
isWorkspaceValidationFailedRun,
readWorkspaceValidationPayloadFromRun,
} from "./workspaces.js";
import { admitExplicitContinuationRetry } from "../explicit-native-continuation.js";
import {
CONVERSATION_CONTINUATION_POLICY,
hasConversationContinuationPolicy,
} from "../conversation-continuation.js";
import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";
import {
executionFailureRetryCount,
executionRetryAttemptCount,
accountingForScheduledRetry,
} from "../execution-recovery-attempt.js";
import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js";
import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js";
import { randomUUID } from "node:crypto";
import {
and,
asc,
desc,
eq,
inArray,
ne,
notInArray,
or,
sql,
} from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
agentWakeupRequests,
executionWorkspaces,
heartbeatRuns,
issues,
} from "@paperclipai/db";
import {
HttpError,
notFound,
} from "../../errors.js";
import { logger } from "../../middleware/logger.js";
import { parseObject } from "../../adapters/utils.js";
import { logActivity } from "../activity-log.js";
import { readContinuationAttempt } from "../recovery/index.js";
import { withRecoveryContext } from "../recovery/status-only-context.js";
import {
MAX_TURN_CONTINUATION_RETRY_REASON,
WORKSPACE_BUSY_RETRY_REASON,
AI_CONNECTION_BUSY_RETRY_REASON,
AI_CONNECTION_POOL_WAIT_RETRY_REASON,
INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
isNonAssigneeWorkspaceBusyRetry,
isResolvedInteractionContinuationWakeContext,
} from "../../modules/run-dispatch/index.js";
import type { AppendHeartbeatRunEventInput } from "../heartbeat-run-events.js";
import type { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js";
import type { createHeartbeatRunState } from "./run-state.js";
import type { createHeartbeatRunPreparation } from "./run-preparation.js";
import type { createRunDispatch, PostCommitEffect } from "../../modules/run-dispatch/index.js";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
type PlanApprovalResumeFailure = {
run: HeartbeatRun;
issueId: string | null;
attempt: number;
maxAttempts: number;
};
/** The retry module owns scheduling writes; the service owns lifecycle effects. */
export interface HeartbeatRetryDependencies {
hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
runDispatch: Pick<ReturnType<typeof createRunDispatch>,
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
getRun: (
runId: string,
options?: Parameters<HeartbeatRunState["getRun"]>[1],
) => Promise<Awaited<ReturnType<HeartbeatRunState["getRun"]>> | null | undefined>;
resolveSessionBeforeForWakeup: HeartbeatRunState["resolveSessionBeforeForWakeup"];
resolveResponsibleUserIdForRunContext: ReturnType<typeof createHeartbeatRunPreparation>["resolveResponsibleUserIdForRunContext"];
getAgentInvokability: (agent: typeof agents.$inferSelect | null | undefined) => ReturnType<typeof evaluateAgentInvokabilityFromDb>;
appendRunEvent: (run: HeartbeatRun, event: {
eventType: string;
stream?: "system" | "stdout" | "stderr";
level?: "info" | "warn" | "error";
color?: string;
message?: string;
payload?: Record<string, unknown>;
retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"];
}) => Promise<unknown>;
escalatePlanApprovalResumeFailureNeedsAttention: (input: PlanApprovalResumeFailure) => Promise<string | null>;
recordPlanApprovalResumeFailureRetry: (input: PlanApprovalResumeFailure & { retryRunId: string | null }) => Promise<string | null>;
setRunStatusIfRunning: (runId: string, status: string, patch?: Partial<typeof heartbeatRuns.$inferInsert>) => Promise<{
updated: boolean;
run: HeartbeatRun | null;
}>;
setWakeupStatus: (wakeupRequestId: string | null | undefined, status: string, patch?: Partial<typeof agentWakeupRequests.$inferInsert>) => Promise<void>;
releaseIssueExecutionAndPromote: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<unknown>;
finalizeAgentStatus: (
agentId: string,
outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
failureReason?: string | null,
options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },
) => Promise<void>;
getWorktreeExecutionCutoff: () => Promise<Date | null>;
applyRunDispatchPostCommitEffects: (effects: PostCommitEffect[]) => void;
}
export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [
30_000, 60_000,
] as const;
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
export function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean {
return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated;
}
export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS =
BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2;
const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [
"scheduled_retry",
"queued",
"running",
] as const;
export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry";
export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy";
export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000;
export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000;
// Preserve the one-hour shared-workspace holder cutoff independently of the
// informational output-silence warnings. Warning sooner must not let another
// run overtake a quiet holder and mutate its shared workspace.
export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = 60 * 60 * 1000;
// Issue-level executionWorkspaceSettings.mode values that unambiguously opt an
// issue's runs out of the shared project workspace, and therefore out of
// shared-workspace serialization ("isolated" is the legacy alias
// parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any
// other value — including agent_default and an absent mode — may still resolve
// to the shared workspace and counts as a holder.
const ISOLATED_EXECUTION_WORKSPACE_MODES = [
"isolated_workspace",
"operator_branch",
"isolated",
] as const;
type CodexTransientFallbackMode =
| "same_session"
| "safer_invocation"
| "fresh_session"
| "fresh_session_safer_invocation";
export interface SharedWorkspaceHolder {
runId: string;
agentId: string;
issueId: string;
issueIdentifier: string | null;
}
export interface ExecutionWorkspaceHolder extends SharedWorkspaceHolder {
// Silent past WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS but still running.
stale: boolean;
}
// Pre-dispatch gate outcome: another running run currently holds the issue's
// shared project workspace, or the shared execution workspace it reuses. Not a
// failure — the run is parked as a bounded scheduled retry and re-attempted
// once the holder finishes, so two agents never mutate the same working tree
// concurrently.
export class WorkspaceBusyDeferral extends Error {
code = WORKSPACE_BUSY_ERROR_CODE;
holder: SharedWorkspaceHolder;
projectWorkspaceId: string | null;
executionWorkspaceId: string | null;
deferralAttempt: number;
wasIssueAssignee: boolean;
constructor(input: {
holder: SharedWorkspaceHolder;
projectWorkspaceId?: string | null;
executionWorkspaceId?: string | null;
deferralAttempt: number;
wasIssueAssignee: boolean;
}) {
super(
`Shared ${input.executionWorkspaceId ? "execution" : "project"} workspace is busy: run ${input.holder.runId} (issue ${
input.holder.issueIdentifier ?? input.holder.issueId
}) is still running`,
);
this.name = "WorkspaceBusyDeferral";
this.holder = input.holder;
this.projectWorkspaceId = input.projectWorkspaceId ?? null;
this.executionWorkspaceId = input.executionWorkspaceId ?? null;
this.deferralAttempt = input.deferralAttempt;
this.wasIssueAssignee = input.wasIssueAssignee;
}
}
export function isWorkspaceBusyDeferral(
error: unknown,
): error is WorkspaceBusyDeferral {
return error instanceof WorkspaceBusyDeferral;
}
export function computeWorkspaceBusyRetryDelayMs(
random: () => number = Math.random,
) {
const jitter = Math.min(Math.max(random(), 0), 1);
return (
WORKSPACE_BUSY_RETRY_BASE_DELAY_MS +
Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS)
);
}
function resolveCodexTransientFallbackMode(
attempt: number,
): CodexTransientFallbackMode {
if (attempt <= 1) return "same_session";
if (attempt === 2) return "safer_invocation";
if (attempt === 3) return "fresh_session";
return "fresh_session_safer_invocation";
}
export function readHeartbeatRunErrorFamily(
run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
) {
const resultJson = parseObject(run.resultJson);
const persistedFamily = readNonEmptyString(resultJson.errorFamily);
if (persistedFamily) return persistedFamily;
if (run.errorCode === "provider_quota") {
return "provider_quota";
}
if (
run.errorCode === "codex_transient_upstream" ||
run.errorCode === "claude_transient_upstream" ||
run.errorCode === "codex_harness_crash"
) {
return "transient_upstream";
}
return null;
}
function readTransientRetryNotBeforeFromRun(
run: Pick<typeof heartbeatRuns.$inferSelect, "resultJson">,
) {
const resultJson = parseObject(run.resultJson);
const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore;
if (!(
typeof value === "string" ||
typeof value === "number" ||
value instanceof Date
)) {
return null;
}
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? null : parsed;
}
export function readTransientRecoveryContractFromRun(
run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
) {
const errorFamily = readHeartbeatRunErrorFamily(run);
return errorFamily === "transient_upstream" ||
errorFamily === "provider_quota"
? {
errorFamily,
retryNotBefore: readTransientRetryNotBeforeFromRun(run),
}
: null;
}
function isSpawnLikeFailureMessage(value: unknown) {
if (typeof value !== "string") return false;
return /failed to start command|spawn\b|\bENOENT\b/i.test(value);
}
// A sandbox provider plugin's worker can be briefly down during its own
// restart window (e.g. a rolling deploy of the plugin worker process). Lease
// acquisition fails immediately in that window, but the condition is
// transient and self-healing, so it must be treated as retryable
// infrastructure rather than a terminal setup failure. See
// resolveSandboxProviderPlugin's "worker_unavailable" message in
// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is
// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is
// not running.'
//
// This is anchored on both "is installed via plugin" and "but its worker is
// not running" so it does not also match plugin-environment-driver.ts's
// unrelated, permanent "provider not installed" message ('Sandbox provider
// "X" is not installed or its plugin worker is not running.'), which
// coincidentally contains the same "worker is not running" substring but
// describes a terminal condition that must not be retried.
function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) {
if (typeof value !== "string") return false;
return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test(
value,
);
}
function isRetryableInteractionContinuationInfrastructureFailure(
run: Pick<
typeof heartbeatRuns.$inferSelect,
"error" | "errorCode" | "resultJson"
>,
) {
// `controller_lease_lost` is only recorded before provider dispatch, so the
// retry cannot repeat started work.
if (
run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE ||
run.errorCode === "process_lost" ||
run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE
) {
return true;
}
if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed")
return false;
const resultJson = parseObject(run.resultJson);
return (
isSpawnLikeFailureMessage(run.error) ||
isSpawnLikeFailureMessage(resultJson.errorMessage) ||
isSpawnLikeFailureMessage(resultJson.message) ||
isSandboxProviderWorkerUnavailableFailureMessage(run.error) ||
isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) ||
isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message)
);
}
export function computeBoundedTransientHeartbeatRetrySchedule(
attempt: number,
now = new Date(),
random: () => number = Math.random,
) {
if (!Number.isInteger(attempt) || attempt <= 0) return null;
const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1];
if (typeof baseDelayMs !== "number") return null;
const delayMs = infrastructureRetryDelayMs(attempt, random());
return {
attempt,
baseDelayMs,
delayMs,
dueAt: new Date(now.getTime() + delayMs),
maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
};
}
export function normalizeAgentNameKey(value: string | null | undefined) {
if (typeof value !== "string") return null;
const normalized = value.trim().toLowerCase();
return normalized.length > 0 ? normalized : null;
}
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
/** Retry persistence uses the service database and explicit lifecycle callbacks. */
export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDependencies) {
const {
hasCaseHandoffForRun,
appendRunEvent,
escalatePlanApprovalResumeFailureNeedsAttention,
getAgentInvokability,
runDispatch,
resolveSessionBeforeForWakeup,
resolveResponsibleUserIdForRunContext,
recordPlanApprovalResumeFailureRetry,
setRunStatusIfRunning,
setWakeupStatus,
getRun,
getAgent,
releaseIssueExecutionAndPromote,
finalizeAgentStatus,
getWorktreeExecutionCutoff,
applyRunDispatchPostCommitEffects,
} = dependencies;
async function scheduleBoundedRetryForRun(
run: typeof heartbeatRuns.$inferSelect,
agent: typeof agents.$inferSelect,
opts?: {
now?: Date;
random?: () => number;
retryReason?: string;
wakeReason?: string;
maxAttempts?: number;
delayMs?: number;
},
) {
if (await hasCaseHandoffForRun(run)) return { outcome: "not_scheduled" as const,
reason: "The durable case handoff owns successor work.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) };
if (["openai_dot", "muse"].includes(String(parseObject(agent.adapterConfig).provider))
|| ["openai_dot", "muse"].includes(String(parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind))) {
return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) };
}
if (run.errorCode === "provider_tool_definition_invalid") {
return { outcome: "not_scheduled" as const,
reason: "Repair the invalid tool definitions before starting a new attempt.",
issueId: readNonEmptyString(run.contextSnapshot?.issueId) };
}
if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) &&
run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) {
return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.",
errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) };
}
const now = opts?.now ?? new Date();
const retryReason =
opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
const wakeReason =
opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
const maxAttempts = Math.max(
0,
Math.floor(
opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
),
);
const consumedAttempts = executionRetryAttemptCount(run, retryReason);
const nextAttempt = consumedAttempts + 1;
const computedBaseSchedule =
opts?.delayMs != null
? nextAttempt <= maxAttempts
? {
attempt: nextAttempt,
baseDelayMs: Math.max(0, Math.floor(opts.delayMs)),
delayMs: Math.max(0, Math.floor(opts.delayMs)),
dueAt: new Date(
now.getTime() + Math.max(0, Math.floor(opts.delayMs)),
),
maxAttempts,
}
: null
: nextAttempt <= maxAttempts
? computeBoundedTransientHeartbeatRetrySchedule(
nextAttempt,
now,
opts?.random,
)
: null;
const baseSchedule = computedBaseSchedule
? { ...computedBaseSchedule, maxAttempts }
: null;
const transientRecovery =
retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON
? readTransientRecoveryContractFromRun(run)
: null;
const codexTransientFallbackMode =
agent.adapterType === "codex_local" &&
transientRecovery?.errorFamily === "transient_upstream"
? resolveCodexTransientFallbackMode(nextAttempt)
: null;
const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null;
const contextSnapshot = parseObject(run.contextSnapshot);
// A retry inherits the durable authorization scope of its source run. Do
// not promote an untrusted or legacy contextSnapshot.issueId into a new
// issue binding: that could either violate the FK or misclassify history.
const issueId = run.scopeKind === "issue" ? run.issueId : null;
if (!baseSchedule) {
const exhaustion = {
retryReason,
scheduledRetryAttempt: consumedAttempts,
maxAttempts,
};
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`,
payload: exhaustion,
retryExhaustion: exhaustion,
});
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
await escalatePlanApprovalResumeFailureNeedsAttention({
run,
issueId,
attempt: Math.min(
consumedAttempts,
maxAttempts,
),
maxAttempts,
}).catch((error) => {
logger.warn(
{ err: error, runId: run.id, issueId },
"failed to escalate exhausted plan-approval resume failure",
);
});
}
return {
outcome: "retry_exhausted" as const,
attempt: nextAttempt,
maxAttempts,
};
}
if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) {
return {
outcome: "not_scheduled" as const,
reason:
"Reconcile the previous execution before retrying; safe provider recovery is unavailable.",
errorCode: "legacy_execution_requires_reconciliation" as const,
issueId: readNonEmptyString(run.contextSnapshot?.issueId),
};
}
if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) {
const invokability = await getAgentInvokability(agent);
if (!invokability.invokable) {
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message:
"Scheduled retry suppressed because the agent is not invokable",
payload: {
retryReason,
scheduledRetryAttempt: nextAttempt,
maxAttempts,
reason: invokability.reason,
invalidOrgChain: invokability.invalidOrgChain,
...invokability.details,
},
});
return {
outcome: "not_scheduled" as const,
reason:
"Scheduled retry suppressed because the agent is not invokable",
errorCode: "agent_not_invokable" as const,
issueId,
};
}
}
const schedule =
transientRetryNotBefore &&
transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime()
? {
...baseSchedule,
dueAt: transientRetryNotBefore,
delayMs: Math.max(
0,
transientRetryNotBefore.getTime() - now.getTime(),
),
}
: baseSchedule;
const requiresIssueGate =
isTransientWorkspaceGitScanCode(run.errorCode) ||
hasConversationContinuationPolicy(run.resultJson) ||
(retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) ||
retryReason === MAX_TURN_CONTINUATION_RETRY_REASON ||
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON;
if (requiresIssueGate) {
const gate = await runDispatch.evaluateScheduledRetryGate({
runId: run.id,
companyId: run.companyId,
retryReasonOverride: retryReason,
now,
});
if (!gate.allowed) {
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message: gate.reason,
payload: {
retryReason,
scheduledRetryAttempt: nextAttempt,
maxAttempts,
...gate.details,
},
});
return {
outcome: "not_scheduled" as const,
reason: gate.reason,
errorCode: gate.errorCode,
issueId: gate.issueId,
};
}
}
const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
const interactionContinuationPayload =
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON
? {
mutation: "interaction",
interactionId: readNonEmptyString(contextSnapshot.interactionId),
interactionKind: readNonEmptyString(
contextSnapshot.interactionKind,
),
interactionStatus: readNonEmptyString(
contextSnapshot.interactionStatus,
),
continuationPolicy: readNonEmptyString(
contextSnapshot.continuationPolicy,
),
}
: {};
const workspaceValidationRetryPayload =
retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON &&
isWorkspaceValidationFailedRun(run)
? readWorkspaceValidationPayloadFromRun(run)
: null;
const shouldQuarantineWorkspaceForRetry =
workspaceValidationRetryPayload !== null &&
Object.keys(workspaceValidationRetryPayload).length > 0;
const retryContextSnapshot: Record<string, unknown> = withRecoveryContext(
{
...contextSnapshot,
executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt),
retryOfRunId: run.id,
wakeReason,
retryReason,
...(retryReason === WORKSPACE_BUSY_RETRY_REASON
? {
failureRetriesBeforeWorkspaceWait:
executionFailureRetryCount(run),
}
: {}),
...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON)
? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) }
: {}),
...(shouldQuarantineWorkspaceForRetry
? {
workspaceValidationRecovery: {
strategy: "quarantine_failed_workspace_and_retry_clean",
sourceRunId: run.id,
reason:
readNonEmptyString(workspaceValidationRetryPayload?.reason) ??
WORKSPACE_VALIDATION_FAILURE_CODE,
fingerprint: readNonEmptyString(
workspaceValidationRetryPayload?.fingerprint,
),
failedExecutionWorkspaceId: readNonEmptyString(
workspaceValidationRetryPayload?.executionWorkspaceId,
),
},
}
: {}),
...(transientRecovery
? { errorFamily: transientRecovery.errorFamily }
: {}),
scheduledRetryAttempt: schedule.attempt,
scheduledRetryAt: schedule.dueAt.toISOString(),
...(transientRetryNotBefore
? { transientRetryNotBefore: transientRetryNotBefore.toISOString() }
: {}),
...(transientRecovery?.errorFamily === "provider_quota" &&
transientRetryNotBefore
? {
providerQuotaRetryNotBefore:
transientRetryNotBefore.toISOString(),
}
: {}),
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
},
"normal_model",
);
const responsibleUserId = await resolveResponsibleUserIdForRunContext(
run,
retryContextSnapshot,
);
const continuationRetryIdempotencyKey =
retryReason === MAX_TURN_CONTINUATION_RETRY_REASON
? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
: retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON
? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}`
: null;
type ScheduledRetryTransactionResult =
| {
outcome: "scheduled";
run: typeof heartbeatRuns.$inferSelect;
reusedExisting: boolean;
}
| {
outcome: "not_scheduled";
reason: string;
errorCode:
| "issue_not_found"
| "issue_reassigned"
| "issue_cancelled"
| "issue_terminal_status"
| "issue_not_in_progress"
| "continuation_user_authorization_missing"
| "issue_execution_lock_changed";
issueId: string | null;
details: Record<string, unknown>;
};
const scheduleResult = await db.transaction(
async (tx): Promise<ScheduledRetryTransactionResult> => {
// All automatic failure paths share the same predecessor claim. A
// duplicate monitor, restart sweep or wake must reuse its successor.
if (
retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON &&
retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON
) {
if (issueId)
await tx.execute(
sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`,
);
await tx.execute(
sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`,
);
const [existing] = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, run.companyId),
eq(heartbeatRuns.retryOfRunId, run.id),
),
)
.limit(1);
if (existing)
return {
outcome: "scheduled",
run: existing,
reusedExisting: true,
};
}
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
if (issueId) {
await tx.execute(
sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`,
);
} else {
await tx.execute(
sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`,
);
}
const existingContinuation = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, run.companyId),
eq(heartbeatRuns.retryOfRunId, run.id),
eq(heartbeatRuns.scheduledRetryReason, retryReason),
eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt),
inArray(heartbeatRuns.status, [
...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES,
]),
issueId
? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`
: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`,
),
)
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
.limit(1)
.then((rows) => rows[0] ?? null);
if (existingContinuation) {
if (existingContinuation.wakeupRequestId) {
const existingWakeup = await tx
.select({ coalescedCount: agentWakeupRequests.coalescedCount })
.from(agentWakeupRequests)
.where(
eq(
agentWakeupRequests.id,
existingContinuation.wakeupRequestId,
),
)
.then((rows) => rows[0] ?? null);
await tx
.update(agentWakeupRequests)
.set({
coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1,
updatedAt: now,
})
.where(
eq(
agentWakeupRequests.id,
existingContinuation.wakeupRequestId,
),
);
}
return {
outcome: "scheduled",
run: existingContinuation,
reusedExisting: true,
};
}
}
if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) {
if (issueId) {
await tx.execute(
sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`,
);
} else {
await tx.execute(
sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`,
);
}
const existingContinuation = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, run.companyId),
eq(heartbeatRuns.retryOfRunId, run.id),
eq(heartbeatRuns.scheduledRetryReason, retryReason),
eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt),
inArray(heartbeatRuns.status, [
...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES,
]),
issueId
? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`
: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`,
),
)
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
.limit(1)
.then((rows) => rows[0] ?? null);
if (existingContinuation) {
if (existingContinuation.wakeupRequestId) {
const existingWakeup = await tx
.select({ coalescedCount: agentWakeupRequests.coalescedCount })
.from(agentWakeupRequests)
.where(
eq(
agentWakeupRequests.id,
existingContinuation.wakeupRequestId,
),
)
.then((rows) => rows[0] ?? null);
await tx
.update(agentWakeupRequests)
.set({
coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1,
updatedAt: now,
})
.where(
eq(
agentWakeupRequests.id,
existingContinuation.wakeupRequestId,
),
);
}
return {
outcome: "scheduled",
run: existingContinuation,
reusedExisting: true,
};
}
if (issueId) {
const lockedIssue = await tx
.select({
id: issues.id,
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
executionRunId: issues.executionRunId,
})
.from(issues)
.where(
and(
eq(issues.id, issueId),
eq(issues.companyId, run.companyId),
),
)
.then((rows) => rows[0] ?? null);
if (!lockedIssue) {
return {
outcome: "not_scheduled",
reason:
"Scheduled max-turn continuation suppressed because the target issue no longer exists",
errorCode: "issue_not_found",
issueId,
details: { issueId },
};
}
if (lockedIssue.assigneeAgentId !== run.agentId) {
return {
outcome: "not_scheduled",
reason:
"Scheduled max-turn continuation suppressed because issue ownership changed",
errorCode: "issue_reassigned",
issueId,
details: {
issueId,
previousAssigneeAgentId: run.agentId,
currentAssigneeAgentId: lockedIssue.assigneeAgentId,
},
};
}
if (
lockedIssue.status === "cancelled" ||
lockedIssue.status === "done"
) {
return {
outcome: "not_scheduled",
reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`,
errorCode:
lockedIssue.status === "cancelled"
? "issue_cancelled"
: "issue_terminal_status",
issueId,
details: { issueId, currentStatus: lockedIssue.status },
};
}
if (lockedIssue.status !== "in_progress") {
return {
outcome: "not_scheduled",
reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`,
errorCode: "issue_not_in_progress",
issueId,
details: {
issueId,
currentStatus: lockedIssue.status,
requiredStatus: "in_progress",
},
};
}
if (lockedIssue.executionRunId !== run.id) {
return {
outcome: "not_scheduled",
reason:
"Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run",
errorCode: "issue_execution_lock_changed",
issueId,
details: {
issueId,
expectedExecutionRunId: run.id,
currentExecutionRunId: lockedIssue.executionRunId,
},
};
}
}
}
if (
(retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId &&
!isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot)
) {
// The issue row is locked above. Recheck after the preflight gate so
// cancellation or recovery cannot leave a successor without its lock.
const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId })
.from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)));
if (lockedIssue?.executionRunId !== run.id) {
return {
outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed",
reason: "Subscription retry suppressed because the task execution lock changed",
details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null },
};
}
}
const scheduledRunId = randomUUID();
if (contextSnapshot.explicitUserContinuation) {
const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({
db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId,
parentRunId: run.id, successorRunId: scheduledRunId, now,
}) : null;
if (!continuation) return {
outcome: "not_scheduled", issueId,
errorCode: "continuation_user_authorization_missing",
reason: "The automatic retry could not revalidate the original user continuation.",
details: {},
};
retryContextSnapshot.explicitUserContinuation = continuation;
retryContextSnapshot.previousRunId = continuation.previousRunId;
}
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
companyId: run.companyId,
agentId: run.agentId,
source: "automation",
triggerDetail: "system",
reason: wakeReason,
payload: withRecoveryContext(
{
...(issueId ? { issueId } : {}),
retryOfRunId: run.id,
...interactionContinuationPayload,
retryReason,
...(transientRecovery
? { errorFamily: transientRecovery.errorFamily }
: {}),
scheduledRetryAttempt: schedule.attempt,
scheduledRetryAt: schedule.dueAt.toISOString(),
...(transientRetryNotBefore
? {
transientRetryNotBefore:
transientRetryNotBefore.toISOString(),
}
: {}),
...(transientRecovery?.errorFamily === "provider_quota" &&
transientRetryNotBefore
? {
providerQuotaRetryNotBefore:
transientRetryNotBefore.toISOString(),
}
: {}),
...(codexTransientFallbackMode
? { codexTransientFallbackMode }
: {}),
},
"normal_model",
),
status: "queued",
requestedByActorType: "system",
requestedByActorId: null,
idempotencyKey: continuationRetryIdempotencyKey,
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
const scheduledRun = await tx
.insert(heartbeatRuns)
.values({
id: scheduledRunId,
companyId: run.companyId,
agentId: run.agentId,
scopeKind: run.scopeKind,
issueId,
invocationSource: "automation",
triggerDetail: "system",
status: "scheduled_retry",
wakeupRequestId: wakeupRequest.id,
contextSnapshot: retryContextSnapshot,
...(hasConversationContinuationPolicy(run.resultJson)
? { resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } } : {}),
responsibleUserId,
sessionIdBefore: sessionBefore,
retryOfRunId: run.id,
scheduledRetryAt: schedule.dueAt,
scheduledRetryAttempt: schedule.attempt,
scheduledRetryReason: retryReason,
continuationAttempt: readContinuationAttempt(
retryContextSnapshot.livenessContinuationAttempt,
),
updatedAt: now,
})
.returning()
.then((rows) => rows[0]);
await tx
.update(agentWakeupRequests)
.set({
runId: scheduledRun.id,
updatedAt: now,
})
.where(eq(agentWakeupRequests.id, wakeupRequest.id));
let detachWorkspaceFromIssue = false;
if (issueId && shouldQuarantineWorkspaceForRetry) {
const issueWorkspace = await tx
.select({
id: issues.id,
companyId: issues.companyId,
executionWorkspaceId: issues.executionWorkspaceId,
})
.from(issues)
.where(
and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)),
)
.for("update")
.then((rows) => rows[0] ?? null);
const failedExecutionWorkspaceId =
readNonEmptyString(
workspaceValidationRetryPayload?.executionWorkspaceId,
) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId);
if (issueWorkspace && failedExecutionWorkspaceId) {
const failedWorkspace = await tx
.select({
id: executionWorkspaces.id,
companyId: executionWorkspaces.companyId,
sourceIssueId: executionWorkspaces.sourceIssueId,
status: executionWorkspaces.status,
metadata: executionWorkspaces.metadata,
})
.from(executionWorkspaces)
.where(
and(
eq(executionWorkspaces.id, failedExecutionWorkspaceId),
eq(executionWorkspaces.companyId, run.companyId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
const workspaceBelongsToIssue = failedWorkspace
? failedWorkspace.sourceIssueId === issueId
: false;
if (
failedWorkspace &&
workspaceBelongsToIssue &&
issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId
) {
const existingMetadata = parseObject(failedWorkspace.metadata);
const quarantine = {
reason: WORKSPACE_VALIDATION_FAILURE_CODE,
retryReason,
sourceRunId: run.id,
retryRunId: scheduledRun.id,
issueId,
sourceIssueId: failedWorkspace.sourceIssueId ?? null,
quarantinedAt: now.toISOString(),
workspaceValidation: workspaceValidationRetryPayload ?? {},
};
await tx
.update(executionWorkspaces)
.set({
status: "archived",
closedAt: now,
cleanupEligibleAt: null,
cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE,
metadata: {
...existingMetadata,
workspaceValidationQuarantine: quarantine,
},
updatedAt: now,
})
.where(
and(
eq(executionWorkspaces.id, failedWorkspace.id),
eq(executionWorkspaces.companyId, run.companyId),
),
);
await logActivity(tx as unknown as Db, {
companyId: run.companyId,
actorType: "system",
actorId: "heartbeat",
agentId: run.agentId,
runId: run.id,
action: "execution_workspace.workspace_validation_quarantined",
entityType: "execution_workspace",
entityId: failedWorkspace.id,
details: quarantine,
});
detachWorkspaceFromIssue =
issueWorkspace.executionWorkspaceId ===
failedExecutionWorkspaceId;
}
}
}
if (issueId) {
await tx
.update(issues)
.set({
executionRunId: scheduledRun.id,
checkoutRunId: sql`case when ${issues.checkoutRunId} = ${run.id} then null else ${issues.checkoutRunId} end`,
executionAgentNameKey: normalizeAgentNameKey(agent.name),
executionLockedAt: now,
...(detachWorkspaceFromIssue
? {
executionWorkspaceId: null,
executionWorkspacePreference: null,
}
: {}),
updatedAt: now,
})
.where(
and(
eq(issues.id, issueId),
eq(issues.companyId, run.companyId),
eq(issues.executionRunId, run.id),
),
);
}
return {
outcome: "scheduled",
run: scheduledRun,
reusedExisting: false,
};
},
);
if (scheduleResult.outcome === "not_scheduled") {
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message: scheduleResult.reason,
payload: {
retryReason,
scheduledRetryAttempt: nextAttempt,
maxAttempts,
...scheduleResult.details,
},
});
return {
outcome: "not_scheduled" as const,
reason: scheduleResult.reason,
errorCode: scheduleResult.errorCode,
issueId: scheduleResult.issueId,
};
}
const retryRun = scheduleResult.run;
const dueAt = retryRun.scheduledRetryAt
? new Date(retryRun.scheduledRetryAt)
: schedule.dueAt;
if (scheduleResult.reusedExisting) {
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "info",
message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`,
payload: {
retryRunId: retryRun.id,
retryReason,
idempotencyKey: continuationRetryIdempotencyKey,
scheduledRetryAttempt: retryRun.scheduledRetryAttempt,
scheduledRetryAt: dueAt.toISOString(),
},
});
return {
outcome: "scheduled" as const,
run: retryRun,
dueAt,
attempt: retryRun.scheduledRetryAttempt,
maxAttempts: schedule.maxAttempts,
reusedExisting: true,
};
}
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`,
payload: {
retryRunId: retryRun.id,
retryReason,
...(transientRecovery
? { errorFamily: transientRecovery.errorFamily }
: {}),
scheduledRetryAttempt: schedule.attempt,
scheduledRetryAt: schedule.dueAt.toISOString(),
baseDelayMs: schedule.baseDelayMs,
delayMs: schedule.delayMs,
...(transientRetryNotBefore
? { transientRetryNotBefore: transientRetryNotBefore.toISOString() }
: {}),
...(transientRecovery?.errorFamily === "provider_quota" &&
transientRetryNotBefore
? {
providerQuotaRetryNotBefore:
transientRetryNotBefore.toISOString(),
}
: {}),
...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),
},
});
if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) {
await recordPlanApprovalResumeFailureRetry({
run,
issueId,
retryRunId: retryRun.id,
attempt: schedule.attempt,
maxAttempts: schedule.maxAttempts,
}).catch((error) => {
logger.warn(
{ err: error, runId: run.id, issueId, retryRunId: retryRun.id },
"failed to record plan-approval resume retry failure",
);
});
}
return {
outcome: "scheduled" as const,
run: retryRun,
dueAt,
attempt: schedule.attempt,
maxAttempts: schedule.maxAttempts,
};
}
// Finds a running heartbeat run (other than the caller's) whose context
// issue shares the same project workspace, i.e. the run that currently
// "holds" the shared working tree. Runs that have been silent past
// WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS do not count — a zombie holder must
// not park other work forever. This cutoff is independent of informational
// silence warnings. When isolated workspaces are enabled, holders whose
// issue explicitly opted into an isolated workspace never touch the shared
// tree, so they are excluded; a NULL/agent_default mode may resolve to the
// shared tree and counts as a holder (over-serializing is the safe
// direction). When the isolated-workspaces experiment is off, every run
// resolves to the shared tree, so no holder is excluded.
async function findSharedWorkspaceHolder(input: {
companyId: string;
projectWorkspaceId: string;
excludeIssueId: string;
excludeRunId: string;
honorIsolatedWorkspaceModes: boolean;
now?: Date;
}): Promise<SharedWorkspaceHolder | null> {
const staleCutoff = new Date(
(input.now ?? new Date()).getTime() -
WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS,
);
return await db
.select({
runId: heartbeatRuns.id,
agentId: heartbeatRuns.agentId,
issueId: sql<string>`${issues.id}::text`,
issueIdentifier: issues.identifier,
})
.from(heartbeatRuns)
.innerJoin(
issues,
and(
eq(issues.companyId, heartbeatRuns.companyId),
sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot} ->> 'issueId'`,
),
)
.where(
and(
eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.status, "running"),
ne(heartbeatRuns.id, input.excludeRunId),
// Last observed activity: output beats start beats creation. A run
// that started recently but has not written output yet is live.
sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`,
eq(issues.projectWorkspaceId, input.projectWorkspaceId),
ne(sql`${issues.id}::text`, input.excludeIssueId),
...(input.honorIsolatedWorkspaceModes
? [
or(
// Covers both a NULL settings blob and a blob without a mode
// key; either may still resolve to the shared workspace.
sql`${issues.executionWorkspaceSettings} ->> 'mode' is null`,
notInArray(
sql`${issues.executionWorkspaceSettings} ->> 'mode'`,
[...ISOLATED_EXECUTION_WORKSPACE_MODES],
),
),
]
: []),
),
)
.orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
.limit(1)
.then((rows) => rows[0] ?? null);
}
// Finds a running run (other than the caller's) on an issue bound to the
// same reused execution workspace, typically a parent run that is still
// active after delegating a child that inherited its workspace. Unlike
// findSharedWorkspaceHolder, a holder silent past
// WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS is still returned (flagged stale):
// silence does not prove the holder stopped, so the caller must not treat
// the folder as free. A live holder is preferred over a stale one.
async function findExecutionWorkspaceHolder(input: {
companyId: string;
executionWorkspaceId: string;
excludeRunId: string;
now?: Date;
}): Promise<ExecutionWorkspaceHolder | null> {
const staleCutoff = new Date(
(input.now ?? new Date()).getTime() -
WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS,
);
const lastActivityBefore = sql<boolean>`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz`;
return await db
.select({
runId: heartbeatRuns.id,
agentId: heartbeatRuns.agentId,
issueId: sql<string>`${issues.id}::text`,
issueIdentifier: issues.identifier,
stale: lastActivityBefore,
})
.from(heartbeatRuns)
.innerJoin(
issues,
and(
eq(issues.companyId, heartbeatRuns.companyId),
eq(issues.executionWorkspaceId, input.executionWorkspaceId),
or(
eq(issues.executionRunId, heartbeatRuns.id),
sql`${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`,
),
),
)
.where(
and(
eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.status, "running"),
ne(heartbeatRuns.id, input.excludeRunId),
),
)
.orderBy(asc(lastActivityBefore), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id))
.limit(1)
.then((rows) => rows[0] ?? null);
}
// Credential rotation can briefly contend with a fresh runtime read. Keep
// the task on its automatic pre-provider retry path while the lock clears.
async function finalizeAiConnectionBusyDeferral(
run: typeof heartbeatRuns.$inferSelect,
error: HttpError,
wasIssueAssignee: boolean,
) {
const now = new Date();
const cancelled = await setRunStatusIfRunning(run.id, "cancelled", {
error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now,
resultJson: {
executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false },
cancellation: {
source: "control_plane",
expected: true,
initiator: { type: "system" },
reason: "Waiting for shared AI credentials",
recordedAt: now.toISOString(),
},
},
contextSnapshot: {
...parseObject(run.contextSnapshot),
aiConnectionBusyDeferredWhileAssignee: wasIssueAssignee,
},
});
if (!cancelled.updated) return;
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined);
const cancelledRun = cancelled.run ?? await getRun(run.id);
const agent = await getAgent(run.agentId);
let scheduled = false;
try {
if (cancelledRun && agent) {
const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, {
now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry",
maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1,
delayMs: computeWorkspaceBusyRetryDelayMs(),
});
scheduled = retry.outcome === "scheduled";
await appendRunEvent(cancelledRun, {
eventType: "lifecycle", stream: "system", level: "info",
message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.",
payload: { retryScheduled: scheduled },
});
}
} finally {
try {
if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun);
} finally {
await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) });
}
}
}
// Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch
// gate: cancel the run (contention is not a failure), schedule a
// workspace_busy retry, and leave the agent idle. The issue execution lock
// transfers to the scheduled retry run inside scheduleBoundedRetryForRun, so
// the issue keeps an active execution path and recovery leaves it alone.
// Deferral has no attempt ceiling — the retry keeps rescheduling while a
// live holder exists, and holder staleness (not a counter) is what prevents
// waiting on a zombie. If no retry could be scheduled (agent no longer
// invokable), the lock is released so the issue does not strand on a
// cancelled run.
async function finalizeWorkspaceBusyDeferral(
run: typeof heartbeatRuns.$inferSelect,
deferral: WorkspaceBusyDeferral,
) {
const now = new Date();
const cancelWrite = await setRunStatusIfRunning(run.id, "cancelled", {
error: deferral.message,
errorCode: WORKSPACE_BUSY_ERROR_CODE,
finishedAt: now,
resultJson: {
executionRecovery: {
kind: "workspace_wait",
providerWorkStarted: false,
},
cancellation: {
source: "control_plane",
expected: true,
initiator: { type: "system" },
reason: `Waiting for the shared ${deferral.executionWorkspaceId ? "execution" : "project"} workspace`,
recordedAt: now.toISOString(),
},
workspaceBusy: {
projectWorkspaceId: deferral.projectWorkspaceId,
executionWorkspaceId: deferral.executionWorkspaceId,
holderRunId: deferral.holder.runId,
holderIssueId: deferral.holder.issueId,
deferralAttempt: deferral.deferralAttempt,
},
},
// Recorded on the run (and inherited by the scheduled retry's context)
// so the retry promotion gate can tell a non-assignee wake — where an
// assignee mismatch is the expected state — from a reassignment race.
contextSnapshot: {
...parseObject(run.contextSnapshot),
workspaceBusyDeferredWhileAssignee: deferral.wasIssueAssignee,
},
});
if (!cancelWrite.updated) {
logger.info(
{ runId: run.id, currentStatus: cancelWrite.run?.status ?? null },
"skipping workspace-busy deferral finalization because the run already left running state",
);
return;
}
await setWakeupStatus(run.wakeupRequestId, "cancelled", {
finishedAt: now,
error: deferral.message,
}).catch(() => undefined);
const cancelledRun =
cancelWrite.run ?? (await getRun(run.id).catch(() => null));
const agentRow = await getAgent(run.agentId).catch(() => null);
let scheduleOutcome: string | null = null;
if (cancelledRun && agentRow) {
const scheduleResult = await scheduleBoundedRetryForRun(
cancelledRun,
agentRow,
{
now,
retryReason: WORKSPACE_BUSY_RETRY_REASON,
wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON,
// Always admit the next attempt: workspace-busy deferral is bounded by
// holder liveness, not by an attempt counter.
maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1,
delayMs: computeWorkspaceBusyRetryDelayMs(),
},
).catch((scheduleErr) => {
logger.error(
{ err: scheduleErr, runId: run.id },
"failed to schedule workspace-busy retry after deferral",
);
return null;
});
scheduleOutcome = scheduleResult?.outcome ?? null;
}
if (cancelledRun) {
await appendRunEvent(cancelledRun, {
eventType: "lifecycle",
stream: "system",
level: "info",
message:
scheduleOutcome === "scheduled"
? `Deferred: ${deferral.message}. Retry ${deferral.deferralAttempt + 1} scheduled; the run waits for the workspace to free.`
: `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`,
payload: {
projectWorkspaceId: deferral.projectWorkspaceId,
executionWorkspaceId: deferral.executionWorkspaceId,
holderRunId: deferral.holder.runId,
holderIssueId: deferral.holder.issueId,
deferralAttempt: deferral.deferralAttempt,
retryScheduled: scheduleOutcome === "scheduled",
},
}).catch(() => undefined);
}
if (cancelledRun && scheduleOutcome !== "scheduled") {
await releaseIssueExecutionAndPromote(cancelledRun).catch(
(releaseErr) => {
logger.error(
{ err: releaseErr, runId: run.id },
"failed to release issue execution after workspace-busy deferral",
);
},
);
}
await finalizeAgentStatus(run.agentId, "cancelled", null, {
wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
}).catch(() => undefined);
}
async function scheduleInteractionContinuationInfrastructureRetryIfEligible(
run: typeof heartbeatRuns.$inferSelect,
agent: typeof agents.$inferSelect,
) {
if (await hasCaseHandoffForRun(run)) return null;
if (!run.wakeupRequestId) return null;
if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot))
return null;
if (!isRetryableInteractionContinuationInfrastructureFailure(run)) {
const context = parseObject(run.contextSnapshot);
const issueId = readNonEmptyString(context.issueId);
await escalatePlanApprovalResumeFailureNeedsAttention({
run,
issueId,
attempt: Math.min(
run.scheduledRetryAttempt ??
INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
),
maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
}).catch((error) => {
logger.warn(
{ err: error, runId: run.id, issueId },
"failed to escalate non-retryable plan-approval resume failure",
);
});
return null;
}
return scheduleBoundedRetryForRun(run, agent, {
retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS,
});
}
async function promoteDueScheduledRetries(now = new Date()) {
const cutoff = await getWorktreeExecutionCutoff();
const result = await runDispatch.promoteDueScheduledRetries({
now,
cutoff,
});
applyRunDispatchPostCommitEffects(result.postCommitEffects);
return { promoted: result.promoted, runIds: result.runIds };
}
async function getIssueRetryRun(
companyId: string,
issueId: string,
statuses: Array<"scheduled_retry" | "queued" | "running" | "cancelled">,
) {
if (statuses.length === 0) return null;
return db
.select({
run: heartbeatRuns,
agentName: agents.name,
})
.from(heartbeatRuns)
.innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
.where(
and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.status, statuses),
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
sql`${heartbeatRuns.retryOfRunId} is not null`,
),
)
.orderBy(
desc(heartbeatRuns.updatedAt),
desc(heartbeatRuns.createdAt),
desc(heartbeatRuns.id),
)
.limit(1)
.then((rows) => rows[0] ?? null);
}
function summarizeIssueScheduledRetryRun(row: {
run: typeof heartbeatRuns.$inferSelect;
agentName: string | null;
}) {
return {
runId: row.run.id,
status: row.run.status as
"scheduled_retry" | "queued" | "running" | "cancelled",
agentId: row.run.agentId,
agentName: row.agentName,
retryOfRunId: row.run.retryOfRunId,
scheduledRetryAt: row.run.scheduledRetryAt,
scheduledRetryAttempt: row.run.scheduledRetryAttempt,
scheduledRetryReason: row.run.scheduledRetryReason,
error: row.run.error,
errorCode: row.run.errorCode,
};
}
async function retryScheduledRetryNow(input: {
issueId: string;
actor?: {
actorType?: "user" | "agent" | "system";
actorId?: string | null;
};
now?: Date;
}) {
const now = input.now ?? new Date();
const issue = await db
.select({ id: issues.id, companyId: issues.companyId })
.from(issues)
.where(eq(issues.id, input.issueId))
.then((rows) => rows[0] ?? null);
if (!issue) throw notFound("Issue not found");
const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [
"scheduled_retry",
]);
if (!scheduled) {
const alreadyPromoted = await getIssueRetryRun(
issue.companyId,
issue.id,
["queued", "running"],
);
if (alreadyPromoted) {
return {
outcome: "already_promoted" as const,
message: "Scheduled retry was already promoted",
scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted),
};
}
return {
outcome: "no_scheduled_retry" as const,
message: "No live scheduled retry exists for this issue",
scheduledRetry: null,
};
}
const contextSnapshot = {
...parseObject(scheduled.run.contextSnapshot),
scheduledRetryAt: now.toISOString(),
retryNowRequestedAt: now.toISOString(),
retryNowRequestedByActorType: input.actor?.actorType ?? null,
retryNowRequestedByActorId: input.actor?.actorId ?? null,
};
const updated = await db.transaction(async (tx) => {
const row = await tx
.update(heartbeatRuns)
.set({
scheduledRetryAt: now,
contextSnapshot,
updatedAt: now,
})
.where(
and(
eq(heartbeatRuns.id, scheduled.run.id),
eq(heartbeatRuns.status, "scheduled_retry"),
),
)
.returning()
.then((rows) => rows[0] ?? null);
if (!row) return null;
if (row.wakeupRequestId) {
const wakeupPayload = {
...parseObject(
await tx
.select({ payload: agentWakeupRequests.payload })
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, row.wakeupRequestId))
.then((rows) => rows[0]?.payload ?? null),
),
scheduledRetryAt: now.toISOString(),
retryNowRequestedAt: now.toISOString(),
};
await tx
.update(agentWakeupRequests)
.set({
payload: wakeupPayload,
updatedAt: now,
})
.where(eq(agentWakeupRequests.id, row.wakeupRequestId));
}
return row;
});
if (!updated) {
const alreadyPromoted = await getIssueRetryRun(
issue.companyId,
issue.id,
["queued", "running"],
);
if (alreadyPromoted) {
return {
outcome: "already_promoted" as const,
message: "Scheduled retry was already promoted",
scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted),
};
}
return {
outcome: "no_scheduled_retry" as const,
message: "No live scheduled retry exists for this issue",
scheduledRetry: null,
};
}
await appendRunEvent(updated, {
eventType: "lifecycle",
stream: "system",
level: "info",
message: "Scheduled retry was requested to run now",
payload: {
issueId: issue.id,
scheduledRetryAttempt: updated.scheduledRetryAttempt,
scheduledRetryAt: updated.scheduledRetryAt
? new Date(updated.scheduledRetryAt).toISOString()
: null,
scheduledRetryReason: updated.scheduledRetryReason,
requestedByActorType: input.actor?.actorType ?? null,
requestedByActorId: input.actor?.actorId ?? null,
},
});
const promotion = await runDispatch.promoteScheduledRetry({
runId: updated.id,
companyId: updated.companyId,
now,
});
if (promotion.outcome === "promoted") {
applyRunDispatchPostCommitEffects(promotion.postCommitEffects);
}
// Promotion can preserve this row as a cleanup wait. Read that exact run,
// not an older cancelled retry or the pre-promotion schedule.
const currentRun = await getRun(updated.id);
const scheduledRetry = currentRun
? summarizeIssueScheduledRetryRun({
run: currentRun,
agentName: scheduled.agentName,
})
: null;
if (currentRun?.status === "scheduled_retry") {
return {
outcome: "waiting" as const,
message: parseObject(currentRun.resultJson?.executionWait).cause === "execution_owner_active"
? "Waiting for execution cleanup. Paperclip will retry automatically once cleanup finishes."
: "The retry remains scheduled. Paperclip will check again at the scheduled time.",
scheduledRetry,
};
}
if (promotion.outcome === "promoted") {
return {
outcome: "promoted" as const,
message: "Scheduled retry was promoted to the queued run pool",
scheduledRetry,
};
}
if (promotion.outcome === "gate_suppressed") {
return {
outcome: "gate_suppressed" as const,
message: promotion.reason,
scheduledRetry,
};
}
if (currentRun && ["queued", "running"].includes(currentRun.status)) {
return {
outcome: "already_promoted" as const,
message: "Scheduled retry was already promoted",
scheduledRetry,
};
}
return {
outcome: "no_scheduled_retry" as const,
message: "No live scheduled retry exists for this issue",
scheduledRetry: null,
};
}
function timerClaimWasFirstHeartbeat(
run: Pick<typeof heartbeatRuns.$inferSelect, "contextSnapshot">,
): true | undefined {
return parseObject(run.contextSnapshot).timerClaimWasFirstHeartbeat === true
? true
: undefined;
}
return {
scheduleBoundedRetryForRun,
timerClaimWasFirstHeartbeat,
scheduleInteractionContinuationInfrastructureRetryIfEligible,
findSharedWorkspaceHolder,
findExecutionWorkspaceHolder,
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
retryScheduledRetryNow,
scheduleBoundedRetry: async (
runId: string,
opts?: {
now?: Date;
random?: () => number;
retryReason?: string;
wakeReason?: string;
maxAttempts?: number;
delayMs?: number;
},
) => {
const run = await getRun(runId, { unsafeFullResultJson: true });
if (!run) return { outcome: "missing_run" as const };
const agent = await getAgent(run.agentId);
if (!agent) return { outcome: "missing_agent" as const };
return scheduleBoundedRetryForRun(run, agent, opts);
},
};
}