Skip to content

The open-source app everyone uses to manage agents at work

refactor(heartbeat): extract restart recovery and lease cleanup (#15622)

Dotta04de3132c202
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index 709eb2cb93..9736a2e7be 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -1263,6 +1263,15 @@ run events, issue-lock release, plan-resume reporting, and worktree cutoffs. It
re-exports the existing retry helpers and workspace-busy error class. Keep retry
policy changes separate from this extraction.
+Restart recovery and lease cleanup are in `server/src/services/heartbeat/recovery.ts`.
+It owns hot-restart snapshots and adoption, native restart recovery, shutdown
+draining, orphaned-run reaping, and active/pending-cleanup lease sweeps.
+`createHeartbeatRecovery` binds the service database and explicit lifecycle
+callbacks without starting work. The service supplies its shutdown flag callback
+and shared execution sets so separate service instances keep the same ownership
+and shutdown barriers. Cleanup single-flight state stays at module scope.
+Keep recovery policy changes separate from retry scheduling and execution changes.
+
## Wake Context Delivery
Built-in adapters deliver wake context through the run prompt, including structured
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index a4f6421e26..6d47fc4795 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -7974,7 +7974,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
expect(next?.contextSnapshot?.wakeCommentIds).toEqual([pending!.id, go!.id]);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferred!.id)))[0]).toMatchObject({ status: "coalesced", runId: next!.id });
- await vi.waitFor(async () => expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"));
+ await heartbeat.drainActiveRunExecutions();
+ expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");
});
it.each(["dedicated deferred donor", "non-coalescing recipient", "persistent agent conversation"] as const)(
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index a3a8969166..5673162940 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -1,3 +1,10 @@
+import {
+ HEARTBEAT_RUN_TERMINAL_STATUSES,
+ DETACHED_PROCESS_ERROR_CODE,
+ isProcessAlive,
+ CANCELLABLE_HEARTBEAT_RUN_STATUSES,
+ createHeartbeatRecovery,
+} from "./heartbeat/recovery.js";
import {
BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
normalizeAgentNameKey,
@@ -238,7 +245,7 @@ import {
NativeWorkspaceFinalizationOwnershipLostError,
type NativeWorkspaceFinalizationOwnership,
} from "./native-runtime/native-workspace-finalization-ownership.js";
-import { hasStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
+
import { reserveRunBudget } from "./budget-reservations.js";
import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js";
import { createRunUsageRecorder } from "./usage-receipts.js";
@@ -257,9 +264,18 @@ import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-
import { getConversationConfirmationContext } from "./conversation-confirmation-context.js";
import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";
import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";
-import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLease, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js";
+import {
+ legacyControllerBootId,
+ legacyControllerClaim,
+ renewLegacyControllerLease,
+ watchLegacyControllerLease,
+} from "./legacy-controller-lease.js";
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
-import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
+import {
+ hasRemoteTerminationReceipt,
+ remoteExecutionHasStopped,
+ stoppedRemoteCleanupScopes,
+} from "./remote-execution-termination.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
import {
admitExplicitNativeContinuation,
@@ -436,13 +452,9 @@ import {
buildNativeExecutionWithCheckpoint,
buildNativeRuntimeContext,
cancelNativeSession,
- claimNativeRestartRecoveries,
closeWarmNativeSessionsForEnvironment,
- closeIdleWarmNativeSessionsForRestart,
reserveWarmNativeInstructionDirectory,
- currentNativeControllerIdentity,
dispatchNativeSessionResumptions,
- detachNativeSessionsForRestart,
ensureNativeCompletionContract,
executePaperclipNativeSession,
finalizeNativeRun,
@@ -461,13 +473,9 @@ import {
nativeToolContractFingerprintForTarget,
prepareNativeSessionBootstrapPersistence,
prepareNativeWorkspaceSync,
- readNativeWorkspaceSyncReference,
recordNativeFinalizationFailure,
type NativeRestartRecoveryClaim,
rebindNativeSessionCheckpoint,
- reconcileNativeFinalizations,
- reconcileRetainedNativeSessionCleanup,
- reconcileRetainedNativeSessionCleanups,
resolveHeartbeatNativeRuntimeMode,
} from "./native-runtime/index.js";
import {
@@ -484,7 +492,6 @@ import {
type NativeRunHistoricalSpan,
} from "./native-runtime/native-run-trace.js";
import {
- drainRetainedRunnerdMaintenanceOperations,
describeRunnerdNativeSessionBackend,
parseNativeExecutionInput,
type NativeExecutionInput,
@@ -539,8 +546,7 @@ import {
} from "./agent-task-run-telemetry.js";
import { readAiConnectionConfigurationFailure } from "./ai-connection-configuration-failure.js";
import { reportRunFailure } from "./run-failure-report.js";
-import { performance } from "node:perf_hooks";
-import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js";
+
import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";
import { companySkillService } from "./company-skills.js";
import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";
@@ -808,15 +814,7 @@ import {
touchHeartbeatRunRuntimeStatus,
} from "./heartbeat-run-runtime-status.js";
import {
- findMissingHotRestartSnapshotRunIds,
- readHotRestartIntent,
readProcessStartedAt,
- removeHotRestartIntent,
- shouldHonorHotRestartIntentForProcess,
- writeHotRestartReport,
- writeHotRestartShutdownSnapshot,
- type HotRestartIntentRun,
- type HotRestartReportRun,
} from "./hot-restart.js";
import {
assertLowTrustRuntimeServicesAllowed,
@@ -825,7 +823,7 @@ import {
import { resolveAndRetainRunTrustPreset } from "./run-trust-preset.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
-import { serverVersion } from "../version.js";
+
import { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain, readTaskDrain } from "./task-admission.js";
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
@@ -861,123 +859,16 @@ const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
];
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
-
const ACCEPTED_PLAN_CONVERSION_SKILL_KEY =
"paperclipai/paperclip/paperclip-converting-plans-to-tasks";
-
-
-const DETACHED_PROCESS_ERROR_CODE = "process_detached";
-const NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE =
- "Native execution ownership could not be verified; automatic recovery is blocked";
-// The reaper sweeps at most this many pending_cleanup leases per tick.
-const PENDING_CLEANUP_SWEEP_PAGE_SIZE = 20;
-const pendingCleanupAttemptsInFlight = new Set<string>();
-// Escalate and slow cleanup after this many attempts; never abandon a live lease.
-const PENDING_CLEANUP_SWEEP_ATTEMPT_CAP = 5;
-// The reaper stores its retry state under these keys in the lease metadata.
-const PENDING_CLEANUP_ATTEMPTS_METADATA_KEY = "pendingCleanupRetryAttempts";
-const PENDING_CLEANUP_CAP_WARNED_METADATA_KEY = "pendingCleanupRetryCapWarned";
-// The reaper sweeps at most this many orphaned active leases per tick.
-const ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE = 20;
-
-// A provider or plugin destroy rejection can carry a bearer credential, a
-// signed URL, or provider response detail in its name, code, message, cause, or
-// stack. The exception fields cross the server boundary, so they are not a
-// trusted enum. The pending_cleanup sweep logs never read the exception. Each
-// catch site logs a constant, locally generated `errorKind` instead.
-const PENDING_CLEANUP_RETRY_ERROR_KIND = "destroy_failed";
-const PENDING_CLEANUP_SWEEP_ERROR_KIND = "sweep_failed";
-const ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND = "orphaned_active_lease_sweep_failed";
-
-// Read the stored retry attempt count as a safe value, directly in SQL. A
-// provider can write a malformed value under the attempts key. The type guard
-// makes any non-number value read as zero. The reader computes as numeric and
-// never casts to int, so a finite number outside the 32-bit range (for example
-// 1e300) never throws. The reader clamps a negative value to zero and a positive
-// value to the attempt cap. One malformed lease therefore never aborts the page
-// sweep. This matches the TypeScript reader `readPendingCleanupRetryAttempts`,
-// which clamps to the same range. The claim predicate compares the two readers,
-// so both must yield the same value for every input.
-function pendingCleanupAttemptsSql() {
- return sql`
- case
- when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}) = 'number'
- then least(
- greatest(
- floor((${environmentLeases.metadata} ->> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY})::numeric),
- 0
- ),
- ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}
- )
- else 0
- end`;
-}
-
-function pendingCleanupRetryDueSql(explicitRetry = false) {
- // Ownership renewal and retry cooldown are separate clocks. A late renewal
- // cannot change when a completed attempt may retry. Older claims used the
- // retry field for both clocks, so retain that fallback until they settle.
- const deadline = sql`case
- when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'
- and ${environmentLeases.metadata} ? 'pendingCleanupLeaseExpiresAtMs'
- then ${environmentLeases.metadata}->'pendingCleanupLeaseExpiresAtMs'
- else ${environmentLeases.metadata}->'pendingCleanupRetryAfterMs' end`;
- return sql`case
- when ${explicitRetry} and coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true' then true
- when jsonb_typeof(${deadline}) = 'number'
- then (${deadline})::numeric <= ${Date.now()}
- or (${deadline})::numeric > ${Date.now() + 30 * 60_000 + 1_000}
- else true end`;
-}
-
-// Choose the `jsonb_set` target root. A provider can write a scalar or array
-// metadata root. `jsonb_set` fails on a non-object root, so the reader uses the
-// stored metadata only when its root is an object. A NULL, scalar, or array root
-// reads as an empty object. `jsonb_typeof(NULL)` is NULL, so the else branch also
-// covers a NULL root.
-function pendingCleanupMetadataObjectSql() {
- return sql`case when jsonb_typeof(${environmentLeases.metadata}) = 'object' then ${environmentLeases.metadata} else '{}'::jsonb end`;
-}
-
-// Read the stored cap-warned flag as a safe boolean, directly in SQL. A
-// malformed value reads as false, so the boolean cast never throws.
-function pendingCleanupCapWarnedSql() {
- return sql`coalesce(
- case
- when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}) = 'boolean'
- then (${environmentLeases.metadata} ->> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY})::boolean
- else false
- end,
- false
- )`;
-}
-
-
-
-
-
-
-
const execFile = promisify(execFileCallback);
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [
"queued",
"running",
"scheduled_retry",
] as const;
-const CANCELLABLE_HEARTBEAT_RUN_STATUSES = [
- "queued",
- "running",
- "scheduled_retry",
-] as const;
-const NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY = "nativeQuestionCancellation";
-const HEARTBEAT_RUN_TERMINAL_STATUSES = [
- "succeeded",
- "interrupted",
- "failed",
- "cancelled",
- "timed_out",
-] as const;
+
const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = [
"failed",
"cancelled",
@@ -1186,16 +1077,6 @@ const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([
"issue_recovery_action_restored",
]);
-const SESSIONED_LOCAL_ADAPTERS = new Set([
- "claude_local",
- "codex_local",
- "cursor",
- "gemini_local",
- "hermes_local",
- "kimi_local",
- "opencode_local",
- "pi_local",
-]);
// Routes and the scheduler construct separate heartbeatService instances, but
// they must agree on in-process adapter executions when reaping stale runs.
const activeRunExecutions = new Set<string>();
@@ -1633,10 +1514,6 @@ function isSameTaskScope(left: string | null, right: string | null) {
return (left ?? null) === (right ?? null);
}
-function isTrackedLocalChildProcessAdapter(adapterType: string) {
- return SESSIONED_LOCAL_ADAPTERS.has(adapterType);
-}
-
function isHeartbeatRunTerminalStatus(
status: string | null | undefined,
): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {
@@ -1914,23 +1791,6 @@ function buildRunEventRuntimeProgress(input: {
};
}
-// A positive liveness check means some process currently owns the PID.
-// On Linux, PIDs can be recycled, so this is a best-effort signal rather
-// than proof that the original child is still alive.
-function isProcessAlive(pid: number | null | undefined) {
- if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)
- return false;
- try {
- process.kill(pid, 0);
- return true;
- } catch (error) {
- const code = (error as NodeJS.ErrnoException | undefined)?.code;
- if (code === "EPERM") return true;
- if (code === "ESRCH") return false;
- return false;
- }
-}
-
export async function persistHeartbeatRunProcessMetadata(
db: Db,
runId: string,
@@ -1990,65 +1850,6 @@ async function terminateHeartbeatRunProcess(input: {
);
}
-function buildProcessLossMessage(
- run: {
- processPid: number | null;
- processGroupId: number | null;
- },
- options?: { descendantOnly?: boolean },
-) {
- if (options?.descendantOnly && run.processGroupId) {
- return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`;
- }
- if (run.processPid) {
- return `Process lost -- child pid ${run.processPid} is no longer running`;
- }
- if (run.processGroupId) {
- return `Process lost -- process group ${run.processGroupId} is no longer running`;
- }
- return "Process lost -- server may have restarted";
-}
-
-function readHotRestartAdoptionMetadata(
- resultJson: Record<string, unknown> | null | undefined,
-) {
- const result = parseObject(resultJson);
- const hotRestart = parseObject(result.hotRestart);
- if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string")
- return null;
- return hotRestart;
-}
-
-function mergeHotRestartAdoptionResultJson(
- resultJson: Record<string, unknown> | null | undefined,
- input: {
- adoptedAt: Date;
- previousServerPid: number;
- newServerPid: number;
- previousServerVersion: string | null;
- newServerVersion: string;
- processPid: number | null;
- processGroupId: number | null;
- },
-) {
- const result = parseObject(resultJson);
- const existing = parseObject(result.hotRestart);
- return {
- ...result,
- hotRestart: {
- ...existing,
- adopted: true,
- adoptedAt: input.adoptedAt.toISOString(),
- previousServerPid: input.previousServerPid,
- newServerPid: input.newServerPid,
- previousServerVersion: input.previousServerVersion,
- newServerVersion: input.newServerVersion,
- processPid: input.processPid,
- processGroupId: input.processGroupId,
- },
- };
-}
-
type SkillTestHeartbeatCompletion = {
outcome: "failed" | "cancelled";
error: string | null;
@@ -2456,6 +2257,49 @@ export function heartbeatService(
applyRunDispatchPostCommitEffects,
});
+ const {
+ sweepPendingCleanupLeases,
+ markNativeOwnershipUnverified,
+ prepareHotRestartShutdown,
+ reconcileHotRestartAdoption,
+ recoverNativeRunsAfterRestart,
+ reapOrphanedRuns,
+ sweepOrphanedActiveLeases,
+ drainRunningRunsForShutdown,
+ } = createHeartbeatRecovery(db, {
+ enterShutdown: () => { shutdownInProgress = true; },
+ scheduleBoundedRetryForRun,
+ appendRunEvent,
+ environmentRuntime,
+ scheduleNativeSessionResumeDispatch,
+ getRun,
+ executeRun,
+ activeRunExecutionPromises,
+ cancelHeartbeatNativeRun,
+ terminateHeartbeatRunProcess,
+ setRunStatusIfRunning,
+ mergeRunStopMetadataForAgent,
+ setWakeupStatus,
+ classifyAndPersistRunLiveness,
+ releaseEnvironmentLeasesForRun,
+ releaseIssueExecutionAndPromote,
+ finalizeAgentStatus,
+ timerClaimWasFirstHeartbeat,
+ environmentsSvc,
+ acknowledgeRemoteStop,
+ resumeRemoteStopComments,
+ setRunStatusFromLive,
+ instructionCopies,
+ dispatchPendingNativeStatusWakeups,
+ runtimeEnv,
+ cancelRunInternal,
+ activeRunExecutions,
+ setRunStatus,
+ getAgent,
+ scheduleInteractionContinuationInfrastructureRetryIfEligible,
+ startNextQueuedRunForAgent,
+ });
+
// Applies the post-commit effects a run-dispatch operation returns, on a
// best-effort basis, exactly as this service publishes them for every
// other run write. A publish failure never rolls back the write that
@@ -6025,1231 +5869,352 @@ export function heartbeatService(
return { outcome: "retry_exhausted" as const, queuedRun: null };
}
- async function enqueueProcessLossRetry(
- run: typeof heartbeatRuns.$inferSelect,
- agent: typeof agents.$inferSelect,
- now: Date,
- ) {
- // Completion deliveries own their durable bounded retry and reply identity.
- // A second process-loss retry would compete for the same outbox input.
- if (run.contextSnapshot?.wakeReason === CHAT_COMPLETION_WAKE_REASON &&
- Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds)) return null;
- // Native sessions have their own fenced same-run controller. Legacy
- // bootstrap recovery shares the durable delay and incident counter with
- // transient retries; process loss must not open a second retry budget.
- if (run.runtimeMode === "native" || await legacyExecutionNeedsReconciliationWithEvidence(db, run))
- return null;
- const scheduled = await scheduleBoundedRetryForRun(run, agent, { now });
- return scheduled.outcome === "scheduled" ? scheduled.run : null;
- }
+ function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) {
+ const runtimeConfig = parseObject(agent.runtimeConfig);
+ const heartbeat = parseObject(runtimeConfig.heartbeat);
- function toHotRestartIntentRun(input: {
- run: typeof heartbeatRuns.$inferSelect;
- adapterType: string;
- }): HotRestartIntentRun {
- const context = parseObject(input.run.contextSnapshot);
return {
- runId: input.run.id,
- companyId: input.run.companyId,
- agentId: input.run.agentId,
- adapterType: input.adapterType,
- status: input.run.status,
- processPid: input.run.processPid ?? null,
- processGroupId: input.run.processGroupId ?? null,
- issueId: readNonEmptyString(context.issueId),
- runtimeMode: input.run.runtimeMode,
- nativeSessionId: input.run.nativeSessionId,
- runnerInstanceId: input.run.runnerInstanceId,
- processStartedAt: input.run.processStartedAt?.toISOString() ?? null,
+ enabled: asBoolean(heartbeat.enabled, false),
+ intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),
+ wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent),
+ // A Dot binding has one external turn. Competing assignments must retain
+ // their queue position instead of claiming a second run that cannot bind.
+ maxConcurrentRuns: agent.adapterType === "paperclip_runner" &&
+ parseObject(agent.adapterConfig).provider === "openai_dot"
+ ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
+ skipTimerWhenNoActionableWork: asBoolean(
+ heartbeat.skipTimerWhenNoActionableWork ??
+ heartbeat.requireActionableTimerWork ??
+ heartbeat.issueOnlyTimer,
+ false,
+ ),
+ maxDailyRuns: normalizeOptionalNonNegativeInteger(
+ heartbeat.maxDailyRuns ??
+ heartbeat.dailyRunLimit ??
+ heartbeat.dailyRunCap ??
+ heartbeat.maxRunsPerDay,
+ ),
+ maxDailyCostCents: normalizeOptionalNonNegativeInteger(
+ heartbeat.maxDailyCostCents ??
+ heartbeat.dailyCostCentsLimit ??
+ heartbeat.dailySpendCentsLimit ??
+ heartbeat.dailyBudgetCents,
+ ),
};
}
- function isServerStdioBoundHotRestartRun(input: {
- run: typeof heartbeatRuns.$inferSelect;
- adapterType: string;
- adapterConfig: unknown;
- }) {
- const context = parseObject(input.run.contextSnapshot);
- if (
- context.processTopology === "server_stdio" ||
- context.executionEngine === "acp"
- ) {
- return true;
- }
- if (
- context.processTopology === "detached" ||
- context.executionEngine === "cli"
- ) {
- return false;
- }
- if (
- !["claude_local", "codex_local", "gemini_local"].includes(
- input.adapterType,
- )
- ) {
- return false;
- }
- return (
- readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli"
+ function normalizeOptionalNonNegativeInteger(value: unknown) {
+ if (value === null || value === undefined || value === "") return null;
+ const normalized = Math.floor(asNumber(value, 0));
+ return normalized >= 0 ? normalized : null;
+ }
+
+ function currentUtcDayWindow(now = new Date()) {
+ const start = new Date(
+ Date.UTC(
+ now.getUTCFullYear(),
+ now.getUTCMonth(),
+ now.getUTCDate(),
+ 0,
+ 0,
+ 0,
+ 0,
+ ),
+ );
+ const end = new Date(
+ Date.UTC(
+ now.getUTCFullYear(),
+ now.getUTCMonth(),
+ now.getUTCDate() + 1,
+ 0,
+ 0,
+ 0,
+ 0,
+ ),
);
+ return { start, end };
}
- async function prepareHotRestartShutdown(
- signal: "SIGINT" | "SIGTERM",
- now = new Date(),
+ async function getHeartbeatDailyCapBlock(
+ agent: typeof agents.$inferSelect,
+ policy: ReturnType<typeof parseHeartbeatPolicy>,
+ options: {
+ checkRunCap?: boolean;
+ checkCostCap?: boolean;
+ excludeRunId?: string | null;
+ } = {},
+ client: Pick<Db, "select"> = db,
) {
- shutdownInProgress = true;
- const idleSessions = await closeIdleWarmNativeSessionsForRestart();
- if (idleSessions.failed > 0) {
- logger.warn({ idleSessions }, "idle native sessions could not checkpoint before controller shutdown");
- }
- let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;
- try {
- intent = await readHotRestartIntent();
- } catch (err) {
- logger.warn(
- { err },
- "failed to read hot-restart intent; falling back to normal shutdown drain",
- );
- return {
- mode: "read_error" as const,
- skipDrain: false as const,
- activeRunIds: [] as string[],
- };
+ const checkRunCap = options.checkRunCap ?? true;
+ const checkCostCap = options.checkCostCap ?? true;
+ const { start, end } = currentUtcDayWindow();
+ if (checkRunCap && policy.maxDailyRuns !== null) {
+ const conditions = [
+ eq(heartbeatRuns.companyId, agent.companyId),
+ eq(heartbeatRuns.agentId, agent.id),
+ gte(heartbeatRuns.startedAt, start),
+ lt(heartbeatRuns.startedAt, end),
+ notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]),
+ ];
+ if (options.excludeRunId) {
+ conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`);
+ }
+ const [row] = await client
+ .select({ total: sql<number>`count(*)::integer` })
+ .from(heartbeatRuns)
+ .where(and(...conditions));
+ const observed = Number(row?.total ?? 0);
+ if (observed >= policy.maxDailyRuns) {
+ return {
+ reason: "heartbeat.daily_run_limit",
+ observed,
+ limit: policy.maxDailyRuns,
+ };
+ }
}
- if (!intent)
- return {
- mode: "not_requested" as const,
- skipDrain: false as const,
- activeRunIds: [] as string[],
- };
- if (intent.drainRequired)
- return {
- mode: "drain_required" as const,
- skipDrain: false as const,
- activeRunIds: [] as string[],
- };
- if (!shouldHonorHotRestartIntentForProcess(intent)) {
- logger.warn(
- { expectedPid: intent.previousServerPid, currentPid: process.pid },
- "hot-restart intent targets a different server pid; falling back to normal shutdown drain",
- );
- return {
- mode: "pid_mismatch" as const,
- skipDrain: false as const,
- activeRunIds: [] as string[],
- };
+ if (checkCostCap && policy.maxDailyCostCents !== null) {
+ const [row] = await client
+ .select({
+ total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
+ })
+ .from(costEvents)
+ .where(
+ and(
+ eq(costEvents.companyId, agent.companyId),
+ eq(costEvents.agentId, agent.id),
+ gte(costEvents.occurredAt, start),
+ lt(costEvents.occurredAt, end),
+ ),
+ );
+ const observed = Number(row?.total ?? 0);
+ if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) {
+ return {
+ reason: "heartbeat.daily_cost_limit",
+ observed,
+ limit: policy.maxDailyCostCents,
+ };
+ }
}
- const activeRuns = await db
- .select({
- run: heartbeatRuns,
- adapterType: agents.adapterType,
- adapterConfig: agents.adapterConfig,
- })
- .from(heartbeatRuns)
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
- .where(eq(heartbeatRuns.status, "running"));
- const snapshotRuns = activeRuns.map(toHotRestartIntentRun);
- const intentWithVersion = {
- ...intent,
- previousServerVersion: intent.previousServerVersion ?? serverVersion,
- };
-
- const serverStdioRuns = activeRuns.filter(isServerStdioBoundHotRestartRun);
- if (serverStdioRuns.length > 0) {
- const activeServerStdioRunIds = serverStdioRuns.map(({ run }) => run.id);
- await writeHotRestartShutdownSnapshot({
- intent: intentWithVersion,
- signal,
- activeRuns: snapshotRuns,
- drainReason: "active_acp_run",
- drainRunIds: activeServerStdioRunIds,
- capturedAt: now,
- });
-
- logger.warn(
- {
- signal,
- previousServerPid: intent.previousServerPid,
- activeRunIds: snapshotRuns.map((run) => run.runId),
- activeServerStdioRunIds,
- drainReason: "active_acp_run",
- },
- "server-stdio agent run prevents hot-restart adoption; using graceful drain and retry",
- );
-
- return {
- mode: "acp_drain_required" as const,
- skipDrain: false as const,
- activeRunIds: snapshotRuns.map((run) => run.runId),
- activeAcpRunIds: activeServerStdioRunIds,
- drainRunIds: activeServerStdioRunIds,
- drainReason: "active_acp_run" as const,
- };
- }
+ return null;
+ }
- await writeHotRestartShutdownSnapshot({
- intent: intentWithVersion,
- signal,
- activeRuns: snapshotRuns,
- capturedAt: now,
- });
-
- for (const { run } of activeRuns) {
- await appendRunEvent(run, {
- eventType: "lifecycle",
- stream: "system",
- level: "info",
- message:
- "Hot restart requested; leaving child process alive for startup adoption",
- payload: {
- signal,
- previousServerPid: intent.previousServerPid,
- previousServerVersion: intentWithVersion.previousServerVersion,
- processPid: run.processPid ?? null,
- processGroupId: run.processGroupId ?? null,
- },
- });
- }
-
- const nativeRunIds = activeRuns
- .filter(
- ({ run, adapterType }) =>
- adapterType === "paperclip_runner" &&
- isNativeSessionId(run.nativeSessionId),
- )
- .map(({ run }) => run.id);
- const detachedNativeSessions =
- await detachNativeSessionsForRestart(nativeRunIds);
-
- logger.info(
- {
- signal,
- previousServerPid: intent.previousServerPid,
- activeRunIds: snapshotRuns.map((run) => run.runId),
- detachedNativeSessions,
+ async function cancelQueuedRunForHeartbeatDailyCap(
+ run: typeof heartbeatRuns.$inferSelect,
+ dailyCapBlock: NonNullable<
+ Awaited<ReturnType<typeof getHeartbeatDailyCapBlock>>
+ >,
+ ) {
+ const now = new Date();
+ const reason =
+ "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation";
+ const cancelled = await setRunStatus(run.id, "cancelled", {
+ finishedAt: now,
+ error: reason,
+ errorCode: dailyCapBlock.reason,
+ resultJson: {
+ ...parseObject(run.resultJson),
+ stopReason: dailyCapBlock.reason,
+ observed: dailyCapBlock.observed,
+ limit: dailyCapBlock.limit,
+ effectiveTimeoutSec: 0,
+ timeoutConfigured: false,
+ timeoutSource: "heartbeat_daily_cap_gate",
+ timeoutFired: false,
},
- "hot-restart shutdown snapshot captured; skipping graceful run drain",
- );
-
- return {
- mode: "hot_restart" as const,
- skipDrain: true as const,
- activeRunIds: snapshotRuns.map((run) => run.runId),
- };
- }
-
- async function reconcileHotRestartAdoption(now = new Date()) {
- let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;
- try {
- intent = await readHotRestartIntent();
- } catch (err) {
- logger.warn(
- { err },
- "failed to read hot-restart intent on startup; skipping adoption",
- );
- return {
- mode: "read_error" as const,
- adoptedRunIds: [] as string[],
- finalizedWhileDownRunIds: [] as string[],
- lostRunIds: [] as string[],
- skippedRunIds: [] as string[],
- };
- }
- if (!intent) {
- return {
- mode: "not_requested" as const,
- adoptedRunIds: [] as string[],
- finalizedWhileDownRunIds: [] as string[],
- lostRunIds: [] as string[],
- skippedRunIds: [] as string[],
- };
- }
-
- if (!intent.shutdownSnapshot) {
- const log = intent.drainRequired
- ? logger.info.bind(logger)
- : logger.warn.bind(logger);
- log(
- {
- previousServerPid: intent.previousServerPid,
- preflightActiveRunIds: intent.preflightActiveRunIds,
- drainReason: intent.drainReason ?? null,
- },
- intent.drainRequired
- ? "drain-required restart intent has no adoption snapshot"
- : "hot-restart intent present but shutdown snapshot is missing; no runs can be adopted",
- );
- }
- const candidates = intent.shutdownSnapshot?.activeRuns ?? [];
- const missingSnapshotRunIds = findMissingHotRestartSnapshotRunIds(intent);
- const reconciliationRunIds = [
- ...new Set([
- ...candidates.map((run) => run.runId),
- ...missingSnapshotRunIds,
- ]),
- ];
- const currentRows =
- reconciliationRunIds.length > 0
- ? await db
- .select({
- run: heartbeatRuns,
- adapterType: agents.adapterType,
- })
- .from(heartbeatRuns)
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
- .where(inArray(heartbeatRuns.id, reconciliationRunIds))
- : [];
- const currentByRunId = new Map(currentRows.map((row) => [row.run.id, row]));
-
- const reportRuns: HotRestartReportRun[] = [];
- const adoptedRunIds: string[] = [];
- const finalizedWhileDownRunIds: string[] = [];
- const lostRunIds: string[] = [];
- const skippedRunIds: string[] = [];
-
- const classify = (
- candidate: HotRestartIntentRun,
- classification: HotRestartReportRun["classification"],
- reason: string,
- patch?: Partial<HotRestartIntentRun>,
- ) => {
- const run = {
- ...candidate,
- ...patch,
- classification,
- reason,
- } satisfies HotRestartReportRun;
- reportRuns.push(run);
- if (classification === "adopted") adoptedRunIds.push(candidate.runId);
- else if (classification === "finalized_while_down")
- finalizedWhileDownRunIds.push(candidate.runId);
- else if (classification === "lost") lostRunIds.push(candidate.runId);
- else skippedRunIds.push(candidate.runId);
- };
-
- for (const runId of missingSnapshotRunIds) {
- const current = currentByRunId.get(runId);
- if (!current) {
- finalizedWhileDownRunIds.push(runId);
- continue;
- }
-
- const candidate = toHotRestartIntentRun(current);
- if (current.run.status !== "running") {
- classify(
- candidate,
- "finalized_while_down",
- `run_status_${current.run.status}`,
- );
- } else {
- classify(candidate, "lost", "missing_shutdown_snapshot");
- }
- }
-
- if (lostRunIds.length > 0) {
- logger.error(
- { previousServerPid: intent.previousServerPid, lostRunIds },
- "hot-restart shutdown snapshot omitted live preflight runs; reporting them as lost",
- );
- }
-
- for (const candidate of candidates) {
- const current = currentByRunId.get(candidate.runId);
- if (!current) {
- classify(candidate, "finalized_while_down", "run_row_missing");
- continue;
- }
-
- const { run, adapterType } = current;
- const patch = {
- adapterType,
- status: run.status,
- processPid: run.processPid ?? candidate.processPid,
- processGroupId: run.processGroupId ?? candidate.processGroupId,
- };
-
- if (run.status !== "running") {
- classify(
- candidate,
- "finalized_while_down",
- `run_status_${run.status}`,
- patch,
- );
- continue;
- }
-
- const hasSelectiveAcpDrain =
- intent.drainReason === "active_acp_run" &&
- (intent.drainRunIds?.length ?? 0) > 0;
- if (
- hasSelectiveAcpDrain &&
- intent.drainRunIds?.includes(candidate.runId)
- ) {
- // A selective ACP drain is expected to persist a terminal row before
- // the new server starts. If the process was terminated but that write
- // failed, surface the run as lost instead of hiding it as an expected
- // drain skip.
- classify(candidate, "lost", "selective_drain_not_finalized", patch);
- continue;
- }
- if (intent.drainRequired && !hasSelectiveAcpDrain) {
- classify(candidate, "skipped", "drain_required", patch);
- continue;
- }
-
- if (run.runtimeMode === "native" && adapterType === "paperclip_runner") {
- classify(candidate, "skipped", "native_restart_recovery_owned", patch);
- continue;
- }
-
- if (!isTrackedLocalChildProcessAdapter(adapterType)) {
- classify(
- candidate,
- "skipped",
- "adapter_not_local_child_process",
- patch,
- );
- continue;
- }
-
- const processPid = run.processPid ?? candidate.processPid;
- const processGroupId = run.processGroupId ?? candidate.processGroupId;
- const processPidAlive = isProcessAlive(processPid);
- const processGroupAlive = isProcessGroupAlive(processGroupId);
- if (!processPid && !processGroupId) {
- classify(candidate, "lost", "missing_process_metadata", patch);
- continue;
- }
- if (!processPidAlive && !processGroupAlive) {
- classify(candidate, "lost", "process_not_alive", patch);
- continue;
- }
-
- const resultJson = mergeHotRestartAdoptionResultJson(
- parseObject(run.resultJson),
- {
- adoptedAt: now,
- previousServerPid: intent.previousServerPid,
- newServerPid: process.pid,
- previousServerVersion: intent.previousServerVersion,
- newServerVersion: serverVersion,
- processPid,
- processGroupId,
- },
- );
- const updated = await db
- .update(heartbeatRuns)
- .set({
- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(resultJson, true),
- error:
- run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error,
- errorCode:
- run.errorCode === DETACHED_PROCESS_ERROR_CODE
- ? null
- : run.errorCode,
- updatedAt: now,
- })
- .where(
- and(
- eq(heartbeatRuns.id, run.id),
- eq(heartbeatRuns.status, "running"),
- ),
- )
- .returning()
- .then((rows) => rows[0] ?? null);
-
- if (!updated) {
- const latest = await db
- .select({ status: heartbeatRuns.status })
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, run.id))
- .then((rows) => rows[0] ?? null);
- if (latest && latest.status !== "running") {
- classify(
- candidate,
- "finalized_while_down",
- `run_status_${latest.status}`,
- patch,
- );
- } else {
- classify(candidate, "lost", "adoption_update_not_applied", patch);
- }
- continue;
- }
-
- await appendRunEvent(updated, {
- eventType: "lifecycle",
- stream: "system",
- level: "info",
- message: "Adopted live child process after hot restart",
- payload: {
- previousServerPid: intent.previousServerPid,
- newServerPid: process.pid,
- previousServerVersion: intent.previousServerVersion,
- newServerVersion: serverVersion,
- processPid,
- processGroupId,
- },
- });
- classify(
- candidate,
- "adopted",
- processPidAlive ? "process_pid_alive" : "process_group_alive",
- patch,
- );
- }
+ });
+ if (!cancelled) return null;
- const report = await writeHotRestartReport({
- version: 1,
- requestedAt: intent.requestedAt,
- completedAt: now.toISOString(),
- drainRequired: intent.drainRequired,
- drainReason:
- intent.drainReason ?? (intent.drainRequired ? "requested" : null),
- previousServerPid: intent.previousServerPid,
- newServerPid: process.pid,
- previousServerVersion: intent.previousServerVersion,
- newServerVersion: serverVersion,
- adoptedRunIds,
- finalizedWhileDownRunIds,
- lostRunIds,
- skippedRunIds,
- runs: reportRuns,
+ await setWakeupStatus(run.wakeupRequestId, "skipped", {
+ finishedAt: now,
+ error: reason,
});
- await removeHotRestartIntent(undefined, intent);
- logger.info(
- {
- previousServerPid: report.previousServerPid,
- newServerPid: report.newServerPid,
- adoptedRunIds,
- finalizedWhileDownRunIds,
- lostRunIds,
- missingSnapshotRunIds,
- skippedRunIds,
+ await appendRunEvent(cancelled, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: reason,
+ payload: {
+ reason: dailyCapBlock.reason,
+ observed: dailyCapBlock.observed,
+ limit: dailyCapBlock.limit,
},
- "hot-restart adoption report written",
- );
+ });
- return {
- mode: "reported" as const,
- adoptedRunIds,
- finalizedWhileDownRunIds,
- lostRunIds,
- skippedRunIds,
- };
+ await releaseIssueExecutionAndPromote(cancelled, {
+ suppressImmediateRecovery: true,
+ });
+
+ return cancelled;
}
- async function recoverNativeRunsAfterRestart(now = new Date()) {
- // A result committed before the old controller stopped outranks process
- // recovery. Finish its durable workspace/status suffix before deciding
- // whether any provider authority needs to be reopened.
- await reconcileNativeFinalizations(db, undefined, {
- environmentRuntime,
- onWorkspaceSettled: settleRecoveredNativeWorkspace,
- });
- scheduleRetainedNativeSessionCleanup();
- const intent = await readHotRestartIntent().catch((error) => {
- logger.warn(
- { err: error },
- "failed to read hot-restart intent before native startup recovery",
- );
- return null;
- });
- const restartKind = intent ? ("hot" as const) : ("hard" as const);
- const previousStartedAt = intent?.previousServerStartedAt
- ? new Date(intent.previousServerStartedAt)
- : null;
- const scheduledNativeRetries = await db
- .select({
- runId: nativeRunFinalizations.runId,
- nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
- })
- .from(nativeRunFinalizations)
- .innerJoin(
- heartbeatRuns,
- eq(heartbeatRuns.id, nativeRunFinalizations.runId),
- )
+ async function hasActionableTimerWork(agent: typeof agents.$inferSelect) {
+ const row = await db
+ .select({ id: issues.id })
+ .from(issues)
.where(
and(
- eq(heartbeatRuns.runtimeMode, "native"),
- inArray(heartbeatRuns.status, ["running", "failed"]),
- isNull(nativeRunFinalizations.resultId),
- eq(nativeRunFinalizations.phase, "retryable_failure"),
- gt(nativeRunFinalizations.nextAttemptAt, now),
+ eq(issues.companyId, agent.companyId),
+ eq(issues.assigneeAgentId, agent.id),
+ isNull(issues.assigneeUserId),
+ isNull(issues.hiddenAt),
+ inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]),
+ isNull(issues.conversationAgentId),
+ nonIdleSlackIssueCondition(),
),
- );
- for (const scheduled of scheduledNativeRetries) {
- if (scheduled.nextAttemptAt) {
- scheduleNativeSessionResumeDispatch(
- scheduled.runId,
- scheduled.nextAttemptAt,
- );
- }
- }
- const dispositions = await claimNativeRestartRecoveries({
- db,
- restartKind,
- recoveryRequestId: intent?.recoveryRequestId ?? null,
- coordinatedPreviousController: intent
- ? {
- pid: intent.previousServerPid,
- processStartedAt:
- previousStartedAt && !Number.isNaN(previousStartedAt.getTime())
- ? previousStartedAt
- : null,
- }
- : null,
- now,
- });
-
- const claims = dispositions.filter(
- (disposition): disposition is NativeRestartRecoveryClaim =>
- disposition.kind === "reattach_existing_runner" ||
- disposition.kind === "reattach_remote_runner" ||
- disposition.kind === "resume_dead_runner" ||
- disposition.kind === "bootstrap_incomplete",
- );
- for (const disposition of dispositions) {
- const run = await getRun(disposition.runId);
- if (run) {
- const isClaim =
- disposition.kind === "reattach_existing_runner" ||
- disposition.kind === "reattach_remote_runner" ||
- disposition.kind === "resume_dead_runner" ||
- disposition.kind === "bootstrap_incomplete";
- await appendRunEvent(run, {
- eventType: "native.recovery.transition",
- stream: "system",
- level: disposition.kind === "blocked" ? "warn" : "info",
- message:
- disposition.kind === "reattach_existing_runner" ||
- disposition.kind === "reattach_remote_runner"
- ? "Recovering the existing native runner process after server restart"
- : disposition.kind === "resume_dead_runner"
- ? "Resuming the durable native provider session after runner process loss"
- : disposition.kind === "bootstrap_incomplete"
- ? "Restarting an incomplete native runner bootstrap on the same heartbeat run"
- : disposition.kind === "awaiting_evidence"
- ? "Native restart recovery is waiting for safe ownership evidence"
- : disposition.kind === "already_finalized"
- ? "Native restart recovery found an already-finalized result"
- : "Native restart recovery blocked ambiguous or conflicting ownership",
- payload: {
- restartKind: isClaim ? disposition.restartKind : restartKind,
- recoveryRequestId: isClaim
- ? disposition.recoveryRequestId
- : (intent?.recoveryRequestId ?? null),
- runnerDisposition: disposition.kind,
- ...(isClaim
- ? {
- controllerGeneration: disposition.controllerGeneration,
- providerAttempt: disposition.providerAttempt,
- }
- : { reason: disposition.reason }),
- ...(disposition.kind === "reattach_existing_runner"
- ? {
- processPid: disposition.process.pid,
- processGroupId: disposition.process.processGroupId,
- processStartedAt: disposition.process.startedAt,
- }
- : {}),
- },
- });
- }
- }
- for (const claim of claims) {
- const execution = executeRun(claim.runId, {
- nativeLeaseOwner: claim.leaseOwner,
- nativeRestartRecovery: claim,
- }).catch((error) => {
- logger.error(
- { err: error, runId: claim.runId, disposition: claim.kind },
- "native restart recovery execution failed",
- );
- });
- activeRunExecutionPromises.add(execution);
- void execution.finally(() =>
- activeRunExecutionPromises.delete(execution),
- );
- }
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ return Boolean(row);
+ }
- return {
- restartKind,
- claims,
- dispositions,
- scheduledRetryRunIds: scheduledNativeRetries.map((entry) => entry.runId),
- awaitingEvidenceRunIds: dispositions
- .filter((entry) => entry.kind === "awaiting_evidence")
- .map((entry) => entry.runId),
- blockedRunIds: dispositions
- .filter((entry) => entry.kind === "blocked")
- .map((entry) => entry.runId),
- };
+ async function markTimerHeartbeatChecked(
+ agentId: string,
+ source: WakeupOptions["source"],
+ ) {
+ if (source !== "timer") return;
+ await db
+ .update(agents)
+ .set({
+ lastHeartbeatAt: new Date(),
+ updatedAt: new Date(),
+ })
+ .where(eq(agents.id, agentId));
}
- async function drainRunningRunsForShutdown(
- signal: "SIGINT" | "SIGTERM",
- now = new Date(),
- runIds: readonly string[] | null = null,
+ async function claimDueTimerHeartbeat(
+ agent: typeof agents.$inferSelect,
+ now: Date,
+ intervalSec: number,
) {
- const selectedRunIds = runIds ? [...new Set(runIds)] : null;
- if (selectedRunIds?.length === 0) {
- return {
- interrupted: 0,
- interruptedRunIds: [],
- retryRunIds: [],
- restartSuspendedRunIds: [],
- };
- }
- const activeRuns = await db
- .select({
- run: heartbeatRuns,
- agent: agents,
+ const dueBefore = new Date(now.getTime() - intervalSec * 1000);
+ const claimed = await db
+ .update(agents)
+ .set({
+ lastHeartbeatAt: now,
+ updatedAt: now,
})
- .from(heartbeatRuns)
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
.where(
- selectedRunIds
- ? and(
- eq(heartbeatRuns.status, "running"),
- inArray(heartbeatRuns.id, selectedRunIds),
- )
- : eq(heartbeatRuns.status, "running"),
- );
-
- const interruptedRunIds: string[] = [];
- const retryRunIds: string[] = [];
- const restartSuspendedRunIds: string[] = [];
-
- for (const { run, agent } of activeRuns) {
- // Shutdown owns only this boot's legacy executions. Expired foreign
- // owners belong to the reaper, not another container's drain.
- if (run.runtimeMode === "legacy" && run.controllerBootId &&
- run.controllerBootId !== legacyControllerBootId) continue;
- if (isNativeRunnerOwnershipHeld(run)) continue;
- if (
- run.runtimeMode === "native" &&
- agent.adapterType === "paperclip_runner"
- ) {
- // A graceful shutdown relinquishes controller authority just like a
- // hot restart. Leaving the old event consumer attached lets its
- // finalizer interrupt/suspend Claude while the next server is adopting
- // the same turn.
- await detachNativeSessionsForRestart([run.id]);
- const recoveryHistoryEntry = JSON.stringify({
- at: now.toISOString(),
- restartKind: "graceful",
- disposition: "restart_suspended",
- reason: signal,
- processPid: run.processPid,
- processStartedAt: run.processStartedAt?.toISOString() ?? null,
- });
- await db
- .update(nativeRunFinalizations)
- .set({
- recoveryState: "awaiting_runner_reattach",
- recoveryHistory: sql`(
- select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb)
- from jsonb_array_elements(
- coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
- || jsonb_build_array(${recoveryHistoryEntry}::jsonb)
- ) with ordinality as history(item, ordinal)
- where ordinal > greatest(
- jsonb_array_length(
- coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
- || jsonb_build_array(${recoveryHistoryEntry}::jsonb)
- ) - 20,
- 0
- )
- )`,
- updatedAt: now,
- })
- .where(
+ and(
+ eq(agents.id, agent.id),
+ eq(agents.companyId, agent.companyId),
+ or(
+ lte(agents.lastHeartbeatAt, dueBefore),
and(
- eq(nativeRunFinalizations.runId, run.id),
- isNull(nativeRunFinalizations.resultId),
+ isNull(agents.lastHeartbeatAt),
+ lte(agents.createdAt, dueBefore),
),
- );
- await appendRunEvent(run, {
- eventType: "native.recovery.transition",
- stream: "system",
- level: "info",
- message:
- "Server shutdown suspended native controller ownership without cancelling provider work",
- payload: {
- restartKind: "graceful",
- signal,
- runnerDisposition: "awaiting_runner_reattach",
- processPid: run.processPid,
- processGroupId: run.processGroupId,
- processStartedAt: run.processStartedAt?.toISOString() ?? null,
- retryRunCreated: false,
- },
- });
- restartSuspendedRunIds.push(run.id);
- continue;
- }
- const message = `Interrupted by graceful server shutdown (${signal})`;
- const running = runningProcesses.get(run.id);
- try {
- if (run.runtimeMode === "native") {
- await cancelHeartbeatNativeRun({
- db,
- runId: run.id,
- reason: message,
- runtimeMode: run.runtimeMode,
- });
- }
- if (running) {
- await terminateHeartbeatRunProcess({
- pid: running.child.pid,
- processGroupId: running.processGroupId,
- graceMs: Math.max(1, running.graceSec) * 1000,
- });
- }
- } finally {
- runningProcesses.delete(run.id);
- }
-
- const persistedCancellationResult =
- run.runtimeMode === "native"
- ? await getRun(run.id).then((current) =>
- parseObject(current?.resultJson),
- )
- : parseObject(run.resultJson);
-
- const interruptedStatus = await setRunStatusIfRunning(
- run.id,
- "interrupted",
- {
- finishedAt: now,
- error: message,
- errorCode: "server_shutdown_interrupted",
- signal,
- resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {
- conversationContinuationEligible: await runUsedConversationAdapter(db, run),
- resultJson: persistedCancellationResult,
- errorCode: "server_shutdown_interrupted",
- errorMessage: message,
- }),
- },
- );
- if (!interruptedStatus.updated || !interruptedStatus.run) continue;
- let interrupted = interruptedStatus.run;
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
- finishedAt: now,
- error: null,
- });
- interrupted =
- (await classifyAndPersistRunLiveness(
- interrupted,
- parseObject(interrupted.resultJson),
- )) ?? interrupted;
-
- await releaseEnvironmentLeasesForRun({
- runId: interrupted.id,
- companyId: interrupted.companyId,
- agentId: interrupted.agentId,
- status: interrupted.status,
- failureReason: interrupted.error ?? undefined,
- });
-
- const retry = await enqueueProcessLossRetry(interrupted, agent, now);
- if (!retry) {
- await releaseIssueExecutionAndPromote(interrupted);
- } else {
- retryRunIds.push(retry.id);
- }
-
- await appendRunEvent(interrupted, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message,
- payload: {
- signal,
- ...(run.processPid ? { processPid: run.processPid } : {}),
- ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
- ...(retry ? { retryRunId: retry.id } : {}),
- },
- });
-
- await finalizeAgentStatus(run.agentId, "interrupted", message, {
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
- });
- interruptedRunIds.push(interrupted.id);
- }
-
- if (interruptedRunIds.length > 0) {
- logger.warn(
- {
- signal,
- interrupted: interruptedRunIds.length,
- interruptedRunIds,
- retryRunIds,
- },
- "interrupted running heartbeat runs for graceful shutdown",
- );
- }
-
- return {
- interrupted: interruptedRunIds.length,
- interruptedRunIds,
- retryRunIds,
- restartSuspendedRunIds,
- };
+ ),
+ ),
+ )
+ .returning({ id: agents.id })
+ .then((rows) => rows[0] ?? null);
+ if (!claimed) return null;
+ return { wasFirstHeartbeat: !agent.lastHeartbeatAt };
}
- function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) {
+ function parseMaxTurnContinuationPolicy(
+ agent: typeof agents.$inferSelect,
+ ): MaxTurnContinuationPolicy {
const runtimeConfig = parseObject(agent.runtimeConfig);
const heartbeat = parseObject(runtimeConfig.heartbeat);
+ const configured = parseObject(heartbeat.maxTurnContinuation);
+ const rawMaxAttempts = Math.floor(
+ asNumber(
+ configured.maxAttempts,
+ MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,
+ ),
+ );
+ const rawDelayMs = Math.floor(
+ asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),
+ );
return {
- enabled: asBoolean(heartbeat.enabled, false),
- intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),
- wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent),
- // A Dot binding has one external turn. Competing assignments must retain
- // their queue position instead of claiming a second run that cannot bind.
- maxConcurrentRuns: agent.adapterType === "paperclip_runner" &&
- parseObject(agent.adapterConfig).provider === "openai_dot"
- ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
- skipTimerWhenNoActionableWork: asBoolean(
- heartbeat.skipTimerWhenNoActionableWork ??
- heartbeat.requireActionableTimerWork ??
- heartbeat.issueOnlyTimer,
- false,
- ),
- maxDailyRuns: normalizeOptionalNonNegativeInteger(
- heartbeat.maxDailyRuns ??
- heartbeat.dailyRunLimit ??
- heartbeat.dailyRunCap ??
- heartbeat.maxRunsPerDay,
+ enabled: asBoolean(configured.enabled, true),
+ maxAttempts: Math.max(
+ 0,
+ Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),
),
- maxDailyCostCents: normalizeOptionalNonNegativeInteger(
- heartbeat.maxDailyCostCents ??
- heartbeat.dailyCostCentsLimit ??
- heartbeat.dailySpendCentsLimit ??
- heartbeat.dailyBudgetCents,
+ delayMs: Math.max(
+ 0,
+ Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),
),
};
}
- function normalizeOptionalNonNegativeInteger(value: unknown) {
- if (value === null || value === undefined || value === "") return null;
- const normalized = Math.floor(asNumber(value, 0));
- return normalized >= 0 ? normalized : null;
+ function issueRunPriorityRank(priority: string | null | undefined) {
+ switch (priority) {
+ case "critical":
+ return 0;
+ case "high":
+ return 1;
+ case "medium":
+ return 2;
+ case "low":
+ return 3;
+ default:
+ return 4;
+ }
}
- function currentUtcDayWindow(now = new Date()) {
- const start = new Date(
- Date.UTC(
- now.getUTCFullYear(),
- now.getUTCMonth(),
- now.getUTCDate(),
- 0,
- 0,
- 0,
- 0,
- ),
- );
- const end = new Date(
- Date.UTC(
- now.getUTCFullYear(),
- now.getUTCMonth(),
- now.getUTCDate() + 1,
- 0,
- 0,
- 0,
- 0,
+ async function listQueuedRunDependencyReadiness(
+ companyId: string,
+ queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
+ ) {
+ const issueIds = [
+ ...new Set(
+ queuedRuns
+ .map((run) =>
+ readNonEmptyString(parseObject(run.contextSnapshot).issueId),
+ )
+ .filter((issueId): issueId is string => Boolean(issueId)),
),
- );
- return { start, end };
+ ];
+ if (issueIds.length === 0) {
+ return new Map<
+ string,
+ Awaited<ReturnType<typeof issuesSvc.getDependencyReadiness>>
+ >();
+ }
+ return issuesSvc.listDependencyReadiness(companyId, issueIds);
}
- async function getHeartbeatDailyCapBlock(
- agent: typeof agents.$inferSelect,
- policy: ReturnType<typeof parseHeartbeatPolicy>,
- options: {
- checkRunCap?: boolean;
- checkCostCap?: boolean;
- excludeRunId?: string | null;
- } = {},
- client: Pick<Db, "select"> = db,
+ async function countRunningRunsForAgent(agentId: string) {
+ const [{ count }] = await db
+ .select({ count: sql<number>`count(*)` })
+ .from(heartbeatRuns)
+ .where(
+ and(
+ eq(heartbeatRuns.agentId, agentId),
+ eq(heartbeatRuns.status, "running"),
+ ),
+ );
+ return Number(count ?? 0);
+ }
+
+ async function withChatControlRecoveryGate(
+ run: typeof heartbeatRuns.$inferSelect,
+ stage: "claim" | "dispatch",
+ onClear: (tx: Db) => Promise<typeof heartbeatRuns.$inferSelect | null>,
+ nativeRecovery = false,
) {
- const checkRunCap = options.checkRunCap ?? true;
- const checkCostCap = options.checkCostCap ?? true;
- const { start, end } = currentUtcDayWindow();
- if (checkRunCap && policy.maxDailyRuns !== null) {
- const conditions = [
- eq(heartbeatRuns.companyId, agent.companyId),
- eq(heartbeatRuns.agentId, agent.id),
- gte(heartbeatRuns.startedAt, start),
- lt(heartbeatRuns.startedAt, end),
- notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]),
- ];
- if (options.excludeRunId) {
- conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`);
- }
- const [row] = await client
- .select({ total: sql<number>`count(*)::integer` })
- .from(heartbeatRuns)
- .where(and(...conditions));
- const observed = Number(row?.total ?? 0);
- if (observed >= policy.maxDailyRuns) {
- return {
- reason: "heartbeat.daily_run_limit",
- observed,
- limit: policy.maxDailyRuns,
- };
- }
- }
-
- if (checkCostCap && policy.maxDailyCostCents !== null) {
- const [row] = await client
- .select({
- total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
- })
- .from(costEvents)
- .where(
- and(
- eq(costEvents.companyId, agent.companyId),
- eq(costEvents.agentId, agent.id),
- gte(costEvents.occurredAt, start),
- lt(costEvents.occurredAt, end),
- ),
- );
- const observed = Number(row?.total ?? 0);
- if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) {
- return {
- reason: "heartbeat.daily_cost_limit",
- observed,
- limit: policy.maxDailyCostCents,
- };
- }
- }
-
- return null;
- }
-
- async function cancelQueuedRunForHeartbeatDailyCap(
- run: typeof heartbeatRuns.$inferSelect,
- dailyCapBlock: NonNullable<
- Awaited<ReturnType<typeof getHeartbeatDailyCapBlock>>
- >,
- ) {
- const now = new Date();
- const reason =
- "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation";
- const cancelled = await setRunStatus(run.id, "cancelled", {
- finishedAt: now,
- error: reason,
- errorCode: dailyCapBlock.reason,
- resultJson: {
- ...parseObject(run.resultJson),
- stopReason: dailyCapBlock.reason,
- observed: dailyCapBlock.observed,
- limit: dailyCapBlock.limit,
- effectiveTimeoutSec: 0,
- timeoutConfigured: false,
- timeoutSource: "heartbeat_daily_cap_gate",
- timeoutFired: false,
- },
- });
- if (!cancelled) return null;
-
- await setWakeupStatus(run.wakeupRequestId, "skipped", {
- finishedAt: now,
- error: reason,
- });
-
- await appendRunEvent(cancelled, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message: reason,
- payload: {
- reason: dailyCapBlock.reason,
- observed: dailyCapBlock.observed,
- limit: dailyCapBlock.limit,
- },
- });
-
- await releaseIssueExecutionAndPromote(cancelled, {
- suppressImmediateRecovery: true,
- });
-
- return cancelled;
- }
-
- async function hasActionableTimerWork(agent: typeof agents.$inferSelect) {
- const row = await db
- .select({ id: issues.id })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, agent.companyId),
- eq(issues.assigneeAgentId, agent.id),
- isNull(issues.assigneeUserId),
- isNull(issues.hiddenAt),
- inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]),
- isNull(issues.conversationAgentId),
- nonIdleSlackIssueCondition(),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
- return Boolean(row);
- }
-
- async function markTimerHeartbeatChecked(
- agentId: string,
- source: WakeupOptions["source"],
- ) {
- if (source !== "timer") return;
- await db
- .update(agents)
- .set({
- lastHeartbeatAt: new Date(),
- updatedAt: new Date(),
- })
- .where(eq(agents.id, agentId));
- }
-
- async function claimDueTimerHeartbeat(
- agent: typeof agents.$inferSelect,
- now: Date,
- intervalSec: number,
- ) {
- const dueBefore = new Date(now.getTime() - intervalSec * 1000);
- const claimed = await db
- .update(agents)
- .set({
- lastHeartbeatAt: now,
- updatedAt: now,
- })
- .where(
- and(
- eq(agents.id, agent.id),
- eq(agents.companyId, agent.companyId),
- or(
- lte(agents.lastHeartbeatAt, dueBefore),
- and(
- isNull(agents.lastHeartbeatAt),
- lte(agents.createdAt, dueBefore),
- ),
- ),
- ),
- )
- .returning({ id: agents.id })
- .then((rows) => rows[0] ?? null);
- if (!claimed) return null;
- return { wasFirstHeartbeat: !agent.lastHeartbeatAt };
- }
-
- function parseMaxTurnContinuationPolicy(
- agent: typeof agents.$inferSelect,
- ): MaxTurnContinuationPolicy {
- const runtimeConfig = parseObject(agent.runtimeConfig);
- const heartbeat = parseObject(runtimeConfig.heartbeat);
- const configured = parseObject(heartbeat.maxTurnContinuation);
- const rawMaxAttempts = Math.floor(
- asNumber(
- configured.maxAttempts,
- MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,
- ),
- );
- const rawDelayMs = Math.floor(
- asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),
- );
-
- return {
- enabled: asBoolean(configured.enabled, true),
- maxAttempts: Math.max(
- 0,
- Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),
- ),
- delayMs: Math.max(
- 0,
- Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),
- ),
- };
- }
-
- function issueRunPriorityRank(priority: string | null | undefined) {
- switch (priority) {
- case "critical":
- return 0;
- case "high":
- return 1;
- case "medium":
- return 2;
- case "low":
- return 3;
- default:
- return 4;
- }
- }
-
- async function listQueuedRunDependencyReadiness(
- companyId: string,
- queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
- ) {
- const issueIds = [
- ...new Set(
- queuedRuns
- .map((run) =>
- readNonEmptyString(parseObject(run.contextSnapshot).issueId),
- )
- .filter((issueId): issueId is string => Boolean(issueId)),
- ),
- ];
- if (issueIds.length === 0) {
- return new Map<
- string,
- Awaited<ReturnType<typeof issuesSvc.getDependencyReadiness>>
- >();
- }
- return issuesSvc.listDependencyReadiness(companyId, issueIds);
- }
-
- async function countRunningRunsForAgent(agentId: string) {
- const [{ count }] = await db
- .select({ count: sql<number>`count(*)` })
- .from(heartbeatRuns)
- .where(
- and(
- eq(heartbeatRuns.agentId, agentId),
- eq(heartbeatRuns.status, "running"),
- ),
- );
- return Number(count ?? 0);
- }
-
- async function withChatControlRecoveryGate(
- run: typeof heartbeatRuns.$inferSelect,
- stage: "claim" | "dispatch",
- onClear: (tx: Db) => Promise<typeof heartbeatRuns.$inferSelect | null>,
- nativeRecovery = false,
- ) {
- const issueId = readNonEmptyString(
- parseObject(run.contextSnapshot).issueId,
- );
- if (!issueId || run.invocationSource !== "automation") return onClear(db);
- await options.beforeChatControlRecoveryCheck?.({
- runId: run.id,
- issueId,
- stage,
- });
- if (stage === "dispatch" && nativeRecovery) {
- // Protected historical/already-admitted owners did not previously take
- // this fence's locks. A fresh exact read keeps unrelated Board contention
- // from turning their recovery into a new admission failure.
- const [current] = await db
- .select()
+ const issueId = readNonEmptyString(
+ parseObject(run.contextSnapshot).issueId,
+ );
+ if (!issueId || run.invocationSource !== "automation") return onClear(db);
+ await options.beforeChatControlRecoveryCheck?.({
+ runId: run.id,
+ issueId,
+ stage,
+ });
+ if (stage === "dispatch" && nativeRecovery) {
+ // Protected historical/already-admitted owners did not previously take
+ // this fence's locks. A fresh exact read keeps unrelated Board contention
+ // from turning their recovery into a new admission failure.
+ const [current] = await db
+ .select()
.from(heartbeatRuns)
.where(
and(
@@ -8178,1574 +7143,428 @@ export function heartbeatService(
if (released.wakeupRequestId) {
await tx
- .update(agentWakeupRequests)
- .set({ status: "queued", claimedAt: null, updatedAt: now })
- .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));
- }
-
- const context = parseObject(released.contextSnapshot);
- const issueId = readNonEmptyString(context.issueId);
- if (issueId) {
- await tx
- .update(issues)
- .set({
- executionRunId: null,
- executionAgentNameKey: null,
- executionLockedAt: null,
- updatedAt: now,
- })
- .where(
- and(
- eq(issues.id, issueId),
- eq(issues.companyId, released.companyId),
- eq(issues.executionRunId, released.id),
- ),
- );
- }
- });
- }
-
- async function cancelQueuedRunForBlockedDependencies(
- run: typeof heartbeatRuns.$inferSelect,
- issueId: string,
- unresolvedBlockerIssueIds: string[],
- ) {
- const now = new Date();
- const reason =
- "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve";
- const cancelled = await setRunStatus(run.id, "cancelled", {
- finishedAt: now,
- error: reason,
- errorCode: "issue_dependencies_blocked",
- resultJson: {
- ...parseObject(run.resultJson),
- stopReason: "issue_dependencies_blocked",
- effectiveTimeoutSec: 0,
- timeoutConfigured: false,
- timeoutSource: "dependency_gate",
- timeoutFired: false,
- },
- });
- if (!cancelled) return null;
-
- await setWakeupStatus(run.wakeupRequestId, "skipped", {
- finishedAt: now,
- error: reason,
- });
-
- await db
- .update(issues)
- .set({
- executionRunId: null,
- executionAgentNameKey: null,
- executionLockedAt: null,
- updatedAt: now,
- })
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.id, issueId),
- eq(issues.executionRunId, run.id),
- ),
- );
-
- await appendRunEvent(cancelled, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message: reason,
- payload: {
- issueId,
- unresolvedBlockerIssueIds,
- },
- });
-
- return cancelled;
- }
-
- function truncateAgentErrorReason(
- reason: string | null | undefined,
- ): string | null {
- if (!reason) return null;
- const trimmed = reason.trim();
- if (!trimmed) return null;
- return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;
- }
-
- async function finalizeAgentStatus(
- agentId: string,
- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
- failureReason?: string | null,
- options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },
- ) {
- const existing = await getAgent(agentId);
- if (!existing) return;
-
- if (existing.status === "paused" || existing.status === "terminated") {
- return;
- }
-
- const isFirstHeartbeat =
- options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;
-
- const runningCount = await countRunningRunsForAgent(agentId);
- const nextStatus =
- runningCount > 0
- ? "running"
- : outcome === "succeeded" ||
- outcome === "interrupted" ||
- outcome === "cancelled" ||
- (outcome === "failed" && options?.keepIdleOnFailure)
- ? "idle"
- : "error";
-
- const updated = await db
- .update(agents)
- .set({
- status: nextStatus,
- // Persist a human-readable reason on the agent record when it enters
- // error so operators see it on the agent page without digging into run
- // events; clear it whenever the agent leaves error.
- errorReason:
- nextStatus === "error"
- ? truncateAgentErrorReason(failureReason)
- : null,
- lastHeartbeatAt: new Date(),
- updatedAt: new Date(),
- })
- .where(eq(agents.id, agentId))
- .returning()
- .then((rows) => rows[0] ?? null);
-
- if (isFirstHeartbeat && updated) {
- const tc = getTelemetryClient();
- if (tc)
- trackAgentFirstHeartbeat(tc, {
- agentRole: updated.role,
- agentId: updated.id,
- });
- }
-
- if (updated) {
- publishLiveEvent({
- companyId: updated.companyId,
- type: "agent.status",
- payload: {
- agentId: updated.id,
- status: updated.status,
- lastHeartbeatAt: updated.lastHeartbeatAt
- ? new Date(updated.lastHeartbeatAt).toISOString()
- : null,
- outcome,
- },
- });
- }
- }
-
- function mergeRunStopMetadataForAgent(
- agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
- options?: {
- resultJson?: Record<string, unknown> | null;
- conversationContinuationEligible?: boolean;
- errorCode?: string | null;
- errorMessage?: string | null;
- },
- ) {
- const stopMetadata = buildHeartbeatRunStopMetadata({
- adapterType: agent.adapterType,
- adapterConfig: parseObject(agent.adapterConfig),
- outcome,
- errorCode: options?.errorCode ?? null,
- errorMessage: options?.errorMessage ?? null,
- });
- const result = mergeHeartbeatRunStopMetadata(
- options?.resultJson ?? null,
- stopMetadata,
- );
- const cancellationAcknowledged =
- parseObject(result?.executionCancellation).state === "acknowledged";
- return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&
- (outcome !== "cancelled" || cancellationAcknowledged) &&
- isConversationAdapter(agent.adapterType)
- ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }
- : result;
- }
-
- function countValue(value: unknown) {
- const parsed = Number(value ?? 0);
- return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
- }
-
- function dateValue(value: unknown) {
- if (value instanceof Date)
- return Number.isNaN(value.getTime()) ? null : value;
- if (typeof value === "string" || typeof value === "number") {
- const parsed = new Date(value);
- return Number.isNaN(parsed.getTime()) ? null : parsed;
- }
- return null;
- }
-
- function latestDate(...values: unknown[]) {
- let latest: Date | null = null;
- for (const value of values) {
- const parsed = dateValue(value);
- if (!parsed) continue;
- if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
- }
- return latest;
- }
-
- async function buildRunLivenessInput(
- run: typeof heartbeatRuns.$inferSelect,
- resultJson: Record<string, unknown> | null | undefined,
- ): Promise<RunLivenessClassificationInput> {
- const context = parseObject(run.contextSnapshot);
- const contextIssueId = readNonEmptyString(context.issueId);
- const continuationAttempt = asNumber(
- context.continuationAttempt,
- run.continuationAttempt ?? 0,
- );
-
- const issue = contextIssueId
- ? await db
- .select({
- status: issues.status,
- title: issues.title,
- description: issues.description,
- workMode: issues.workMode,
- })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.id, contextIssueId),
- ),
- )
- .then((rows) => rows[0] ?? null)
- : null;
-
- const [commentStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
- })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, run.companyId),
- eq(issueComments.issueId, contextIssueId),
- eq(issueComments.createdByRunId, run.id),
- isNull(issueComments.deletedAt),
- ),
- )
- : [{ count: 0, latestAt: null }];
-
- const issueCommentBodies = contextIssueId
- ? await db
- .select({ body: issueComments.body })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, run.companyId),
- eq(issueComments.issueId, contextIssueId),
- eq(issueComments.createdByRunId, run.id),
- ),
- )
- .orderBy(desc(issueComments.createdAt), desc(issueComments.id))
- .limit(5)
- .then((rows) => rows.reverse().map((row) => row.body))
- : [];
-
- const continuationSummary = contextIssueId
- ? await getIssueContinuationSummaryDocument(db, contextIssueId)
- : null;
-
- const [documentStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
- latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
- })
- .from(documentRevisions)
- .innerJoin(
- issueDocuments,
- eq(documentRevisions.documentId, issueDocuments.documentId),
- )
- .where(
- and(
- eq(documentRevisions.companyId, run.companyId),
- eq(documentRevisions.createdByRunId, run.id),
- eq(issueDocuments.companyId, run.companyId),
- eq(issueDocuments.issueId, contextIssueId),
- sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
- ),
- )
- : [{ count: 0, planCount: 0, latestAt: null }];
+ .update(agentWakeupRequests)
+ .set({ status: "queued", claimedAt: null, updatedAt: now })
+ .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));
+ }
- const [workProductStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
+ const context = parseObject(released.contextSnapshot);
+ const issueId = readNonEmptyString(context.issueId);
+ if (issueId) {
+ await tx
+ .update(issues)
+ .set({
+ executionRunId: null,
+ executionAgentNameKey: null,
+ executionLockedAt: null,
+ updatedAt: now,
})
- .from(issueWorkProducts)
.where(
and(
- eq(issueWorkProducts.companyId, run.companyId),
- eq(issueWorkProducts.issueId, contextIssueId),
- eq(issueWorkProducts.createdByRunId, run.id),
+ eq(issues.id, issueId),
+ eq(issues.companyId, released.companyId),
+ eq(issues.executionRunId, released.id),
),
- )
- : [{ count: 0, latestAt: null }];
-
- const [workspaceOperationStats] = await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
- })
- .from(workspaceOperations)
- .where(
- and(
- eq(workspaceOperations.companyId, run.companyId),
- eq(workspaceOperations.heartbeatRunId, run.id),
- ),
- );
-
- const [activityStats] = await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
- })
- .from(activityLog)
- .where(
- and(
- eq(activityLog.companyId, run.companyId),
- eq(activityLog.runId, run.id),
- notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),
- ),
- );
-
- const [eventStats] = await db
- .select({
- count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
- latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
- })
- .from(heartbeatRunEvents)
- .where(
- and(
- eq(heartbeatRunEvents.companyId, run.companyId),
- eq(heartbeatRunEvents.runId, run.id),
- ),
- );
-
- return {
- runStatus: run.status,
- issue,
- resultJson: resultJson ?? run.resultJson ?? null,
- issueCommentBodies,
- continuationSummaryBody: continuationSummary?.body ?? null,
- stdoutExcerpt: run.stdoutExcerpt ?? null,
- stderrExcerpt: run.stderrExcerpt ?? null,
- error: run.error ?? null,
- errorCode: run.errorCode ?? null,
- continuationAttempt,
- evidence: {
- issueCommentsCreated: countValue(commentStats?.count),
- documentRevisionsCreated: countValue(documentStats?.count),
- planDocumentRevisionsCreated: countValue(documentStats?.planCount),
- workProductsCreated: countValue(workProductStats?.count),
- workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
- activityEventsCreated: countValue(activityStats?.count),
- toolOrActionEventsCreated: countValue(eventStats?.count),
- latestEvidenceAt: latestDate(
- commentStats?.latestAt,
- documentStats?.latestAt,
- workProductStats?.latestAt,
- workspaceOperationStats?.latestAt,
- activityStats?.latestAt,
- eventStats?.latestAt,
- ),
- },
- };
+ );
+ }
+ });
}
- async function classifyAndPersistRunLiveness(
+ async function cancelQueuedRunForBlockedDependencies(
run: typeof heartbeatRuns.$inferSelect,
- resultJson?: Record<string, unknown> | null,
+ issueId: string,
+ unresolvedBlockerIssueIds: string[],
) {
- const authRepair = run.status === "failed"
- ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {
- logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");
- return null;
- })
- : null;
- const classification = classifyRunLiveness({
- ...await buildRunLivenessInput(run, resultJson),
- authenticationRepairRequested: Boolean(authRepair?.interactionId),
+ const now = new Date();
+ const reason =
+ "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve";
+ const cancelled = await setRunStatus(run.id, "cancelled", {
+ finishedAt: now,
+ error: reason,
+ errorCode: "issue_dependencies_blocked",
+ resultJson: {
+ ...parseObject(run.resultJson),
+ stopReason: "issue_dependencies_blocked",
+ effectiveTimeoutSec: 0,
+ timeoutConfigured: false,
+ timeoutSource: "dependency_gate",
+ timeoutFired: false,
+ },
});
- return db
- .update(heartbeatRuns)
- .set({
- livenessState: classification.livenessState,
- livenessReason: classification.livenessReason,
- continuationAttempt: classification.continuationAttempt,
- lastUsefulActionAt: classification.lastUsefulActionAt,
- nextAction: classification.nextAction,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, run.id))
- .returning()
- .then((rows) => rows[0] ?? null);
- }
-
- // Clamp the stored attempt count to the range [0, cap]. The SQL reader
- // `pendingCleanupAttemptsSql` clamps to the same range, so both readers yield
- // the same value for every input. The claim predicate compares the two values,
- // so this alignment lets the claim match for a malformed lease.
- function readPendingCleanupRetryAttempts(
- metadata: Record<string, unknown>,
- ): number {
- const value = metadata[PENDING_CLEANUP_ATTEMPTS_METADATA_KEY];
- if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)
- return 0;
- return Math.min(Math.floor(value), PENDING_CLEANUP_SWEEP_ATTEMPT_CAP);
- }
+ if (!cancelled) return null;
- // Atomically claim one retry attempt on a pending_cleanup lease. The update
- // only matches when the lease is still pending_cleanup and its stored attempt
- // count still equals `expectedAttempts`. Two concurrent sweeps read the same
- // count, but Postgres serializes the two updates on the row and only the first
- // matches the guard. The loser gets zero rows and skips the lease. The persisted in-flight deadline also prevents a later tick
- // from starting another attempt while this one is still running.
- // Returns the attempt identity only for the sweep that won the claim.
- //
- // The update patches cleanup ownership fields without writing a copied
- // metadata object, so a concurrent write to an unrelated metadata key
- // survives. The guard reads the stored count through the safe SQL reader, so a
- // malformed value never throws.
- async function claimPendingCleanupRetryAttempt(
- leaseId: string,
- expectedAttempts: number,
- manualAttempt?: { previousId: unknown },
- ): Promise<string | null> {
- const now = new Date();
- const attemptId = randomUUID();
- const claimed = await db
- .update(environmentLeases)
- .set({
- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}], to_jsonb(${expectedAttempts + 1}::int), true)
- || ${JSON.stringify({ ...(manualAttempt ? { pendingCleanupManualAttemptId: randomUUID() } : {}),
- pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true,
- pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000,
- })}::jsonb`,
- lastUsedAt: now,
- updatedAt: now,
- })
- .where(
- and(
- eq(environmentLeases.id, leaseId),
- eq(environmentLeases.status, "pending_cleanup"),
- sql`${pendingCleanupAttemptsSql()} = ${expectedAttempts}`,
- manualAttempt ? sql`coalesce(${environmentLeases.metadata}->'pendingCleanupManualAttemptId', 'null'::jsonb) is not distinct from ${JSON.stringify(manualAttempt.previousId ?? null)}::jsonb` : undefined,
- pendingCleanupRetryDueSql(Boolean(manualAttempt)),
- ),
- )
- .returning({ id: environmentLeases.id });
- return claimed.length > 0 ? attemptId : null;
- }
+ await setWakeupStatus(run.wakeupRequestId, "skipped", {
+ finishedAt: now,
+ error: reason,
+ });
- // Atomically claim the one-time cap warning for a lease. The update only
- // matches when the lease is still pending_cleanup, its stored attempt count is
- // at or above the cap, and it has not yet carried the warned flag. Two
- // concurrent sweeps that both reach the cap race here, but only one update
- // sets the flag and returns a row. The loser skips the warning. This keeps the
- // warning to one log line per lease.
- //
- // The status and cap predicates are the last line of defense. They stop a warn
- // flag write to a lease that left pending_cleanup or dropped below the cap
- // between the read and this claim. The update writes only the warned key with
- // `jsonb_set`, so a concurrent write to an unrelated metadata key survives.
- async function claimPendingCleanupCapWarning(
- leaseId: string,
- ): Promise<boolean> {
- const now = new Date();
- const claimed = await db
- .update(environmentLeases)
+ await db
+ .update(issues)
.set({
- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}], to_jsonb(true), true)`,
+ executionRunId: null,
+ executionAgentNameKey: null,
+ executionLockedAt: null,
updatedAt: now,
})
.where(
and(
- eq(environmentLeases.id, leaseId),
- eq(environmentLeases.status, "pending_cleanup"),
- sql`${pendingCleanupAttemptsSql()} >= ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}`,
- sql`${pendingCleanupCapWarnedSql()} = false`,
- ),
- )
- .returning({ id: environmentLeases.id });
- return claimed.length > 0;
- }
-
- // Defer a pending_cleanup lease whose provider plugin is not ready this tick.
- // The sweep reads one page of the oldest rows, ordered by `updatedAt`. A lease
- // that the sweep only skips keeps its old `updatedAt`, so it stays the oldest
- // and refills the page on every tick. That starves a newer lease whose
- // provider is ready. The defer bumps `updatedAt` to now, so the unavailable
- // lease moves to the back of the queue and a ready lease takes its page slot.
- // The defer never writes the attempt count, so a long provider outage never
- // consumes a finite retry. The status guard keeps the write on a lease that is
- // still pending_cleanup.
- async function deferPendingCleanupLease(leaseId: string): Promise<void> {
- const now = new Date();
- await db
- .update(environmentLeases)
- .set({ updatedAt: now })
- .where(
- and(
- eq(environmentLeases.id, leaseId),
- eq(environmentLeases.status, "pending_cleanup"),
- ),
- );
- }
-
- // Move a guarded orphan candidate to the back of the sweep order. The
- // shared-resource guard below can skip a row on every tick as long as the
- // other owner stays live, retained, or pending cleanup. The select orders
- // by `updatedAt` ascending and takes only the oldest page, so an untouched
- // skipped row keeps refilling that same page and blocks every row behind
- // it. The bump pushes the row past the fixed-size page, so the next tick
- // reaches the rows behind it. It costs the row one extra backoff wait,
- // which is safe because the guard means a physical sandbox still exists.
- async function deferOrphanedActiveLease(leaseId: string): Promise<void> {
- const now = new Date();
- await db
- .update(environmentLeases)
- .set({ updatedAt: now })
- .where(
- and(
- eq(environmentLeases.id, leaseId),
- eq(environmentLeases.status, "active"),
+ eq(issues.companyId, run.companyId),
+ eq(issues.id, issueId),
+ eq(issues.executionRunId, run.id),
),
);
- }
-
- // An active lease is reachable only while its heartbeat run keeps the
- // running status. The reaper writes the run status and the lease release as
- // two separate statements, so a restart between them can leave a terminal
- // run and an active lease. `releaseRunLeases` also skips a lease whose
- // environment row is gone, so that lease stays active too. No later query
- // finds either lease, because every production query selects an active
- // lease by its environment or by its heartbeat run id, never by age. This
- // sweep finds both stranded classes and moves each lease to pending_cleanup,
- // so the existing pending_cleanup sweep tears the sandbox down from the
- // data already on the lease row.
- async function sweepOrphanedActiveLeases(opts: {
- backoffMs: number;
- }): Promise<{ recovered: number }> {
- const cutoff = new Date(Date.now() - opts.backoffMs);
-
- const rows = await db
- .select({ lease: environmentLeases })
- .from(environmentLeases)
- .leftJoin(
- heartbeatRuns,
- eq(environmentLeases.heartbeatRunId, heartbeatRuns.id),
- )
- .where(
- and(
- eq(environmentLeases.status, "active"),
- or(
- isNull(environmentLeases.heartbeatRunId),
- inArray(heartbeatRuns.status, [
- ...HEARTBEAT_RUN_TERMINAL_STATUSES,
- ]),
- ),
- lte(environmentLeases.updatedAt, cutoff),
- ),
- )
- .orderBy(asc(environmentLeases.updatedAt))
- .limit(ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE);
-
- let recovered = 0;
- for (const { lease } of rows) {
- // A provider resource id names one physical sandbox. A different lease
- // row can still hold that same resource in a live status, so this sweep
- // must not tear down a sandbox that a different lease still owns.
- if (lease.provider && lease.providerLeaseId) {
- const [otherOwner] = await db
- .select({ id: environmentLeases.id })
- .from(environmentLeases)
- .where(
- and(
- ne(environmentLeases.id, lease.id),
- eq(environmentLeases.provider, lease.provider),
- eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
- inArray(environmentLeases.status, [
- "active",
- "retained",
- "pending_cleanup",
- ]),
- ),
- )
- .limit(1);
- if (otherOwner) {
- // Defer this row so the fixed-size page reaches the rows behind
- // it next tick, instead of re-selecting the same guarded rows
- // forever.
- await deferOrphanedActiveLease(lease.id);
- continue;
- }
- }
- // Keep the row's existing updatedAt value. The select above already
- // proved the row is older than the backoff cutoff, so the
- // pending_cleanup sweep can accept the same row in this same tick. A
- // fresh timestamp here would push the row inside that sweep's own
- // backoff window and delay the teardown by one full tick.
- const flipped = await db
- .update(environmentLeases)
- .set({
- status: "pending_cleanup",
- failureReason: "orphaned_active_lease_recovered",
- })
- .where(
- and(
- eq(environmentLeases.id, lease.id),
- eq(environmentLeases.status, "active"),
- ),
- )
- .returning({ id: environmentLeases.id });
- if (flipped.length > 0) recovered += 1;
- }
+ await appendRunEvent(cancelled, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: reason,
+ payload: {
+ issueId,
+ unresolvedBlockerIssueIds,
+ },
+ });
- return { recovered };
+ return cancelled;
}
- // Retry the leases stranded in "pending_cleanup". A failed destroy leaves a
- // lease in that state forever without this sweep. The reaper tick runs the
- // sweep. The backoff equals the reaper staleness threshold, so a lease waits
- // for that period between attempts. The sweep reads and writes the attempt
- // count in the lease metadata. It warns once when a lease reaches the attempt
- // threshold, then continues with slower retries. Live attempts renew their
- // cleanup claim; after controller loss, exact-resource destruction may repeat.
- async function sweepPendingCleanupLeases(opts?: {
- backoffMs?: number;
- /** One cleanup attempt per explicit user Retry, for this failed run only.
- * A later user Retry may bypass the cooldown after a provider failure,
- * but cannot take over an in-flight cleanup attempt.
- */
- explicitRetry?: { companyId: string; runId: string; actorId: string; reason?: "retry_failed_run" | "queued_comment_interrupt" };
- }): Promise<{
- swept: number;
- destroyed: number;
- capped: number;
- }> {
- const backoffMs = opts?.backoffMs ?? 0;
- const now = new Date();
- const cutoff = new Date(now.getTime() - backoffMs);
+ function truncateAgentErrorReason(
+ reason: string | null | undefined,
+ ): string | null {
+ if (!reason) return null;
+ const trimmed = reason.trim();
+ if (!trimmed) return null;
+ return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;
+ }
- // Flush the in-process orphan-cleanup buffer first. A failed acquire buffers
- // an orphan there when every synchronous pending-cleanup write failed after a
- // failed teardown. The flush re-inserts each buffered record, so a durable
- // `pending_cleanup` row lands once the database recovers. The flush runs
- // before the read below, so this same tick tears down a freshly-landed row.
- try {
- const flushed = opts?.explicitRetry ? null : await environmentRuntime.flushDeferredOrphanCleanups?.();
- if (flushed && (flushed.recovered > 0 || flushed.pending > 0)) {
- logger.info(
- { recovered: flushed.recovered, pending: flushed.pending },
- "flushed the in-process orphan sandbox cleanup buffer to the database",
- );
- }
- } catch {
- // A flush failure never stops the sweep. The buffer keeps the orphan for a
- // later tick, and the database rows below still need this sweep. The caught
- // exception never enters the log, because a write error can carry a
- // credential in its message, code, cause, or stack.
- logger.warn(
- "orphan sandbox cleanup buffer flush failed; the sweep continues",
- );
- }
+ async function finalizeAgentStatus(
+ agentId: string,
+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
+ failureReason?: string | null,
+ options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },
+ ) {
+ const existing = await getAgent(agentId);
+ if (!existing) return;
- const rows = await db
- .select()
- .from(environmentLeases)
- .where(
- and(
- eq(environmentLeases.status, "pending_cleanup"),
- opts?.explicitRetry ? eq(environmentLeases.companyId, opts.explicitRetry.companyId) : undefined,
- opts?.explicitRetry ? eq(environmentLeases.heartbeatRunId, opts.explicitRetry.runId) : undefined,
- pendingCleanupRetryDueSql(Boolean(opts?.explicitRetry)),
- backoffMs > 0 ? lte(environmentLeases.updatedAt, cutoff) : undefined,
- ),
- )
- .orderBy(asc(environmentLeases.updatedAt))
- .limit(PENDING_CLEANUP_SWEEP_PAGE_SIZE);
+ if (existing.status === "paused" || existing.status === "terminated") {
+ return;
+ }
- let destroyed = 0;
- let capped = 0;
- for (const row of rows) {
- if (pendingCleanupAttemptsInFlight.has(row.id)) continue;
- const metadata = { ...(row.metadata ?? {}) } as Record<string, unknown>;
- const attempts = readPendingCleanupRetryAttempts(metadata);
+ const isFirstHeartbeat =
+ options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;
- const environment = row.environmentId
- ? await environmentsSvc.getById(row.environmentId)
- : null;
- const lease = await environmentsSvc.getLeaseById(row.id);
- if (!lease) continue;
-
- // An orphan ephemeral lease keeps its provider, its provider lease id, and
- // its sandbox config in the lease row. A failed acquire records it, and its
- // environment row may be gone or foreign-bound. A reuse_by_environment lease
- // whose environment a delete removed keeps the same recorded data, because
- // the schema sets the environment reference to null on delete and preserves
- // the row. Both leases tear down from the recorded lease data through
- // `retryPendingSandboxTeardown`, which never reads the environment row. So
- // the sweep uses that path whenever the lease is an orphan ephemeral lease
- // or its environment row is gone. A reuse_by_environment lease whose
- // environment still exists tears down through `destroyRunLease`. That
- // path uses the provider and configuration recorded on the lease first;
- // the environment is lifecycle context and only a legacy fallback.
- const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";
- const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);
-
- // Do not consume a finite cleanup attempt while the provider plugin is
- // briefly unavailable. A plugin worker restart, a plugin reload, or a
- // plugin reinstall makes the provider unavailable for a short window. The
- // plugin can be missing or not ready in that window. A teardown then throws,
- // and the atomic claim below would count that throw against the cap, so a
- // long restart or reload could exhaust the retries and strand a live
- // sandbox. So probe the provider first, and defer the lease this tick when
- // the provider is not ready. The sweep preserves the pending_cleanup row,
- // and a later sweep retries after the provider recovers. The probe reports
- // ready only for a permanent condition (a missing provider string, a
- // built-in provider, or no worker manager), so a genuine teardown failure
- // still runs, throws, and counts toward the cap. A runtime with no probe
- // method treats the lease as ready, so the sweep keeps its earlier
- // behavior.
- const workerReady = environmentRuntime.isPendingCleanupWorkerReady
- ? await environmentRuntime.isPendingCleanupWorkerReady({
- environment,
- lease,
- })
- : true;
- if (!workerReady) {
- // Move the unavailable lease to the back of the sweep queue. Otherwise
- // the oldest unavailable rows refill the page on every tick and starve a
- // newer lease that has a ready provider. The defer bumps `updatedAt`
- // only, so it consumes no finite retry attempt.
- await deferPendingCleanupLease(row.id);
- continue;
- }
+ const runningCount = await countRunningRunsForAgent(agentId);
+ const nextStatus =
+ runningCount > 0
+ ? "running"
+ : outcome === "succeeded" ||
+ outcome === "interrupted" ||
+ outcome === "cancelled" ||
+ (outcome === "failed" && options?.keepIdleOnFailure)
+ ? "idle"
+ : "error";
- // Atomically claim the attempt before the retry. Only the winning sweep
- // increments the count and tears the sandbox down, so an overlapping sweep
- // cannot start another attempt while this one holds the cleanup lease.
- // The deadline survives a server restart; the counter saturates at the
- // escalation threshold while attempt identities remain unique.
- const claimed = await claimPendingCleanupRetryAttempt(row.id, attempts,
- opts?.explicitRetry ? { previousId: metadata.pendingCleanupManualAttemptId } : undefined);
- if (!claimed) continue;
- pendingCleanupAttemptsInFlight.add(row.id);
- lease.metadata = { ...lease.metadata, pendingCleanupAttemptId: claimed };
- let activeRenewal: Promise<void> | null = null;
- const renewal = setInterval(() => {
- if (activeRenewal) return;
- activeRenewal = db.update(environmentLeases).set({
- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, '{pendingCleanupLeaseExpiresAtMs}', to_jsonb(${Date.now() + 15 * 60_000}::bigint))`,
- }).where(and(eq(environmentLeases.id, row.id), eq(environmentLeases.status, "pending_cleanup"),
- sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`,
- sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`))
- .then(() => {})
- .catch(() => logger.warn({ leaseId: row.id }, "cleanup ownership renewal failed"))
- .finally(() => { activeRenewal = null; });
- }, 30_000);
- renewal.unref();
+ const updated = await db
+ .update(agents)
+ .set({
+ status: nextStatus,
+ // Persist a human-readable reason on the agent record when it enters
+ // error so operators see it on the agent page without digging into run
+ // events; clear it whenever the agent leaves error.
+ errorReason:
+ nextStatus === "error"
+ ? truncateAgentErrorReason(failureReason)
+ : null,
+ lastHeartbeatAt: new Date(),
+ updatedAt: new Date(),
+ })
+ .where(eq(agents.id, agentId))
+ .returning()
+ .then((rows) => rows[0] ?? null);
- try {
- if (opts?.explicitRetry) await logActivity(db, {
- companyId: row.companyId, actorType: "user", actorId: opts.explicitRetry.actorId,
- action: "environment_lease.cleanup_retried", entityType: "environment_lease", entityId: row.id,
- runId: opts.explicitRetry.runId, details: { attempt: attempts + 1, reason: opts.explicitRetry.reason ?? "retry_failed_run" },
+ if (isFirstHeartbeat && updated) {
+ const tc = getTelemetryClient();
+ if (tc)
+ trackAgentFirstHeartbeat(tc, {
+ agentRole: updated.role,
+ agentId: updated.id,
});
- if (useRecordedTeardown) {
- // Tear the sandbox down from the recorded provider config and the
- // cleanup-authorized secret versions. Preserve any provider receipt;
- // a completed retry must grant the same evidence as initial cleanup.
- const receipt = await environmentRuntime.retryPendingSandboxTeardown({
- environment,
- lease,
- });
- const released = hasStopOnlyCleanup(lease)
- ? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt })
- : await environmentsSvc.releaseLease(lease.id, "expired", {
- expectedPendingCleanupAttemptId: claimed,
- cleanupStatus: "success",
- failureReason: "pending_cleanup_retry",
- remoteExecutionTermination: remoteTerminationReceipt(lease, receipt),
- });
- if (released) destroyed += 1;
- } else if (environment) {
- const result = await environmentRuntime.destroyRunLease({
- environment,
- lease,
- failureReason: "pending_cleanup_retry",
- });
- if (result && result.status !== "pending_cleanup") {
- destroyed += 1;
- }
- }
- } catch {
- // The recorded-data teardown throws on failure, so revert the lease to
- // pending_cleanup for a later sweep. The claimed attempt still counts
- // for backoff, so requests stay bounded. The `destroyRunLease`
- // path reverts the lease itself, so this revert only runs for the
- // recorded-data teardown path.
- if (useRecordedTeardown) {
- await environmentsSvc.releaseLease(lease.id, "pending_cleanup", {
- expectedPendingCleanupAttemptId: claimed,
- cleanupStatus: "failed",
- failureReason: "pending_cleanup_retry",
- });
- }
- // Log a constant errorKind only. The exception can carry a credential in
- // its name, code, message, cause, or stack, so the sweep never reads it.
- logger.warn(
- {
- errorKind: PENDING_CLEANUP_RETRY_ERROR_KIND,
- leaseId: row.id,
- environmentId: row.environmentId,
- attempts: attempts + 1,
- },
- "pending_cleanup lease retry failed",
- );
- } finally {
- clearInterval(renewal);
- // A hung renewal must not retain process-local cleanup ownership.
- // Late renewals are attempt-fenced and never write the retry cooldown.
- pendingCleanupAttemptsInFlight.delete(row.id);
- }
- if (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP) {
- capped += 1;
- // Warn once, then continue automatic cleanup with backoff. The atomic claim
- // keeps the warning to one log line even when two sweeps overlap.
- if (metadata[PENDING_CLEANUP_CAP_WARNED_METADATA_KEY] !== true) {
- const warned = await claimPendingCleanupCapWarning(row.id);
- if (warned) {
- logger.warn(
- { leaseId: row.id, environmentId: row.environmentId, attempts },
- "environment lease needs operator attention; automatic cleanup continues with backoff",
- );
- }
- }
- }
- // Persist the cooldown independently of process memory. A crash before
- // this write leaves the bounded in-flight lease for a later sweep.
- await db.update(environmentLeases).set({
- metadata: sql`${pendingCleanupMetadataObjectSql()} || ${JSON.stringify({
- pendingCleanupInFlight: false,
- pendingCleanupRetryAfterMs: Date.now() + (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP
- ? 30 * 60_000 : Math.min(30 * 60_000, Math.max(30_000, backoffMs))),
- })}::jsonb`,
- }).where(and(eq(environmentLeases.id, lease.id),
- sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`));
- if (lease.heartbeatRunId) {
- // Delivery failure must not revert successful provider cleanup. A new
- // message can still use the persisted receipt on its next admission.
- await (async () => {
- await acknowledgeRemoteStop(lease.heartbeatRunId!, lease.companyId);
- const run = await getRun(lease.heartbeatRunId!);
- if (run) await resumeRemoteStopComments(run);
- })().catch(() => logger.warn({ leaseId: lease.id }, "could not reconsider messages after cleanup retry"));
- }
}
- return { swept: rows.length, destroyed, capped };
+ if (updated) {
+ publishLiveEvent({
+ companyId: updated.companyId,
+ type: "agent.status",
+ payload: {
+ agentId: updated.id,
+ status: updated.status,
+ lastHeartbeatAt: updated.lastHeartbeatAt
+ ? new Date(updated.lastHeartbeatAt).toISOString()
+ : null,
+ outcome,
+ },
+ });
+ }
}
- async function markNativeOwnershipUnverified(
- run: typeof heartbeatRuns.$inferSelect,
- evidence: {
- reason:
- | "live_process_identifier"
- | "observed_owner_unverified"
- | "adopted_runner_authentication_timeout"
- | "native_chat_workspace_scope_mismatch";
- processPidAlive?: boolean;
- processGroupAlive?: boolean;
+ function mergeRunStopMetadataForAgent(
+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
+ options?: {
+ resultJson?: Record<string, unknown> | null;
+ conversationContinuationEligible?: boolean;
+ errorCode?: string | null;
+ errorMessage?: string | null;
},
) {
- const durableOwnershipHold =
- evidence.reason === "adopted_runner_authentication_timeout" ||
- evidence.reason === "native_chat_workspace_scope_mismatch";
- if (
- run.errorCode === NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE &&
- run.error === NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE &&
- (!durableOwnershipHold || isNativeRunnerOwnershipHeld(run))
- )
- return run;
- const blockedStatus = run.status === "failed" ? "failed" : "running";
- const blockedWrite = await setRunStatusFromLive(
- run.id,
- blockedStatus,
- [blockedStatus],
- {
- error: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,
- errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
- ...(durableOwnershipHold
- ? {
- nativePhase: "terminal_failure",
- nativePhaseUpdatedAt: new Date(),
- }
- : {}),
- },
- );
- if (!blockedWrite.updated || !blockedWrite.run) {
- return blockedWrite.run ?? run;
- }
- const blocked = blockedWrite.run;
- await appendRunEvent(blocked, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,
- payload: {
- reason: evidence.reason,
- ...(evidence.processPidAlive === true ? { processPidAlive: true } : {}),
- ...(evidence.processGroupAlive === true
- ? { processGroupAlive: true }
- : {}),
- },
+ const stopMetadata = buildHeartbeatRunStopMetadata({
+ adapterType: agent.adapterType,
+ adapterConfig: parseObject(agent.adapterConfig),
+ outcome,
+ errorCode: options?.errorCode ?? null,
+ errorMessage: options?.errorMessage ?? null,
});
- return blocked;
+ const result = mergeHeartbeatRunStopMetadata(
+ options?.resultJson ?? null,
+ stopMetadata,
+ );
+ const cancellationAcknowledged =
+ parseObject(result?.executionCancellation).state === "acknowledged";
+ return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&
+ (outcome !== "cancelled" || cancellationAcknowledged) &&
+ isConversationAdapter(agent.adapterType)
+ ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }
+ : result;
}
- async function settleRecoveredNativeWorkspace(input: {
- runId: string;
- companyId: string;
- agentId: string;
- succeeded: boolean;
- }) {
- const settledRun = await getRun(input.runId);
- const workspaceSyncReference = readNativeWorkspaceSyncReference(
- parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
- );
- await releaseEnvironmentLeasesForRun({
- runId: input.runId,
- companyId: input.companyId,
- agentId: input.agentId,
- status: settledRun?.status,
- failureReason: settledRun?.error ?? undefined,
- providerResourceDisposition: input.succeeded && (!parseObject(settledRun?.resultJson).workspaceExportRetry || workspaceSyncReference?.resourceDisposition === "destroy")
- ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")
- : "stop_and_retain",
- });
- await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);
- await finalizeAgentStatus(
- input.agentId,
- input.succeeded ? "succeeded" : "failed",
- input.succeeded
- ? null
- : (settledRun?.error ?? "native_workspace_sync_out_failed"),
- {
- wasFirstHeartbeat: settledRun
- ? timerClaimWasFirstHeartbeat(settledRun)
- : undefined,
- },
- ).catch(() => undefined);
+ function countValue(value: unknown) {
+ const parsed = Number(value ?? 0);
+ return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
}
- function scheduleRetainedNativeSessionCleanup() {
- // The per-database sweep joins startup and periodic callers. One bounded
- // control-only repair must not hold up unrelated provider ingress or the
- // entire orphan reaper, but shutdown must still await its physical owner.
- const cleanup = reconcileRetainedNativeSessionCleanups(db, {
- cleanup: (input) => reconcileRetainedNativeSessionCleanup(db, input),
- onError: (error, runId) => {
- logger.warn(
- { err: error, runId },
- "retained native session cleanup failed",
- );
- },
- })
- .then(() => undefined)
- .catch((error) => {
- logger.warn({ err: error }, "retained native cleanup discovery failed");
- })
- // The bounded maintenance attempt may fail before an already-started
- // database callback settles. Keep shutdown ownership until the original
- // operations finish; their timeout cannot authorize closing the database.
- .finally(() => drainRetainedRunnerdMaintenanceOperations());
- activeRunExecutionPromises.add(cleanup);
- void cleanup.finally(() => activeRunExecutionPromises.delete(cleanup));
+ function dateValue(value: unknown) {
+ if (value instanceof Date)
+ return Number.isNaN(value.getTime()) ? null : value;
+ if (typeof value === "string" || typeof value === "number") {
+ const parsed = new Date(value);
+ return Number.isNaN(parsed.getTime()) ? null : parsed;
+ }
+ return null;
}
- async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
- const staleThresholdMs = opts?.staleThresholdMs ?? 0;
- const now = new Date();
- // Recovery never launches a provider or infers stopped ownership from
- // terminal status. Uncaptured local copies require durable stop evidence.
- await instructionCopies.recoverStopped().catch(error => {
- logger.warn({ err: error }, "failed to recover stopped instruction copies");
- });
- await instructionCopies.recoverCaptured().catch(error => {
- logger.warn({ err: error }, "failed to retry captured instruction revisions");
- });
+ function latestDate(...values: unknown[]) {
+ let latest: Date | null = null;
+ for (const value of values) {
+ const parsed = dateValue(value);
+ if (!parsed) continue;
+ if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
+ }
+ return latest;
+ }
- // Complete persisted native results before generic orphan recovery. The
- // reconciler reads the durable workspace barrier and persisted runtime
- // mode, never the current feature flag.
- await reconcileNativeFinalizations(db, undefined, {
- environmentRuntime,
- onWorkspaceSettled: settleRecoveredNativeWorkspace,
- }).catch((error) => {
- logger.warn(
- { err: error },
- "failed to reconcile persisted native finalizations before orphan reaping",
- );
- });
- scheduleRetainedNativeSessionCleanup();
- await dispatchPendingNativeStatusWakeups().catch((error) => {
- logger.warn(
- { err: error },
- "failed to dispatch persisted native status wake intents before orphan reaping",
- );
- });
+ async function buildRunLivenessInput(
+ run: typeof heartbeatRuns.$inferSelect,
+ resultJson: Record<string, unknown> | null | undefined,
+ ): Promise<RunLivenessClassificationInput> {
+ const context = parseObject(run.contextSnapshot);
+ const contextIssueId = readNonEmptyString(context.issueId);
+ const continuationAttempt = asNumber(
+ context.continuationAttempt,
+ run.continuationAttempt ?? 0,
+ );
- // A retryable native run can retain process identifiers from the failed
- // attempt. Inspect them before the recovery claim: a live identifier is
- // unowned and blocks recovery, while identifiers that are all dead can be
- // cleared with a compare-and-set so the explicit retryable failure becomes
- // claimable in this same sweep.
- const retryableNativeProcesses = await db
- .select({ run: heartbeatRuns })
- .from(heartbeatRuns)
- .innerJoin(
- nativeRunFinalizations,
- eq(nativeRunFinalizations.runId, heartbeatRuns.id),
- )
- .where(
- and(
- inArray(heartbeatRuns.status, ["running", "failed"]),
- eq(heartbeatRuns.runtimeMode, "native"),
- eq(nativeRunFinalizations.phase, "retryable_failure"),
- isNull(nativeRunFinalizations.resultId),
- ),
- );
- const claimableNativeRunIds = new Set<string>();
- for (const { run } of retryableNativeProcesses) {
- if (isNativeRunnerOwnershipHeld(run)) continue;
- if (!run.processPid && !run.processGroupId) {
- claimableNativeRunIds.add(run.id);
- continue;
- }
- const processPidAlive =
- !!run.processPid && isProcessAlive(run.processPid);
- const processGroupAlive =
- !!run.processGroupId && isProcessGroupAlive(run.processGroupId);
- if (processPidAlive || processGroupAlive) {
- await markNativeOwnershipUnverified(run, {
- reason: "live_process_identifier",
- processPidAlive,
- processGroupAlive,
- });
- continue;
- }
- const cleared = await db.transaction(async tx => {
- const cleared = await tx
- .update(heartbeatRuns)
- .set({
- processPid: null,
- processGroupId: null,
- processStartedAt: null,
- updatedAt: now,
+ const issue = contextIssueId
+ ? await db
+ .select({
+ status: issues.status,
+ title: issues.title,
+ description: issues.description,
+ workMode: issues.workMode,
+ })
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, run.companyId),
+ eq(issues.id, contextIssueId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null)
+ : null;
+
+ const [commentStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
+ })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, run.companyId),
+ eq(issueComments.issueId, contextIssueId),
+ eq(issueComments.createdByRunId, run.id),
+ isNull(issueComments.deletedAt),
+ ),
+ )
+ : [{ count: 0, latestAt: null }];
+
+ const issueCommentBodies = contextIssueId
+ ? await db
+ .select({ body: issueComments.body })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, run.companyId),
+ eq(issueComments.issueId, contextIssueId),
+ eq(issueComments.createdByRunId, run.id),
+ ),
+ )
+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id))
+ .limit(5)
+ .then((rows) => rows.reverse().map((row) => row.body))
+ : [];
+
+ const continuationSummary = contextIssueId
+ ? await getIssueContinuationSummaryDocument(db, contextIssueId)
+ : null;
+
+ const [documentStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
+ latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
})
+ .from(documentRevisions)
+ .innerJoin(
+ issueDocuments,
+ eq(documentRevisions.documentId, issueDocuments.documentId),
+ )
.where(
and(
- eq(heartbeatRuns.id, run.id),
- eq(heartbeatRuns.runtimeMode, "native"),
- run.processPid === null
- ? isNull(heartbeatRuns.processPid)
- : eq(heartbeatRuns.processPid, run.processPid),
- run.processGroupId === null
- ? isNull(heartbeatRuns.processGroupId)
- : eq(heartbeatRuns.processGroupId, run.processGroupId),
- run.processStartedAt === null
- ? isNull(heartbeatRuns.processStartedAt)
- : eq(heartbeatRuns.processStartedAt, run.processStartedAt),
+ eq(documentRevisions.companyId, run.companyId),
+ eq(documentRevisions.createdByRunId, run.id),
+ eq(issueDocuments.companyId, run.companyId),
+ eq(issueDocuments.issueId, contextIssueId),
+ sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
),
)
- .returning({ id: heartbeatRuns.id })
- .then((rows) => rows[0] ?? null);
- if (cleared) await recordNativeLocalProcessStop(tx as unknown as Db, run);
- return cleared;
- });
- if (cleared) claimableNativeRunIds.add(cleared.id);
- }
+ : [{ count: 0, planCount: 0, latestAt: null }];
- // An explicit result-less retryable failure resumes on the original run.
- // The database lease is claimed before dispatch so concurrent service
- // instances cannot open competing recoveries; executeRun receives the exact
- // claimed owner. Expired `observed` ownership never enters this set.
- const nativeResumeClaims =
- claimableNativeRunIds.size === 0
- ? []
- : await dispatchNativeSessionResumptions({
- db,
- runnerInstanceId:
- runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat",
- now,
- runIds: [...claimableNativeRunIds],
- dispatch: (claim) => {
- const execution = executeRun(claim.runId, {
- nativeLeaseOwner: claim.leaseOwner,
- }).catch((error) => {
- logger.error(
- { err: error, runId: claim.runId },
- "persisted native session resume failed",
- );
- });
- activeRunExecutionPromises.add(execution);
- void execution.finally(() =>
- activeRunExecutionPromises.delete(execution),
- );
- },
- }).catch((error) => {
- logger.warn(
- { err: error },
- "failed to claim persisted native session resumptions",
- );
- return [];
- });
- const resumedRunIds = new Set(
- nativeResumeClaims.map((claim) => claim.runId),
- );
+ const [workProductStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
+ })
+ .from(issueWorkProducts)
+ .where(
+ and(
+ eq(issueWorkProducts.companyId, run.companyId),
+ eq(issueWorkProducts.issueId, contextIssueId),
+ eq(issueWorkProducts.createdByRunId, run.id),
+ ),
+ )
+ : [{ count: 0, latestAt: null }];
- // A terminal issue transition writes this intent in the same transaction
- // that closes the question's task, even when its card is retained. Consume
- // it before generic orphan recovery so a restart preserves cancellation.
- const cancellationRequests = await db
+ const [workspaceOperationStats] = await db
.select({
- id: heartbeatRuns.id,
- contextSnapshot: heartbeatRuns.contextSnapshot,
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
})
- .from(heartbeatRuns)
+ .from(workspaceOperations)
.where(
and(
- inArray(heartbeatRuns.status, [
- ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
- ]),
- sql`${heartbeatRuns.contextSnapshot} -> ${NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY} is not null`,
+ eq(workspaceOperations.companyId, run.companyId),
+ eq(workspaceOperations.heartbeatRunId, run.id),
),
);
- for (const request of cancellationRequests) {
- const marker = parseObject(
- parseObject(request.contextSnapshot)[
- NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY
- ],
- );
- const issueId = readNonEmptyString(marker.issueId);
- const issueStatus = readNonEmptyString(marker.issueStatus);
- const interactionId = readNonEmptyString(marker.interactionId);
- const kind = readNonEmptyString(marker.kind);
- const reason =
- kind === "interaction_withdrawn"
- ? "Question withdrawn while waiting for operator input"
- : kind === "interaction_cancelled"
- ? "Cancelled while waiting for operator input"
- : "Task closed while waiting for operator input";
- try {
- await cancelRunInternal(request.id, reason, {
- resultJson: {
- ...(kind === "interaction_withdrawn" && interactionId
- ? { withdrawnInteractionId: interactionId }
- : {}),
- ...(kind === "interaction_cancelled" && interactionId
- ? { cancelledInteractionId: interactionId }
- : {}),
- ...((!kind || kind === "issue_terminal") && issueStatus
- ? { cancelledByIssueStatus: issueStatus }
- : {}),
- ...(issueId ? { cancelledIssueId: issueId } : {}),
- },
- });
- } catch (err) {
- // Keep the marker intact for the next startup/periodic sweep.
- logger.warn(
- { err, runId: request.id },
- "native question cancellation recovery attempt failed",
- );
- }
- }
- // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
- const activeRuns = await db
+ const [activityStats] = await db
.select({
- run: heartbeatRuns,
- adapterType: agents.adapterType,
- adapterConfig: agents.adapterConfig,
- nativeCoordinatorPhase: nativeRunFinalizations.phase,
- nativeRecoveryState: nativeRunFinalizations.recoveryState,
- nativeControllerBootId: nativeRunFinalizations.controllerBootId,
- nativeControllerPid: nativeRunFinalizations.controllerPid,
- nativeControllerProcessStartedAt:
- nativeRunFinalizations.controllerProcessStartedAt,
- nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
})
- .from(heartbeatRuns)
- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
- .leftJoin(
- nativeRunFinalizations,
- eq(nativeRunFinalizations.runId, heartbeatRuns.id),
- )
- .where(eq(heartbeatRuns.status, "running"));
-
- const monitorIssueIds = [
- ...new Set(
- activeRuns.flatMap(({ run }) => {
- const runContext = parseObject(run.contextSnapshot);
- if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due")
- return [];
- const issueId = readNonEmptyString(runContext.issueId);
- return issueId ? [issueId] : [];
- }),
- ),
- ];
- const monitorIssues =
- monitorIssueIds.length > 0
- ? await db
- .select({
- id: issues.id,
- companyId: issues.companyId,
- monitorNextCheckAt: issues.monitorNextCheckAt,
- })
- .from(issues)
- .where(inArray(issues.id, monitorIssueIds))
- : [];
- const monitorNextCheckAtByIssue = new Map(
- monitorIssues.map((issue) => [
- `${issue.companyId}:${issue.id}`,
- issue.monitorNextCheckAt,
- ]),
- );
-
- const reaped: string[] = [];
- const currentNativeController = await currentNativeControllerIdentity();
-
- for (const {
- run,
- adapterType,
- adapterConfig,
- nativeCoordinatorPhase,
- nativeRecoveryState,
- nativeControllerBootId,
- nativeControllerPid,
- nativeControllerProcessStartedAt,
- nativeControllerLeaseExpiresAt,
- } of activeRuns) {
- // Authentication timeout requires an explicit ownership resolution, not
- // repeated reattachment or a process-gone guess on subsequent sweeps.
- if (isNativeRunnerOwnershipHeld(run)) continue;
- const nativeRun = run.runtimeMode === "native";
- const nativeProcessPidAlive =
- nativeRun && !!run.processPid && isProcessAlive(run.processPid);
- const nativeProcessGroupAlive =
- nativeRun &&
- !!run.processGroupId &&
- isProcessGroupAlive(run.processGroupId);
- const coordinatorOwnedByCurrentController =
- nativeRun &&
- nativeControllerBootId === currentNativeController.bootId &&
- nativeControllerPid === currentNativeController.pid &&
- nativeControllerProcessStartedAt?.getTime() ===
- currentNativeController.processStartedAt.getTime() &&
- !!nativeControllerLeaseExpiresAt &&
- nativeControllerLeaseExpiresAt.getTime() > now.getTime();
- const locallyTracked =
- runningProcesses.has(run.id) ||
- activeRunExecutions.has(run.id) ||
- coordinatorOwnedByCurrentController;
- if (
- nativeRun &&
- ([
- "awaiting_evidence",
- "awaiting_runner_reattach",
- "resuming_session",
- "bootstrap_incomplete",
- ].includes(nativeRecoveryState ?? "") ||
- nativeCoordinatorPhase === "retryable_failure")
- ) {
- continue;
- }
- const observedOwnerUnverified =
- nativeRun &&
- (nativeCoordinatorPhase === "observed" ||
- (nativeCoordinatorPhase === null &&
- run.nativePhase === "observed")) &&
- !resumedRunIds.has(run.id) &&
- !locallyTracked;
- // Persisted numeric process identifiers prove only that some process is
- // alive, not that Paperclip still owns it. Likewise an observed native
- // coordinator without a live in-process execution has no durable proof
- // that its prior provider owner stopped. Keep both cases running but
- // blocked: never signal, finalize, or retry them automatically. This gate
- // intentionally precedes resumedRunIds so a claim cannot bypass the
- // ownership check.
- if (
- !locallyTracked &&
- (nativeProcessPidAlive ||
- nativeProcessGroupAlive ||
- observedOwnerUnverified)
- ) {
- await markNativeOwnershipUnverified(run, {
- reason:
- nativeProcessPidAlive || nativeProcessGroupAlive
- ? "live_process_identifier"
- : "observed_owner_unverified",
- processPidAlive: nativeProcessPidAlive,
- processGroupAlive: nativeProcessGroupAlive,
- });
- continue;
- }
- if (resumedRunIds.has(run.id)) continue;
- if (locallyTracked) continue;
- if (await hasLiveLegacyController(db, run)) continue;
-
- // Apply staleness threshold to avoid false positives
- if (staleThresholdMs > 0) {
- const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
- if (now.getTime() - refTime < staleThresholdMs) continue;
- }
-
- const currentAdapterTracksLocalChild =
- isTrackedLocalChildProcessAdapter(adapterType);
- const tracksLegacyLocalChild =
- run.runtimeMode !== "native" && currentAdapterTracksLocalChild;
- // Native runner processes also persist child metadata, but they must not
- // inherit legacy retry or termination authority. Use their PID/group only
- // for a read-only liveness check so a lost in-memory handle cannot cause
- // overlapping provider/tool execution while that child is still alive.
- const checksPersistedChildLiveness =
- currentAdapterTracksLocalChild || run.runtimeMode === "native";
- const processPidAlive =
- checksPersistedChildLiveness &&
- run.processPid &&
- isProcessAlive(run.processPid);
- const processGroupAlive =
- checksPersistedChildLiveness &&
- run.processGroupId &&
- isProcessGroupAlive(run.processGroupId);
- if (
- (processPidAlive || processGroupAlive) &&
- readHotRestartAdoptionMetadata(parseObject(run.resultJson))
- ) {
- continue;
- }
- if (processPidAlive || processGroupAlive) {
- if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) {
- const detachedMessage = processPidAlive
- ? `Lost in-memory process handle, but child pid ${run.processPid} is still alive`
- : `Lost in-memory process handle, but persisted process group ${run.processGroupId} is still alive`;
- const detachedRun = await setRunStatus(run.id, "running", {
- error: detachedMessage,
- errorCode: DETACHED_PROCESS_ERROR_CODE,
- });
- if (detachedRun) {
- await appendRunEvent(detachedRun, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message: detachedMessage,
- payload: {
- processPid: run.processPid ?? null,
- processGroupId: run.processGroupId ?? null,
- ownedProcessHandle: false,
- },
- });
- }
- }
- continue;
- }
-
- const runContext = parseObject(run.contextSnapshot);
- const monitorIssueId = readNonEmptyString(runContext.issueId);
- const monitorNextCheckAt = monitorIssueId
- ? monitorNextCheckAtByIssue.get(`${run.companyId}:${monitorIssueId}`)
- : undefined;
- const monitorDispatchLostWithoutFutureWake =
- readNonEmptyString(runContext.wakeReason) === "issue_monitor_due" &&
- monitorNextCheckAt !== undefined &&
- (!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime());
- const shouldRetry =
- (run.processLossRetryCount ?? 0) < 1 &&
- ((tracksLegacyLocalChild &&
- (!!run.processPid || !!run.processGroupId)) ||
- monitorDispatchLostWithoutFutureWake);
- if (!(await revokeExpiredLegacyController(db, run))) continue;
- const baseMessage = buildProcessLossMessage(run);
- const processLossDiagnostic = buildProcessLossDiagnostic({
- run,
- nowMs: Date.now(),
- observerStartedAtMs: performance.timeOrigin,
- checksPersistedChildLiveness,
- retryEligible: shouldRetry,
- });
- const conversationContinuationEligible = await runUsedConversationAdapter(db, run);
-
- const failureWrite = await setRunStatusFromLive(
- run.id,
- "failed",
- ["running"],
- {
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
- errorCode: "process_lost",
- finishedAt: now,
- resultJson: (() => {
- const result = mergeRunStopMetadataForAgent(
- { adapterType, adapterConfig },
- "failed",
- {
- conversationContinuationEligible,
- resultJson: { ...parseObject(run.resultJson), processLossDiagnostic },
- errorCode: "process_lost",
- errorMessage: shouldRetry
- ? `${baseMessage}; retrying once`
- : baseMessage,
- },
- );
- return result;
- })(),
- },
- );
- if (!failureWrite.updated || !failureWrite.run) continue;
- let finalizedRun: typeof heartbeatRuns.$inferSelect | null =
- failureWrite.run;
- await setWakeupStatus(run.wakeupRequestId, "failed", {
- finishedAt: now,
- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
- });
- if (!finalizedRun) finalizedRun = await getRun(run.id);
- if (!finalizedRun) continue;
- finalizedRun =
- (await classifyAndPersistRunLiveness(
- finalizedRun,
- parseObject(finalizedRun.resultJson),
- )) ?? finalizedRun;
- await releaseEnvironmentLeasesForRun({
- runId: finalizedRun.id,
- companyId: finalizedRun.companyId,
- agentId: finalizedRun.agentId,
- status: finalizedRun.status,
- failureReason: finalizedRun.error ?? undefined,
- });
-
- let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;
- const retryAgent = await getAgent(run.agentId);
- if (shouldRetry) {
- if (retryAgent) {
- retriedRun = await enqueueProcessLossRetry(
- finalizedRun,
- retryAgent,
- now,
- );
- }
- } else if (retryAgent) {
- const scheduled =
- await scheduleInteractionContinuationInfrastructureRetryIfEligible(
- finalizedRun,
- retryAgent,
- );
- retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null;
- }
-
- if (!retriedRun) {
- await releaseIssueExecutionAndPromote(finalizedRun);
- }
-
- await appendRunEvent(finalizedRun, {
- eventType: "lifecycle",
- stream: "system",
- level: "error",
- message: shouldRetry
- ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
- : baseMessage,
- payload: {
- ...(run.processPid ? { processPid: run.processPid } : {}),
- ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
- ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
- },
- });
-
- await finalizeAgentStatus(run.agentId, "failed", baseMessage, {
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
- });
- await startNextQueuedRunForAgent(run.agentId);
- runningProcesses.delete(run.id);
- reaped.push(run.id);
- }
-
- if (reaped.length > 0) {
- logger.warn(
- { reapedCount: reaped.length, runIds: reaped },
- "reaped orphaned heartbeat runs",
+ .from(activityLog)
+ .where(
+ and(
+ eq(activityLog.companyId, run.companyId),
+ eq(activityLog.runId, run.id),
+ notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),
+ ),
);
- }
- // Recover an active lease whose run already ended before the same-tick
- // pending_cleanup sweep, so this tick can stop the recovered sandbox.
- // Isolate the sweep so its failure never hides the reaper result.
- try {
- const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({
- backoffMs: staleThresholdMs,
- });
- if (orphanedActiveLeaseSweep.recovered > 0) {
- logger.warn(
- { recovered: orphanedActiveLeaseSweep.recovered },
- "recovered orphaned active environment leases",
- );
- }
- } catch {
- // Log a constant errorKind only. The exception can carry a credential in
- // its name, code, message, cause, or stack, so the sweep never reads it.
- logger.error(
- { errorKind: ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND },
- "orphaned active environment lease sweep failed",
+ const [eventStats] = await db
+ .select({
+ count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
+ latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
+ })
+ .from(heartbeatRunEvents)
+ .where(
+ and(
+ eq(heartbeatRunEvents.companyId, run.companyId),
+ eq(heartbeatRunEvents.runId, run.id),
+ ),
);
- }
- // Retry stranded pending_cleanup leases on the same tick. Isolate the sweep
- // so its failure never hides the reaper result. The backoff equals the
- // reaper staleness threshold.
- try {
- const sweep = await sweepPendingCleanupLeases({
- backoffMs: staleThresholdMs,
- });
- if (sweep.destroyed > 0 || sweep.capped > 0) {
- logger.warn(
- {
- destroyed: sweep.destroyed,
- capped: sweep.capped,
- swept: sweep.swept,
- },
- "swept pending_cleanup environment leases",
- );
- }
- } catch {
- // Log a constant errorKind only. The exception can carry a credential in
- // its name, code, message, cause, or stack, so the sweep never reads it.
- logger.error(
- { errorKind: PENDING_CLEANUP_SWEEP_ERROR_KIND },
- "pending_cleanup lease sweep failed",
- );
- }
+ return {
+ runStatus: run.status,
+ issue,
+ resultJson: resultJson ?? run.resultJson ?? null,
+ issueCommentBodies,
+ continuationSummaryBody: continuationSummary?.body ?? null,
+ stdoutExcerpt: run.stdoutExcerpt ?? null,
+ stderrExcerpt: run.stderrExcerpt ?? null,
+ error: run.error ?? null,
+ errorCode: run.errorCode ?? null,
+ continuationAttempt,
+ evidence: {
+ issueCommentsCreated: countValue(commentStats?.count),
+ documentRevisionsCreated: countValue(documentStats?.count),
+ planDocumentRevisionsCreated: countValue(documentStats?.planCount),
+ workProductsCreated: countValue(workProductStats?.count),
+ workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
+ activityEventsCreated: countValue(activityStats?.count),
+ toolOrActionEventsCreated: countValue(eventStats?.count),
+ latestEvidenceAt: latestDate(
+ commentStats?.latestAt,
+ documentStats?.latestAt,
+ workProductStats?.latestAt,
+ workspaceOperationStats?.latestAt,
+ activityStats?.latestAt,
+ eventStats?.latestAt,
+ ),
+ },
+ };
+ }
- return { reaped: reaped.length, runIds: reaped };
+ async function classifyAndPersistRunLiveness(
+ run: typeof heartbeatRuns.$inferSelect,
+ resultJson?: Record<string, unknown> | null,
+ ) {
+ const authRepair = run.status === "failed"
+ ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {
+ logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");
+ return null;
+ })
+ : null;
+ const classification = classifyRunLiveness({
+ ...await buildRunLivenessInput(run, resultJson),
+ authenticationRepairRequested: Boolean(authRepair?.interactionId),
+ });
+ return db
+ .update(heartbeatRuns)
+ .set({
+ livenessState: classification.livenessState,
+ livenessReason: classification.livenessReason,
+ continuationAttempt: classification.continuationAttempt,
+ lastUsefulActionAt: classification.lastUsefulActionAt,
+ nextAction: classification.nextAction,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, run.id))
+ .returning()
+ .then((rows) => rows[0] ?? null);
}
async function resumeQueuedRuns() {
diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts
new file mode 100644
index 0000000000..a925edfaa7
--- /dev/null
+++ b/server/src/services/heartbeat/recovery.test.ts
@@ -0,0 +1,268 @@
+import { randomUUID } from "node:crypto";
+import type { Db } from "@paperclipai/db";
+import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
+import { eq } from "drizzle-orm";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import { environmentRuntimeService } from "../environment-runtime.js";
+import { environmentService } from "../environments.js";
+import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "./recovery.js";
+import type { HotRestartIntent } from "../hot-restart.js";
+import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js";
+import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js";
+
+type Run = typeof heartbeatRuns.$inferSelect;
+
+const native = vi.hoisted(() => ({
+ closeIdle: vi.fn(async () => ({ closed: 0, failed: 0 })),
+ detach: vi.fn(async () => 0),
+ finalizations: vi.fn(async (_db: Db, _ids?: string[], _options?: unknown) => undefined),
+ cleanups: vi.fn(async () => undefined),
+ drainMaintenance: vi.fn<() => Promise<void>>(async () => undefined),
+ claims: vi.fn<() => Promise<NativeRestartRecoveryDisposition[]>>(async () => []),
+}));
+const hotRestart = vi.hoisted(() => ({
+ read: vi.fn<() => Promise<HotRestartIntent | null>>(async () => null),
+}));
+vi.mock("../hot-restart.js", async (importOriginal) => ({
+ ...await importOriginal<typeof import("../hot-restart.js")>(),
+ readHotRestartIntent: hotRestart.read,
+}));
+vi.mock("../native-runtime/index.js", async (importOriginal) => ({
+ ...await importOriginal<typeof import("../native-runtime/index.js")>(),
+ closeIdleWarmNativeSessionsForRestart: native.closeIdle,
+ detachNativeSessionsForRestart: native.detach,
+ reconcileNativeFinalizations: native.finalizations,
+ reconcileRetainedNativeSessionCleanups: native.cleanups,
+ claimNativeRestartRecoveries: native.claims,
+}));
+vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({
+ ...await importOriginal<typeof import("../../vendor/paperclip-runner/index.js")>(),
+ drainRetainedRunnerdMaintenanceOperations: native.drainMaintenance,
+}));
+
+function callbacks(db: Db) {
+ return {
+ enterShutdown: vi.fn(),
+ getRun: vi.fn<HeartbeatRecoveryDependencies["getRun"]>(async () => null),
+ getAgent: vi.fn<HeartbeatRecoveryDependencies["getAgent"]>(async () => null),
+ appendRunEvent: vi.fn<HeartbeatRecoveryDependencies["appendRunEvent"]>(async () => undefined),
+ setRunStatusIfRunning: vi.fn<HeartbeatRecoveryDependencies["setRunStatusIfRunning"]>(async () => ({ updated: false, run: null })),
+ setRunStatusFromLive: vi.fn<HeartbeatRecoveryDependencies["setRunStatusFromLive"]>(async () => ({ updated: false, run: null })),
+ setRunStatus: vi.fn<HeartbeatRecoveryDependencies["setRunStatus"]>(async () => null),
+ setWakeupStatus: vi.fn(async () => undefined),
+ releaseIssueExecutionAndPromote: vi.fn(async () => undefined),
+ finalizeAgentStatus: vi.fn(async () => undefined),
+ environmentRuntime: environmentRuntimeService(db),
+ environmentsSvc: environmentService(db),
+ instructionCopies: {
+ recoverStopped: vi.fn(async () => 0),
+ recoverCaptured: vi.fn(async () => 0),
+ },
+ runtimeEnv: {},
+ activeRunExecutions: new Set<string>(),
+ activeRunExecutionPromises: new Set<Promise<void>>(),
+ scheduleBoundedRetryForRun: vi.fn<HeartbeatRecoveryDependencies["scheduleBoundedRetryForRun"]>(async () => ({ outcome: "retry_exhausted", retryReason: "transient_failure", attempt: 1, maxAttempts: 0 })),
+ scheduleInteractionContinuationInfrastructureRetryIfEligible: vi.fn<HeartbeatRecoveryDependencies["scheduleInteractionContinuationInfrastructureRetryIfEligible"]>(async () => null),
+ timerClaimWasFirstHeartbeat: vi.fn(() => undefined),
+ executeRun: vi.fn<HeartbeatRecoveryDependencies["executeRun"]>(async () => undefined),
+ scheduleNativeSessionResumeDispatch: vi.fn(),
+ cancelHeartbeatNativeRun: vi.fn(async () => undefined),
+ terminateHeartbeatRunProcess: vi.fn(async () => undefined),
+ mergeRunStopMetadataForAgent: vi.fn<HeartbeatRecoveryDependencies["mergeRunStopMetadataForAgent"]>((_agent, _outcome, options) => options?.resultJson ?? null),
+ classifyAndPersistRunLiveness: vi.fn<HeartbeatRecoveryDependencies["classifyAndPersistRunLiveness"]>(async run => run),
+ releaseEnvironmentLeasesForRun: vi.fn(async () => undefined),
+ acknowledgeRemoteStop: vi.fn(async () => undefined),
+ resumeRemoteStopComments: vi.fn(async () => undefined),
+ dispatchPendingNativeStatusWakeups: vi.fn(async () => undefined),
+ cancelRunInternal: vi.fn(async () => undefined),
+ startNextQueuedRunForAgent: vi.fn(async () => undefined),
+ } satisfies HeartbeatRecoveryDependencies;
+}
+
+function guardedDatabase() {
+ const access = vi.fn(() => { throw new Error("Unexpected recovery database access"); });
+ return { db: new Proxy({}, { get: access }) as Db, access };
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ hotRestart.read.mockResolvedValue(null);
+ native.closeIdle.mockResolvedValue({ closed: 0, failed: 0 });
+ native.finalizations.mockResolvedValue(undefined);
+ native.cleanups.mockResolvedValue(undefined);
+ native.drainMaintenance.mockResolvedValue(undefined);
+ native.claims.mockResolvedValue([]);
+});
+
+describe("heartbeat recovery module callbacks", () => {
+ it("constructs without database access, shutdown, or background work", () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ createHeartbeatRecovery(database.db, deps);
+ expect(database.access).not.toHaveBeenCalled();
+ for (const callback of Object.values(deps)) if (typeof callback === "function") expect(callback).not.toHaveBeenCalled();
+ for (const callback of Object.values(native)) expect(callback).not.toHaveBeenCalled();
+ });
+
+ it.each(["not_requested", "read_error"] as const)("enters shutdown before checkpointing for %s", async (mode) => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ if (mode === "read_error") hotRestart.read.mockRejectedValueOnce(new Error("unreadable intent"));
+ const recovery = createHeartbeatRecovery(database.db, deps);
+ expect(await recovery.prepareHotRestartShutdown("SIGTERM")).toEqual({ mode, skipDrain: false, activeRunIds: [] });
+ expect(native.closeIdle).toHaveBeenCalledAfter(deps.enterShutdown);
+ expect(hotRestart.read).toHaveBeenCalledAfter(native.closeIdle);
+ expect(database.access).not.toHaveBeenCalled();
+ });
+
+ it("does not adopt or query runs without a restart intent", async () => {
+ const database = guardedDatabase();
+ expect(await createHeartbeatRecovery(database.db, callbacks(database.db)).reconcileHotRestartAdoption()).toEqual({
+ mode: "not_requested", adoptedRunIds: [], finalizedWhileDownRunIds: [], lostRunIds: [], skippedRunIds: [],
+ });
+ expect(database.access).not.toHaveBeenCalled();
+ });
+
+ it("does not broaden an empty selective shutdown drain to every run", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ expect(await createHeartbeatRecovery(database.db, deps).drainRunningRunsForShutdown("SIGINT", new Date(), [])).toEqual({
+ interrupted: 0, interruptedRunIds: [], retryRunIds: [], restartSuspendedRunIds: [],
+ });
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled();
+ expect(deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
+ });
+
+ it.each(["running", "failed"] as const)("keeps a %s ownership hold when a status race wins", async (status) => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { id: "run", runtimeMode: "native", status } as Run;
+ const winner = { ...run, status: "cancelled" };
+ deps.setRunStatusFromLive.mockResolvedValue({ updated: false, run: winner });
+ expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "live_process_identifier" })).toBe(winner);
+ expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", status, [status], expect.objectContaining({ errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE }));
+ expect(deps.appendRunEvent).not.toHaveBeenCalled();
+ expect(database.access).not.toHaveBeenCalled();
+ });
+
+ it("persists an authentication ownership hold before its event without terminating or retrying", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { id: "run", runtimeMode: "native", status: "running" } as Run;
+ const blocked = { ...run, nativePhase: "terminal_failure", errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE };
+ deps.setRunStatusFromLive.mockResolvedValue({ updated: true, run: blocked });
+ expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "adopted_runner_authentication_timeout" })).toBe(blocked);
+ expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", "running", ["running"], expect.objectContaining({ nativePhase: "terminal_failure" }));
+ expect(deps.appendRunEvent).toHaveBeenCalledAfter(deps.setRunStatusFromLive);
+ expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled();
+ expect(deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
+ expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
+ expect(database.access).not.toHaveBeenCalled();
+ });
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+const describePostgres = support.supported ? describe : describe.skip;
+if (!support.supported) console.warn(`Skipping recovery module database tests: ${support.reason}`);
+
+describePostgres("heartbeat recovery module ownership", () => {
+ let db: Db;
+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+ beforeAll(async () => {
+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-recovery-module-");
+ db = createDb(database.connectionString);
+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+ afterAll(async () => { await database?.cleanup(); });
+
+ async function seedRun() {
+ const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), issueId = randomUUID();
+ await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `T${companyId.slice(0, 6)}` });
+ await db.insert(agents).values({ id: agentId, companyId, name: "Recovery", role: "engineer", adapterType: "process" });
+ await db.insert(issues).values({ id: issueId, companyId, title: "Recover this run", status: "in_progress", assigneeAgentId: agentId });
+ return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];
+ }
+
+ it("uses the shared live-execution set after construction across two factories", async () => {
+ const run = await seedRun();
+ const deps = callbacks(db);
+ const first = createHeartbeatRecovery(db, deps);
+ const second = createHeartbeatRecovery(db, deps);
+ deps.activeRunExecutions.add(run.id);
+ expect(await first.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
+ expect(await second.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
+ await Promise.all([...deps.activeRunExecutionPromises]);
+ await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
+ });
+
+ it("settles persisted native results before claiming restart authority and tracks the exact execution", async () => {
+ const deps = callbacks(db);
+ const claim: NativeRestartRecoveryClaim = { kind: "bootstrap_incomplete", runId: randomUUID(), leaseOwner: "claimed-owner", controllerGeneration: 3, providerAttempt: 2, restartKind: "hard", recoveryRequestId: null };
+ native.claims.mockResolvedValue([claim]);
+ let finish!: () => void;
+ const execution = new Promise<void>(resolve => { finish = resolve; });
+ deps.executeRun.mockReturnValue(execution);
+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
+ expect(result.claims).toEqual([claim]);
+ expect(native.claims).toHaveBeenCalledAfter(native.finalizations);
+ expect(deps.executeRun).toHaveBeenCalledWith(claim.runId, { nativeLeaseOwner: claim.leaseOwner, nativeRestartRecovery: claim });
+ expect(deps.activeRunExecutionPromises.size).toBeGreaterThan(0);
+ finish();
+ await Promise.all([...deps.activeRunExecutionPromises]);
+ await vi.waitFor(() => expect(deps.activeRunExecutionPromises.size).toBe(0));
+ });
+
+ it("does not claim or execute recovery when persisted finalization fails", async () => {
+ const deps = callbacks(db);
+ native.finalizations.mockRejectedValueOnce(new Error("result settlement failed"));
+ await expect(createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart()).rejects.toThrow("result settlement failed");
+ expect(native.claims).not.toHaveBeenCalled();
+ expect(deps.executeRun).not.toHaveBeenCalled();
+ expect(native.cleanups).not.toHaveBeenCalled();
+ });
+
+ it("keeps blocked and ambiguous ownership out of execution", async () => {
+ const deps = callbacks(db);
+ const dispositions: NativeRestartRecoveryDisposition[] = [
+ { kind: "blocked", runId: randomUUID(), reason: "conflicting_owner" },
+ { kind: "awaiting_evidence", runId: randomUUID(), reason: "missing_stop_evidence" },
+ ];
+ native.claims.mockResolvedValue(dispositions);
+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
+ expect(result.blockedRunIds).toEqual([dispositions[0].runId]);
+ expect(result.awaitingEvidenceRunIds).toEqual([dispositions[1].runId]);
+ expect(result.claims).toEqual([]);
+ expect(deps.executeRun).not.toHaveBeenCalled();
+ await Promise.all([...deps.activeRunExecutionPromises]);
+ });
+
+ it("rearms the persisted native retry deadline through the service callback", async () => {
+ const run = await seedRun();
+ const nextAttemptAt = new Date(Date.now() + 60_000);
+ await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status: "failed" }).where(eq(heartbeatRuns.id, run.id));
+ await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt });
+ const deps = callbacks(db);
+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
+ expect(result.scheduledRetryRunIds).toEqual([run.id]);
+ expect(deps.scheduleNativeSessionResumeDispatch).toHaveBeenCalledWith(run.id, nextAttemptAt);
+ expect(deps.executeRun).not.toHaveBeenCalled();
+ await Promise.all([...deps.activeRunExecutionPromises]);
+ await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
+ await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
+ });
+
+ it("retains shutdown ownership until background native cleanup physically settles", async () => {
+ const deps = callbacks(db);
+ let settle!: () => void;
+ native.drainMaintenance.mockReturnValueOnce(new Promise<void>(resolve => { settle = resolve; }));
+ await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
+ await vi.waitFor(() => expect(native.drainMaintenance).toHaveBeenCalled());
+ expect(deps.activeRunExecutionPromises.size).toBe(1);
+ settle();
+ await Promise.all([...deps.activeRunExecutionPromises]);
+ await vi.waitFor(() => expect(deps.activeRunExecutionPromises.size).toBe(0));
+ });
+});
diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts
new file mode 100644
index 0000000000..4798251a1f
--- /dev/null
+++ b/server/src/services/heartbeat/recovery.ts
@@ -0,0 +1,2423 @@
+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
+import { CHAT_COMPLETION_WAKE_REASON } from "../chat-completion-delivery.js";
+import {
+ hasStopOnlyCleanup,
+ settleStopOnlyCleanup,
+} from "../sandbox-stop-and-retain.js";
+import { recordNativeLocalProcessStop } from "../native-local-process-stop.js";
+import {
+ legacyControllerBootId,
+ hasLiveLegacyController,
+ revokeExpiredLegacyController,
+} from "../legacy-controller-lease.js";
+import { remoteTerminationReceipt } from "../remote-execution-termination.js";
+import { runUsedConversationAdapter } from "../conversation-continuation.js";
+import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";
+import { randomUUID } from "node:crypto";
+import {
+ and,
+ asc,
+ eq,
+ gt,
+ inArray,
+ isNull,
+ lte,
+ ne,
+ or,
+ sql,
+} from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import {
+ agents,
+ environmentLeases,
+ heartbeatRuns,
+ issues,
+ nativeRunFinalizations,
+} from "@paperclipai/db";
+import { logger } from "../../middleware/logger.js";
+import {
+ claimNativeRestartRecoveries,
+ closeIdleWarmNativeSessionsForRestart,
+ currentNativeControllerIdentity,
+ dispatchNativeSessionResumptions,
+ detachNativeSessionsForRestart,
+ isNativeSessionId,
+ readNativeWorkspaceSyncReference,
+ type NativeRestartRecoveryClaim,
+ reconcileNativeFinalizations,
+ reconcileRetainedNativeSessionCleanup,
+ reconcileRetainedNativeSessionCleanups,
+} from "../native-runtime/index.js";
+import { drainRetainedRunnerdMaintenanceOperations } from "../../vendor/paperclip-runner/index.js";
+import { runningProcesses } from "../../adapters/index.js";
+import { parseObject } from "../../adapters/utils.js";
+import {
+ isNativeRunnerOwnershipHeld,
+ NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
+} from "../native-runtime/native-runner-ownership.js";
+import { performance } from "node:perf_hooks";
+import { buildProcessLossDiagnostic } from "../process-loss-diagnostics.js";
+import { logActivity } from "../activity-log.js";
+import { releaseRuntimeServicesForRun } from "../workspace-runtime.js";
+import { isProcessGroupAlive } from "../local-service-supervisor.js";
+import {
+ findMissingHotRestartSnapshotRunIds,
+ readHotRestartIntent,
+ removeHotRestartIntent,
+ shouldHonorHotRestartIntentForProcess,
+ writeHotRestartReport,
+ writeHotRestartShutdownSnapshot,
+ type HotRestartIntentRun,
+ type HotRestartReportRun,
+} from "../hot-restart.js";
+import { serverVersion } from "../../version.js";
+
+import type { environmentService } from "../environments.js";
+import type { environmentRuntimeService, ProviderResourceDisposition } from "../environment-runtime.js";
+import type { agentInstructionWorkingCopyService } from "../agent-instruction-working-copies.js";
+import type { createHeartbeatRunState } from "./run-state.js";
+import type { HeartbeatRetryDependencies, createHeartbeatRetries } from "./retries.js";
+
+type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
+type HeartbeatRetries = ReturnType<typeof createHeartbeatRetries>;
+
+/** Recovery owns reconciliation; the service retains execution and lifecycle effects. */
+export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDependencies,
+ "getAgent" | "appendRunEvent" | "setRunStatusIfRunning" |
+ "setWakeupStatus" | "releaseIssueExecutionAndPromote" | "finalizeAgentStatus"> {
+ enterShutdown: () => void;
+ getRun: (runId: string, options?: Parameters<ReturnType<typeof createHeartbeatRunState>["getRun"]>[1]) => Promise<
+ Awaited<ReturnType<ReturnType<typeof createHeartbeatRunState>["getRun"]>> | null
+ >;
+ environmentRuntime: ReturnType<typeof environmentRuntimeService>;
+ environmentsSvc: Pick<ReturnType<typeof environmentService>, "getById" | "getLeaseById" | "releaseLease">;
+ instructionCopies: Pick<ReturnType<typeof agentInstructionWorkingCopyService>, "recoverStopped" | "recoverCaptured">;
+ runtimeEnv: Record<string, string | undefined>;
+ // Preserve the module-wide sets shared by scheduler and route service instances.
+ activeRunExecutions: Set<string>;
+ activeRunExecutionPromises: Set<Promise<void>>;
+ scheduleBoundedRetryForRun: HeartbeatRetries["scheduleBoundedRetryForRun"];
+ scheduleInteractionContinuationInfrastructureRetryIfEligible: HeartbeatRetries["scheduleInteractionContinuationInfrastructureRetryIfEligible"];
+ timerClaimWasFirstHeartbeat: HeartbeatRetries["timerClaimWasFirstHeartbeat"];
+ executeRun: (runId: string, options?: {
+ nativeLeaseOwner?: string;
+ nativeRestartRecovery?: NativeRestartRecoveryClaim;
+ }) => Promise<void>;
+ scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date) => void;
+ cancelHeartbeatNativeRun: (input: {
+ db: Db; runId: string; reason: string; runtimeMode: string | null;
+ }) => Promise<unknown>;
+ terminateHeartbeatRunProcess: (input: {
+ pid: number | null | undefined;
+ processGroupId: number | null | undefined;
+ graceMs?: number;
+ }) => Promise<void>;
+ setRunStatus: (runId: string, status: string, patch?: Partial<typeof heartbeatRuns.$inferInsert>) => Promise<HeartbeatRun | null>;
+ setRunStatusFromLive: (runId: string, status: string, fromStatuses: string[], patch?: Partial<typeof heartbeatRuns.$inferInsert>) => Promise<{
+ updated: boolean;
+ run: HeartbeatRun | null;
+ }>;
+ mergeRunStopMetadataForAgent: (
+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
+ options?: {
+ resultJson?: Record<string, unknown> | null;
+ conversationContinuationEligible?: boolean;
+ errorCode?: string | null;
+ errorMessage?: string | null;
+ },
+ ) => Record<string, unknown> | null;
+ classifyAndPersistRunLiveness: (run: HeartbeatRun, resultJson?: Record<string, unknown> | null) => Promise<HeartbeatRun | null>;
+ releaseEnvironmentLeasesForRun: (input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ status: string | null | undefined;
+ failureReason?: string | null;
+ providerResourceDisposition?: ProviderResourceDisposition;
+ }) => Promise<void>;
+ acknowledgeRemoteStop: (runId: string, companyId: string) => Promise<void>;
+ resumeRemoteStopComments: (run: HeartbeatRun) => Promise<void>;
+ dispatchPendingNativeStatusWakeups: () => Promise<unknown>;
+ cancelRunInternal: (runId: string, reason?: string, options?: { resultJson?: Record<string, unknown> }) => Promise<unknown>;
+ startNextQueuedRunForAgent: (agentId: string) => Promise<unknown>;
+}
+
+export const DETACHED_PROCESS_ERROR_CODE = "process_detached";
+
+const NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE =
+ "Native execution ownership could not be verified; automatic recovery is blocked";
+
+// The reaper sweeps at most this many pending_cleanup leases per tick.
+const PENDING_CLEANUP_SWEEP_PAGE_SIZE = 20;
+
+const pendingCleanupAttemptsInFlight = new Set<string>();
+
+// Escalate and slow cleanup after this many attempts; never abandon a live lease.
+const PENDING_CLEANUP_SWEEP_ATTEMPT_CAP = 5;
+
+// The reaper stores its retry state under these keys in the lease metadata.
+const PENDING_CLEANUP_ATTEMPTS_METADATA_KEY = "pendingCleanupRetryAttempts";
+
+const PENDING_CLEANUP_CAP_WARNED_METADATA_KEY = "pendingCleanupRetryCapWarned";
+
+// The reaper sweeps at most this many orphaned active leases per tick.
+const ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE = 20;
+
+// A provider or plugin destroy rejection can carry a bearer credential, a
+// signed URL, or provider response detail in its name, code, message, cause, or
+// stack. The exception fields cross the server boundary, so they are not a
+// trusted enum. The pending_cleanup sweep logs never read the exception. Each
+// catch site logs a constant, locally generated `errorKind` instead.
+const PENDING_CLEANUP_RETRY_ERROR_KIND = "destroy_failed";
+
+const PENDING_CLEANUP_SWEEP_ERROR_KIND = "sweep_failed";
+
+const ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND = "orphaned_active_lease_sweep_failed";
+
+// Read the stored retry attempt count as a safe value, directly in SQL. A
+// provider can write a malformed value under the attempts key. The type guard
+// makes any non-number value read as zero. The reader computes as numeric and
+// never casts to int, so a finite number outside the 32-bit range (for example
+// 1e300) never throws. The reader clamps a negative value to zero and a positive
+// value to the attempt cap. One malformed lease therefore never aborts the page
+// sweep. This matches the TypeScript reader `readPendingCleanupRetryAttempts`,
+// which clamps to the same range. The claim predicate compares the two readers,
+// so both must yield the same value for every input.
+function pendingCleanupAttemptsSql() {
+ return sql`
+ case
+ when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}) = 'number'
+ then least(
+ greatest(
+ floor((${environmentLeases.metadata} ->> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY})::numeric),
+ 0
+ ),
+ ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}
+ )
+ else 0
+ end`;
+}
+
+function pendingCleanupRetryDueSql(explicitRetry = false) {
+ // Ownership renewal and retry cooldown are separate clocks. A late renewal
+ // cannot change when a completed attempt may retry. Older claims used the
+ // retry field for both clocks, so retain that fallback until they settle.
+ const deadline = sql`case
+ when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'
+ and ${environmentLeases.metadata} ? 'pendingCleanupLeaseExpiresAtMs'
+ then ${environmentLeases.metadata}->'pendingCleanupLeaseExpiresAtMs'
+ else ${environmentLeases.metadata}->'pendingCleanupRetryAfterMs' end`;
+ return sql`case
+ when ${explicitRetry} and coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true' then true
+ when jsonb_typeof(${deadline}) = 'number'
+ then (${deadline})::numeric <= ${Date.now()}
+ or (${deadline})::numeric > ${Date.now() + 30 * 60_000 + 1_000}
+ else true end`;
+}
+
+// Choose the `jsonb_set` target root. A provider can write a scalar or array
+// metadata root. `jsonb_set` fails on a non-object root, so the reader uses the
+// stored metadata only when its root is an object. A NULL, scalar, or array root
+// reads as an empty object. `jsonb_typeof(NULL)` is NULL, so the else branch also
+// covers a NULL root.
+function pendingCleanupMetadataObjectSql() {
+ return sql`case when jsonb_typeof(${environmentLeases.metadata}) = 'object' then ${environmentLeases.metadata} else '{}'::jsonb end`;
+}
+
+// Read the stored cap-warned flag as a safe boolean, directly in SQL. A
+// malformed value reads as false, so the boolean cast never throws.
+function pendingCleanupCapWarnedSql() {
+ return sql`coalesce(
+ case
+ when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}) = 'boolean'
+ then (${environmentLeases.metadata} ->> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY})::boolean
+ else false
+ end,
+ false
+ )`;
+}
+
+export const CANCELLABLE_HEARTBEAT_RUN_STATUSES = [
+ "queued",
+ "running",
+ "scheduled_retry",
+] as const;
+
+const NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY = "nativeQuestionCancellation";
+
+export const HEARTBEAT_RUN_TERMINAL_STATUSES = [
+ "succeeded",
+ "interrupted",
+ "failed",
+ "cancelled",
+ "timed_out",
+] as const;
+
+const SESSIONED_LOCAL_ADAPTERS = new Set([
+ "claude_local",
+ "codex_local",
+ "cursor",
+ "gemini_local",
+ "hermes_local",
+ "kimi_local",
+ "opencode_local",
+ "pi_local",
+]);
+
+function isTrackedLocalChildProcessAdapter(adapterType: string) {
+ return SESSIONED_LOCAL_ADAPTERS.has(adapterType);
+}
+
+// A positive liveness check means some process currently owns the PID.
+// On Linux, PIDs can be recycled, so this is a best-effort signal rather
+// than proof that the original child is still alive.
+export function isProcessAlive(pid: number | null | undefined) {
+ if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)
+ return false;
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException | undefined)?.code;
+ if (code === "EPERM") return true;
+ if (code === "ESRCH") return false;
+ return false;
+ }
+}
+
+function buildProcessLossMessage(
+ run: {
+ processPid: number | null;
+ processGroupId: number | null;
+ },
+ options?: { descendantOnly?: boolean },
+) {
+ if (options?.descendantOnly && run.processGroupId) {
+ return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`;
+ }
+ if (run.processPid) {
+ return `Process lost -- child pid ${run.processPid} is no longer running`;
+ }
+ if (run.processGroupId) {
+ return `Process lost -- process group ${run.processGroupId} is no longer running`;
+ }
+ return "Process lost -- server may have restarted";
+}
+
+function readHotRestartAdoptionMetadata(
+ resultJson: Record<string, unknown> | null | undefined,
+) {
+ const result = parseObject(resultJson);
+ const hotRestart = parseObject(result.hotRestart);
+ if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string")
+ return null;
+ return hotRestart;
+}
+
+function mergeHotRestartAdoptionResultJson(
+ resultJson: Record<string, unknown> | null | undefined,
+ input: {
+ adoptedAt: Date;
+ previousServerPid: number;
+ newServerPid: number;
+ previousServerVersion: string | null;
+ newServerVersion: string;
+ processPid: number | null;
+ processGroupId: number | null;
+ },
+) {
+ const result = parseObject(resultJson);
+ const existing = parseObject(result.hotRestart);
+ return {
+ ...result,
+ hotRestart: {
+ ...existing,
+ adopted: true,
+ adoptedAt: input.adoptedAt.toISOString(),
+ previousServerPid: input.previousServerPid,
+ newServerPid: input.newServerPid,
+ previousServerVersion: input.previousServerVersion,
+ newServerVersion: input.newServerVersion,
+ processPid: input.processPid,
+ processGroupId: input.processGroupId,
+ },
+ };
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+/** Restart recovery shares the service lifecycle callbacks and execution ownership. */
+export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryDependencies) {
+ const {
+ enterShutdown,
+ scheduleBoundedRetryForRun,
+ appendRunEvent,
+ environmentRuntime,
+ scheduleNativeSessionResumeDispatch,
+ getRun,
+ executeRun,
+ activeRunExecutionPromises,
+ cancelHeartbeatNativeRun,
+ terminateHeartbeatRunProcess,
+ setRunStatusIfRunning,
+ mergeRunStopMetadataForAgent,
+ setWakeupStatus,
+ classifyAndPersistRunLiveness,
+ releaseEnvironmentLeasesForRun,
+ releaseIssueExecutionAndPromote,
+ finalizeAgentStatus,
+ timerClaimWasFirstHeartbeat,
+ environmentsSvc,
+ acknowledgeRemoteStop,
+ resumeRemoteStopComments,
+ setRunStatusFromLive,
+ instructionCopies,
+ dispatchPendingNativeStatusWakeups,
+ runtimeEnv,
+ cancelRunInternal,
+ activeRunExecutions,
+ setRunStatus,
+ getAgent,
+ scheduleInteractionContinuationInfrastructureRetryIfEligible,
+ startNextQueuedRunForAgent,
+ } = dependencies;
+
+ async function enqueueProcessLossRetry(
+ run: typeof heartbeatRuns.$inferSelect,
+ agent: typeof agents.$inferSelect,
+ now: Date,
+ ) {
+ // Completion deliveries own their durable bounded retry and reply identity.
+ // A second process-loss retry would compete for the same outbox input.
+ if (run.contextSnapshot?.wakeReason === CHAT_COMPLETION_WAKE_REASON &&
+ Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds)) return null;
+ // Native sessions have their own fenced same-run controller. Legacy
+ // bootstrap recovery shares the durable delay and incident counter with
+ // transient retries; process loss must not open a second retry budget.
+ if (run.runtimeMode === "native" || await legacyExecutionNeedsReconciliationWithEvidence(db, run))
+ return null;
+ const scheduled = await scheduleBoundedRetryForRun(run, agent, { now });
+ return scheduled.outcome === "scheduled" ? scheduled.run : null;
+ }
+
+ function toHotRestartIntentRun(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ adapterType: string;
+ }): HotRestartIntentRun {
+ const context = parseObject(input.run.contextSnapshot);
+ return {
+ runId: input.run.id,
+ companyId: input.run.companyId,
+ agentId: input.run.agentId,
+ adapterType: input.adapterType,
+ status: input.run.status,
+ processPid: input.run.processPid ?? null,
+ processGroupId: input.run.processGroupId ?? null,
+ issueId: readNonEmptyString(context.issueId),
+ runtimeMode: input.run.runtimeMode,
+ nativeSessionId: input.run.nativeSessionId,
+ runnerInstanceId: input.run.runnerInstanceId,
+ processStartedAt: input.run.processStartedAt?.toISOString() ?? null,
+ };
+ }
+
+ function isServerStdioBoundHotRestartRun(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ adapterType: string;
+ adapterConfig: unknown;
+ }) {
+ const context = parseObject(input.run.contextSnapshot);
+ if (
+ context.processTopology === "server_stdio" ||
+ context.executionEngine === "acp"
+ ) {
+ return true;
+ }
+ if (
+ context.processTopology === "detached" ||
+ context.executionEngine === "cli"
+ ) {
+ return false;
+ }
+ if (
+ !["claude_local", "codex_local", "gemini_local"].includes(
+ input.adapterType,
+ )
+ ) {
+ return false;
+ }
+ return (
+ readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli"
+ );
+ }
+
+ async function prepareHotRestartShutdown(
+ signal: "SIGINT" | "SIGTERM",
+ now = new Date(),
+ ) {
+ enterShutdown();
+ const idleSessions = await closeIdleWarmNativeSessionsForRestart();
+ if (idleSessions.failed > 0) {
+ logger.warn({ idleSessions }, "idle native sessions could not checkpoint before controller shutdown");
+ }
+ let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;
+ try {
+ intent = await readHotRestartIntent();
+ } catch (err) {
+ logger.warn(
+ { err },
+ "failed to read hot-restart intent; falling back to normal shutdown drain",
+ );
+ return {
+ mode: "read_error" as const,
+ skipDrain: false as const,
+ activeRunIds: [] as string[],
+ };
+ }
+
+ if (!intent)
+ return {
+ mode: "not_requested" as const,
+ skipDrain: false as const,
+ activeRunIds: [] as string[],
+ };
+ if (intent.drainRequired)
+ return {
+ mode: "drain_required" as const,
+ skipDrain: false as const,
+ activeRunIds: [] as string[],
+ };
+ if (!shouldHonorHotRestartIntentForProcess(intent)) {
+ logger.warn(
+ { expectedPid: intent.previousServerPid, currentPid: process.pid },
+ "hot-restart intent targets a different server pid; falling back to normal shutdown drain",
+ );
+ return {
+ mode: "pid_mismatch" as const,
+ skipDrain: false as const,
+ activeRunIds: [] as string[],
+ };
+ }
+
+ const activeRuns = await db
+ .select({
+ run: heartbeatRuns,
+ adapterType: agents.adapterType,
+ adapterConfig: agents.adapterConfig,
+ })
+ .from(heartbeatRuns)
+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+ .where(eq(heartbeatRuns.status, "running"));
+ const snapshotRuns = activeRuns.map(toHotRestartIntentRun);
+ const intentWithVersion = {
+ ...intent,
+ previousServerVersion: intent.previousServerVersion ?? serverVersion,
+ };
+
+ const serverStdioRuns = activeRuns.filter(isServerStdioBoundHotRestartRun);
+ if (serverStdioRuns.length > 0) {
+ const activeServerStdioRunIds = serverStdioRuns.map(({ run }) => run.id);
+ await writeHotRestartShutdownSnapshot({
+ intent: intentWithVersion,
+ signal,
+ activeRuns: snapshotRuns,
+ drainReason: "active_acp_run",
+ drainRunIds: activeServerStdioRunIds,
+ capturedAt: now,
+ });
+
+ logger.warn(
+ {
+ signal,
+ previousServerPid: intent.previousServerPid,
+ activeRunIds: snapshotRuns.map((run) => run.runId),
+ activeServerStdioRunIds,
+ drainReason: "active_acp_run",
+ },
+ "server-stdio agent run prevents hot-restart adoption; using graceful drain and retry",
+ );
+
+ return {
+ mode: "acp_drain_required" as const,
+ skipDrain: false as const,
+ activeRunIds: snapshotRuns.map((run) => run.runId),
+ activeAcpRunIds: activeServerStdioRunIds,
+ drainRunIds: activeServerStdioRunIds,
+ drainReason: "active_acp_run" as const,
+ };
+ }
+
+ await writeHotRestartShutdownSnapshot({
+ intent: intentWithVersion,
+ signal,
+ activeRuns: snapshotRuns,
+ capturedAt: now,
+ });
+
+ for (const { run } of activeRuns) {
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "info",
+ message:
+ "Hot restart requested; leaving child process alive for startup adoption",
+ payload: {
+ signal,
+ previousServerPid: intent.previousServerPid,
+ previousServerVersion: intentWithVersion.previousServerVersion,
+ processPid: run.processPid ?? null,
+ processGroupId: run.processGroupId ?? null,
+ },
+ });
+ }
+
+ const nativeRunIds = activeRuns
+ .filter(
+ ({ run, adapterType }) =>
+ adapterType === "paperclip_runner" &&
+ isNativeSessionId(run.nativeSessionId),
+ )
+ .map(({ run }) => run.id);
+ const detachedNativeSessions =
+ await detachNativeSessionsForRestart(nativeRunIds);
+
+ logger.info(
+ {
+ signal,
+ previousServerPid: intent.previousServerPid,
+ activeRunIds: snapshotRuns.map((run) => run.runId),
+ detachedNativeSessions,
+ },
+ "hot-restart shutdown snapshot captured; skipping graceful run drain",
+ );
+
+ return {
+ mode: "hot_restart" as const,
+ skipDrain: true as const,
+ activeRunIds: snapshotRuns.map((run) => run.runId),
+ };
+ }
+
+ async function reconcileHotRestartAdoption(now = new Date()) {
+ let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;
+ try {
+ intent = await readHotRestartIntent();
+ } catch (err) {
+ logger.warn(
+ { err },
+ "failed to read hot-restart intent on startup; skipping adoption",
+ );
+ return {
+ mode: "read_error" as const,
+ adoptedRunIds: [] as string[],
+ finalizedWhileDownRunIds: [] as string[],
+ lostRunIds: [] as string[],
+ skippedRunIds: [] as string[],
+ };
+ }
+ if (!intent) {
+ return {
+ mode: "not_requested" as const,
+ adoptedRunIds: [] as string[],
+ finalizedWhileDownRunIds: [] as string[],
+ lostRunIds: [] as string[],
+ skippedRunIds: [] as string[],
+ };
+ }
+
+ if (!intent.shutdownSnapshot) {
+ const log = intent.drainRequired
+ ? logger.info.bind(logger)
+ : logger.warn.bind(logger);
+ log(
+ {
+ previousServerPid: intent.previousServerPid,
+ preflightActiveRunIds: intent.preflightActiveRunIds,
+ drainReason: intent.drainReason ?? null,
+ },
+ intent.drainRequired
+ ? "drain-required restart intent has no adoption snapshot"
+ : "hot-restart intent present but shutdown snapshot is missing; no runs can be adopted",
+ );
+ }
+ const candidates = intent.shutdownSnapshot?.activeRuns ?? [];
+ const missingSnapshotRunIds = findMissingHotRestartSnapshotRunIds(intent);
+ const reconciliationRunIds = [
+ ...new Set([
+ ...candidates.map((run) => run.runId),
+ ...missingSnapshotRunIds,
+ ]),
+ ];
+ const currentRows =
+ reconciliationRunIds.length > 0
+ ? await db
+ .select({
+ run: heartbeatRuns,
+ adapterType: agents.adapterType,
+ })
+ .from(heartbeatRuns)
+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+ .where(inArray(heartbeatRuns.id, reconciliationRunIds))
+ : [];
+ const currentByRunId = new Map(currentRows.map((row) => [row.run.id, row]));
+
+ const reportRuns: HotRestartReportRun[] = [];
+ const adoptedRunIds: string[] = [];
+ const finalizedWhileDownRunIds: string[] = [];
+ const lostRunIds: string[] = [];
+ const skippedRunIds: string[] = [];
+
+ const classify = (
+ candidate: HotRestartIntentRun,
+ classification: HotRestartReportRun["classification"],
+ reason: string,
+ patch?: Partial<HotRestartIntentRun>,
+ ) => {
+ const run = {
+ ...candidate,
+ ...patch,
+ classification,
+ reason,
+ } satisfies HotRestartReportRun;
+ reportRuns.push(run);
+ if (classification === "adopted") adoptedRunIds.push(candidate.runId);
+ else if (classification === "finalized_while_down")
+ finalizedWhileDownRunIds.push(candidate.runId);
+ else if (classification === "lost") lostRunIds.push(candidate.runId);
+ else skippedRunIds.push(candidate.runId);
+ };
+
+ for (const runId of missingSnapshotRunIds) {
+ const current = currentByRunId.get(runId);
+ if (!current) {
+ finalizedWhileDownRunIds.push(runId);
+ continue;
+ }
+
+ const candidate = toHotRestartIntentRun(current);
+ if (current.run.status !== "running") {
+ classify(
+ candidate,
+ "finalized_while_down",
+ `run_status_${current.run.status}`,
+ );
+ } else {
+ classify(candidate, "lost", "missing_shutdown_snapshot");
+ }
+ }
+
+ if (lostRunIds.length > 0) {
+ logger.error(
+ { previousServerPid: intent.previousServerPid, lostRunIds },
+ "hot-restart shutdown snapshot omitted live preflight runs; reporting them as lost",
+ );
+ }
+
+ for (const candidate of candidates) {
+ const current = currentByRunId.get(candidate.runId);
+ if (!current) {
+ classify(candidate, "finalized_while_down", "run_row_missing");
+ continue;
+ }
+
+ const { run, adapterType } = current;
+ const patch = {
+ adapterType,
+ status: run.status,
+ processPid: run.processPid ?? candidate.processPid,
+ processGroupId: run.processGroupId ?? candidate.processGroupId,
+ };
+
+ if (run.status !== "running") {
+ classify(
+ candidate,
+ "finalized_while_down",
+ `run_status_${run.status}`,
+ patch,
+ );
+ continue;
+ }
+
+ const hasSelectiveAcpDrain =
+ intent.drainReason === "active_acp_run" &&
+ (intent.drainRunIds?.length ?? 0) > 0;
+ if (
+ hasSelectiveAcpDrain &&
+ intent.drainRunIds?.includes(candidate.runId)
+ ) {
+ // A selective ACP drain is expected to persist a terminal row before
+ // the new server starts. If the process was terminated but that write
+ // failed, surface the run as lost instead of hiding it as an expected
+ // drain skip.
+ classify(candidate, "lost", "selective_drain_not_finalized", patch);
+ continue;
+ }
+ if (intent.drainRequired && !hasSelectiveAcpDrain) {
+ classify(candidate, "skipped", "drain_required", patch);
+ continue;
+ }
+
+ if (run.runtimeMode === "native" && adapterType === "paperclip_runner") {
+ classify(candidate, "skipped", "native_restart_recovery_owned", patch);
+ continue;
+ }
+
+ if (!isTrackedLocalChildProcessAdapter(adapterType)) {
+ classify(
+ candidate,
+ "skipped",
+ "adapter_not_local_child_process",
+ patch,
+ );
+ continue;
+ }
+
+ const processPid = run.processPid ?? candidate.processPid;
+ const processGroupId = run.processGroupId ?? candidate.processGroupId;
+ const processPidAlive = isProcessAlive(processPid);
+ const processGroupAlive = isProcessGroupAlive(processGroupId);
+ if (!processPid && !processGroupId) {
+ classify(candidate, "lost", "missing_process_metadata", patch);
+ continue;
+ }
+ if (!processPidAlive && !processGroupAlive) {
+ classify(candidate, "lost", "process_not_alive", patch);
+ continue;
+ }
+
+ const resultJson = mergeHotRestartAdoptionResultJson(
+ parseObject(run.resultJson),
+ {
+ adoptedAt: now,
+ previousServerPid: intent.previousServerPid,
+ newServerPid: process.pid,
+ previousServerVersion: intent.previousServerVersion,
+ newServerVersion: serverVersion,
+ processPid,
+ processGroupId,
+ },
+ );
+ const updated = await db
+ .update(heartbeatRuns)
+ .set({
+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(resultJson, true),
+ error:
+ run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error,
+ errorCode:
+ run.errorCode === DETACHED_PROCESS_ERROR_CODE
+ ? null
+ : run.errorCode,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, run.id),
+ eq(heartbeatRuns.status, "running"),
+ ),
+ )
+ .returning()
+ .then((rows) => rows[0] ?? null);
+
+ if (!updated) {
+ const latest = await db
+ .select({ status: heartbeatRuns.status })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, run.id))
+ .then((rows) => rows[0] ?? null);
+ if (latest && latest.status !== "running") {
+ classify(
+ candidate,
+ "finalized_while_down",
+ `run_status_${latest.status}`,
+ patch,
+ );
+ } else {
+ classify(candidate, "lost", "adoption_update_not_applied", patch);
+ }
+ continue;
+ }
+
+ await appendRunEvent(updated, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "info",
+ message: "Adopted live child process after hot restart",
+ payload: {
+ previousServerPid: intent.previousServerPid,
+ newServerPid: process.pid,
+ previousServerVersion: intent.previousServerVersion,
+ newServerVersion: serverVersion,
+ processPid,
+ processGroupId,
+ },
+ });
+ classify(
+ candidate,
+ "adopted",
+ processPidAlive ? "process_pid_alive" : "process_group_alive",
+ patch,
+ );
+ }
+
+ const report = await writeHotRestartReport({
+ version: 1,
+ requestedAt: intent.requestedAt,
+ completedAt: now.toISOString(),
+ drainRequired: intent.drainRequired,
+ drainReason:
+ intent.drainReason ?? (intent.drainRequired ? "requested" : null),
+ previousServerPid: intent.previousServerPid,
+ newServerPid: process.pid,
+ previousServerVersion: intent.previousServerVersion,
+ newServerVersion: serverVersion,
+ adoptedRunIds,
+ finalizedWhileDownRunIds,
+ lostRunIds,
+ skippedRunIds,
+ runs: reportRuns,
+ });
+ await removeHotRestartIntent(undefined, intent);
+
+ logger.info(
+ {
+ previousServerPid: report.previousServerPid,
+ newServerPid: report.newServerPid,
+ adoptedRunIds,
+ finalizedWhileDownRunIds,
+ lostRunIds,
+ missingSnapshotRunIds,
+ skippedRunIds,
+ },
+ "hot-restart adoption report written",
+ );
+
+ return {
+ mode: "reported" as const,
+ adoptedRunIds,
+ finalizedWhileDownRunIds,
+ lostRunIds,
+ skippedRunIds,
+ };
+ }
+
+ async function recoverNativeRunsAfterRestart(now = new Date()) {
+ // A result committed before the old controller stopped outranks process
+ // recovery. Finish its durable workspace/status suffix before deciding
+ // whether any provider authority needs to be reopened.
+ await reconcileNativeFinalizations(db, undefined, {
+ environmentRuntime,
+ onWorkspaceSettled: settleRecoveredNativeWorkspace,
+ });
+ scheduleRetainedNativeSessionCleanup();
+ const intent = await readHotRestartIntent().catch((error) => {
+ logger.warn(
+ { err: error },
+ "failed to read hot-restart intent before native startup recovery",
+ );
+ return null;
+ });
+ const restartKind = intent ? ("hot" as const) : ("hard" as const);
+ const previousStartedAt = intent?.previousServerStartedAt
+ ? new Date(intent.previousServerStartedAt)
+ : null;
+ const scheduledNativeRetries = await db
+ .select({
+ runId: nativeRunFinalizations.runId,
+ nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
+ })
+ .from(nativeRunFinalizations)
+ .innerJoin(
+ heartbeatRuns,
+ eq(heartbeatRuns.id, nativeRunFinalizations.runId),
+ )
+ .where(
+ and(
+ eq(heartbeatRuns.runtimeMode, "native"),
+ inArray(heartbeatRuns.status, ["running", "failed"]),
+ isNull(nativeRunFinalizations.resultId),
+ eq(nativeRunFinalizations.phase, "retryable_failure"),
+ gt(nativeRunFinalizations.nextAttemptAt, now),
+ ),
+ );
+ for (const scheduled of scheduledNativeRetries) {
+ if (scheduled.nextAttemptAt) {
+ scheduleNativeSessionResumeDispatch(
+ scheduled.runId,
+ scheduled.nextAttemptAt,
+ );
+ }
+ }
+ const dispositions = await claimNativeRestartRecoveries({
+ db,
+ restartKind,
+ recoveryRequestId: intent?.recoveryRequestId ?? null,
+ coordinatedPreviousController: intent
+ ? {
+ pid: intent.previousServerPid,
+ processStartedAt:
+ previousStartedAt && !Number.isNaN(previousStartedAt.getTime())
+ ? previousStartedAt
+ : null,
+ }
+ : null,
+ now,
+ });
+
+ const claims = dispositions.filter(
+ (disposition): disposition is NativeRestartRecoveryClaim =>
+ disposition.kind === "reattach_existing_runner" ||
+ disposition.kind === "reattach_remote_runner" ||
+ disposition.kind === "resume_dead_runner" ||
+ disposition.kind === "bootstrap_incomplete",
+ );
+ for (const disposition of dispositions) {
+ const run = await getRun(disposition.runId);
+ if (run) {
+ const isClaim =
+ disposition.kind === "reattach_existing_runner" ||
+ disposition.kind === "reattach_remote_runner" ||
+ disposition.kind === "resume_dead_runner" ||
+ disposition.kind === "bootstrap_incomplete";
+ await appendRunEvent(run, {
+ eventType: "native.recovery.transition",
+ stream: "system",
+ level: disposition.kind === "blocked" ? "warn" : "info",
+ message:
+ disposition.kind === "reattach_existing_runner" ||
+ disposition.kind === "reattach_remote_runner"
+ ? "Recovering the existing native runner process after server restart"
+ : disposition.kind === "resume_dead_runner"
+ ? "Resuming the durable native provider session after runner process loss"
+ : disposition.kind === "bootstrap_incomplete"
+ ? "Restarting an incomplete native runner bootstrap on the same heartbeat run"
+ : disposition.kind === "awaiting_evidence"
+ ? "Native restart recovery is waiting for safe ownership evidence"
+ : disposition.kind === "already_finalized"
+ ? "Native restart recovery found an already-finalized result"
+ : "Native restart recovery blocked ambiguous or conflicting ownership",
+ payload: {
+ restartKind: isClaim ? disposition.restartKind : restartKind,
+ recoveryRequestId: isClaim
+ ? disposition.recoveryRequestId
+ : (intent?.recoveryRequestId ?? null),
+ runnerDisposition: disposition.kind,
+ ...(isClaim
+ ? {
+ controllerGeneration: disposition.controllerGeneration,
+ providerAttempt: disposition.providerAttempt,
+ }
+ : { reason: disposition.reason }),
+ ...(disposition.kind === "reattach_existing_runner"
+ ? {
+ processPid: disposition.process.pid,
+ processGroupId: disposition.process.processGroupId,
+ processStartedAt: disposition.process.startedAt,
+ }
+ : {}),
+ },
+ });
+ }
+ }
+ for (const claim of claims) {
+ const execution = executeRun(claim.runId, {
+ nativeLeaseOwner: claim.leaseOwner,
+ nativeRestartRecovery: claim,
+ }).catch((error) => {
+ logger.error(
+ { err: error, runId: claim.runId, disposition: claim.kind },
+ "native restart recovery execution failed",
+ );
+ });
+ activeRunExecutionPromises.add(execution);
+ void execution.finally(() =>
+ activeRunExecutionPromises.delete(execution),
+ );
+ }
+
+ return {
+ restartKind,
+ claims,
+ dispositions,
+ scheduledRetryRunIds: scheduledNativeRetries.map((entry) => entry.runId),
+ awaitingEvidenceRunIds: dispositions
+ .filter((entry) => entry.kind === "awaiting_evidence")
+ .map((entry) => entry.runId),
+ blockedRunIds: dispositions
+ .filter((entry) => entry.kind === "blocked")
+ .map((entry) => entry.runId),
+ };
+ }
+
+ async function drainRunningRunsForShutdown(
+ signal: "SIGINT" | "SIGTERM",
+ now = new Date(),
+ runIds: readonly string[] | null = null,
+ ) {
+ const selectedRunIds = runIds ? [...new Set(runIds)] : null;
+ if (selectedRunIds?.length === 0) {
+ return {
+ interrupted: 0,
+ interruptedRunIds: [],
+ retryRunIds: [],
+ restartSuspendedRunIds: [],
+ };
+ }
+ const activeRuns = await db
+ .select({
+ run: heartbeatRuns,
+ agent: agents,
+ })
+ .from(heartbeatRuns)
+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+ .where(
+ selectedRunIds
+ ? and(
+ eq(heartbeatRuns.status, "running"),
+ inArray(heartbeatRuns.id, selectedRunIds),
+ )
+ : eq(heartbeatRuns.status, "running"),
+ );
+
+ const interruptedRunIds: string[] = [];
+ const retryRunIds: string[] = [];
+ const restartSuspendedRunIds: string[] = [];
+
+ for (const { run, agent } of activeRuns) {
+ // Shutdown owns only this boot's legacy executions. Expired foreign
+ // owners belong to the reaper, not another container's drain.
+ if (run.runtimeMode === "legacy" && run.controllerBootId &&
+ run.controllerBootId !== legacyControllerBootId) continue;
+ if (isNativeRunnerOwnershipHeld(run)) continue;
+ if (
+ run.runtimeMode === "native" &&
+ agent.adapterType === "paperclip_runner"
+ ) {
+ // A graceful shutdown relinquishes controller authority just like a
+ // hot restart. Leaving the old event consumer attached lets its
+ // finalizer interrupt/suspend Claude while the next server is adopting
+ // the same turn.
+ await detachNativeSessionsForRestart([run.id]);
+ const recoveryHistoryEntry = JSON.stringify({
+ at: now.toISOString(),
+ restartKind: "graceful",
+ disposition: "restart_suspended",
+ reason: signal,
+ processPid: run.processPid,
+ processStartedAt: run.processStartedAt?.toISOString() ?? null,
+ });
+ await db
+ .update(nativeRunFinalizations)
+ .set({
+ recoveryState: "awaiting_runner_reattach",
+ recoveryHistory: sql`(
+ select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb)
+ from jsonb_array_elements(
+ coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
+ || jsonb_build_array(${recoveryHistoryEntry}::jsonb)
+ ) with ordinality as history(item, ordinal)
+ where ordinal > greatest(
+ jsonb_array_length(
+ coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)
+ || jsonb_build_array(${recoveryHistoryEntry}::jsonb)
+ ) - 20,
+ 0
+ )
+ )`,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(nativeRunFinalizations.runId, run.id),
+ isNull(nativeRunFinalizations.resultId),
+ ),
+ );
+ await appendRunEvent(run, {
+ eventType: "native.recovery.transition",
+ stream: "system",
+ level: "info",
+ message:
+ "Server shutdown suspended native controller ownership without cancelling provider work",
+ payload: {
+ restartKind: "graceful",
+ signal,
+ runnerDisposition: "awaiting_runner_reattach",
+ processPid: run.processPid,
+ processGroupId: run.processGroupId,
+ processStartedAt: run.processStartedAt?.toISOString() ?? null,
+ retryRunCreated: false,
+ },
+ });
+ restartSuspendedRunIds.push(run.id);
+ continue;
+ }
+ const message = `Interrupted by graceful server shutdown (${signal})`;
+ const running = runningProcesses.get(run.id);
+ try {
+ if (run.runtimeMode === "native") {
+ await cancelHeartbeatNativeRun({
+ db,
+ runId: run.id,
+ reason: message,
+ runtimeMode: run.runtimeMode,
+ });
+ }
+ if (running) {
+ await terminateHeartbeatRunProcess({
+ pid: running.child.pid,
+ processGroupId: running.processGroupId,
+ graceMs: Math.max(1, running.graceSec) * 1000,
+ });
+ }
+ } finally {
+ runningProcesses.delete(run.id);
+ }
+
+ const persistedCancellationResult =
+ run.runtimeMode === "native"
+ ? await getRun(run.id).then((current) =>
+ parseObject(current?.resultJson),
+ )
+ : parseObject(run.resultJson);
+
+ const interruptedStatus = await setRunStatusIfRunning(
+ run.id,
+ "interrupted",
+ {
+ finishedAt: now,
+ error: message,
+ errorCode: "server_shutdown_interrupted",
+ signal,
+ resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {
+ conversationContinuationEligible: await runUsedConversationAdapter(db, run),
+ resultJson: persistedCancellationResult,
+ errorCode: "server_shutdown_interrupted",
+ errorMessage: message,
+ }),
+ },
+ );
+ if (!interruptedStatus.updated || !interruptedStatus.run) continue;
+ let interrupted = interruptedStatus.run;
+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {
+ finishedAt: now,
+ error: null,
+ });
+ interrupted =
+ (await classifyAndPersistRunLiveness(
+ interrupted,
+ parseObject(interrupted.resultJson),
+ )) ?? interrupted;
+
+ await releaseEnvironmentLeasesForRun({
+ runId: interrupted.id,
+ companyId: interrupted.companyId,
+ agentId: interrupted.agentId,
+ status: interrupted.status,
+ failureReason: interrupted.error ?? undefined,
+ });
+
+ const retry = await enqueueProcessLossRetry(interrupted, agent, now);
+ if (!retry) {
+ await releaseIssueExecutionAndPromote(interrupted);
+ } else {
+ retryRunIds.push(retry.id);
+ }
+
+ await appendRunEvent(interrupted, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message,
+ payload: {
+ signal,
+ ...(run.processPid ? { processPid: run.processPid } : {}),
+ ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
+ ...(retry ? { retryRunId: retry.id } : {}),
+ },
+ });
+
+ await finalizeAgentStatus(run.agentId, "interrupted", message, {
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ });
+ interruptedRunIds.push(interrupted.id);
+ }
+
+ if (interruptedRunIds.length > 0) {
+ logger.warn(
+ {
+ signal,
+ interrupted: interruptedRunIds.length,
+ interruptedRunIds,
+ retryRunIds,
+ },
+ "interrupted running heartbeat runs for graceful shutdown",
+ );
+ }
+
+ return {
+ interrupted: interruptedRunIds.length,
+ interruptedRunIds,
+ retryRunIds,
+ restartSuspendedRunIds,
+ };
+ }
+
+ // Clamp the stored attempt count to the range [0, cap]. The SQL reader
+ // `pendingCleanupAttemptsSql` clamps to the same range, so both readers yield
+ // the same value for every input. The claim predicate compares the two values,
+ // so this alignment lets the claim match for a malformed lease.
+ function readPendingCleanupRetryAttempts(
+ metadata: Record<string, unknown>,
+ ): number {
+ const value = metadata[PENDING_CLEANUP_ATTEMPTS_METADATA_KEY];
+ if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)
+ return 0;
+ return Math.min(Math.floor(value), PENDING_CLEANUP_SWEEP_ATTEMPT_CAP);
+ }
+
+ // Atomically claim one retry attempt on a pending_cleanup lease. The update
+ // only matches when the lease is still pending_cleanup and its stored attempt
+ // count still equals `expectedAttempts`. Two concurrent sweeps read the same
+ // count, but Postgres serializes the two updates on the row and only the first
+ // matches the guard. The loser gets zero rows and skips the lease. The persisted in-flight deadline also prevents a later tick
+ // from starting another attempt while this one is still running.
+ // Returns the attempt identity only for the sweep that won the claim.
+ //
+ // The update patches cleanup ownership fields without writing a copied
+ // metadata object, so a concurrent write to an unrelated metadata key
+ // survives. The guard reads the stored count through the safe SQL reader, so a
+ // malformed value never throws.
+ async function claimPendingCleanupRetryAttempt(
+ leaseId: string,
+ expectedAttempts: number,
+ manualAttempt?: { previousId: unknown },
+ ): Promise<string | null> {
+ const now = new Date();
+ const attemptId = randomUUID();
+ const claimed = await db
+ .update(environmentLeases)
+ .set({
+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}], to_jsonb(${expectedAttempts + 1}::int), true)
+ || ${JSON.stringify({ ...(manualAttempt ? { pendingCleanupManualAttemptId: randomUUID() } : {}),
+ pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true,
+ pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000,
+ })}::jsonb`,
+ lastUsedAt: now,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(environmentLeases.id, leaseId),
+ eq(environmentLeases.status, "pending_cleanup"),
+ sql`${pendingCleanupAttemptsSql()} = ${expectedAttempts}`,
+ manualAttempt ? sql`coalesce(${environmentLeases.metadata}->'pendingCleanupManualAttemptId', 'null'::jsonb) is not distinct from ${JSON.stringify(manualAttempt.previousId ?? null)}::jsonb` : undefined,
+ pendingCleanupRetryDueSql(Boolean(manualAttempt)),
+ ),
+ )
+ .returning({ id: environmentLeases.id });
+ return claimed.length > 0 ? attemptId : null;
+ }
+
+ // Atomically claim the one-time cap warning for a lease. The update only
+ // matches when the lease is still pending_cleanup, its stored attempt count is
+ // at or above the cap, and it has not yet carried the warned flag. Two
+ // concurrent sweeps that both reach the cap race here, but only one update
+ // sets the flag and returns a row. The loser skips the warning. This keeps the
+ // warning to one log line per lease.
+ //
+ // The status and cap predicates are the last line of defense. They stop a warn
+ // flag write to a lease that left pending_cleanup or dropped below the cap
+ // between the read and this claim. The update writes only the warned key with
+ // `jsonb_set`, so a concurrent write to an unrelated metadata key survives.
+ async function claimPendingCleanupCapWarning(
+ leaseId: string,
+ ): Promise<boolean> {
+ const now = new Date();
+ const claimed = await db
+ .update(environmentLeases)
+ .set({
+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}], to_jsonb(true), true)`,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(environmentLeases.id, leaseId),
+ eq(environmentLeases.status, "pending_cleanup"),
+ sql`${pendingCleanupAttemptsSql()} >= ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}`,
+ sql`${pendingCleanupCapWarnedSql()} = false`,
+ ),
+ )
+ .returning({ id: environmentLeases.id });
+ return claimed.length > 0;
+ }
+
+ // Defer a pending_cleanup lease whose provider plugin is not ready this tick.
+ // The sweep reads one page of the oldest rows, ordered by `updatedAt`. A lease
+ // that the sweep only skips keeps its old `updatedAt`, so it stays the oldest
+ // and refills the page on every tick. That starves a newer lease whose
+ // provider is ready. The defer bumps `updatedAt` to now, so the unavailable
+ // lease moves to the back of the queue and a ready lease takes its page slot.
+ // The defer never writes the attempt count, so a long provider outage never
+ // consumes a finite retry. The status guard keeps the write on a lease that is
+ // still pending_cleanup.
+ async function deferPendingCleanupLease(leaseId: string): Promise<void> {
+ const now = new Date();
+ await db
+ .update(environmentLeases)
+ .set({ updatedAt: now })
+ .where(
+ and(
+ eq(environmentLeases.id, leaseId),
+ eq(environmentLeases.status, "pending_cleanup"),
+ ),
+ );
+ }
+
+ // Move a guarded orphan candidate to the back of the sweep order. The
+ // shared-resource guard below can skip a row on every tick as long as the
+ // other owner stays live, retained, or pending cleanup. The select orders
+ // by `updatedAt` ascending and takes only the oldest page, so an untouched
+ // skipped row keeps refilling that same page and blocks every row behind
+ // it. The bump pushes the row past the fixed-size page, so the next tick
+ // reaches the rows behind it. It costs the row one extra backoff wait,
+ // which is safe because the guard means a physical sandbox still exists.
+ async function deferOrphanedActiveLease(leaseId: string): Promise<void> {
+ const now = new Date();
+ await db
+ .update(environmentLeases)
+ .set({ updatedAt: now })
+ .where(
+ and(
+ eq(environmentLeases.id, leaseId),
+ eq(environmentLeases.status, "active"),
+ ),
+ );
+ }
+
+ // An active lease is reachable only while its heartbeat run keeps the
+ // running status. The reaper writes the run status and the lease release as
+ // two separate statements, so a restart between them can leave a terminal
+ // run and an active lease. `releaseRunLeases` also skips a lease whose
+ // environment row is gone, so that lease stays active too. No later query
+ // finds either lease, because every production query selects an active
+ // lease by its environment or by its heartbeat run id, never by age. This
+ // sweep finds both stranded classes and moves each lease to pending_cleanup,
+ // so the existing pending_cleanup sweep tears the sandbox down from the
+ // data already on the lease row.
+ async function sweepOrphanedActiveLeases(opts: {
+ backoffMs: number;
+ }): Promise<{ recovered: number }> {
+ const cutoff = new Date(Date.now() - opts.backoffMs);
+
+ const rows = await db
+ .select({ lease: environmentLeases })
+ .from(environmentLeases)
+ .leftJoin(
+ heartbeatRuns,
+ eq(environmentLeases.heartbeatRunId, heartbeatRuns.id),
+ )
+ .where(
+ and(
+ eq(environmentLeases.status, "active"),
+ or(
+ isNull(environmentLeases.heartbeatRunId),
+ inArray(heartbeatRuns.status, [
+ ...HEARTBEAT_RUN_TERMINAL_STATUSES,
+ ]),
+ ),
+ lte(environmentLeases.updatedAt, cutoff),
+ ),
+ )
+ .orderBy(asc(environmentLeases.updatedAt))
+ .limit(ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE);
+
+ let recovered = 0;
+ for (const { lease } of rows) {
+ // A provider resource id names one physical sandbox. A different lease
+ // row can still hold that same resource in a live status, so this sweep
+ // must not tear down a sandbox that a different lease still owns.
+ if (lease.provider && lease.providerLeaseId) {
+ const [otherOwner] = await db
+ .select({ id: environmentLeases.id })
+ .from(environmentLeases)
+ .where(
+ and(
+ ne(environmentLeases.id, lease.id),
+ eq(environmentLeases.provider, lease.provider),
+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
+ inArray(environmentLeases.status, [
+ "active",
+ "retained",
+ "pending_cleanup",
+ ]),
+ ),
+ )
+ .limit(1);
+ if (otherOwner) {
+ // Defer this row so the fixed-size page reaches the rows behind
+ // it next tick, instead of re-selecting the same guarded rows
+ // forever.
+ await deferOrphanedActiveLease(lease.id);
+ continue;
+ }
+ }
+
+ // Keep the row's existing updatedAt value. The select above already
+ // proved the row is older than the backoff cutoff, so the
+ // pending_cleanup sweep can accept the same row in this same tick. A
+ // fresh timestamp here would push the row inside that sweep's own
+ // backoff window and delay the teardown by one full tick.
+ const flipped = await db
+ .update(environmentLeases)
+ .set({
+ status: "pending_cleanup",
+ failureReason: "orphaned_active_lease_recovered",
+ })
+ .where(
+ and(
+ eq(environmentLeases.id, lease.id),
+ eq(environmentLeases.status, "active"),
+ ),
+ )
+ .returning({ id: environmentLeases.id });
+ if (flipped.length > 0) recovered += 1;
+ }
+
+ return { recovered };
+ }
+
+ // Retry the leases stranded in "pending_cleanup". A failed destroy leaves a
+ // lease in that state forever without this sweep. The reaper tick runs the
+ // sweep. The backoff equals the reaper staleness threshold, so a lease waits
+ // for that period between attempts. The sweep reads and writes the attempt
+ // count in the lease metadata. It warns once when a lease reaches the attempt
+ // threshold, then continues with slower retries. Live attempts renew their
+ // cleanup claim; after controller loss, exact-resource destruction may repeat.
+ async function sweepPendingCleanupLeases(opts?: {
+ backoffMs?: number;
+ /** One cleanup attempt per explicit user Retry, for this failed run only.
+ * A later user Retry may bypass the cooldown after a provider failure,
+ * but cannot take over an in-flight cleanup attempt.
+ */
+ explicitRetry?: { companyId: string; runId: string; actorId: string; reason?: "retry_failed_run" | "queued_comment_interrupt" };
+ }): Promise<{
+ swept: number;
+ destroyed: number;
+ capped: number;
+ }> {
+ const backoffMs = opts?.backoffMs ?? 0;
+ const now = new Date();
+ const cutoff = new Date(now.getTime() - backoffMs);
+
+ // Flush the in-process orphan-cleanup buffer first. A failed acquire buffers
+ // an orphan there when every synchronous pending-cleanup write failed after a
+ // failed teardown. The flush re-inserts each buffered record, so a durable
+ // `pending_cleanup` row lands once the database recovers. The flush runs
+ // before the read below, so this same tick tears down a freshly-landed row.
+ try {
+ const flushed = opts?.explicitRetry ? null : await environmentRuntime.flushDeferredOrphanCleanups?.();
+ if (flushed && (flushed.recovered > 0 || flushed.pending > 0)) {
+ logger.info(
+ { recovered: flushed.recovered, pending: flushed.pending },
+ "flushed the in-process orphan sandbox cleanup buffer to the database",
+ );
+ }
+ } catch {
+ // A flush failure never stops the sweep. The buffer keeps the orphan for a
+ // later tick, and the database rows below still need this sweep. The caught
+ // exception never enters the log, because a write error can carry a
+ // credential in its message, code, cause, or stack.
+ logger.warn(
+ "orphan sandbox cleanup buffer flush failed; the sweep continues",
+ );
+ }
+
+ const rows = await db
+ .select()
+ .from(environmentLeases)
+ .where(
+ and(
+ eq(environmentLeases.status, "pending_cleanup"),
+ opts?.explicitRetry ? eq(environmentLeases.companyId, opts.explicitRetry.companyId) : undefined,
+ opts?.explicitRetry ? eq(environmentLeases.heartbeatRunId, opts.explicitRetry.runId) : undefined,
+ pendingCleanupRetryDueSql(Boolean(opts?.explicitRetry)),
+ backoffMs > 0 ? lte(environmentLeases.updatedAt, cutoff) : undefined,
+ ),
+ )
+ .orderBy(asc(environmentLeases.updatedAt))
+ .limit(PENDING_CLEANUP_SWEEP_PAGE_SIZE);
+
+ let destroyed = 0;
+ let capped = 0;
+ for (const row of rows) {
+ if (pendingCleanupAttemptsInFlight.has(row.id)) continue;
+ const metadata = { ...(row.metadata ?? {}) } as Record<string, unknown>;
+ const attempts = readPendingCleanupRetryAttempts(metadata);
+
+ const environment = row.environmentId
+ ? await environmentsSvc.getById(row.environmentId)
+ : null;
+ const lease = await environmentsSvc.getLeaseById(row.id);
+ if (!lease) continue;
+
+ // An orphan ephemeral lease keeps its provider, its provider lease id, and
+ // its sandbox config in the lease row. A failed acquire records it, and its
+ // environment row may be gone or foreign-bound. A reuse_by_environment lease
+ // whose environment a delete removed keeps the same recorded data, because
+ // the schema sets the environment reference to null on delete and preserves
+ // the row. Both leases tear down from the recorded lease data through
+ // `retryPendingSandboxTeardown`, which never reads the environment row. So
+ // the sweep uses that path whenever the lease is an orphan ephemeral lease
+ // or its environment row is gone. A reuse_by_environment lease whose
+ // environment still exists tears down through `destroyRunLease`. That
+ // path uses the provider and configuration recorded on the lease first;
+ // the environment is lifecycle context and only a legacy fallback.
+ const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";
+ const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);
+
+ // Do not consume a finite cleanup attempt while the provider plugin is
+ // briefly unavailable. A plugin worker restart, a plugin reload, or a
+ // plugin reinstall makes the provider unavailable for a short window. The
+ // plugin can be missing or not ready in that window. A teardown then throws,
+ // and the atomic claim below would count that throw against the cap, so a
+ // long restart or reload could exhaust the retries and strand a live
+ // sandbox. So probe the provider first, and defer the lease this tick when
+ // the provider is not ready. The sweep preserves the pending_cleanup row,
+ // and a later sweep retries after the provider recovers. The probe reports
+ // ready only for a permanent condition (a missing provider string, a
+ // built-in provider, or no worker manager), so a genuine teardown failure
+ // still runs, throws, and counts toward the cap. A runtime with no probe
+ // method treats the lease as ready, so the sweep keeps its earlier
+ // behavior.
+ const workerReady = environmentRuntime.isPendingCleanupWorkerReady
+ ? await environmentRuntime.isPendingCleanupWorkerReady({
+ environment,
+ lease,
+ })
+ : true;
+ if (!workerReady) {
+ // Move the unavailable lease to the back of the sweep queue. Otherwise
+ // the oldest unavailable rows refill the page on every tick and starve a
+ // newer lease that has a ready provider. The defer bumps `updatedAt`
+ // only, so it consumes no finite retry attempt.
+ await deferPendingCleanupLease(row.id);
+ continue;
+ }
+
+ // Atomically claim the attempt before the retry. Only the winning sweep
+ // increments the count and tears the sandbox down, so an overlapping sweep
+ // cannot start another attempt while this one holds the cleanup lease.
+ // The deadline survives a server restart; the counter saturates at the
+ // escalation threshold while attempt identities remain unique.
+ const claimed = await claimPendingCleanupRetryAttempt(row.id, attempts,
+ opts?.explicitRetry ? { previousId: metadata.pendingCleanupManualAttemptId } : undefined);
+ if (!claimed) continue;
+ pendingCleanupAttemptsInFlight.add(row.id);
+ lease.metadata = { ...lease.metadata, pendingCleanupAttemptId: claimed };
+ let activeRenewal: Promise<void> | null = null;
+ const renewal = setInterval(() => {
+ if (activeRenewal) return;
+ activeRenewal = db.update(environmentLeases).set({
+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, '{pendingCleanupLeaseExpiresAtMs}', to_jsonb(${Date.now() + 15 * 60_000}::bigint))`,
+ }).where(and(eq(environmentLeases.id, row.id), eq(environmentLeases.status, "pending_cleanup"),
+ sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`,
+ sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`))
+ .then(() => {})
+ .catch(() => logger.warn({ leaseId: row.id }, "cleanup ownership renewal failed"))
+ .finally(() => { activeRenewal = null; });
+ }, 30_000);
+ renewal.unref();
+
+ try {
+ if (opts?.explicitRetry) await logActivity(db, {
+ companyId: row.companyId, actorType: "user", actorId: opts.explicitRetry.actorId,
+ action: "environment_lease.cleanup_retried", entityType: "environment_lease", entityId: row.id,
+ runId: opts.explicitRetry.runId, details: { attempt: attempts + 1, reason: opts.explicitRetry.reason ?? "retry_failed_run" },
+ });
+ if (useRecordedTeardown) {
+ // Tear the sandbox down from the recorded provider config and the
+ // cleanup-authorized secret versions. Preserve any provider receipt;
+ // a completed retry must grant the same evidence as initial cleanup.
+ const receipt = await environmentRuntime.retryPendingSandboxTeardown({
+ environment,
+ lease,
+ });
+ const released = hasStopOnlyCleanup(lease)
+ ? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt })
+ : await environmentsSvc.releaseLease(lease.id, "expired", {
+ expectedPendingCleanupAttemptId: claimed,
+ cleanupStatus: "success",
+ failureReason: "pending_cleanup_retry",
+ remoteExecutionTermination: remoteTerminationReceipt(lease, receipt),
+ });
+ if (released) destroyed += 1;
+ } else if (environment) {
+ const result = await environmentRuntime.destroyRunLease({
+ environment,
+ lease,
+ failureReason: "pending_cleanup_retry",
+ });
+ if (result && result.status !== "pending_cleanup") {
+ destroyed += 1;
+ }
+ }
+ } catch {
+ // The recorded-data teardown throws on failure, so revert the lease to
+ // pending_cleanup for a later sweep. The claimed attempt still counts
+ // for backoff, so requests stay bounded. The `destroyRunLease`
+ // path reverts the lease itself, so this revert only runs for the
+ // recorded-data teardown path.
+ if (useRecordedTeardown) {
+ await environmentsSvc.releaseLease(lease.id, "pending_cleanup", {
+ expectedPendingCleanupAttemptId: claimed,
+ cleanupStatus: "failed",
+ failureReason: "pending_cleanup_retry",
+ });
+ }
+ // Log a constant errorKind only. The exception can carry a credential in
+ // its name, code, message, cause, or stack, so the sweep never reads it.
+ logger.warn(
+ {
+ errorKind: PENDING_CLEANUP_RETRY_ERROR_KIND,
+ leaseId: row.id,
+ environmentId: row.environmentId,
+ attempts: attempts + 1,
+ },
+ "pending_cleanup lease retry failed",
+ );
+ } finally {
+ clearInterval(renewal);
+ // A hung renewal must not retain process-local cleanup ownership.
+ // Late renewals are attempt-fenced and never write the retry cooldown.
+ pendingCleanupAttemptsInFlight.delete(row.id);
+ }
+ if (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP) {
+ capped += 1;
+ // Warn once, then continue automatic cleanup with backoff. The atomic claim
+ // keeps the warning to one log line even when two sweeps overlap.
+ if (metadata[PENDING_CLEANUP_CAP_WARNED_METADATA_KEY] !== true) {
+ const warned = await claimPendingCleanupCapWarning(row.id);
+ if (warned) {
+ logger.warn(
+ { leaseId: row.id, environmentId: row.environmentId, attempts },
+ "environment lease needs operator attention; automatic cleanup continues with backoff",
+ );
+ }
+ }
+ }
+ // Persist the cooldown independently of process memory. A crash before
+ // this write leaves the bounded in-flight lease for a later sweep.
+ await db.update(environmentLeases).set({
+ metadata: sql`${pendingCleanupMetadataObjectSql()} || ${JSON.stringify({
+ pendingCleanupInFlight: false,
+ pendingCleanupRetryAfterMs: Date.now() + (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP
+ ? 30 * 60_000 : Math.min(30 * 60_000, Math.max(30_000, backoffMs))),
+ })}::jsonb`,
+ }).where(and(eq(environmentLeases.id, lease.id),
+ sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`));
+ if (lease.heartbeatRunId) {
+ // Delivery failure must not revert successful provider cleanup. A new
+ // message can still use the persisted receipt on its next admission.
+ await (async () => {
+ await acknowledgeRemoteStop(lease.heartbeatRunId!, lease.companyId);
+ const run = await getRun(lease.heartbeatRunId!);
+ if (run) await resumeRemoteStopComments(run);
+ })().catch(() => logger.warn({ leaseId: lease.id }, "could not reconsider messages after cleanup retry"));
+ }
+ }
+
+ return { swept: rows.length, destroyed, capped };
+ }
+
+ async function markNativeOwnershipUnverified(
+ run: typeof heartbeatRuns.$inferSelect,
+ evidence: {
+ reason:
+ | "live_process_identifier"
+ | "observed_owner_unverified"
+ | "adopted_runner_authentication_timeout"
+ | "native_chat_workspace_scope_mismatch";
+ processPidAlive?: boolean;
+ processGroupAlive?: boolean;
+ },
+ ) {
+ const durableOwnershipHold =
+ evidence.reason === "adopted_runner_authentication_timeout" ||
+ evidence.reason === "native_chat_workspace_scope_mismatch";
+ if (
+ run.errorCode === NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE &&
+ run.error === NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE &&
+ (!durableOwnershipHold || isNativeRunnerOwnershipHeld(run))
+ )
+ return run;
+ const blockedStatus = run.status === "failed" ? "failed" : "running";
+ const blockedWrite = await setRunStatusFromLive(
+ run.id,
+ blockedStatus,
+ [blockedStatus],
+ {
+ error: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,
+ errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
+ ...(durableOwnershipHold
+ ? {
+ nativePhase: "terminal_failure",
+ nativePhaseUpdatedAt: new Date(),
+ }
+ : {}),
+ },
+ );
+ if (!blockedWrite.updated || !blockedWrite.run) {
+ return blockedWrite.run ?? run;
+ }
+ const blocked = blockedWrite.run;
+ await appendRunEvent(blocked, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,
+ payload: {
+ reason: evidence.reason,
+ ...(evidence.processPidAlive === true ? { processPidAlive: true } : {}),
+ ...(evidence.processGroupAlive === true
+ ? { processGroupAlive: true }
+ : {}),
+ },
+ });
+ return blocked;
+ }
+
+ async function settleRecoveredNativeWorkspace(input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ succeeded: boolean;
+ }) {
+ const settledRun = await getRun(input.runId);
+ const workspaceSyncReference = readNativeWorkspaceSyncReference(
+ parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
+ );
+ await releaseEnvironmentLeasesForRun({
+ runId: input.runId,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ status: settledRun?.status,
+ failureReason: settledRun?.error ?? undefined,
+ providerResourceDisposition: input.succeeded && (!parseObject(settledRun?.resultJson).workspaceExportRetry || workspaceSyncReference?.resourceDisposition === "destroy")
+ ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")
+ : "stop_and_retain",
+ });
+ await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);
+ await finalizeAgentStatus(
+ input.agentId,
+ input.succeeded ? "succeeded" : "failed",
+ input.succeeded
+ ? null
+ : (settledRun?.error ?? "native_workspace_sync_out_failed"),
+ {
+ wasFirstHeartbeat: settledRun
+ ? timerClaimWasFirstHeartbeat(settledRun)
+ : undefined,
+ },
+ ).catch(() => undefined);
+ }
+
+ function scheduleRetainedNativeSessionCleanup() {
+ // The per-database sweep joins startup and periodic callers. One bounded
+ // control-only repair must not hold up unrelated provider ingress or the
+ // entire orphan reaper, but shutdown must still await its physical owner.
+ const cleanup = reconcileRetainedNativeSessionCleanups(db, {
+ cleanup: (input) => reconcileRetainedNativeSessionCleanup(db, input),
+ onError: (error, runId) => {
+ logger.warn(
+ { err: error, runId },
+ "retained native session cleanup failed",
+ );
+ },
+ })
+ .then(() => undefined)
+ .catch((error) => {
+ logger.warn({ err: error }, "retained native cleanup discovery failed");
+ })
+ // The bounded maintenance attempt may fail before an already-started
+ // database callback settles. Keep shutdown ownership until the original
+ // operations finish; their timeout cannot authorize closing the database.
+ .finally(() => drainRetainedRunnerdMaintenanceOperations());
+ activeRunExecutionPromises.add(cleanup);
+ void cleanup.finally(() => activeRunExecutionPromises.delete(cleanup));
+ }
+
+ async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
+ const staleThresholdMs = opts?.staleThresholdMs ?? 0;
+ const now = new Date();
+ // Recovery never launches a provider or infers stopped ownership from
+ // terminal status. Uncaptured local copies require durable stop evidence.
+ await instructionCopies.recoverStopped().catch(error => {
+ logger.warn({ err: error }, "failed to recover stopped instruction copies");
+ });
+ await instructionCopies.recoverCaptured().catch(error => {
+ logger.warn({ err: error }, "failed to retry captured instruction revisions");
+ });
+
+ // Complete persisted native results before generic orphan recovery. The
+ // reconciler reads the durable workspace barrier and persisted runtime
+ // mode, never the current feature flag.
+ await reconcileNativeFinalizations(db, undefined, {
+ environmentRuntime,
+ onWorkspaceSettled: settleRecoveredNativeWorkspace,
+ }).catch((error) => {
+ logger.warn(
+ { err: error },
+ "failed to reconcile persisted native finalizations before orphan reaping",
+ );
+ });
+ scheduleRetainedNativeSessionCleanup();
+ await dispatchPendingNativeStatusWakeups().catch((error) => {
+ logger.warn(
+ { err: error },
+ "failed to dispatch persisted native status wake intents before orphan reaping",
+ );
+ });
+
+ // A retryable native run can retain process identifiers from the failed
+ // attempt. Inspect them before the recovery claim: a live identifier is
+ // unowned and blocks recovery, while identifiers that are all dead can be
+ // cleared with a compare-and-set so the explicit retryable failure becomes
+ // claimable in this same sweep.
+ const retryableNativeProcesses = await db
+ .select({ run: heartbeatRuns })
+ .from(heartbeatRuns)
+ .innerJoin(
+ nativeRunFinalizations,
+ eq(nativeRunFinalizations.runId, heartbeatRuns.id),
+ )
+ .where(
+ and(
+ inArray(heartbeatRuns.status, ["running", "failed"]),
+ eq(heartbeatRuns.runtimeMode, "native"),
+ eq(nativeRunFinalizations.phase, "retryable_failure"),
+ isNull(nativeRunFinalizations.resultId),
+ ),
+ );
+ const claimableNativeRunIds = new Set<string>();
+ for (const { run } of retryableNativeProcesses) {
+ if (isNativeRunnerOwnershipHeld(run)) continue;
+ if (!run.processPid && !run.processGroupId) {
+ claimableNativeRunIds.add(run.id);
+ continue;
+ }
+ const processPidAlive =
+ !!run.processPid && isProcessAlive(run.processPid);
+ const processGroupAlive =
+ !!run.processGroupId && isProcessGroupAlive(run.processGroupId);
+ if (processPidAlive || processGroupAlive) {
+ await markNativeOwnershipUnverified(run, {
+ reason: "live_process_identifier",
+ processPidAlive,
+ processGroupAlive,
+ });
+ continue;
+ }
+ const cleared = await db.transaction(async tx => {
+ const cleared = await tx
+ .update(heartbeatRuns)
+ .set({
+ processPid: null,
+ processGroupId: null,
+ processStartedAt: null,
+ updatedAt: now,
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, run.id),
+ eq(heartbeatRuns.runtimeMode, "native"),
+ run.processPid === null
+ ? isNull(heartbeatRuns.processPid)
+ : eq(heartbeatRuns.processPid, run.processPid),
+ run.processGroupId === null
+ ? isNull(heartbeatRuns.processGroupId)
+ : eq(heartbeatRuns.processGroupId, run.processGroupId),
+ run.processStartedAt === null
+ ? isNull(heartbeatRuns.processStartedAt)
+ : eq(heartbeatRuns.processStartedAt, run.processStartedAt),
+ ),
+ )
+ .returning({ id: heartbeatRuns.id })
+ .then((rows) => rows[0] ?? null);
+ if (cleared) await recordNativeLocalProcessStop(tx as unknown as Db, run);
+ return cleared;
+ });
+ if (cleared) claimableNativeRunIds.add(cleared.id);
+ }
+
+ // An explicit result-less retryable failure resumes on the original run.
+ // The database lease is claimed before dispatch so concurrent service
+ // instances cannot open competing recoveries; executeRun receives the exact
+ // claimed owner. Expired `observed` ownership never enters this set.
+ const nativeResumeClaims =
+ claimableNativeRunIds.size === 0
+ ? []
+ : await dispatchNativeSessionResumptions({
+ db,
+ runnerInstanceId:
+ runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat",
+ now,
+ runIds: [...claimableNativeRunIds],
+ dispatch: (claim) => {
+ const execution = executeRun(claim.runId, {
+ nativeLeaseOwner: claim.leaseOwner,
+ }).catch((error) => {
+ logger.error(
+ { err: error, runId: claim.runId },
+ "persisted native session resume failed",
+ );
+ });
+ activeRunExecutionPromises.add(execution);
+ void execution.finally(() =>
+ activeRunExecutionPromises.delete(execution),
+ );
+ },
+ }).catch((error) => {
+ logger.warn(
+ { err: error },
+ "failed to claim persisted native session resumptions",
+ );
+ return [];
+ });
+ const resumedRunIds = new Set(
+ nativeResumeClaims.map((claim) => claim.runId),
+ );
+
+ // A terminal issue transition writes this intent in the same transaction
+ // that closes the question's task, even when its card is retained. Consume
+ // it before generic orphan recovery so a restart preserves cancellation.
+ const cancellationRequests = await db
+ .select({
+ id: heartbeatRuns.id,
+ contextSnapshot: heartbeatRuns.contextSnapshot,
+ })
+ .from(heartbeatRuns)
+ .where(
+ and(
+ inArray(heartbeatRuns.status, [
+ ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
+ ]),
+ sql`${heartbeatRuns.contextSnapshot} -> ${NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY} is not null`,
+ ),
+ );
+ for (const request of cancellationRequests) {
+ const marker = parseObject(
+ parseObject(request.contextSnapshot)[
+ NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY
+ ],
+ );
+ const issueId = readNonEmptyString(marker.issueId);
+ const issueStatus = readNonEmptyString(marker.issueStatus);
+ const interactionId = readNonEmptyString(marker.interactionId);
+ const kind = readNonEmptyString(marker.kind);
+ const reason =
+ kind === "interaction_withdrawn"
+ ? "Question withdrawn while waiting for operator input"
+ : kind === "interaction_cancelled"
+ ? "Cancelled while waiting for operator input"
+ : "Task closed while waiting for operator input";
+ try {
+ await cancelRunInternal(request.id, reason, {
+ resultJson: {
+ ...(kind === "interaction_withdrawn" && interactionId
+ ? { withdrawnInteractionId: interactionId }
+ : {}),
+ ...(kind === "interaction_cancelled" && interactionId
+ ? { cancelledInteractionId: interactionId }
+ : {}),
+ ...((!kind || kind === "issue_terminal") && issueStatus
+ ? { cancelledByIssueStatus: issueStatus }
+ : {}),
+ ...(issueId ? { cancelledIssueId: issueId } : {}),
+ },
+ });
+ } catch (err) {
+ // Keep the marker intact for the next startup/periodic sweep.
+ logger.warn(
+ { err, runId: request.id },
+ "native question cancellation recovery attempt failed",
+ );
+ }
+ }
+
+ // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
+ const activeRuns = await db
+ .select({
+ run: heartbeatRuns,
+ adapterType: agents.adapterType,
+ adapterConfig: agents.adapterConfig,
+ nativeCoordinatorPhase: nativeRunFinalizations.phase,
+ nativeRecoveryState: nativeRunFinalizations.recoveryState,
+ nativeControllerBootId: nativeRunFinalizations.controllerBootId,
+ nativeControllerPid: nativeRunFinalizations.controllerPid,
+ nativeControllerProcessStartedAt:
+ nativeRunFinalizations.controllerProcessStartedAt,
+ nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
+ })
+ .from(heartbeatRuns)
+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))
+ .leftJoin(
+ nativeRunFinalizations,
+ eq(nativeRunFinalizations.runId, heartbeatRuns.id),
+ )
+ .where(eq(heartbeatRuns.status, "running"));
+
+ const monitorIssueIds = [
+ ...new Set(
+ activeRuns.flatMap(({ run }) => {
+ const runContext = parseObject(run.contextSnapshot);
+ if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due")
+ return [];
+ const issueId = readNonEmptyString(runContext.issueId);
+ return issueId ? [issueId] : [];
+ }),
+ ),
+ ];
+ const monitorIssues =
+ monitorIssueIds.length > 0
+ ? await db
+ .select({
+ id: issues.id,
+ companyId: issues.companyId,
+ monitorNextCheckAt: issues.monitorNextCheckAt,
+ })
+ .from(issues)
+ .where(inArray(issues.id, monitorIssueIds))
+ : [];
+ const monitorNextCheckAtByIssue = new Map(
+ monitorIssues.map((issue) => [
+ `${issue.companyId}:${issue.id}`,
+ issue.monitorNextCheckAt,
+ ]),
+ );
+
+ const reaped: string[] = [];
+ const currentNativeController = await currentNativeControllerIdentity();
+
+ for (const {
+ run,
+ adapterType,
+ adapterConfig,
+ nativeCoordinatorPhase,
+ nativeRecoveryState,
+ nativeControllerBootId,
+ nativeControllerPid,
+ nativeControllerProcessStartedAt,
+ nativeControllerLeaseExpiresAt,
+ } of activeRuns) {
+ // Authentication timeout requires an explicit ownership resolution, not
+ // repeated reattachment or a process-gone guess on subsequent sweeps.
+ if (isNativeRunnerOwnershipHeld(run)) continue;
+ const nativeRun = run.runtimeMode === "native";
+ const nativeProcessPidAlive =
+ nativeRun && !!run.processPid && isProcessAlive(run.processPid);
+ const nativeProcessGroupAlive =
+ nativeRun &&
+ !!run.processGroupId &&
+ isProcessGroupAlive(run.processGroupId);
+ const coordinatorOwnedByCurrentController =
+ nativeRun &&
+ nativeControllerBootId === currentNativeController.bootId &&
+ nativeControllerPid === currentNativeController.pid &&
+ nativeControllerProcessStartedAt?.getTime() ===
+ currentNativeController.processStartedAt.getTime() &&
+ !!nativeControllerLeaseExpiresAt &&
+ nativeControllerLeaseExpiresAt.getTime() > now.getTime();
+ const locallyTracked =
+ runningProcesses.has(run.id) ||
+ activeRunExecutions.has(run.id) ||
+ coordinatorOwnedByCurrentController;
+ if (
+ nativeRun &&
+ ([
+ "awaiting_evidence",
+ "awaiting_runner_reattach",
+ "resuming_session",
+ "bootstrap_incomplete",
+ ].includes(nativeRecoveryState ?? "") ||
+ nativeCoordinatorPhase === "retryable_failure")
+ ) {
+ continue;
+ }
+ const observedOwnerUnverified =
+ nativeRun &&
+ (nativeCoordinatorPhase === "observed" ||
+ (nativeCoordinatorPhase === null &&
+ run.nativePhase === "observed")) &&
+ !resumedRunIds.has(run.id) &&
+ !locallyTracked;
+ // Persisted numeric process identifiers prove only that some process is
+ // alive, not that Paperclip still owns it. Likewise an observed native
+ // coordinator without a live in-process execution has no durable proof
+ // that its prior provider owner stopped. Keep both cases running but
+ // blocked: never signal, finalize, or retry them automatically. This gate
+ // intentionally precedes resumedRunIds so a claim cannot bypass the
+ // ownership check.
+ if (
+ !locallyTracked &&
+ (nativeProcessPidAlive ||
+ nativeProcessGroupAlive ||
+ observedOwnerUnverified)
+ ) {
+ await markNativeOwnershipUnverified(run, {
+ reason:
+ nativeProcessPidAlive || nativeProcessGroupAlive
+ ? "live_process_identifier"
+ : "observed_owner_unverified",
+ processPidAlive: nativeProcessPidAlive,
+ processGroupAlive: nativeProcessGroupAlive,
+ });
+ continue;
+ }
+ if (resumedRunIds.has(run.id)) continue;
+ if (locallyTracked) continue;
+ if (await hasLiveLegacyController(db, run)) continue;
+
+ // Apply staleness threshold to avoid false positives
+ if (staleThresholdMs > 0) {
+ const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
+ if (now.getTime() - refTime < staleThresholdMs) continue;
+ }
+
+ const currentAdapterTracksLocalChild =
+ isTrackedLocalChildProcessAdapter(adapterType);
+ const tracksLegacyLocalChild =
+ run.runtimeMode !== "native" && currentAdapterTracksLocalChild;
+ // Native runner processes also persist child metadata, but they must not
+ // inherit legacy retry or termination authority. Use their PID/group only
+ // for a read-only liveness check so a lost in-memory handle cannot cause
+ // overlapping provider/tool execution while that child is still alive.
+ const checksPersistedChildLiveness =
+ currentAdapterTracksLocalChild || run.runtimeMode === "native";
+ const processPidAlive =
+ checksPersistedChildLiveness &&
+ run.processPid &&
+ isProcessAlive(run.processPid);
+ const processGroupAlive =
+ checksPersistedChildLiveness &&
+ run.processGroupId &&
+ isProcessGroupAlive(run.processGroupId);
+ if (
+ (processPidAlive || processGroupAlive) &&
+ readHotRestartAdoptionMetadata(parseObject(run.resultJson))
+ ) {
+ continue;
+ }
+ if (processPidAlive || processGroupAlive) {
+ if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) {
+ const detachedMessage = processPidAlive
+ ? `Lost in-memory process handle, but child pid ${run.processPid} is still alive`
+ : `Lost in-memory process handle, but persisted process group ${run.processGroupId} is still alive`;
+ const detachedRun = await setRunStatus(run.id, "running", {
+ error: detachedMessage,
+ errorCode: DETACHED_PROCESS_ERROR_CODE,
+ });
+ if (detachedRun) {
+ await appendRunEvent(detachedRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: detachedMessage,
+ payload: {
+ processPid: run.processPid ?? null,
+ processGroupId: run.processGroupId ?? null,
+ ownedProcessHandle: false,
+ },
+ });
+ }
+ }
+ continue;
+ }
+
+ const runContext = parseObject(run.contextSnapshot);
+ const monitorIssueId = readNonEmptyString(runContext.issueId);
+ const monitorNextCheckAt = monitorIssueId
+ ? monitorNextCheckAtByIssue.get(`${run.companyId}:${monitorIssueId}`)
+ : undefined;
+ const monitorDispatchLostWithoutFutureWake =
+ readNonEmptyString(runContext.wakeReason) === "issue_monitor_due" &&
+ monitorNextCheckAt !== undefined &&
+ (!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime());
+ const shouldRetry =
+ (run.processLossRetryCount ?? 0) < 1 &&
+ ((tracksLegacyLocalChild &&
+ (!!run.processPid || !!run.processGroupId)) ||
+ monitorDispatchLostWithoutFutureWake);
+ if (!(await revokeExpiredLegacyController(db, run))) continue;
+ const baseMessage = buildProcessLossMessage(run);
+ const processLossDiagnostic = buildProcessLossDiagnostic({
+ run,
+ nowMs: Date.now(),
+ observerStartedAtMs: performance.timeOrigin,
+ checksPersistedChildLiveness,
+ retryEligible: shouldRetry,
+ });
+ const conversationContinuationEligible = await runUsedConversationAdapter(db, run);
+
+ const failureWrite = await setRunStatusFromLive(
+ run.id,
+ "failed",
+ ["running"],
+ {
+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+ errorCode: "process_lost",
+ finishedAt: now,
+ resultJson: (() => {
+ const result = mergeRunStopMetadataForAgent(
+ { adapterType, adapterConfig },
+ "failed",
+ {
+ conversationContinuationEligible,
+ resultJson: { ...parseObject(run.resultJson), processLossDiagnostic },
+ errorCode: "process_lost",
+ errorMessage: shouldRetry
+ ? `${baseMessage}; retrying once`
+ : baseMessage,
+ },
+ );
+ return result;
+ })(),
+ },
+ );
+ if (!failureWrite.updated || !failureWrite.run) continue;
+ let finalizedRun: typeof heartbeatRuns.$inferSelect | null =
+ failureWrite.run;
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
+ finishedAt: now,
+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,
+ });
+ if (!finalizedRun) finalizedRun = await getRun(run.id);
+ if (!finalizedRun) continue;
+ finalizedRun =
+ (await classifyAndPersistRunLiveness(
+ finalizedRun,
+ parseObject(finalizedRun.resultJson),
+ )) ?? finalizedRun;
+ await releaseEnvironmentLeasesForRun({
+ runId: finalizedRun.id,
+ companyId: finalizedRun.companyId,
+ agentId: finalizedRun.agentId,
+ status: finalizedRun.status,
+ failureReason: finalizedRun.error ?? undefined,
+ });
+
+ let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;
+ const retryAgent = await getAgent(run.agentId);
+ if (shouldRetry) {
+ if (retryAgent) {
+ retriedRun = await enqueueProcessLossRetry(
+ finalizedRun,
+ retryAgent,
+ now,
+ );
+ }
+ } else if (retryAgent) {
+ const scheduled =
+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(
+ finalizedRun,
+ retryAgent,
+ );
+ retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null;
+ }
+
+ if (!retriedRun) {
+ await releaseIssueExecutionAndPromote(finalizedRun);
+ }
+
+ await appendRunEvent(finalizedRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "error",
+ message: shouldRetry
+ ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()
+ : baseMessage,
+ payload: {
+ ...(run.processPid ? { processPid: run.processPid } : {}),
+ ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),
+ ...(retriedRun ? { retryRunId: retriedRun.id } : {}),
+ },
+ });
+
+ await finalizeAgentStatus(run.agentId, "failed", baseMessage, {
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ });
+ await startNextQueuedRunForAgent(run.agentId);
+ runningProcesses.delete(run.id);
+ reaped.push(run.id);
+ }
+
+ if (reaped.length > 0) {
+ logger.warn(
+ { reapedCount: reaped.length, runIds: reaped },
+ "reaped orphaned heartbeat runs",
+ );
+ }
+
+ // Recover an active lease whose run already ended before the same-tick
+ // pending_cleanup sweep, so this tick can stop the recovered sandbox.
+ // Isolate the sweep so its failure never hides the reaper result.
+ try {
+ const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({
+ backoffMs: staleThresholdMs,
+ });
+ if (orphanedActiveLeaseSweep.recovered > 0) {
+ logger.warn(
+ { recovered: orphanedActiveLeaseSweep.recovered },
+ "recovered orphaned active environment leases",
+ );
+ }
+ } catch {
+ // Log a constant errorKind only. The exception can carry a credential in
+ // its name, code, message, cause, or stack, so the sweep never reads it.
+ logger.error(
+ { errorKind: ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND },
+ "orphaned active environment lease sweep failed",
+ );
+ }
+
+ // Retry stranded pending_cleanup leases on the same tick. Isolate the sweep
+ // so its failure never hides the reaper result. The backoff equals the
+ // reaper staleness threshold.
+ try {
+ const sweep = await sweepPendingCleanupLeases({
+ backoffMs: staleThresholdMs,
+ });
+ if (sweep.destroyed > 0 || sweep.capped > 0) {
+ logger.warn(
+ {
+ destroyed: sweep.destroyed,
+ capped: sweep.capped,
+ swept: sweep.swept,
+ },
+ "swept pending_cleanup environment leases",
+ );
+ }
+ } catch {
+ // Log a constant errorKind only. The exception can carry a credential in
+ // its name, code, message, cause, or stack, so the sweep never reads it.
+ logger.error(
+ { errorKind: PENDING_CLEANUP_SWEEP_ERROR_KIND },
+ "pending_cleanup lease sweep failed",
+ );
+ }
+
+ return { reaped: reaped.length, runIds: reaped };
+ }
+
+ return {
+ sweepPendingCleanupLeases,
+ markNativeOwnershipUnverified,
+ prepareHotRestartShutdown,
+ reconcileHotRestartAdoption,
+ recoverNativeRunsAfterRestart,
+ reapOrphanedRuns,
+ sweepOrphanedActiveLeases,
+ drainRunningRunsForShutdown,
+ };
+}