The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(heartbeat): extract restart recovery and lease cleanup (#15622)
Dotta04de3132c202
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 709eb2cb93..9736a2e7be 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1263,6 +1263,15 @@ run events, issue-lock release, plan-resume reporting, and worktree cutoffs. Itre-exports the existing retry helpers and workspace-busy error class. Keep retrypolicy changes separate from this extraction.+Restart recovery and lease cleanup are in `server/src/services/heartbeat/recovery.ts`.+It owns hot-restart snapshots and adoption, native restart recovery, shutdown+draining, orphaned-run reaping, and active/pending-cleanup lease sweeps.+`createHeartbeatRecovery` binds the service database and explicit lifecycle+callbacks without starting work. The service supplies its shutdown flag callback+and shared execution sets so separate service instances keep the same ownership+and shutdown barriers. Cleanup single-flight state stays at module scope.+Keep recovery policy changes separate from retry scheduling and execution changes.+## Wake Context DeliveryBuilt-in adapters deliver wake context through the run prompt, including structureddiff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex a4f6421e26..6d47fc4795 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -7974,7 +7974,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {});expect(next?.contextSnapshot?.wakeCommentIds).toEqual([pending!.id, go!.id]);expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferred!.id)))[0]).toMatchObject({ status: "coalesced", runId: next!.id });- await vi.waitFor(async () => expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"));+ await heartbeat.drainActiveRunExecutions();+ expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");});it.each(["dedicated deferred donor", "non-coalescing recipient", "persistent agent conversation"] as const)(diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex a3a8969166..5673162940 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,3 +1,10 @@+import {+ HEARTBEAT_RUN_TERMINAL_STATUSES,+ DETACHED_PROCESS_ERROR_CODE,+ isProcessAlive,+ CANCELLABLE_HEARTBEAT_RUN_STATUSES,+ createHeartbeatRecovery,+} from "./heartbeat/recovery.js";import {BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,normalizeAgentNameKey,@@ -238,7 +245,7 @@ import {NativeWorkspaceFinalizationOwnershipLostError,type NativeWorkspaceFinalizationOwnership,} from "./native-runtime/native-workspace-finalization-ownership.js";-import { hasStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js";+import { reserveRunBudget } from "./budget-reservations.js";import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js";import { createRunUsageRecorder } from "./usage-receipts.js";@@ -257,9 +264,18 @@ import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-import { getConversationConfirmationContext } from "./conversation-confirmation-context.js";import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";-import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLease, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js";+import {+ legacyControllerBootId,+ legacyControllerClaim,+ renewLegacyControllerLease,+ watchLegacyControllerLease,+} from "./legacy-controller-lease.js";import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";-import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";+import {+ hasRemoteTerminationReceipt,+ remoteExecutionHasStopped,+ stoppedRemoteCleanupScopes,+} from "./remote-execution-termination.js";import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";import {admitExplicitNativeContinuation,@@ -436,13 +452,9 @@ import {buildNativeExecutionWithCheckpoint,buildNativeRuntimeContext,cancelNativeSession,- claimNativeRestartRecoveries,closeWarmNativeSessionsForEnvironment,- closeIdleWarmNativeSessionsForRestart,reserveWarmNativeInstructionDirectory,- currentNativeControllerIdentity,dispatchNativeSessionResumptions,- detachNativeSessionsForRestart,ensureNativeCompletionContract,executePaperclipNativeSession,finalizeNativeRun,@@ -461,13 +473,9 @@ import {nativeToolContractFingerprintForTarget,prepareNativeSessionBootstrapPersistence,prepareNativeWorkspaceSync,- readNativeWorkspaceSyncReference,recordNativeFinalizationFailure,type NativeRestartRecoveryClaim,rebindNativeSessionCheckpoint,- reconcileNativeFinalizations,- reconcileRetainedNativeSessionCleanup,- reconcileRetainedNativeSessionCleanups,resolveHeartbeatNativeRuntimeMode,} from "./native-runtime/index.js";import {@@ -484,7 +492,6 @@ import {type NativeRunHistoricalSpan,} from "./native-runtime/native-run-trace.js";import {- drainRetainedRunnerdMaintenanceOperations,describeRunnerdNativeSessionBackend,parseNativeExecutionInput,type NativeExecutionInput,@@ -539,8 +546,7 @@ import {} from "./agent-task-run-telemetry.js";import { readAiConnectionConfigurationFailure } from "./ai-connection-configuration-failure.js";import { reportRunFailure } from "./run-failure-report.js";-import { performance } from "node:perf_hooks";-import { buildProcessLossDiagnostic } from "./process-loss-diagnostics.js";+import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";import { companySkillService } from "./company-skills.js";import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";@@ -808,15 +814,7 @@ import {touchHeartbeatRunRuntimeStatus,} from "./heartbeat-run-runtime-status.js";import {- findMissingHotRestartSnapshotRunIds,- readHotRestartIntent,readProcessStartedAt,- removeHotRestartIntent,- shouldHonorHotRestartIntentForProcess,- writeHotRestartReport,- writeHotRestartShutdownSnapshot,- type HotRestartIntentRun,- type HotRestartReportRun,} from "./hot-restart.js";import {assertLowTrustRuntimeServicesAllowed,@@ -825,7 +823,7 @@ import {import { resolveAndRetainRunTrustPreset } from "./run-trust-preset.js";import type { PluginWorkerManager } from "./plugin-worker-manager.js";-import { serverVersion } from "../version.js";+import { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain, readTaskDrain } from "./task-admission.js";const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;@@ -861,123 +859,16 @@ const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [];const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";-const ACCEPTED_PLAN_CONVERSION_SKILL_KEY ="paperclipai/paperclip/paperclip-converting-plans-to-tasks";---const DETACHED_PROCESS_ERROR_CODE = "process_detached";-const NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE =- "Native execution ownership could not be verified; automatic recovery is blocked";-// The reaper sweeps at most this many pending_cleanup leases per tick.-const PENDING_CLEANUP_SWEEP_PAGE_SIZE = 20;-const pendingCleanupAttemptsInFlight = new Set<string>();-// Escalate and slow cleanup after this many attempts; never abandon a live lease.-const PENDING_CLEANUP_SWEEP_ATTEMPT_CAP = 5;-// The reaper stores its retry state under these keys in the lease metadata.-const PENDING_CLEANUP_ATTEMPTS_METADATA_KEY = "pendingCleanupRetryAttempts";-const PENDING_CLEANUP_CAP_WARNED_METADATA_KEY = "pendingCleanupRetryCapWarned";-// The reaper sweeps at most this many orphaned active leases per tick.-const ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE = 20;--// A provider or plugin destroy rejection can carry a bearer credential, a-// signed URL, or provider response detail in its name, code, message, cause, or-// stack. The exception fields cross the server boundary, so they are not a-// trusted enum. The pending_cleanup sweep logs never read the exception. Each-// catch site logs a constant, locally generated `errorKind` instead.-const PENDING_CLEANUP_RETRY_ERROR_KIND = "destroy_failed";-const PENDING_CLEANUP_SWEEP_ERROR_KIND = "sweep_failed";-const ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND = "orphaned_active_lease_sweep_failed";--// Read the stored retry attempt count as a safe value, directly in SQL. A-// provider can write a malformed value under the attempts key. The type guard-// makes any non-number value read as zero. The reader computes as numeric and-// never casts to int, so a finite number outside the 32-bit range (for example-// 1e300) never throws. The reader clamps a negative value to zero and a positive-// value to the attempt cap. One malformed lease therefore never aborts the page-// sweep. This matches the TypeScript reader `readPendingCleanupRetryAttempts`,-// which clamps to the same range. The claim predicate compares the two readers,-// so both must yield the same value for every input.-function pendingCleanupAttemptsSql() {- return sql`- case- when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}) = 'number'- then least(- greatest(- floor((${environmentLeases.metadata} ->> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY})::numeric),- 0- ),- ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}- )- else 0- end`;-}--function pendingCleanupRetryDueSql(explicitRetry = false) {- // Ownership renewal and retry cooldown are separate clocks. A late renewal- // cannot change when a completed attempt may retry. Older claims used the- // retry field for both clocks, so retain that fallback until they settle.- const deadline = sql`case- when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'- and ${environmentLeases.metadata} ? 'pendingCleanupLeaseExpiresAtMs'- then ${environmentLeases.metadata}->'pendingCleanupLeaseExpiresAtMs'- else ${environmentLeases.metadata}->'pendingCleanupRetryAfterMs' end`;- return sql`case- when ${explicitRetry} and coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true' then true- when jsonb_typeof(${deadline}) = 'number'- then (${deadline})::numeric <= ${Date.now()}- or (${deadline})::numeric > ${Date.now() + 30 * 60_000 + 1_000}- else true end`;-}--// Choose the `jsonb_set` target root. A provider can write a scalar or array-// metadata root. `jsonb_set` fails on a non-object root, so the reader uses the-// stored metadata only when its root is an object. A NULL, scalar, or array root-// reads as an empty object. `jsonb_typeof(NULL)` is NULL, so the else branch also-// covers a NULL root.-function pendingCleanupMetadataObjectSql() {- return sql`case when jsonb_typeof(${environmentLeases.metadata}) = 'object' then ${environmentLeases.metadata} else '{}'::jsonb end`;-}--// Read the stored cap-warned flag as a safe boolean, directly in SQL. A-// malformed value reads as false, so the boolean cast never throws.-function pendingCleanupCapWarnedSql() {- return sql`coalesce(- case- when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}) = 'boolean'- then (${environmentLeases.metadata} ->> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY})::boolean- else false- end,- false- )`;-}-------const execFile = promisify(execFileCallback);const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued","running","scheduled_retry",] as const;-const CANCELLABLE_HEARTBEAT_RUN_STATUSES = [- "queued",- "running",- "scheduled_retry",-] as const;-const NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY = "nativeQuestionCancellation";-const HEARTBEAT_RUN_TERMINAL_STATUSES = [- "succeeded",- "interrupted",- "failed",- "cancelled",- "timed_out",-] as const;+const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed","cancelled",@@ -1186,16 +1077,6 @@ const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set(["issue_recovery_action_restored",]);-const SESSIONED_LOCAL_ADAPTERS = new Set([- "claude_local",- "codex_local",- "cursor",- "gemini_local",- "hermes_local",- "kimi_local",- "opencode_local",- "pi_local",-]);// Routes and the scheduler construct separate heartbeatService instances, but// they must agree on in-process adapter executions when reaping stale runs.const activeRunExecutions = new Set<string>();@@ -1633,10 +1514,6 @@ function isSameTaskScope(left: string | null, right: string | null) {return (left ?? null) === (right ?? null);}-function isTrackedLocalChildProcessAdapter(adapterType: string) {- return SESSIONED_LOCAL_ADAPTERS.has(adapterType);-}-function isHeartbeatRunTerminalStatus(status: string | null | undefined,): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {@@ -1914,23 +1791,6 @@ function buildRunEventRuntimeProgress(input: {};}-// A positive liveness check means some process currently owns the PID.-// On Linux, PIDs can be recycled, so this is a best-effort signal rather-// than proof that the original child is still alive.-function isProcessAlive(pid: number | null | undefined) {- if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)- return false;- try {- process.kill(pid, 0);- return true;- } catch (error) {- const code = (error as NodeJS.ErrnoException | undefined)?.code;- if (code === "EPERM") return true;- if (code === "ESRCH") return false;- return false;- }-}-export async function persistHeartbeatRunProcessMetadata(db: Db,runId: string,@@ -1990,65 +1850,6 @@ async function terminateHeartbeatRunProcess(input: {);}-function buildProcessLossMessage(- run: {- processPid: number | null;- processGroupId: number | null;- },- options?: { descendantOnly?: boolean },-) {- if (options?.descendantOnly && run.processGroupId) {- return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`;- }- if (run.processPid) {- return `Process lost -- child pid ${run.processPid} is no longer running`;- }- if (run.processGroupId) {- return `Process lost -- process group ${run.processGroupId} is no longer running`;- }- return "Process lost -- server may have restarted";-}--function readHotRestartAdoptionMetadata(- resultJson: Record<string, unknown> | null | undefined,-) {- const result = parseObject(resultJson);- const hotRestart = parseObject(result.hotRestart);- if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string")- return null;- return hotRestart;-}--function mergeHotRestartAdoptionResultJson(- resultJson: Record<string, unknown> | null | undefined,- input: {- adoptedAt: Date;- previousServerPid: number;- newServerPid: number;- previousServerVersion: string | null;- newServerVersion: string;- processPid: number | null;- processGroupId: number | null;- },-) {- const result = parseObject(resultJson);- const existing = parseObject(result.hotRestart);- return {- ...result,- hotRestart: {- ...existing,- adopted: true,- adoptedAt: input.adoptedAt.toISOString(),- previousServerPid: input.previousServerPid,- newServerPid: input.newServerPid,- previousServerVersion: input.previousServerVersion,- newServerVersion: input.newServerVersion,- processPid: input.processPid,- processGroupId: input.processGroupId,- },- };-}-type SkillTestHeartbeatCompletion = {outcome: "failed" | "cancelled";error: string | null;@@ -2456,6 +2257,49 @@ export function heartbeatService(applyRunDispatchPostCommitEffects,});+ const {+ sweepPendingCleanupLeases,+ markNativeOwnershipUnverified,+ prepareHotRestartShutdown,+ reconcileHotRestartAdoption,+ recoverNativeRunsAfterRestart,+ reapOrphanedRuns,+ sweepOrphanedActiveLeases,+ drainRunningRunsForShutdown,+ } = createHeartbeatRecovery(db, {+ enterShutdown: () => { shutdownInProgress = true; },+ scheduleBoundedRetryForRun,+ appendRunEvent,+ environmentRuntime,+ scheduleNativeSessionResumeDispatch,+ getRun,+ executeRun,+ activeRunExecutionPromises,+ cancelHeartbeatNativeRun,+ terminateHeartbeatRunProcess,+ setRunStatusIfRunning,+ mergeRunStopMetadataForAgent,+ setWakeupStatus,+ classifyAndPersistRunLiveness,+ releaseEnvironmentLeasesForRun,+ releaseIssueExecutionAndPromote,+ finalizeAgentStatus,+ timerClaimWasFirstHeartbeat,+ environmentsSvc,+ acknowledgeRemoteStop,+ resumeRemoteStopComments,+ setRunStatusFromLive,+ instructionCopies,+ dispatchPendingNativeStatusWakeups,+ runtimeEnv,+ cancelRunInternal,+ activeRunExecutions,+ setRunStatus,+ getAgent,+ scheduleInteractionContinuationInfrastructureRetryIfEligible,+ startNextQueuedRunForAgent,+ });+// Applies the post-commit effects a run-dispatch operation returns, on a// best-effort basis, exactly as this service publishes them for every// other run write. A publish failure never rolls back the write that@@ -6025,1231 +5869,352 @@ export function heartbeatService(return { outcome: "retry_exhausted" as const, queuedRun: null };}- async function enqueueProcessLossRetry(- run: typeof heartbeatRuns.$inferSelect,- agent: typeof agents.$inferSelect,- now: Date,- ) {- // Completion deliveries own their durable bounded retry and reply identity.- // A second process-loss retry would compete for the same outbox input.- if (run.contextSnapshot?.wakeReason === CHAT_COMPLETION_WAKE_REASON &&- Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds)) return null;- // Native sessions have their own fenced same-run controller. Legacy- // bootstrap recovery shares the durable delay and incident counter with- // transient retries; process loss must not open a second retry budget.- if (run.runtimeMode === "native" || await legacyExecutionNeedsReconciliationWithEvidence(db, run))- return null;- const scheduled = await scheduleBoundedRetryForRun(run, agent, { now });- return scheduled.outcome === "scheduled" ? scheduled.run : null;- }+ function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) {+ const runtimeConfig = parseObject(agent.runtimeConfig);+ const heartbeat = parseObject(runtimeConfig.heartbeat);- function toHotRestartIntentRun(input: {- run: typeof heartbeatRuns.$inferSelect;- adapterType: string;- }): HotRestartIntentRun {- const context = parseObject(input.run.contextSnapshot);return {- runId: input.run.id,- companyId: input.run.companyId,- agentId: input.run.agentId,- adapterType: input.adapterType,- status: input.run.status,- processPid: input.run.processPid ?? null,- processGroupId: input.run.processGroupId ?? null,- issueId: readNonEmptyString(context.issueId),- runtimeMode: input.run.runtimeMode,- nativeSessionId: input.run.nativeSessionId,- runnerInstanceId: input.run.runnerInstanceId,- processStartedAt: input.run.processStartedAt?.toISOString() ?? null,+ enabled: asBoolean(heartbeat.enabled, false),+ intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),+ wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent),+ // A Dot binding has one external turn. Competing assignments must retain+ // their queue position instead of claiming a second run that cannot bind.+ maxConcurrentRuns: agent.adapterType === "paperclip_runner" &&+ parseObject(agent.adapterConfig).provider === "openai_dot"+ ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),+ skipTimerWhenNoActionableWork: asBoolean(+ heartbeat.skipTimerWhenNoActionableWork ??+ heartbeat.requireActionableTimerWork ??+ heartbeat.issueOnlyTimer,+ false,+ ),+ maxDailyRuns: normalizeOptionalNonNegativeInteger(+ heartbeat.maxDailyRuns ??+ heartbeat.dailyRunLimit ??+ heartbeat.dailyRunCap ??+ heartbeat.maxRunsPerDay,+ ),+ maxDailyCostCents: normalizeOptionalNonNegativeInteger(+ heartbeat.maxDailyCostCents ??+ heartbeat.dailyCostCentsLimit ??+ heartbeat.dailySpendCentsLimit ??+ heartbeat.dailyBudgetCents,+ ),};}- function isServerStdioBoundHotRestartRun(input: {- run: typeof heartbeatRuns.$inferSelect;- adapterType: string;- adapterConfig: unknown;- }) {- const context = parseObject(input.run.contextSnapshot);- if (- context.processTopology === "server_stdio" ||- context.executionEngine === "acp"- ) {- return true;- }- if (- context.processTopology === "detached" ||- context.executionEngine === "cli"- ) {- return false;- }- if (- !["claude_local", "codex_local", "gemini_local"].includes(- input.adapterType,- )- ) {- return false;- }- return (- readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli"+ function normalizeOptionalNonNegativeInteger(value: unknown) {+ if (value === null || value === undefined || value === "") return null;+ const normalized = Math.floor(asNumber(value, 0));+ return normalized >= 0 ? normalized : null;+ }++ function currentUtcDayWindow(now = new Date()) {+ const start = new Date(+ Date.UTC(+ now.getUTCFullYear(),+ now.getUTCMonth(),+ now.getUTCDate(),+ 0,+ 0,+ 0,+ 0,+ ),+ );+ const end = new Date(+ Date.UTC(+ now.getUTCFullYear(),+ now.getUTCMonth(),+ now.getUTCDate() + 1,+ 0,+ 0,+ 0,+ 0,+ ),);+ return { start, end };}- async function prepareHotRestartShutdown(- signal: "SIGINT" | "SIGTERM",- now = new Date(),+ async function getHeartbeatDailyCapBlock(+ agent: typeof agents.$inferSelect,+ policy: ReturnType<typeof parseHeartbeatPolicy>,+ options: {+ checkRunCap?: boolean;+ checkCostCap?: boolean;+ excludeRunId?: string | null;+ } = {},+ client: Pick<Db, "select"> = db,) {- shutdownInProgress = true;- const idleSessions = await closeIdleWarmNativeSessionsForRestart();- if (idleSessions.failed > 0) {- logger.warn({ idleSessions }, "idle native sessions could not checkpoint before controller shutdown");- }- let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;- try {- intent = await readHotRestartIntent();- } catch (err) {- logger.warn(- { err },- "failed to read hot-restart intent; falling back to normal shutdown drain",- );- return {- mode: "read_error" as const,- skipDrain: false as const,- activeRunIds: [] as string[],- };+ const checkRunCap = options.checkRunCap ?? true;+ const checkCostCap = options.checkCostCap ?? true;+ const { start, end } = currentUtcDayWindow();+ if (checkRunCap && policy.maxDailyRuns !== null) {+ const conditions = [+ eq(heartbeatRuns.companyId, agent.companyId),+ eq(heartbeatRuns.agentId, agent.id),+ gte(heartbeatRuns.startedAt, start),+ lt(heartbeatRuns.startedAt, end),+ notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]),+ ];+ if (options.excludeRunId) {+ conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`);+ }+ const [row] = await client+ .select({ total: sql<number>`count(*)::integer` })+ .from(heartbeatRuns)+ .where(and(...conditions));+ const observed = Number(row?.total ?? 0);+ if (observed >= policy.maxDailyRuns) {+ return {+ reason: "heartbeat.daily_run_limit",+ observed,+ limit: policy.maxDailyRuns,+ };+ }}- if (!intent)- return {- mode: "not_requested" as const,- skipDrain: false as const,- activeRunIds: [] as string[],- };- if (intent.drainRequired)- return {- mode: "drain_required" as const,- skipDrain: false as const,- activeRunIds: [] as string[],- };- if (!shouldHonorHotRestartIntentForProcess(intent)) {- logger.warn(- { expectedPid: intent.previousServerPid, currentPid: process.pid },- "hot-restart intent targets a different server pid; falling back to normal shutdown drain",- );- return {- mode: "pid_mismatch" as const,- skipDrain: false as const,- activeRunIds: [] as string[],- };+ if (checkCostCap && policy.maxDailyCostCents !== null) {+ const [row] = await client+ .select({+ total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,+ })+ .from(costEvents)+ .where(+ and(+ eq(costEvents.companyId, agent.companyId),+ eq(costEvents.agentId, agent.id),+ gte(costEvents.occurredAt, start),+ lt(costEvents.occurredAt, end),+ ),+ );+ const observed = Number(row?.total ?? 0);+ if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) {+ return {+ reason: "heartbeat.daily_cost_limit",+ observed,+ limit: policy.maxDailyCostCents,+ };+ }}- const activeRuns = await db- .select({- run: heartbeatRuns,- adapterType: agents.adapterType,- adapterConfig: agents.adapterConfig,- })- .from(heartbeatRuns)- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))- .where(eq(heartbeatRuns.status, "running"));- const snapshotRuns = activeRuns.map(toHotRestartIntentRun);- const intentWithVersion = {- ...intent,- previousServerVersion: intent.previousServerVersion ?? serverVersion,- };-- const serverStdioRuns = activeRuns.filter(isServerStdioBoundHotRestartRun);- if (serverStdioRuns.length > 0) {- const activeServerStdioRunIds = serverStdioRuns.map(({ run }) => run.id);- await writeHotRestartShutdownSnapshot({- intent: intentWithVersion,- signal,- activeRuns: snapshotRuns,- drainReason: "active_acp_run",- drainRunIds: activeServerStdioRunIds,- capturedAt: now,- });-- logger.warn(- {- signal,- previousServerPid: intent.previousServerPid,- activeRunIds: snapshotRuns.map((run) => run.runId),- activeServerStdioRunIds,- drainReason: "active_acp_run",- },- "server-stdio agent run prevents hot-restart adoption; using graceful drain and retry",- );-- return {- mode: "acp_drain_required" as const,- skipDrain: false as const,- activeRunIds: snapshotRuns.map((run) => run.runId),- activeAcpRunIds: activeServerStdioRunIds,- drainRunIds: activeServerStdioRunIds,- drainReason: "active_acp_run" as const,- };- }+ return null;+ }- await writeHotRestartShutdownSnapshot({- intent: intentWithVersion,- signal,- activeRuns: snapshotRuns,- capturedAt: now,- });-- for (const { run } of activeRuns) {- await appendRunEvent(run, {- eventType: "lifecycle",- stream: "system",- level: "info",- message:- "Hot restart requested; leaving child process alive for startup adoption",- payload: {- signal,- previousServerPid: intent.previousServerPid,- previousServerVersion: intentWithVersion.previousServerVersion,- processPid: run.processPid ?? null,- processGroupId: run.processGroupId ?? null,- },- });- }-- const nativeRunIds = activeRuns- .filter(- ({ run, adapterType }) =>- adapterType === "paperclip_runner" &&- isNativeSessionId(run.nativeSessionId),- )- .map(({ run }) => run.id);- const detachedNativeSessions =- await detachNativeSessionsForRestart(nativeRunIds);-- logger.info(- {- signal,- previousServerPid: intent.previousServerPid,- activeRunIds: snapshotRuns.map((run) => run.runId),- detachedNativeSessions,+ async function cancelQueuedRunForHeartbeatDailyCap(+ run: typeof heartbeatRuns.$inferSelect,+ dailyCapBlock: NonNullable<+ Awaited<ReturnType<typeof getHeartbeatDailyCapBlock>>+ >,+ ) {+ const now = new Date();+ const reason =+ "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation";+ const cancelled = await setRunStatus(run.id, "cancelled", {+ finishedAt: now,+ error: reason,+ errorCode: dailyCapBlock.reason,+ resultJson: {+ ...parseObject(run.resultJson),+ stopReason: dailyCapBlock.reason,+ observed: dailyCapBlock.observed,+ limit: dailyCapBlock.limit,+ effectiveTimeoutSec: 0,+ timeoutConfigured: false,+ timeoutSource: "heartbeat_daily_cap_gate",+ timeoutFired: false,},- "hot-restart shutdown snapshot captured; skipping graceful run drain",- );-- return {- mode: "hot_restart" as const,- skipDrain: true as const,- activeRunIds: snapshotRuns.map((run) => run.runId),- };- }-- async function reconcileHotRestartAdoption(now = new Date()) {- let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;- try {- intent = await readHotRestartIntent();- } catch (err) {- logger.warn(- { err },- "failed to read hot-restart intent on startup; skipping adoption",- );- return {- mode: "read_error" as const,- adoptedRunIds: [] as string[],- finalizedWhileDownRunIds: [] as string[],- lostRunIds: [] as string[],- skippedRunIds: [] as string[],- };- }- if (!intent) {- return {- mode: "not_requested" as const,- adoptedRunIds: [] as string[],- finalizedWhileDownRunIds: [] as string[],- lostRunIds: [] as string[],- skippedRunIds: [] as string[],- };- }-- if (!intent.shutdownSnapshot) {- const log = intent.drainRequired- ? logger.info.bind(logger)- : logger.warn.bind(logger);- log(- {- previousServerPid: intent.previousServerPid,- preflightActiveRunIds: intent.preflightActiveRunIds,- drainReason: intent.drainReason ?? null,- },- intent.drainRequired- ? "drain-required restart intent has no adoption snapshot"- : "hot-restart intent present but shutdown snapshot is missing; no runs can be adopted",- );- }- const candidates = intent.shutdownSnapshot?.activeRuns ?? [];- const missingSnapshotRunIds = findMissingHotRestartSnapshotRunIds(intent);- const reconciliationRunIds = [- ...new Set([- ...candidates.map((run) => run.runId),- ...missingSnapshotRunIds,- ]),- ];- const currentRows =- reconciliationRunIds.length > 0- ? await db- .select({- run: heartbeatRuns,- adapterType: agents.adapterType,- })- .from(heartbeatRuns)- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))- .where(inArray(heartbeatRuns.id, reconciliationRunIds))- : [];- const currentByRunId = new Map(currentRows.map((row) => [row.run.id, row]));-- const reportRuns: HotRestartReportRun[] = [];- const adoptedRunIds: string[] = [];- const finalizedWhileDownRunIds: string[] = [];- const lostRunIds: string[] = [];- const skippedRunIds: string[] = [];-- const classify = (- candidate: HotRestartIntentRun,- classification: HotRestartReportRun["classification"],- reason: string,- patch?: Partial<HotRestartIntentRun>,- ) => {- const run = {- ...candidate,- ...patch,- classification,- reason,- } satisfies HotRestartReportRun;- reportRuns.push(run);- if (classification === "adopted") adoptedRunIds.push(candidate.runId);- else if (classification === "finalized_while_down")- finalizedWhileDownRunIds.push(candidate.runId);- else if (classification === "lost") lostRunIds.push(candidate.runId);- else skippedRunIds.push(candidate.runId);- };-- for (const runId of missingSnapshotRunIds) {- const current = currentByRunId.get(runId);- if (!current) {- finalizedWhileDownRunIds.push(runId);- continue;- }-- const candidate = toHotRestartIntentRun(current);- if (current.run.status !== "running") {- classify(- candidate,- "finalized_while_down",- `run_status_${current.run.status}`,- );- } else {- classify(candidate, "lost", "missing_shutdown_snapshot");- }- }-- if (lostRunIds.length > 0) {- logger.error(- { previousServerPid: intent.previousServerPid, lostRunIds },- "hot-restart shutdown snapshot omitted live preflight runs; reporting them as lost",- );- }-- for (const candidate of candidates) {- const current = currentByRunId.get(candidate.runId);- if (!current) {- classify(candidate, "finalized_while_down", "run_row_missing");- continue;- }-- const { run, adapterType } = current;- const patch = {- adapterType,- status: run.status,- processPid: run.processPid ?? candidate.processPid,- processGroupId: run.processGroupId ?? candidate.processGroupId,- };-- if (run.status !== "running") {- classify(- candidate,- "finalized_while_down",- `run_status_${run.status}`,- patch,- );- continue;- }-- const hasSelectiveAcpDrain =- intent.drainReason === "active_acp_run" &&- (intent.drainRunIds?.length ?? 0) > 0;- if (- hasSelectiveAcpDrain &&- intent.drainRunIds?.includes(candidate.runId)- ) {- // A selective ACP drain is expected to persist a terminal row before- // the new server starts. If the process was terminated but that write- // failed, surface the run as lost instead of hiding it as an expected- // drain skip.- classify(candidate, "lost", "selective_drain_not_finalized", patch);- continue;- }- if (intent.drainRequired && !hasSelectiveAcpDrain) {- classify(candidate, "skipped", "drain_required", patch);- continue;- }-- if (run.runtimeMode === "native" && adapterType === "paperclip_runner") {- classify(candidate, "skipped", "native_restart_recovery_owned", patch);- continue;- }-- if (!isTrackedLocalChildProcessAdapter(adapterType)) {- classify(- candidate,- "skipped",- "adapter_not_local_child_process",- patch,- );- continue;- }-- const processPid = run.processPid ?? candidate.processPid;- const processGroupId = run.processGroupId ?? candidate.processGroupId;- const processPidAlive = isProcessAlive(processPid);- const processGroupAlive = isProcessGroupAlive(processGroupId);- if (!processPid && !processGroupId) {- classify(candidate, "lost", "missing_process_metadata", patch);- continue;- }- if (!processPidAlive && !processGroupAlive) {- classify(candidate, "lost", "process_not_alive", patch);- continue;- }-- const resultJson = mergeHotRestartAdoptionResultJson(- parseObject(run.resultJson),- {- adoptedAt: now,- previousServerPid: intent.previousServerPid,- newServerPid: process.pid,- previousServerVersion: intent.previousServerVersion,- newServerVersion: serverVersion,- processPid,- processGroupId,- },- );- const updated = await db- .update(heartbeatRuns)- .set({- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(resultJson, true),- error:- run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error,- errorCode:- run.errorCode === DETACHED_PROCESS_ERROR_CODE- ? null- : run.errorCode,- updatedAt: now,- })- .where(- and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.status, "running"),- ),- )- .returning()- .then((rows) => rows[0] ?? null);-- if (!updated) {- const latest = await db- .select({ status: heartbeatRuns.status })- .from(heartbeatRuns)- .where(eq(heartbeatRuns.id, run.id))- .then((rows) => rows[0] ?? null);- if (latest && latest.status !== "running") {- classify(- candidate,- "finalized_while_down",- `run_status_${latest.status}`,- patch,- );- } else {- classify(candidate, "lost", "adoption_update_not_applied", patch);- }- continue;- }-- await appendRunEvent(updated, {- eventType: "lifecycle",- stream: "system",- level: "info",- message: "Adopted live child process after hot restart",- payload: {- previousServerPid: intent.previousServerPid,- newServerPid: process.pid,- previousServerVersion: intent.previousServerVersion,- newServerVersion: serverVersion,- processPid,- processGroupId,- },- });- classify(- candidate,- "adopted",- processPidAlive ? "process_pid_alive" : "process_group_alive",- patch,- );- }+ });+ if (!cancelled) return null;- const report = await writeHotRestartReport({- version: 1,- requestedAt: intent.requestedAt,- completedAt: now.toISOString(),- drainRequired: intent.drainRequired,- drainReason:- intent.drainReason ?? (intent.drainRequired ? "requested" : null),- previousServerPid: intent.previousServerPid,- newServerPid: process.pid,- previousServerVersion: intent.previousServerVersion,- newServerVersion: serverVersion,- adoptedRunIds,- finalizedWhileDownRunIds,- lostRunIds,- skippedRunIds,- runs: reportRuns,+ await setWakeupStatus(run.wakeupRequestId, "skipped", {+ finishedAt: now,+ error: reason,});- await removeHotRestartIntent(undefined, intent);- logger.info(- {- previousServerPid: report.previousServerPid,- newServerPid: report.newServerPid,- adoptedRunIds,- finalizedWhileDownRunIds,- lostRunIds,- missingSnapshotRunIds,- skippedRunIds,+ await appendRunEvent(cancelled, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: reason,+ payload: {+ reason: dailyCapBlock.reason,+ observed: dailyCapBlock.observed,+ limit: dailyCapBlock.limit,},- "hot-restart adoption report written",- );+ });- return {- mode: "reported" as const,- adoptedRunIds,- finalizedWhileDownRunIds,- lostRunIds,- skippedRunIds,- };+ await releaseIssueExecutionAndPromote(cancelled, {+ suppressImmediateRecovery: true,+ });++ return cancelled;}- async function recoverNativeRunsAfterRestart(now = new Date()) {- // A result committed before the old controller stopped outranks process- // recovery. Finish its durable workspace/status suffix before deciding- // whether any provider authority needs to be reopened.- await reconcileNativeFinalizations(db, undefined, {- environmentRuntime,- onWorkspaceSettled: settleRecoveredNativeWorkspace,- });- scheduleRetainedNativeSessionCleanup();- const intent = await readHotRestartIntent().catch((error) => {- logger.warn(- { err: error },- "failed to read hot-restart intent before native startup recovery",- );- return null;- });- const restartKind = intent ? ("hot" as const) : ("hard" as const);- const previousStartedAt = intent?.previousServerStartedAt- ? new Date(intent.previousServerStartedAt)- : null;- const scheduledNativeRetries = await db- .select({- runId: nativeRunFinalizations.runId,- nextAttemptAt: nativeRunFinalizations.nextAttemptAt,- })- .from(nativeRunFinalizations)- .innerJoin(- heartbeatRuns,- eq(heartbeatRuns.id, nativeRunFinalizations.runId),- )+ async function hasActionableTimerWork(agent: typeof agents.$inferSelect) {+ const row = await db+ .select({ id: issues.id })+ .from(issues).where(and(- eq(heartbeatRuns.runtimeMode, "native"),- inArray(heartbeatRuns.status, ["running", "failed"]),- isNull(nativeRunFinalizations.resultId),- eq(nativeRunFinalizations.phase, "retryable_failure"),- gt(nativeRunFinalizations.nextAttemptAt, now),+ eq(issues.companyId, agent.companyId),+ eq(issues.assigneeAgentId, agent.id),+ isNull(issues.assigneeUserId),+ isNull(issues.hiddenAt),+ inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]),+ isNull(issues.conversationAgentId),+ nonIdleSlackIssueCondition(),),- );- for (const scheduled of scheduledNativeRetries) {- if (scheduled.nextAttemptAt) {- scheduleNativeSessionResumeDispatch(- scheduled.runId,- scheduled.nextAttemptAt,- );- }- }- const dispositions = await claimNativeRestartRecoveries({- db,- restartKind,- recoveryRequestId: intent?.recoveryRequestId ?? null,- coordinatedPreviousController: intent- ? {- pid: intent.previousServerPid,- processStartedAt:- previousStartedAt && !Number.isNaN(previousStartedAt.getTime())- ? previousStartedAt- : null,- }- : null,- now,- });-- const claims = dispositions.filter(- (disposition): disposition is NativeRestartRecoveryClaim =>- disposition.kind === "reattach_existing_runner" ||- disposition.kind === "reattach_remote_runner" ||- disposition.kind === "resume_dead_runner" ||- disposition.kind === "bootstrap_incomplete",- );- for (const disposition of dispositions) {- const run = await getRun(disposition.runId);- if (run) {- const isClaim =- disposition.kind === "reattach_existing_runner" ||- disposition.kind === "reattach_remote_runner" ||- disposition.kind === "resume_dead_runner" ||- disposition.kind === "bootstrap_incomplete";- await appendRunEvent(run, {- eventType: "native.recovery.transition",- stream: "system",- level: disposition.kind === "blocked" ? "warn" : "info",- message:- disposition.kind === "reattach_existing_runner" ||- disposition.kind === "reattach_remote_runner"- ? "Recovering the existing native runner process after server restart"- : disposition.kind === "resume_dead_runner"- ? "Resuming the durable native provider session after runner process loss"- : disposition.kind === "bootstrap_incomplete"- ? "Restarting an incomplete native runner bootstrap on the same heartbeat run"- : disposition.kind === "awaiting_evidence"- ? "Native restart recovery is waiting for safe ownership evidence"- : disposition.kind === "already_finalized"- ? "Native restart recovery found an already-finalized result"- : "Native restart recovery blocked ambiguous or conflicting ownership",- payload: {- restartKind: isClaim ? disposition.restartKind : restartKind,- recoveryRequestId: isClaim- ? disposition.recoveryRequestId- : (intent?.recoveryRequestId ?? null),- runnerDisposition: disposition.kind,- ...(isClaim- ? {- controllerGeneration: disposition.controllerGeneration,- providerAttempt: disposition.providerAttempt,- }- : { reason: disposition.reason }),- ...(disposition.kind === "reattach_existing_runner"- ? {- processPid: disposition.process.pid,- processGroupId: disposition.process.processGroupId,- processStartedAt: disposition.process.startedAt,- }- : {}),- },- });- }- }- for (const claim of claims) {- const execution = executeRun(claim.runId, {- nativeLeaseOwner: claim.leaseOwner,- nativeRestartRecovery: claim,- }).catch((error) => {- logger.error(- { err: error, runId: claim.runId, disposition: claim.kind },- "native restart recovery execution failed",- );- });- activeRunExecutionPromises.add(execution);- void execution.finally(() =>- activeRunExecutionPromises.delete(execution),- );- }+ )+ .limit(1)+ .then((rows) => rows[0] ?? null);+ return Boolean(row);+ }- return {- restartKind,- claims,- dispositions,- scheduledRetryRunIds: scheduledNativeRetries.map((entry) => entry.runId),- awaitingEvidenceRunIds: dispositions- .filter((entry) => entry.kind === "awaiting_evidence")- .map((entry) => entry.runId),- blockedRunIds: dispositions- .filter((entry) => entry.kind === "blocked")- .map((entry) => entry.runId),- };+ async function markTimerHeartbeatChecked(+ agentId: string,+ source: WakeupOptions["source"],+ ) {+ if (source !== "timer") return;+ await db+ .update(agents)+ .set({+ lastHeartbeatAt: new Date(),+ updatedAt: new Date(),+ })+ .where(eq(agents.id, agentId));}- async function drainRunningRunsForShutdown(- signal: "SIGINT" | "SIGTERM",- now = new Date(),- runIds: readonly string[] | null = null,+ async function claimDueTimerHeartbeat(+ agent: typeof agents.$inferSelect,+ now: Date,+ intervalSec: number,) {- const selectedRunIds = runIds ? [...new Set(runIds)] : null;- if (selectedRunIds?.length === 0) {- return {- interrupted: 0,- interruptedRunIds: [],- retryRunIds: [],- restartSuspendedRunIds: [],- };- }- const activeRuns = await db- .select({- run: heartbeatRuns,- agent: agents,+ const dueBefore = new Date(now.getTime() - intervalSec * 1000);+ const claimed = await db+ .update(agents)+ .set({+ lastHeartbeatAt: now,+ updatedAt: now,})- .from(heartbeatRuns)- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)).where(- selectedRunIds- ? and(- eq(heartbeatRuns.status, "running"),- inArray(heartbeatRuns.id, selectedRunIds),- )- : eq(heartbeatRuns.status, "running"),- );-- const interruptedRunIds: string[] = [];- const retryRunIds: string[] = [];- const restartSuspendedRunIds: string[] = [];-- for (const { run, agent } of activeRuns) {- // Shutdown owns only this boot's legacy executions. Expired foreign- // owners belong to the reaper, not another container's drain.- if (run.runtimeMode === "legacy" && run.controllerBootId &&- run.controllerBootId !== legacyControllerBootId) continue;- if (isNativeRunnerOwnershipHeld(run)) continue;- if (- run.runtimeMode === "native" &&- agent.adapterType === "paperclip_runner"- ) {- // A graceful shutdown relinquishes controller authority just like a- // hot restart. Leaving the old event consumer attached lets its- // finalizer interrupt/suspend Claude while the next server is adopting- // the same turn.- await detachNativeSessionsForRestart([run.id]);- const recoveryHistoryEntry = JSON.stringify({- at: now.toISOString(),- restartKind: "graceful",- disposition: "restart_suspended",- reason: signal,- processPid: run.processPid,- processStartedAt: run.processStartedAt?.toISOString() ?? null,- });- await db- .update(nativeRunFinalizations)- .set({- recoveryState: "awaiting_runner_reattach",- recoveryHistory: sql`(- select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb)- from jsonb_array_elements(- coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)- || jsonb_build_array(${recoveryHistoryEntry}::jsonb)- ) with ordinality as history(item, ordinal)- where ordinal > greatest(- jsonb_array_length(- coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)- || jsonb_build_array(${recoveryHistoryEntry}::jsonb)- ) - 20,- 0- )- )`,- updatedAt: now,- })- .where(+ and(+ eq(agents.id, agent.id),+ eq(agents.companyId, agent.companyId),+ or(+ lte(agents.lastHeartbeatAt, dueBefore),and(- eq(nativeRunFinalizations.runId, run.id),- isNull(nativeRunFinalizations.resultId),+ isNull(agents.lastHeartbeatAt),+ lte(agents.createdAt, dueBefore),),- );- await appendRunEvent(run, {- eventType: "native.recovery.transition",- stream: "system",- level: "info",- message:- "Server shutdown suspended native controller ownership without cancelling provider work",- payload: {- restartKind: "graceful",- signal,- runnerDisposition: "awaiting_runner_reattach",- processPid: run.processPid,- processGroupId: run.processGroupId,- processStartedAt: run.processStartedAt?.toISOString() ?? null,- retryRunCreated: false,- },- });- restartSuspendedRunIds.push(run.id);- continue;- }- const message = `Interrupted by graceful server shutdown (${signal})`;- const running = runningProcesses.get(run.id);- try {- if (run.runtimeMode === "native") {- await cancelHeartbeatNativeRun({- db,- runId: run.id,- reason: message,- runtimeMode: run.runtimeMode,- });- }- if (running) {- await terminateHeartbeatRunProcess({- pid: running.child.pid,- processGroupId: running.processGroupId,- graceMs: Math.max(1, running.graceSec) * 1000,- });- }- } finally {- runningProcesses.delete(run.id);- }-- const persistedCancellationResult =- run.runtimeMode === "native"- ? await getRun(run.id).then((current) =>- parseObject(current?.resultJson),- )- : parseObject(run.resultJson);-- const interruptedStatus = await setRunStatusIfRunning(- run.id,- "interrupted",- {- finishedAt: now,- error: message,- errorCode: "server_shutdown_interrupted",- signal,- resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {- conversationContinuationEligible: await runUsedConversationAdapter(db, run),- resultJson: persistedCancellationResult,- errorCode: "server_shutdown_interrupted",- errorMessage: message,- }),- },- );- if (!interruptedStatus.updated || !interruptedStatus.run) continue;- let interrupted = interruptedStatus.run;- await setWakeupStatus(run.wakeupRequestId, "cancelled", {- finishedAt: now,- error: null,- });- interrupted =- (await classifyAndPersistRunLiveness(- interrupted,- parseObject(interrupted.resultJson),- )) ?? interrupted;-- await releaseEnvironmentLeasesForRun({- runId: interrupted.id,- companyId: interrupted.companyId,- agentId: interrupted.agentId,- status: interrupted.status,- failureReason: interrupted.error ?? undefined,- });-- const retry = await enqueueProcessLossRetry(interrupted, agent, now);- if (!retry) {- await releaseIssueExecutionAndPromote(interrupted);- } else {- retryRunIds.push(retry.id);- }-- await appendRunEvent(interrupted, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message,- payload: {- signal,- ...(run.processPid ? { processPid: run.processPid } : {}),- ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),- ...(retry ? { retryRunId: retry.id } : {}),- },- });-- await finalizeAgentStatus(run.agentId, "interrupted", message, {- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),- });- interruptedRunIds.push(interrupted.id);- }-- if (interruptedRunIds.length > 0) {- logger.warn(- {- signal,- interrupted: interruptedRunIds.length,- interruptedRunIds,- retryRunIds,- },- "interrupted running heartbeat runs for graceful shutdown",- );- }-- return {- interrupted: interruptedRunIds.length,- interruptedRunIds,- retryRunIds,- restartSuspendedRunIds,- };+ ),+ ),+ )+ .returning({ id: agents.id })+ .then((rows) => rows[0] ?? null);+ if (!claimed) return null;+ return { wasFirstHeartbeat: !agent.lastHeartbeatAt };}- function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) {+ function parseMaxTurnContinuationPolicy(+ agent: typeof agents.$inferSelect,+ ): MaxTurnContinuationPolicy {const runtimeConfig = parseObject(agent.runtimeConfig);const heartbeat = parseObject(runtimeConfig.heartbeat);+ const configured = parseObject(heartbeat.maxTurnContinuation);+ const rawMaxAttempts = Math.floor(+ asNumber(+ configured.maxAttempts,+ MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,+ ),+ );+ const rawDelayMs = Math.floor(+ asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),+ );return {- enabled: asBoolean(heartbeat.enabled, false),- intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),- wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent),- // A Dot binding has one external turn. Competing assignments must retain- // their queue position instead of claiming a second run that cannot bind.- maxConcurrentRuns: agent.adapterType === "paperclip_runner" &&- parseObject(agent.adapterConfig).provider === "openai_dot"- ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),- skipTimerWhenNoActionableWork: asBoolean(- heartbeat.skipTimerWhenNoActionableWork ??- heartbeat.requireActionableTimerWork ??- heartbeat.issueOnlyTimer,- false,- ),- maxDailyRuns: normalizeOptionalNonNegativeInteger(- heartbeat.maxDailyRuns ??- heartbeat.dailyRunLimit ??- heartbeat.dailyRunCap ??- heartbeat.maxRunsPerDay,+ enabled: asBoolean(configured.enabled, true),+ maxAttempts: Math.max(+ 0,+ Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),),- maxDailyCostCents: normalizeOptionalNonNegativeInteger(- heartbeat.maxDailyCostCents ??- heartbeat.dailyCostCentsLimit ??- heartbeat.dailySpendCentsLimit ??- heartbeat.dailyBudgetCents,+ delayMs: Math.max(+ 0,+ Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),),};}- function normalizeOptionalNonNegativeInteger(value: unknown) {- if (value === null || value === undefined || value === "") return null;- const normalized = Math.floor(asNumber(value, 0));- return normalized >= 0 ? normalized : null;+ function issueRunPriorityRank(priority: string | null | undefined) {+ switch (priority) {+ case "critical":+ return 0;+ case "high":+ return 1;+ case "medium":+ return 2;+ case "low":+ return 3;+ default:+ return 4;+ }}- function currentUtcDayWindow(now = new Date()) {- const start = new Date(- Date.UTC(- now.getUTCFullYear(),- now.getUTCMonth(),- now.getUTCDate(),- 0,- 0,- 0,- 0,- ),- );- const end = new Date(- Date.UTC(- now.getUTCFullYear(),- now.getUTCMonth(),- now.getUTCDate() + 1,- 0,- 0,- 0,- 0,+ async function listQueuedRunDependencyReadiness(+ companyId: string,+ queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,+ ) {+ const issueIds = [+ ...new Set(+ queuedRuns+ .map((run) =>+ readNonEmptyString(parseObject(run.contextSnapshot).issueId),+ )+ .filter((issueId): issueId is string => Boolean(issueId)),),- );- return { start, end };+ ];+ if (issueIds.length === 0) {+ return new Map<+ string,+ Awaited<ReturnType<typeof issuesSvc.getDependencyReadiness>>+ >();+ }+ return issuesSvc.listDependencyReadiness(companyId, issueIds);}- async function getHeartbeatDailyCapBlock(- agent: typeof agents.$inferSelect,- policy: ReturnType<typeof parseHeartbeatPolicy>,- options: {- checkRunCap?: boolean;- checkCostCap?: boolean;- excludeRunId?: string | null;- } = {},- client: Pick<Db, "select"> = db,+ async function countRunningRunsForAgent(agentId: string) {+ const [{ count }] = await db+ .select({ count: sql<number>`count(*)` })+ .from(heartbeatRuns)+ .where(+ and(+ eq(heartbeatRuns.agentId, agentId),+ eq(heartbeatRuns.status, "running"),+ ),+ );+ return Number(count ?? 0);+ }++ async function withChatControlRecoveryGate(+ run: typeof heartbeatRuns.$inferSelect,+ stage: "claim" | "dispatch",+ onClear: (tx: Db) => Promise<typeof heartbeatRuns.$inferSelect | null>,+ nativeRecovery = false,) {- const checkRunCap = options.checkRunCap ?? true;- const checkCostCap = options.checkCostCap ?? true;- const { start, end } = currentUtcDayWindow();- if (checkRunCap && policy.maxDailyRuns !== null) {- const conditions = [- eq(heartbeatRuns.companyId, agent.companyId),- eq(heartbeatRuns.agentId, agent.id),- gte(heartbeatRuns.startedAt, start),- lt(heartbeatRuns.startedAt, end),- notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]),- ];- if (options.excludeRunId) {- conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`);- }- const [row] = await client- .select({ total: sql<number>`count(*)::integer` })- .from(heartbeatRuns)- .where(and(...conditions));- const observed = Number(row?.total ?? 0);- if (observed >= policy.maxDailyRuns) {- return {- reason: "heartbeat.daily_run_limit",- observed,- limit: policy.maxDailyRuns,- };- }- }-- if (checkCostCap && policy.maxDailyCostCents !== null) {- const [row] = await client- .select({- total: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,- })- .from(costEvents)- .where(- and(- eq(costEvents.companyId, agent.companyId),- eq(costEvents.agentId, agent.id),- gte(costEvents.occurredAt, start),- lt(costEvents.occurredAt, end),- ),- );- const observed = Number(row?.total ?? 0);- if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) {- return {- reason: "heartbeat.daily_cost_limit",- observed,- limit: policy.maxDailyCostCents,- };- }- }-- return null;- }-- async function cancelQueuedRunForHeartbeatDailyCap(- run: typeof heartbeatRuns.$inferSelect,- dailyCapBlock: NonNullable<- Awaited<ReturnType<typeof getHeartbeatDailyCapBlock>>- >,- ) {- const now = new Date();- const reason =- "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation";- const cancelled = await setRunStatus(run.id, "cancelled", {- finishedAt: now,- error: reason,- errorCode: dailyCapBlock.reason,- resultJson: {- ...parseObject(run.resultJson),- stopReason: dailyCapBlock.reason,- observed: dailyCapBlock.observed,- limit: dailyCapBlock.limit,- effectiveTimeoutSec: 0,- timeoutConfigured: false,- timeoutSource: "heartbeat_daily_cap_gate",- timeoutFired: false,- },- });- if (!cancelled) return null;-- await setWakeupStatus(run.wakeupRequestId, "skipped", {- finishedAt: now,- error: reason,- });-- await appendRunEvent(cancelled, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message: reason,- payload: {- reason: dailyCapBlock.reason,- observed: dailyCapBlock.observed,- limit: dailyCapBlock.limit,- },- });-- await releaseIssueExecutionAndPromote(cancelled, {- suppressImmediateRecovery: true,- });-- return cancelled;- }-- async function hasActionableTimerWork(agent: typeof agents.$inferSelect) {- const row = await db- .select({ id: issues.id })- .from(issues)- .where(- and(- eq(issues.companyId, agent.companyId),- eq(issues.assigneeAgentId, agent.id),- isNull(issues.assigneeUserId),- isNull(issues.hiddenAt),- inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]),- isNull(issues.conversationAgentId),- nonIdleSlackIssueCondition(),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- return Boolean(row);- }-- async function markTimerHeartbeatChecked(- agentId: string,- source: WakeupOptions["source"],- ) {- if (source !== "timer") return;- await db- .update(agents)- .set({- lastHeartbeatAt: new Date(),- updatedAt: new Date(),- })- .where(eq(agents.id, agentId));- }-- async function claimDueTimerHeartbeat(- agent: typeof agents.$inferSelect,- now: Date,- intervalSec: number,- ) {- const dueBefore = new Date(now.getTime() - intervalSec * 1000);- const claimed = await db- .update(agents)- .set({- lastHeartbeatAt: now,- updatedAt: now,- })- .where(- and(- eq(agents.id, agent.id),- eq(agents.companyId, agent.companyId),- or(- lte(agents.lastHeartbeatAt, dueBefore),- and(- isNull(agents.lastHeartbeatAt),- lte(agents.createdAt, dueBefore),- ),- ),- ),- )- .returning({ id: agents.id })- .then((rows) => rows[0] ?? null);- if (!claimed) return null;- return { wasFirstHeartbeat: !agent.lastHeartbeatAt };- }-- function parseMaxTurnContinuationPolicy(- agent: typeof agents.$inferSelect,- ): MaxTurnContinuationPolicy {- const runtimeConfig = parseObject(agent.runtimeConfig);- const heartbeat = parseObject(runtimeConfig.heartbeat);- const configured = parseObject(heartbeat.maxTurnContinuation);- const rawMaxAttempts = Math.floor(- asNumber(- configured.maxAttempts,- MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,- ),- );- const rawDelayMs = Math.floor(- asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),- );-- return {- enabled: asBoolean(configured.enabled, true),- maxAttempts: Math.max(- 0,- Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),- ),- delayMs: Math.max(- 0,- Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),- ),- };- }-- function issueRunPriorityRank(priority: string | null | undefined) {- switch (priority) {- case "critical":- return 0;- case "high":- return 1;- case "medium":- return 2;- case "low":- return 3;- default:- return 4;- }- }-- async function listQueuedRunDependencyReadiness(- companyId: string,- queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,- ) {- const issueIds = [- ...new Set(- queuedRuns- .map((run) =>- readNonEmptyString(parseObject(run.contextSnapshot).issueId),- )- .filter((issueId): issueId is string => Boolean(issueId)),- ),- ];- if (issueIds.length === 0) {- return new Map<- string,- Awaited<ReturnType<typeof issuesSvc.getDependencyReadiness>>- >();- }- return issuesSvc.listDependencyReadiness(companyId, issueIds);- }-- async function countRunningRunsForAgent(agentId: string) {- const [{ count }] = await db- .select({ count: sql<number>`count(*)` })- .from(heartbeatRuns)- .where(- and(- eq(heartbeatRuns.agentId, agentId),- eq(heartbeatRuns.status, "running"),- ),- );- return Number(count ?? 0);- }-- async function withChatControlRecoveryGate(- run: typeof heartbeatRuns.$inferSelect,- stage: "claim" | "dispatch",- onClear: (tx: Db) => Promise<typeof heartbeatRuns.$inferSelect | null>,- nativeRecovery = false,- ) {- const issueId = readNonEmptyString(- parseObject(run.contextSnapshot).issueId,- );- if (!issueId || run.invocationSource !== "automation") return onClear(db);- await options.beforeChatControlRecoveryCheck?.({- runId: run.id,- issueId,- stage,- });- if (stage === "dispatch" && nativeRecovery) {- // Protected historical/already-admitted owners did not previously take- // this fence's locks. A fresh exact read keeps unrelated Board contention- // from turning their recovery into a new admission failure.- const [current] = await db- .select()+ const issueId = readNonEmptyString(+ parseObject(run.contextSnapshot).issueId,+ );+ if (!issueId || run.invocationSource !== "automation") return onClear(db);+ await options.beforeChatControlRecoveryCheck?.({+ runId: run.id,+ issueId,+ stage,+ });+ if (stage === "dispatch" && nativeRecovery) {+ // Protected historical/already-admitted owners did not previously take+ // this fence's locks. A fresh exact read keeps unrelated Board contention+ // from turning their recovery into a new admission failure.+ const [current] = await db+ .select().from(heartbeatRuns).where(and(@@ -8178,1574 +7143,428 @@ export function heartbeatService(if (released.wakeupRequestId) {await tx- .update(agentWakeupRequests)- .set({ status: "queued", claimedAt: null, updatedAt: now })- .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));- }-- const context = parseObject(released.contextSnapshot);- const issueId = readNonEmptyString(context.issueId);- if (issueId) {- await tx- .update(issues)- .set({- executionRunId: null,- executionAgentNameKey: null,- executionLockedAt: null,- updatedAt: now,- })- .where(- and(- eq(issues.id, issueId),- eq(issues.companyId, released.companyId),- eq(issues.executionRunId, released.id),- ),- );- }- });- }-- async function cancelQueuedRunForBlockedDependencies(- run: typeof heartbeatRuns.$inferSelect,- issueId: string,- unresolvedBlockerIssueIds: string[],- ) {- const now = new Date();- const reason =- "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve";- const cancelled = await setRunStatus(run.id, "cancelled", {- finishedAt: now,- error: reason,- errorCode: "issue_dependencies_blocked",- resultJson: {- ...parseObject(run.resultJson),- stopReason: "issue_dependencies_blocked",- effectiveTimeoutSec: 0,- timeoutConfigured: false,- timeoutSource: "dependency_gate",- timeoutFired: false,- },- });- if (!cancelled) return null;-- await setWakeupStatus(run.wakeupRequestId, "skipped", {- finishedAt: now,- error: reason,- });-- await db- .update(issues)- .set({- executionRunId: null,- executionAgentNameKey: null,- executionLockedAt: null,- updatedAt: now,- })- .where(- and(- eq(issues.companyId, run.companyId),- eq(issues.id, issueId),- eq(issues.executionRunId, run.id),- ),- );-- await appendRunEvent(cancelled, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message: reason,- payload: {- issueId,- unresolvedBlockerIssueIds,- },- });-- return cancelled;- }-- function truncateAgentErrorReason(- reason: string | null | undefined,- ): string | null {- if (!reason) return null;- const trimmed = reason.trim();- if (!trimmed) return null;- return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;- }-- async function finalizeAgentStatus(- agentId: string,- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",- failureReason?: string | null,- options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },- ) {- const existing = await getAgent(agentId);- if (!existing) return;-- if (existing.status === "paused" || existing.status === "terminated") {- return;- }-- const isFirstHeartbeat =- options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;-- const runningCount = await countRunningRunsForAgent(agentId);- const nextStatus =- runningCount > 0- ? "running"- : outcome === "succeeded" ||- outcome === "interrupted" ||- outcome === "cancelled" ||- (outcome === "failed" && options?.keepIdleOnFailure)- ? "idle"- : "error";-- const updated = await db- .update(agents)- .set({- status: nextStatus,- // Persist a human-readable reason on the agent record when it enters- // error so operators see it on the agent page without digging into run- // events; clear it whenever the agent leaves error.- errorReason:- nextStatus === "error"- ? truncateAgentErrorReason(failureReason)- : null,- lastHeartbeatAt: new Date(),- updatedAt: new Date(),- })- .where(eq(agents.id, agentId))- .returning()- .then((rows) => rows[0] ?? null);-- if (isFirstHeartbeat && updated) {- const tc = getTelemetryClient();- if (tc)- trackAgentFirstHeartbeat(tc, {- agentRole: updated.role,- agentId: updated.id,- });- }-- if (updated) {- publishLiveEvent({- companyId: updated.companyId,- type: "agent.status",- payload: {- agentId: updated.id,- status: updated.status,- lastHeartbeatAt: updated.lastHeartbeatAt- ? new Date(updated.lastHeartbeatAt).toISOString()- : null,- outcome,- },- });- }- }-- function mergeRunStopMetadataForAgent(- agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",- options?: {- resultJson?: Record<string, unknown> | null;- conversationContinuationEligible?: boolean;- errorCode?: string | null;- errorMessage?: string | null;- },- ) {- const stopMetadata = buildHeartbeatRunStopMetadata({- adapterType: agent.adapterType,- adapterConfig: parseObject(agent.adapterConfig),- outcome,- errorCode: options?.errorCode ?? null,- errorMessage: options?.errorMessage ?? null,- });- const result = mergeHeartbeatRunStopMetadata(- options?.resultJson ?? null,- stopMetadata,- );- const cancellationAcknowledged =- parseObject(result?.executionCancellation).state === "acknowledged";- return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&- (outcome !== "cancelled" || cancellationAcknowledged) &&- isConversationAdapter(agent.adapterType)- ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }- : result;- }-- function countValue(value: unknown) {- const parsed = Number(value ?? 0);- return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;- }-- function dateValue(value: unknown) {- if (value instanceof Date)- return Number.isNaN(value.getTime()) ? null : value;- if (typeof value === "string" || typeof value === "number") {- const parsed = new Date(value);- return Number.isNaN(parsed.getTime()) ? null : parsed;- }- return null;- }-- function latestDate(...values: unknown[]) {- let latest: Date | null = null;- for (const value of values) {- const parsed = dateValue(value);- if (!parsed) continue;- if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;- }- return latest;- }-- async function buildRunLivenessInput(- run: typeof heartbeatRuns.$inferSelect,- resultJson: Record<string, unknown> | null | undefined,- ): Promise<RunLivenessClassificationInput> {- const context = parseObject(run.contextSnapshot);- const contextIssueId = readNonEmptyString(context.issueId);- const continuationAttempt = asNumber(- context.continuationAttempt,- run.continuationAttempt ?? 0,- );-- const issue = contextIssueId- ? await db- .select({- status: issues.status,- title: issues.title,- description: issues.description,- workMode: issues.workMode,- })- .from(issues)- .where(- and(- eq(issues.companyId, run.companyId),- eq(issues.id, contextIssueId),- ),- )- .then((rows) => rows[0] ?? null)- : null;-- const [commentStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${issueComments.createdAt})`,- })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, run.companyId),- eq(issueComments.issueId, contextIssueId),- eq(issueComments.createdByRunId, run.id),- isNull(issueComments.deletedAt),- ),- )- : [{ count: 0, latestAt: null }];-- const issueCommentBodies = contextIssueId- ? await db- .select({ body: issueComments.body })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, run.companyId),- eq(issueComments.issueId, contextIssueId),- eq(issueComments.createdByRunId, run.id),- ),- )- .orderBy(desc(issueComments.createdAt), desc(issueComments.id))- .limit(5)- .then((rows) => rows.reverse().map((row) => row.body))- : [];-- const continuationSummary = contextIssueId- ? await getIssueContinuationSummaryDocument(db, contextIssueId)- : null;-- const [documentStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,- latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,- })- .from(documentRevisions)- .innerJoin(- issueDocuments,- eq(documentRevisions.documentId, issueDocuments.documentId),- )- .where(- and(- eq(documentRevisions.companyId, run.companyId),- eq(documentRevisions.createdByRunId, run.id),- eq(issueDocuments.companyId, run.companyId),- eq(issueDocuments.issueId, contextIssueId),- sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,- ),- )- : [{ count: 0, planCount: 0, latestAt: null }];+ .update(agentWakeupRequests)+ .set({ status: "queued", claimedAt: null, updatedAt: now })+ .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));+ }- const [workProductStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,+ const context = parseObject(released.contextSnapshot);+ const issueId = readNonEmptyString(context.issueId);+ if (issueId) {+ await tx+ .update(issues)+ .set({+ executionRunId: null,+ executionAgentNameKey: null,+ executionLockedAt: null,+ updatedAt: now,})- .from(issueWorkProducts).where(and(- eq(issueWorkProducts.companyId, run.companyId),- eq(issueWorkProducts.issueId, contextIssueId),- eq(issueWorkProducts.createdByRunId, run.id),+ eq(issues.id, issueId),+ eq(issues.companyId, released.companyId),+ eq(issues.executionRunId, released.id),),- )- : [{ count: 0, latestAt: null }];-- const [workspaceOperationStats] = await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,- })- .from(workspaceOperations)- .where(- and(- eq(workspaceOperations.companyId, run.companyId),- eq(workspaceOperations.heartbeatRunId, run.id),- ),- );-- const [activityStats] = await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${activityLog.createdAt})`,- })- .from(activityLog)- .where(- and(- eq(activityLog.companyId, run.companyId),- eq(activityLog.runId, run.id),- notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),- ),- );-- const [eventStats] = await db- .select({- count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,- latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,- })- .from(heartbeatRunEvents)- .where(- and(- eq(heartbeatRunEvents.companyId, run.companyId),- eq(heartbeatRunEvents.runId, run.id),- ),- );-- return {- runStatus: run.status,- issue,- resultJson: resultJson ?? run.resultJson ?? null,- issueCommentBodies,- continuationSummaryBody: continuationSummary?.body ?? null,- stdoutExcerpt: run.stdoutExcerpt ?? null,- stderrExcerpt: run.stderrExcerpt ?? null,- error: run.error ?? null,- errorCode: run.errorCode ?? null,- continuationAttempt,- evidence: {- issueCommentsCreated: countValue(commentStats?.count),- documentRevisionsCreated: countValue(documentStats?.count),- planDocumentRevisionsCreated: countValue(documentStats?.planCount),- workProductsCreated: countValue(workProductStats?.count),- workspaceOperationsCreated: countValue(workspaceOperationStats?.count),- activityEventsCreated: countValue(activityStats?.count),- toolOrActionEventsCreated: countValue(eventStats?.count),- latestEvidenceAt: latestDate(- commentStats?.latestAt,- documentStats?.latestAt,- workProductStats?.latestAt,- workspaceOperationStats?.latestAt,- activityStats?.latestAt,- eventStats?.latestAt,- ),- },- };+ );+ }+ });}- async function classifyAndPersistRunLiveness(+ async function cancelQueuedRunForBlockedDependencies(run: typeof heartbeatRuns.$inferSelect,- resultJson?: Record<string, unknown> | null,+ issueId: string,+ unresolvedBlockerIssueIds: string[],) {- const authRepair = run.status === "failed"- ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {- logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");- return null;- })- : null;- const classification = classifyRunLiveness({- ...await buildRunLivenessInput(run, resultJson),- authenticationRepairRequested: Boolean(authRepair?.interactionId),+ const now = new Date();+ const reason =+ "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve";+ const cancelled = await setRunStatus(run.id, "cancelled", {+ finishedAt: now,+ error: reason,+ errorCode: "issue_dependencies_blocked",+ resultJson: {+ ...parseObject(run.resultJson),+ stopReason: "issue_dependencies_blocked",+ effectiveTimeoutSec: 0,+ timeoutConfigured: false,+ timeoutSource: "dependency_gate",+ timeoutFired: false,+ },});- return db- .update(heartbeatRuns)- .set({- livenessState: classification.livenessState,- livenessReason: classification.livenessReason,- continuationAttempt: classification.continuationAttempt,- lastUsefulActionAt: classification.lastUsefulActionAt,- nextAction: classification.nextAction,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, run.id))- .returning()- .then((rows) => rows[0] ?? null);- }-- // Clamp the stored attempt count to the range [0, cap]. The SQL reader- // `pendingCleanupAttemptsSql` clamps to the same range, so both readers yield- // the same value for every input. The claim predicate compares the two values,- // so this alignment lets the claim match for a malformed lease.- function readPendingCleanupRetryAttempts(- metadata: Record<string, unknown>,- ): number {- const value = metadata[PENDING_CLEANUP_ATTEMPTS_METADATA_KEY];- if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)- return 0;- return Math.min(Math.floor(value), PENDING_CLEANUP_SWEEP_ATTEMPT_CAP);- }+ if (!cancelled) return null;- // Atomically claim one retry attempt on a pending_cleanup lease. The update- // only matches when the lease is still pending_cleanup and its stored attempt- // count still equals `expectedAttempts`. Two concurrent sweeps read the same- // count, but Postgres serializes the two updates on the row and only the first- // matches the guard. The loser gets zero rows and skips the lease. The persisted in-flight deadline also prevents a later tick- // from starting another attempt while this one is still running.- // Returns the attempt identity only for the sweep that won the claim.- //- // The update patches cleanup ownership fields without writing a copied- // metadata object, so a concurrent write to an unrelated metadata key- // survives. The guard reads the stored count through the safe SQL reader, so a- // malformed value never throws.- async function claimPendingCleanupRetryAttempt(- leaseId: string,- expectedAttempts: number,- manualAttempt?: { previousId: unknown },- ): Promise<string | null> {- const now = new Date();- const attemptId = randomUUID();- const claimed = await db- .update(environmentLeases)- .set({- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}], to_jsonb(${expectedAttempts + 1}::int), true)- || ${JSON.stringify({ ...(manualAttempt ? { pendingCleanupManualAttemptId: randomUUID() } : {}),- pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true,- pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000,- })}::jsonb`,- lastUsedAt: now,- updatedAt: now,- })- .where(- and(- eq(environmentLeases.id, leaseId),- eq(environmentLeases.status, "pending_cleanup"),- sql`${pendingCleanupAttemptsSql()} = ${expectedAttempts}`,- manualAttempt ? sql`coalesce(${environmentLeases.metadata}->'pendingCleanupManualAttemptId', 'null'::jsonb) is not distinct from ${JSON.stringify(manualAttempt.previousId ?? null)}::jsonb` : undefined,- pendingCleanupRetryDueSql(Boolean(manualAttempt)),- ),- )- .returning({ id: environmentLeases.id });- return claimed.length > 0 ? attemptId : null;- }+ await setWakeupStatus(run.wakeupRequestId, "skipped", {+ finishedAt: now,+ error: reason,+ });- // Atomically claim the one-time cap warning for a lease. The update only- // matches when the lease is still pending_cleanup, its stored attempt count is- // at or above the cap, and it has not yet carried the warned flag. Two- // concurrent sweeps that both reach the cap race here, but only one update- // sets the flag and returns a row. The loser skips the warning. This keeps the- // warning to one log line per lease.- //- // The status and cap predicates are the last line of defense. They stop a warn- // flag write to a lease that left pending_cleanup or dropped below the cap- // between the read and this claim. The update writes only the warned key with- // `jsonb_set`, so a concurrent write to an unrelated metadata key survives.- async function claimPendingCleanupCapWarning(- leaseId: string,- ): Promise<boolean> {- const now = new Date();- const claimed = await db- .update(environmentLeases)+ await db+ .update(issues).set({- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}], to_jsonb(true), true)`,+ executionRunId: null,+ executionAgentNameKey: null,+ executionLockedAt: null,updatedAt: now,}).where(and(- eq(environmentLeases.id, leaseId),- eq(environmentLeases.status, "pending_cleanup"),- sql`${pendingCleanupAttemptsSql()} >= ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}`,- sql`${pendingCleanupCapWarnedSql()} = false`,- ),- )- .returning({ id: environmentLeases.id });- return claimed.length > 0;- }-- // Defer a pending_cleanup lease whose provider plugin is not ready this tick.- // The sweep reads one page of the oldest rows, ordered by `updatedAt`. A lease- // that the sweep only skips keeps its old `updatedAt`, so it stays the oldest- // and refills the page on every tick. That starves a newer lease whose- // provider is ready. The defer bumps `updatedAt` to now, so the unavailable- // lease moves to the back of the queue and a ready lease takes its page slot.- // The defer never writes the attempt count, so a long provider outage never- // consumes a finite retry. The status guard keeps the write on a lease that is- // still pending_cleanup.- async function deferPendingCleanupLease(leaseId: string): Promise<void> {- const now = new Date();- await db- .update(environmentLeases)- .set({ updatedAt: now })- .where(- and(- eq(environmentLeases.id, leaseId),- eq(environmentLeases.status, "pending_cleanup"),- ),- );- }-- // Move a guarded orphan candidate to the back of the sweep order. The- // shared-resource guard below can skip a row on every tick as long as the- // other owner stays live, retained, or pending cleanup. The select orders- // by `updatedAt` ascending and takes only the oldest page, so an untouched- // skipped row keeps refilling that same page and blocks every row behind- // it. The bump pushes the row past the fixed-size page, so the next tick- // reaches the rows behind it. It costs the row one extra backoff wait,- // which is safe because the guard means a physical sandbox still exists.- async function deferOrphanedActiveLease(leaseId: string): Promise<void> {- const now = new Date();- await db- .update(environmentLeases)- .set({ updatedAt: now })- .where(- and(- eq(environmentLeases.id, leaseId),- eq(environmentLeases.status, "active"),+ eq(issues.companyId, run.companyId),+ eq(issues.id, issueId),+ eq(issues.executionRunId, run.id),),);- }-- // An active lease is reachable only while its heartbeat run keeps the- // running status. The reaper writes the run status and the lease release as- // two separate statements, so a restart between them can leave a terminal- // run and an active lease. `releaseRunLeases` also skips a lease whose- // environment row is gone, so that lease stays active too. No later query- // finds either lease, because every production query selects an active- // lease by its environment or by its heartbeat run id, never by age. This- // sweep finds both stranded classes and moves each lease to pending_cleanup,- // so the existing pending_cleanup sweep tears the sandbox down from the- // data already on the lease row.- async function sweepOrphanedActiveLeases(opts: {- backoffMs: number;- }): Promise<{ recovered: number }> {- const cutoff = new Date(Date.now() - opts.backoffMs);-- const rows = await db- .select({ lease: environmentLeases })- .from(environmentLeases)- .leftJoin(- heartbeatRuns,- eq(environmentLeases.heartbeatRunId, heartbeatRuns.id),- )- .where(- and(- eq(environmentLeases.status, "active"),- or(- isNull(environmentLeases.heartbeatRunId),- inArray(heartbeatRuns.status, [- ...HEARTBEAT_RUN_TERMINAL_STATUSES,- ]),- ),- lte(environmentLeases.updatedAt, cutoff),- ),- )- .orderBy(asc(environmentLeases.updatedAt))- .limit(ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE);-- let recovered = 0;- for (const { lease } of rows) {- // A provider resource id names one physical sandbox. A different lease- // row can still hold that same resource in a live status, so this sweep- // must not tear down a sandbox that a different lease still owns.- if (lease.provider && lease.providerLeaseId) {- const [otherOwner] = await db- .select({ id: environmentLeases.id })- .from(environmentLeases)- .where(- and(- ne(environmentLeases.id, lease.id),- eq(environmentLeases.provider, lease.provider),- eq(environmentLeases.providerLeaseId, lease.providerLeaseId),- inArray(environmentLeases.status, [- "active",- "retained",- "pending_cleanup",- ]),- ),- )- .limit(1);- if (otherOwner) {- // Defer this row so the fixed-size page reaches the rows behind- // it next tick, instead of re-selecting the same guarded rows- // forever.- await deferOrphanedActiveLease(lease.id);- continue;- }- }- // Keep the row's existing updatedAt value. The select above already- // proved the row is older than the backoff cutoff, so the- // pending_cleanup sweep can accept the same row in this same tick. A- // fresh timestamp here would push the row inside that sweep's own- // backoff window and delay the teardown by one full tick.- const flipped = await db- .update(environmentLeases)- .set({- status: "pending_cleanup",- failureReason: "orphaned_active_lease_recovered",- })- .where(- and(- eq(environmentLeases.id, lease.id),- eq(environmentLeases.status, "active"),- ),- )- .returning({ id: environmentLeases.id });- if (flipped.length > 0) recovered += 1;- }+ await appendRunEvent(cancelled, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: reason,+ payload: {+ issueId,+ unresolvedBlockerIssueIds,+ },+ });- return { recovered };+ return cancelled;}- // Retry the leases stranded in "pending_cleanup". A failed destroy leaves a- // lease in that state forever without this sweep. The reaper tick runs the- // sweep. The backoff equals the reaper staleness threshold, so a lease waits- // for that period between attempts. The sweep reads and writes the attempt- // count in the lease metadata. It warns once when a lease reaches the attempt- // threshold, then continues with slower retries. Live attempts renew their- // cleanup claim; after controller loss, exact-resource destruction may repeat.- async function sweepPendingCleanupLeases(opts?: {- backoffMs?: number;- /** One cleanup attempt per explicit user Retry, for this failed run only.- * A later user Retry may bypass the cooldown after a provider failure,- * but cannot take over an in-flight cleanup attempt.- */- explicitRetry?: { companyId: string; runId: string; actorId: string; reason?: "retry_failed_run" | "queued_comment_interrupt" };- }): Promise<{- swept: number;- destroyed: number;- capped: number;- }> {- const backoffMs = opts?.backoffMs ?? 0;- const now = new Date();- const cutoff = new Date(now.getTime() - backoffMs);+ function truncateAgentErrorReason(+ reason: string | null | undefined,+ ): string | null {+ if (!reason) return null;+ const trimmed = reason.trim();+ if (!trimmed) return null;+ return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;+ }- // Flush the in-process orphan-cleanup buffer first. A failed acquire buffers- // an orphan there when every synchronous pending-cleanup write failed after a- // failed teardown. The flush re-inserts each buffered record, so a durable- // `pending_cleanup` row lands once the database recovers. The flush runs- // before the read below, so this same tick tears down a freshly-landed row.- try {- const flushed = opts?.explicitRetry ? null : await environmentRuntime.flushDeferredOrphanCleanups?.();- if (flushed && (flushed.recovered > 0 || flushed.pending > 0)) {- logger.info(- { recovered: flushed.recovered, pending: flushed.pending },- "flushed the in-process orphan sandbox cleanup buffer to the database",- );- }- } catch {- // A flush failure never stops the sweep. The buffer keeps the orphan for a- // later tick, and the database rows below still need this sweep. The caught- // exception never enters the log, because a write error can carry a- // credential in its message, code, cause, or stack.- logger.warn(- "orphan sandbox cleanup buffer flush failed; the sweep continues",- );- }+ async function finalizeAgentStatus(+ agentId: string,+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",+ failureReason?: string | null,+ options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },+ ) {+ const existing = await getAgent(agentId);+ if (!existing) return;- const rows = await db- .select()- .from(environmentLeases)- .where(- and(- eq(environmentLeases.status, "pending_cleanup"),- opts?.explicitRetry ? eq(environmentLeases.companyId, opts.explicitRetry.companyId) : undefined,- opts?.explicitRetry ? eq(environmentLeases.heartbeatRunId, opts.explicitRetry.runId) : undefined,- pendingCleanupRetryDueSql(Boolean(opts?.explicitRetry)),- backoffMs > 0 ? lte(environmentLeases.updatedAt, cutoff) : undefined,- ),- )- .orderBy(asc(environmentLeases.updatedAt))- .limit(PENDING_CLEANUP_SWEEP_PAGE_SIZE);+ if (existing.status === "paused" || existing.status === "terminated") {+ return;+ }- let destroyed = 0;- let capped = 0;- for (const row of rows) {- if (pendingCleanupAttemptsInFlight.has(row.id)) continue;- const metadata = { ...(row.metadata ?? {}) } as Record<string, unknown>;- const attempts = readPendingCleanupRetryAttempts(metadata);+ const isFirstHeartbeat =+ options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;- const environment = row.environmentId- ? await environmentsSvc.getById(row.environmentId)- : null;- const lease = await environmentsSvc.getLeaseById(row.id);- if (!lease) continue;-- // An orphan ephemeral lease keeps its provider, its provider lease id, and- // its sandbox config in the lease row. A failed acquire records it, and its- // environment row may be gone or foreign-bound. A reuse_by_environment lease- // whose environment a delete removed keeps the same recorded data, because- // the schema sets the environment reference to null on delete and preserves- // the row. Both leases tear down from the recorded lease data through- // `retryPendingSandboxTeardown`, which never reads the environment row. So- // the sweep uses that path whenever the lease is an orphan ephemeral lease- // or its environment row is gone. A reuse_by_environment lease whose- // environment still exists tears down through `destroyRunLease`. That- // path uses the provider and configuration recorded on the lease first;- // the environment is lifecycle context and only a legacy fallback.- const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";- const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);-- // Do not consume a finite cleanup attempt while the provider plugin is- // briefly unavailable. A plugin worker restart, a plugin reload, or a- // plugin reinstall makes the provider unavailable for a short window. The- // plugin can be missing or not ready in that window. A teardown then throws,- // and the atomic claim below would count that throw against the cap, so a- // long restart or reload could exhaust the retries and strand a live- // sandbox. So probe the provider first, and defer the lease this tick when- // the provider is not ready. The sweep preserves the pending_cleanup row,- // and a later sweep retries after the provider recovers. The probe reports- // ready only for a permanent condition (a missing provider string, a- // built-in provider, or no worker manager), so a genuine teardown failure- // still runs, throws, and counts toward the cap. A runtime with no probe- // method treats the lease as ready, so the sweep keeps its earlier- // behavior.- const workerReady = environmentRuntime.isPendingCleanupWorkerReady- ? await environmentRuntime.isPendingCleanupWorkerReady({- environment,- lease,- })- : true;- if (!workerReady) {- // Move the unavailable lease to the back of the sweep queue. Otherwise- // the oldest unavailable rows refill the page on every tick and starve a- // newer lease that has a ready provider. The defer bumps `updatedAt`- // only, so it consumes no finite retry attempt.- await deferPendingCleanupLease(row.id);- continue;- }+ const runningCount = await countRunningRunsForAgent(agentId);+ const nextStatus =+ runningCount > 0+ ? "running"+ : outcome === "succeeded" ||+ outcome === "interrupted" ||+ outcome === "cancelled" ||+ (outcome === "failed" && options?.keepIdleOnFailure)+ ? "idle"+ : "error";- // Atomically claim the attempt before the retry. Only the winning sweep- // increments the count and tears the sandbox down, so an overlapping sweep- // cannot start another attempt while this one holds the cleanup lease.- // The deadline survives a server restart; the counter saturates at the- // escalation threshold while attempt identities remain unique.- const claimed = await claimPendingCleanupRetryAttempt(row.id, attempts,- opts?.explicitRetry ? { previousId: metadata.pendingCleanupManualAttemptId } : undefined);- if (!claimed) continue;- pendingCleanupAttemptsInFlight.add(row.id);- lease.metadata = { ...lease.metadata, pendingCleanupAttemptId: claimed };- let activeRenewal: Promise<void> | null = null;- const renewal = setInterval(() => {- if (activeRenewal) return;- activeRenewal = db.update(environmentLeases).set({- metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, '{pendingCleanupLeaseExpiresAtMs}', to_jsonb(${Date.now() + 15 * 60_000}::bigint))`,- }).where(and(eq(environmentLeases.id, row.id), eq(environmentLeases.status, "pending_cleanup"),- sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`,- sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`))- .then(() => {})- .catch(() => logger.warn({ leaseId: row.id }, "cleanup ownership renewal failed"))- .finally(() => { activeRenewal = null; });- }, 30_000);- renewal.unref();+ const updated = await db+ .update(agents)+ .set({+ status: nextStatus,+ // Persist a human-readable reason on the agent record when it enters+ // error so operators see it on the agent page without digging into run+ // events; clear it whenever the agent leaves error.+ errorReason:+ nextStatus === "error"+ ? truncateAgentErrorReason(failureReason)+ : null,+ lastHeartbeatAt: new Date(),+ updatedAt: new Date(),+ })+ .where(eq(agents.id, agentId))+ .returning()+ .then((rows) => rows[0] ?? null);- try {- if (opts?.explicitRetry) await logActivity(db, {- companyId: row.companyId, actorType: "user", actorId: opts.explicitRetry.actorId,- action: "environment_lease.cleanup_retried", entityType: "environment_lease", entityId: row.id,- runId: opts.explicitRetry.runId, details: { attempt: attempts + 1, reason: opts.explicitRetry.reason ?? "retry_failed_run" },+ if (isFirstHeartbeat && updated) {+ const tc = getTelemetryClient();+ if (tc)+ trackAgentFirstHeartbeat(tc, {+ agentRole: updated.role,+ agentId: updated.id,});- if (useRecordedTeardown) {- // Tear the sandbox down from the recorded provider config and the- // cleanup-authorized secret versions. Preserve any provider receipt;- // a completed retry must grant the same evidence as initial cleanup.- const receipt = await environmentRuntime.retryPendingSandboxTeardown({- environment,- lease,- });- const released = hasStopOnlyCleanup(lease)- ? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt })- : await environmentsSvc.releaseLease(lease.id, "expired", {- expectedPendingCleanupAttemptId: claimed,- cleanupStatus: "success",- failureReason: "pending_cleanup_retry",- remoteExecutionTermination: remoteTerminationReceipt(lease, receipt),- });- if (released) destroyed += 1;- } else if (environment) {- const result = await environmentRuntime.destroyRunLease({- environment,- lease,- failureReason: "pending_cleanup_retry",- });- if (result && result.status !== "pending_cleanup") {- destroyed += 1;- }- }- } catch {- // The recorded-data teardown throws on failure, so revert the lease to- // pending_cleanup for a later sweep. The claimed attempt still counts- // for backoff, so requests stay bounded. The `destroyRunLease`- // path reverts the lease itself, so this revert only runs for the- // recorded-data teardown path.- if (useRecordedTeardown) {- await environmentsSvc.releaseLease(lease.id, "pending_cleanup", {- expectedPendingCleanupAttemptId: claimed,- cleanupStatus: "failed",- failureReason: "pending_cleanup_retry",- });- }- // Log a constant errorKind only. The exception can carry a credential in- // its name, code, message, cause, or stack, so the sweep never reads it.- logger.warn(- {- errorKind: PENDING_CLEANUP_RETRY_ERROR_KIND,- leaseId: row.id,- environmentId: row.environmentId,- attempts: attempts + 1,- },- "pending_cleanup lease retry failed",- );- } finally {- clearInterval(renewal);- // A hung renewal must not retain process-local cleanup ownership.- // Late renewals are attempt-fenced and never write the retry cooldown.- pendingCleanupAttemptsInFlight.delete(row.id);- }- if (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP) {- capped += 1;- // Warn once, then continue automatic cleanup with backoff. The atomic claim- // keeps the warning to one log line even when two sweeps overlap.- if (metadata[PENDING_CLEANUP_CAP_WARNED_METADATA_KEY] !== true) {- const warned = await claimPendingCleanupCapWarning(row.id);- if (warned) {- logger.warn(- { leaseId: row.id, environmentId: row.environmentId, attempts },- "environment lease needs operator attention; automatic cleanup continues with backoff",- );- }- }- }- // Persist the cooldown independently of process memory. A crash before- // this write leaves the bounded in-flight lease for a later sweep.- await db.update(environmentLeases).set({- metadata: sql`${pendingCleanupMetadataObjectSql()} || ${JSON.stringify({- pendingCleanupInFlight: false,- pendingCleanupRetryAfterMs: Date.now() + (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP- ? 30 * 60_000 : Math.min(30 * 60_000, Math.max(30_000, backoffMs))),- })}::jsonb`,- }).where(and(eq(environmentLeases.id, lease.id),- sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`));- if (lease.heartbeatRunId) {- // Delivery failure must not revert successful provider cleanup. A new- // message can still use the persisted receipt on its next admission.- await (async () => {- await acknowledgeRemoteStop(lease.heartbeatRunId!, lease.companyId);- const run = await getRun(lease.heartbeatRunId!);- if (run) await resumeRemoteStopComments(run);- })().catch(() => logger.warn({ leaseId: lease.id }, "could not reconsider messages after cleanup retry"));- }}- return { swept: rows.length, destroyed, capped };+ if (updated) {+ publishLiveEvent({+ companyId: updated.companyId,+ type: "agent.status",+ payload: {+ agentId: updated.id,+ status: updated.status,+ lastHeartbeatAt: updated.lastHeartbeatAt+ ? new Date(updated.lastHeartbeatAt).toISOString()+ : null,+ outcome,+ },+ });+ }}- async function markNativeOwnershipUnverified(- run: typeof heartbeatRuns.$inferSelect,- evidence: {- reason:- | "live_process_identifier"- | "observed_owner_unverified"- | "adopted_runner_authentication_timeout"- | "native_chat_workspace_scope_mismatch";- processPidAlive?: boolean;- processGroupAlive?: boolean;+ function mergeRunStopMetadataForAgent(+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",+ options?: {+ resultJson?: Record<string, unknown> | null;+ conversationContinuationEligible?: boolean;+ errorCode?: string | null;+ errorMessage?: string | null;},) {- const durableOwnershipHold =- evidence.reason === "adopted_runner_authentication_timeout" ||- evidence.reason === "native_chat_workspace_scope_mismatch";- if (- run.errorCode === NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE &&- run.error === NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE &&- (!durableOwnershipHold || isNativeRunnerOwnershipHeld(run))- )- return run;- const blockedStatus = run.status === "failed" ? "failed" : "running";- const blockedWrite = await setRunStatusFromLive(- run.id,- blockedStatus,- [blockedStatus],- {- error: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,- errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,- ...(durableOwnershipHold- ? {- nativePhase: "terminal_failure",- nativePhaseUpdatedAt: new Date(),- }- : {}),- },- );- if (!blockedWrite.updated || !blockedWrite.run) {- return blockedWrite.run ?? run;- }- const blocked = blockedWrite.run;- await appendRunEvent(blocked, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,- payload: {- reason: evidence.reason,- ...(evidence.processPidAlive === true ? { processPidAlive: true } : {}),- ...(evidence.processGroupAlive === true- ? { processGroupAlive: true }- : {}),- },+ const stopMetadata = buildHeartbeatRunStopMetadata({+ adapterType: agent.adapterType,+ adapterConfig: parseObject(agent.adapterConfig),+ outcome,+ errorCode: options?.errorCode ?? null,+ errorMessage: options?.errorMessage ?? null,});- return blocked;+ const result = mergeHeartbeatRunStopMetadata(+ options?.resultJson ?? null,+ stopMetadata,+ );+ const cancellationAcknowledged =+ parseObject(result?.executionCancellation).state === "acknowledged";+ return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&+ (outcome !== "cancelled" || cancellationAcknowledged) &&+ isConversationAdapter(agent.adapterType)+ ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }+ : result;}- async function settleRecoveredNativeWorkspace(input: {- runId: string;- companyId: string;- agentId: string;- succeeded: boolean;- }) {- const settledRun = await getRun(input.runId);- const workspaceSyncReference = readNativeWorkspaceSyncReference(- parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,- );- await releaseEnvironmentLeasesForRun({- runId: input.runId,- companyId: input.companyId,- agentId: input.agentId,- status: settledRun?.status,- failureReason: settledRun?.error ?? undefined,- providerResourceDisposition: input.succeeded && (!parseObject(settledRun?.resultJson).workspaceExportRetry || workspaceSyncReference?.resourceDisposition === "destroy")- ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")- : "stop_and_retain",- });- await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);- await finalizeAgentStatus(- input.agentId,- input.succeeded ? "succeeded" : "failed",- input.succeeded- ? null- : (settledRun?.error ?? "native_workspace_sync_out_failed"),- {- wasFirstHeartbeat: settledRun- ? timerClaimWasFirstHeartbeat(settledRun)- : undefined,- },- ).catch(() => undefined);+ function countValue(value: unknown) {+ const parsed = Number(value ?? 0);+ return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;}- function scheduleRetainedNativeSessionCleanup() {- // The per-database sweep joins startup and periodic callers. One bounded- // control-only repair must not hold up unrelated provider ingress or the- // entire orphan reaper, but shutdown must still await its physical owner.- const cleanup = reconcileRetainedNativeSessionCleanups(db, {- cleanup: (input) => reconcileRetainedNativeSessionCleanup(db, input),- onError: (error, runId) => {- logger.warn(- { err: error, runId },- "retained native session cleanup failed",- );- },- })- .then(() => undefined)- .catch((error) => {- logger.warn({ err: error }, "retained native cleanup discovery failed");- })- // The bounded maintenance attempt may fail before an already-started- // database callback settles. Keep shutdown ownership until the original- // operations finish; their timeout cannot authorize closing the database.- .finally(() => drainRetainedRunnerdMaintenanceOperations());- activeRunExecutionPromises.add(cleanup);- void cleanup.finally(() => activeRunExecutionPromises.delete(cleanup));+ function dateValue(value: unknown) {+ if (value instanceof Date)+ return Number.isNaN(value.getTime()) ? null : value;+ if (typeof value === "string" || typeof value === "number") {+ const parsed = new Date(value);+ return Number.isNaN(parsed.getTime()) ? null : parsed;+ }+ return null;}- async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {- const staleThresholdMs = opts?.staleThresholdMs ?? 0;- const now = new Date();- // Recovery never launches a provider or infers stopped ownership from- // terminal status. Uncaptured local copies require durable stop evidence.- await instructionCopies.recoverStopped().catch(error => {- logger.warn({ err: error }, "failed to recover stopped instruction copies");- });- await instructionCopies.recoverCaptured().catch(error => {- logger.warn({ err: error }, "failed to retry captured instruction revisions");- });+ function latestDate(...values: unknown[]) {+ let latest: Date | null = null;+ for (const value of values) {+ const parsed = dateValue(value);+ if (!parsed) continue;+ if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;+ }+ return latest;+ }- // Complete persisted native results before generic orphan recovery. The- // reconciler reads the durable workspace barrier and persisted runtime- // mode, never the current feature flag.- await reconcileNativeFinalizations(db, undefined, {- environmentRuntime,- onWorkspaceSettled: settleRecoveredNativeWorkspace,- }).catch((error) => {- logger.warn(- { err: error },- "failed to reconcile persisted native finalizations before orphan reaping",- );- });- scheduleRetainedNativeSessionCleanup();- await dispatchPendingNativeStatusWakeups().catch((error) => {- logger.warn(- { err: error },- "failed to dispatch persisted native status wake intents before orphan reaping",- );- });+ async function buildRunLivenessInput(+ run: typeof heartbeatRuns.$inferSelect,+ resultJson: Record<string, unknown> | null | undefined,+ ): Promise<RunLivenessClassificationInput> {+ const context = parseObject(run.contextSnapshot);+ const contextIssueId = readNonEmptyString(context.issueId);+ const continuationAttempt = asNumber(+ context.continuationAttempt,+ run.continuationAttempt ?? 0,+ );- // A retryable native run can retain process identifiers from the failed- // attempt. Inspect them before the recovery claim: a live identifier is- // unowned and blocks recovery, while identifiers that are all dead can be- // cleared with a compare-and-set so the explicit retryable failure becomes- // claimable in this same sweep.- const retryableNativeProcesses = await db- .select({ run: heartbeatRuns })- .from(heartbeatRuns)- .innerJoin(- nativeRunFinalizations,- eq(nativeRunFinalizations.runId, heartbeatRuns.id),- )- .where(- and(- inArray(heartbeatRuns.status, ["running", "failed"]),- eq(heartbeatRuns.runtimeMode, "native"),- eq(nativeRunFinalizations.phase, "retryable_failure"),- isNull(nativeRunFinalizations.resultId),- ),- );- const claimableNativeRunIds = new Set<string>();- for (const { run } of retryableNativeProcesses) {- if (isNativeRunnerOwnershipHeld(run)) continue;- if (!run.processPid && !run.processGroupId) {- claimableNativeRunIds.add(run.id);- continue;- }- const processPidAlive =- !!run.processPid && isProcessAlive(run.processPid);- const processGroupAlive =- !!run.processGroupId && isProcessGroupAlive(run.processGroupId);- if (processPidAlive || processGroupAlive) {- await markNativeOwnershipUnverified(run, {- reason: "live_process_identifier",- processPidAlive,- processGroupAlive,- });- continue;- }- const cleared = await db.transaction(async tx => {- const cleared = await tx- .update(heartbeatRuns)- .set({- processPid: null,- processGroupId: null,- processStartedAt: null,- updatedAt: now,+ const issue = contextIssueId+ ? await db+ .select({+ status: issues.status,+ title: issues.title,+ description: issues.description,+ workMode: issues.workMode,+ })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, run.companyId),+ eq(issues.id, contextIssueId),+ ),+ )+ .then((rows) => rows[0] ?? null)+ : null;++ const [commentStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${issueComments.createdAt})`,+ })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, run.companyId),+ eq(issueComments.issueId, contextIssueId),+ eq(issueComments.createdByRunId, run.id),+ isNull(issueComments.deletedAt),+ ),+ )+ : [{ count: 0, latestAt: null }];++ const issueCommentBodies = contextIssueId+ ? await db+ .select({ body: issueComments.body })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, run.companyId),+ eq(issueComments.issueId, contextIssueId),+ eq(issueComments.createdByRunId, run.id),+ ),+ )+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id))+ .limit(5)+ .then((rows) => rows.reverse().map((row) => row.body))+ : [];++ const continuationSummary = contextIssueId+ ? await getIssueContinuationSummaryDocument(db, contextIssueId)+ : null;++ const [documentStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,+ latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,})+ .from(documentRevisions)+ .innerJoin(+ issueDocuments,+ eq(documentRevisions.documentId, issueDocuments.documentId),+ ).where(and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.runtimeMode, "native"),- run.processPid === null- ? isNull(heartbeatRuns.processPid)- : eq(heartbeatRuns.processPid, run.processPid),- run.processGroupId === null- ? isNull(heartbeatRuns.processGroupId)- : eq(heartbeatRuns.processGroupId, run.processGroupId),- run.processStartedAt === null- ? isNull(heartbeatRuns.processStartedAt)- : eq(heartbeatRuns.processStartedAt, run.processStartedAt),+ eq(documentRevisions.companyId, run.companyId),+ eq(documentRevisions.createdByRunId, run.id),+ eq(issueDocuments.companyId, run.companyId),+ eq(issueDocuments.issueId, contextIssueId),+ sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,),)- .returning({ id: heartbeatRuns.id })- .then((rows) => rows[0] ?? null);- if (cleared) await recordNativeLocalProcessStop(tx as unknown as Db, run);- return cleared;- });- if (cleared) claimableNativeRunIds.add(cleared.id);- }+ : [{ count: 0, planCount: 0, latestAt: null }];- // An explicit result-less retryable failure resumes on the original run.- // The database lease is claimed before dispatch so concurrent service- // instances cannot open competing recoveries; executeRun receives the exact- // claimed owner. Expired `observed` ownership never enters this set.- const nativeResumeClaims =- claimableNativeRunIds.size === 0- ? []- : await dispatchNativeSessionResumptions({- db,- runnerInstanceId:- runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat",- now,- runIds: [...claimableNativeRunIds],- dispatch: (claim) => {- const execution = executeRun(claim.runId, {- nativeLeaseOwner: claim.leaseOwner,- }).catch((error) => {- logger.error(- { err: error, runId: claim.runId },- "persisted native session resume failed",- );- });- activeRunExecutionPromises.add(execution);- void execution.finally(() =>- activeRunExecutionPromises.delete(execution),- );- },- }).catch((error) => {- logger.warn(- { err: error },- "failed to claim persisted native session resumptions",- );- return [];- });- const resumedRunIds = new Set(- nativeResumeClaims.map((claim) => claim.runId),- );+ const [workProductStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,+ })+ .from(issueWorkProducts)+ .where(+ and(+ eq(issueWorkProducts.companyId, run.companyId),+ eq(issueWorkProducts.issueId, contextIssueId),+ eq(issueWorkProducts.createdByRunId, run.id),+ ),+ )+ : [{ count: 0, latestAt: null }];- // A terminal issue transition writes this intent in the same transaction- // that closes the question's task, even when its card is retained. Consume- // it before generic orphan recovery so a restart preserves cancellation.- const cancellationRequests = await db+ const [workspaceOperationStats] = await db.select({- id: heartbeatRuns.id,- contextSnapshot: heartbeatRuns.contextSnapshot,+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,})- .from(heartbeatRuns)+ .from(workspaceOperations).where(and(- inArray(heartbeatRuns.status, [- ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,- ]),- sql`${heartbeatRuns.contextSnapshot} -> ${NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY} is not null`,+ eq(workspaceOperations.companyId, run.companyId),+ eq(workspaceOperations.heartbeatRunId, run.id),),);- for (const request of cancellationRequests) {- const marker = parseObject(- parseObject(request.contextSnapshot)[- NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY- ],- );- const issueId = readNonEmptyString(marker.issueId);- const issueStatus = readNonEmptyString(marker.issueStatus);- const interactionId = readNonEmptyString(marker.interactionId);- const kind = readNonEmptyString(marker.kind);- const reason =- kind === "interaction_withdrawn"- ? "Question withdrawn while waiting for operator input"- : kind === "interaction_cancelled"- ? "Cancelled while waiting for operator input"- : "Task closed while waiting for operator input";- try {- await cancelRunInternal(request.id, reason, {- resultJson: {- ...(kind === "interaction_withdrawn" && interactionId- ? { withdrawnInteractionId: interactionId }- : {}),- ...(kind === "interaction_cancelled" && interactionId- ? { cancelledInteractionId: interactionId }- : {}),- ...((!kind || kind === "issue_terminal") && issueStatus- ? { cancelledByIssueStatus: issueStatus }- : {}),- ...(issueId ? { cancelledIssueId: issueId } : {}),- },- });- } catch (err) {- // Keep the marker intact for the next startup/periodic sweep.- logger.warn(- { err, runId: request.id },- "native question cancellation recovery attempt failed",- );- }- }- // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)- const activeRuns = await db+ const [activityStats] = await db.select({- run: heartbeatRuns,- adapterType: agents.adapterType,- adapterConfig: agents.adapterConfig,- nativeCoordinatorPhase: nativeRunFinalizations.phase,- nativeRecoveryState: nativeRunFinalizations.recoveryState,- nativeControllerBootId: nativeRunFinalizations.controllerBootId,- nativeControllerPid: nativeRunFinalizations.controllerPid,- nativeControllerProcessStartedAt:- nativeRunFinalizations.controllerProcessStartedAt,- nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${activityLog.createdAt})`,})- .from(heartbeatRuns)- .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))- .leftJoin(- nativeRunFinalizations,- eq(nativeRunFinalizations.runId, heartbeatRuns.id),- )- .where(eq(heartbeatRuns.status, "running"));-- const monitorIssueIds = [- ...new Set(- activeRuns.flatMap(({ run }) => {- const runContext = parseObject(run.contextSnapshot);- if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due")- return [];- const issueId = readNonEmptyString(runContext.issueId);- return issueId ? [issueId] : [];- }),- ),- ];- const monitorIssues =- monitorIssueIds.length > 0- ? await db- .select({- id: issues.id,- companyId: issues.companyId,- monitorNextCheckAt: issues.monitorNextCheckAt,- })- .from(issues)- .where(inArray(issues.id, monitorIssueIds))- : [];- const monitorNextCheckAtByIssue = new Map(- monitorIssues.map((issue) => [- `${issue.companyId}:${issue.id}`,- issue.monitorNextCheckAt,- ]),- );-- const reaped: string[] = [];- const currentNativeController = await currentNativeControllerIdentity();-- for (const {- run,- adapterType,- adapterConfig,- nativeCoordinatorPhase,- nativeRecoveryState,- nativeControllerBootId,- nativeControllerPid,- nativeControllerProcessStartedAt,- nativeControllerLeaseExpiresAt,- } of activeRuns) {- // Authentication timeout requires an explicit ownership resolution, not- // repeated reattachment or a process-gone guess on subsequent sweeps.- if (isNativeRunnerOwnershipHeld(run)) continue;- const nativeRun = run.runtimeMode === "native";- const nativeProcessPidAlive =- nativeRun && !!run.processPid && isProcessAlive(run.processPid);- const nativeProcessGroupAlive =- nativeRun &&- !!run.processGroupId &&- isProcessGroupAlive(run.processGroupId);- const coordinatorOwnedByCurrentController =- nativeRun &&- nativeControllerBootId === currentNativeController.bootId &&- nativeControllerPid === currentNativeController.pid &&- nativeControllerProcessStartedAt?.getTime() ===- currentNativeController.processStartedAt.getTime() &&- !!nativeControllerLeaseExpiresAt &&- nativeControllerLeaseExpiresAt.getTime() > now.getTime();- const locallyTracked =- runningProcesses.has(run.id) ||- activeRunExecutions.has(run.id) ||- coordinatorOwnedByCurrentController;- if (- nativeRun &&- ([- "awaiting_evidence",- "awaiting_runner_reattach",- "resuming_session",- "bootstrap_incomplete",- ].includes(nativeRecoveryState ?? "") ||- nativeCoordinatorPhase === "retryable_failure")- ) {- continue;- }- const observedOwnerUnverified =- nativeRun &&- (nativeCoordinatorPhase === "observed" ||- (nativeCoordinatorPhase === null &&- run.nativePhase === "observed")) &&- !resumedRunIds.has(run.id) &&- !locallyTracked;- // Persisted numeric process identifiers prove only that some process is- // alive, not that Paperclip still owns it. Likewise an observed native- // coordinator without a live in-process execution has no durable proof- // that its prior provider owner stopped. Keep both cases running but- // blocked: never signal, finalize, or retry them automatically. This gate- // intentionally precedes resumedRunIds so a claim cannot bypass the- // ownership check.- if (- !locallyTracked &&- (nativeProcessPidAlive ||- nativeProcessGroupAlive ||- observedOwnerUnverified)- ) {- await markNativeOwnershipUnverified(run, {- reason:- nativeProcessPidAlive || nativeProcessGroupAlive- ? "live_process_identifier"- : "observed_owner_unverified",- processPidAlive: nativeProcessPidAlive,- processGroupAlive: nativeProcessGroupAlive,- });- continue;- }- if (resumedRunIds.has(run.id)) continue;- if (locallyTracked) continue;- if (await hasLiveLegacyController(db, run)) continue;-- // Apply staleness threshold to avoid false positives- if (staleThresholdMs > 0) {- const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;- if (now.getTime() - refTime < staleThresholdMs) continue;- }-- const currentAdapterTracksLocalChild =- isTrackedLocalChildProcessAdapter(adapterType);- const tracksLegacyLocalChild =- run.runtimeMode !== "native" && currentAdapterTracksLocalChild;- // Native runner processes also persist child metadata, but they must not- // inherit legacy retry or termination authority. Use their PID/group only- // for a read-only liveness check so a lost in-memory handle cannot cause- // overlapping provider/tool execution while that child is still alive.- const checksPersistedChildLiveness =- currentAdapterTracksLocalChild || run.runtimeMode === "native";- const processPidAlive =- checksPersistedChildLiveness &&- run.processPid &&- isProcessAlive(run.processPid);- const processGroupAlive =- checksPersistedChildLiveness &&- run.processGroupId &&- isProcessGroupAlive(run.processGroupId);- if (- (processPidAlive || processGroupAlive) &&- readHotRestartAdoptionMetadata(parseObject(run.resultJson))- ) {- continue;- }- if (processPidAlive || processGroupAlive) {- if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) {- const detachedMessage = processPidAlive- ? `Lost in-memory process handle, but child pid ${run.processPid} is still alive`- : `Lost in-memory process handle, but persisted process group ${run.processGroupId} is still alive`;- const detachedRun = await setRunStatus(run.id, "running", {- error: detachedMessage,- errorCode: DETACHED_PROCESS_ERROR_CODE,- });- if (detachedRun) {- await appendRunEvent(detachedRun, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message: detachedMessage,- payload: {- processPid: run.processPid ?? null,- processGroupId: run.processGroupId ?? null,- ownedProcessHandle: false,- },- });- }- }- continue;- }-- const runContext = parseObject(run.contextSnapshot);- const monitorIssueId = readNonEmptyString(runContext.issueId);- const monitorNextCheckAt = monitorIssueId- ? monitorNextCheckAtByIssue.get(`${run.companyId}:${monitorIssueId}`)- : undefined;- const monitorDispatchLostWithoutFutureWake =- readNonEmptyString(runContext.wakeReason) === "issue_monitor_due" &&- monitorNextCheckAt !== undefined &&- (!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime());- const shouldRetry =- (run.processLossRetryCount ?? 0) < 1 &&- ((tracksLegacyLocalChild &&- (!!run.processPid || !!run.processGroupId)) ||- monitorDispatchLostWithoutFutureWake);- if (!(await revokeExpiredLegacyController(db, run))) continue;- const baseMessage = buildProcessLossMessage(run);- const processLossDiagnostic = buildProcessLossDiagnostic({- run,- nowMs: Date.now(),- observerStartedAtMs: performance.timeOrigin,- checksPersistedChildLiveness,- retryEligible: shouldRetry,- });- const conversationContinuationEligible = await runUsedConversationAdapter(db, run);-- const failureWrite = await setRunStatusFromLive(- run.id,- "failed",- ["running"],- {- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,- errorCode: "process_lost",- finishedAt: now,- resultJson: (() => {- const result = mergeRunStopMetadataForAgent(- { adapterType, adapterConfig },- "failed",- {- conversationContinuationEligible,- resultJson: { ...parseObject(run.resultJson), processLossDiagnostic },- errorCode: "process_lost",- errorMessage: shouldRetry- ? `${baseMessage}; retrying once`- : baseMessage,- },- );- return result;- })(),- },- );- if (!failureWrite.updated || !failureWrite.run) continue;- let finalizedRun: typeof heartbeatRuns.$inferSelect | null =- failureWrite.run;- await setWakeupStatus(run.wakeupRequestId, "failed", {- finishedAt: now,- error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,- });- if (!finalizedRun) finalizedRun = await getRun(run.id);- if (!finalizedRun) continue;- finalizedRun =- (await classifyAndPersistRunLiveness(- finalizedRun,- parseObject(finalizedRun.resultJson),- )) ?? finalizedRun;- await releaseEnvironmentLeasesForRun({- runId: finalizedRun.id,- companyId: finalizedRun.companyId,- agentId: finalizedRun.agentId,- status: finalizedRun.status,- failureReason: finalizedRun.error ?? undefined,- });-- let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;- const retryAgent = await getAgent(run.agentId);- if (shouldRetry) {- if (retryAgent) {- retriedRun = await enqueueProcessLossRetry(- finalizedRun,- retryAgent,- now,- );- }- } else if (retryAgent) {- const scheduled =- await scheduleInteractionContinuationInfrastructureRetryIfEligible(- finalizedRun,- retryAgent,- );- retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null;- }-- if (!retriedRun) {- await releaseIssueExecutionAndPromote(finalizedRun);- }-- await appendRunEvent(finalizedRun, {- eventType: "lifecycle",- stream: "system",- level: "error",- message: shouldRetry- ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()- : baseMessage,- payload: {- ...(run.processPid ? { processPid: run.processPid } : {}),- ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),- ...(retriedRun ? { retryRunId: retriedRun.id } : {}),- },- });-- await finalizeAgentStatus(run.agentId, "failed", baseMessage, {- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),- });- await startNextQueuedRunForAgent(run.agentId);- runningProcesses.delete(run.id);- reaped.push(run.id);- }-- if (reaped.length > 0) {- logger.warn(- { reapedCount: reaped.length, runIds: reaped },- "reaped orphaned heartbeat runs",+ .from(activityLog)+ .where(+ and(+ eq(activityLog.companyId, run.companyId),+ eq(activityLog.runId, run.id),+ notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),+ ),);- }- // Recover an active lease whose run already ended before the same-tick- // pending_cleanup sweep, so this tick can stop the recovered sandbox.- // Isolate the sweep so its failure never hides the reaper result.- try {- const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({- backoffMs: staleThresholdMs,- });- if (orphanedActiveLeaseSweep.recovered > 0) {- logger.warn(- { recovered: orphanedActiveLeaseSweep.recovered },- "recovered orphaned active environment leases",- );- }- } catch {- // Log a constant errorKind only. The exception can carry a credential in- // its name, code, message, cause, or stack, so the sweep never reads it.- logger.error(- { errorKind: ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND },- "orphaned active environment lease sweep failed",+ const [eventStats] = await db+ .select({+ count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,+ latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,+ })+ .from(heartbeatRunEvents)+ .where(+ and(+ eq(heartbeatRunEvents.companyId, run.companyId),+ eq(heartbeatRunEvents.runId, run.id),+ ),);- }- // Retry stranded pending_cleanup leases on the same tick. Isolate the sweep- // so its failure never hides the reaper result. The backoff equals the- // reaper staleness threshold.- try {- const sweep = await sweepPendingCleanupLeases({- backoffMs: staleThresholdMs,- });- if (sweep.destroyed > 0 || sweep.capped > 0) {- logger.warn(- {- destroyed: sweep.destroyed,- capped: sweep.capped,- swept: sweep.swept,- },- "swept pending_cleanup environment leases",- );- }- } catch {- // Log a constant errorKind only. The exception can carry a credential in- // its name, code, message, cause, or stack, so the sweep never reads it.- logger.error(- { errorKind: PENDING_CLEANUP_SWEEP_ERROR_KIND },- "pending_cleanup lease sweep failed",- );- }+ return {+ runStatus: run.status,+ issue,+ resultJson: resultJson ?? run.resultJson ?? null,+ issueCommentBodies,+ continuationSummaryBody: continuationSummary?.body ?? null,+ stdoutExcerpt: run.stdoutExcerpt ?? null,+ stderrExcerpt: run.stderrExcerpt ?? null,+ error: run.error ?? null,+ errorCode: run.errorCode ?? null,+ continuationAttempt,+ evidence: {+ issueCommentsCreated: countValue(commentStats?.count),+ documentRevisionsCreated: countValue(documentStats?.count),+ planDocumentRevisionsCreated: countValue(documentStats?.planCount),+ workProductsCreated: countValue(workProductStats?.count),+ workspaceOperationsCreated: countValue(workspaceOperationStats?.count),+ activityEventsCreated: countValue(activityStats?.count),+ toolOrActionEventsCreated: countValue(eventStats?.count),+ latestEvidenceAt: latestDate(+ commentStats?.latestAt,+ documentStats?.latestAt,+ workProductStats?.latestAt,+ workspaceOperationStats?.latestAt,+ activityStats?.latestAt,+ eventStats?.latestAt,+ ),+ },+ };+ }- return { reaped: reaped.length, runIds: reaped };+ async function classifyAndPersistRunLiveness(+ run: typeof heartbeatRuns.$inferSelect,+ resultJson?: Record<string, unknown> | null,+ ) {+ const authRepair = run.status === "failed"+ ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {+ logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");+ return null;+ })+ : null;+ const classification = classifyRunLiveness({+ ...await buildRunLivenessInput(run, resultJson),+ authenticationRepairRequested: Boolean(authRepair?.interactionId),+ });+ return db+ .update(heartbeatRuns)+ .set({+ livenessState: classification.livenessState,+ livenessReason: classification.livenessReason,+ continuationAttempt: classification.continuationAttempt,+ lastUsefulActionAt: classification.lastUsefulActionAt,+ nextAction: classification.nextAction,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, run.id))+ .returning()+ .then((rows) => rows[0] ?? null);}async function resumeQueuedRuns() {diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.tsnew file mode 100644index 0000000000..a925edfaa7--- /dev/null+++ b/server/src/services/heartbeat/recovery.test.ts@@ -0,0 +1,268 @@+import { randomUUID } from "node:crypto";+import type { Db } from "@paperclipai/db";+import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";+import { eq } from "drizzle-orm";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import { environmentRuntimeService } from "../environment-runtime.js";+import { environmentService } from "../environments.js";+import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "./recovery.js";+import type { HotRestartIntent } from "../hot-restart.js";+import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js";+import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js";++type Run = typeof heartbeatRuns.$inferSelect;++const native = vi.hoisted(() => ({+ closeIdle: vi.fn(async () => ({ closed: 0, failed: 0 })),+ detach: vi.fn(async () => 0),+ finalizations: vi.fn(async (_db: Db, _ids?: string[], _options?: unknown) => undefined),+ cleanups: vi.fn(async () => undefined),+ drainMaintenance: vi.fn<() => Promise<void>>(async () => undefined),+ claims: vi.fn<() => Promise<NativeRestartRecoveryDisposition[]>>(async () => []),+}));+const hotRestart = vi.hoisted(() => ({+ read: vi.fn<() => Promise<HotRestartIntent | null>>(async () => null),+}));+vi.mock("../hot-restart.js", async (importOriginal) => ({+ ...await importOriginal<typeof import("../hot-restart.js")>(),+ readHotRestartIntent: hotRestart.read,+}));+vi.mock("../native-runtime/index.js", async (importOriginal) => ({+ ...await importOriginal<typeof import("../native-runtime/index.js")>(),+ closeIdleWarmNativeSessionsForRestart: native.closeIdle,+ detachNativeSessionsForRestart: native.detach,+ reconcileNativeFinalizations: native.finalizations,+ reconcileRetainedNativeSessionCleanups: native.cleanups,+ claimNativeRestartRecoveries: native.claims,+}));+vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({+ ...await importOriginal<typeof import("../../vendor/paperclip-runner/index.js")>(),+ drainRetainedRunnerdMaintenanceOperations: native.drainMaintenance,+}));++function callbacks(db: Db) {+ return {+ enterShutdown: vi.fn(),+ getRun: vi.fn<HeartbeatRecoveryDependencies["getRun"]>(async () => null),+ getAgent: vi.fn<HeartbeatRecoveryDependencies["getAgent"]>(async () => null),+ appendRunEvent: vi.fn<HeartbeatRecoveryDependencies["appendRunEvent"]>(async () => undefined),+ setRunStatusIfRunning: vi.fn<HeartbeatRecoveryDependencies["setRunStatusIfRunning"]>(async () => ({ updated: false, run: null })),+ setRunStatusFromLive: vi.fn<HeartbeatRecoveryDependencies["setRunStatusFromLive"]>(async () => ({ updated: false, run: null })),+ setRunStatus: vi.fn<HeartbeatRecoveryDependencies["setRunStatus"]>(async () => null),+ setWakeupStatus: vi.fn(async () => undefined),+ releaseIssueExecutionAndPromote: vi.fn(async () => undefined),+ finalizeAgentStatus: vi.fn(async () => undefined),+ environmentRuntime: environmentRuntimeService(db),+ environmentsSvc: environmentService(db),+ instructionCopies: {+ recoverStopped: vi.fn(async () => 0),+ recoverCaptured: vi.fn(async () => 0),+ },+ runtimeEnv: {},+ activeRunExecutions: new Set<string>(),+ activeRunExecutionPromises: new Set<Promise<void>>(),+ scheduleBoundedRetryForRun: vi.fn<HeartbeatRecoveryDependencies["scheduleBoundedRetryForRun"]>(async () => ({ outcome: "retry_exhausted", retryReason: "transient_failure", attempt: 1, maxAttempts: 0 })),+ scheduleInteractionContinuationInfrastructureRetryIfEligible: vi.fn<HeartbeatRecoveryDependencies["scheduleInteractionContinuationInfrastructureRetryIfEligible"]>(async () => null),+ timerClaimWasFirstHeartbeat: vi.fn(() => undefined),+ executeRun: vi.fn<HeartbeatRecoveryDependencies["executeRun"]>(async () => undefined),+ scheduleNativeSessionResumeDispatch: vi.fn(),+ cancelHeartbeatNativeRun: vi.fn(async () => undefined),+ terminateHeartbeatRunProcess: vi.fn(async () => undefined),+ mergeRunStopMetadataForAgent: vi.fn<HeartbeatRecoveryDependencies["mergeRunStopMetadataForAgent"]>((_agent, _outcome, options) => options?.resultJson ?? null),+ classifyAndPersistRunLiveness: vi.fn<HeartbeatRecoveryDependencies["classifyAndPersistRunLiveness"]>(async run => run),+ releaseEnvironmentLeasesForRun: vi.fn(async () => undefined),+ acknowledgeRemoteStop: vi.fn(async () => undefined),+ resumeRemoteStopComments: vi.fn(async () => undefined),+ dispatchPendingNativeStatusWakeups: vi.fn(async () => undefined),+ cancelRunInternal: vi.fn(async () => undefined),+ startNextQueuedRunForAgent: vi.fn(async () => undefined),+ } satisfies HeartbeatRecoveryDependencies;+}++function guardedDatabase() {+ const access = vi.fn(() => { throw new Error("Unexpected recovery database access"); });+ return { db: new Proxy({}, { get: access }) as Db, access };+}++beforeEach(() => {+ vi.clearAllMocks();+ hotRestart.read.mockResolvedValue(null);+ native.closeIdle.mockResolvedValue({ closed: 0, failed: 0 });+ native.finalizations.mockResolvedValue(undefined);+ native.cleanups.mockResolvedValue(undefined);+ native.drainMaintenance.mockResolvedValue(undefined);+ native.claims.mockResolvedValue([]);+});++describe("heartbeat recovery module callbacks", () => {+ it("constructs without database access, shutdown, or background work", () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ createHeartbeatRecovery(database.db, deps);+ expect(database.access).not.toHaveBeenCalled();+ for (const callback of Object.values(deps)) if (typeof callback === "function") expect(callback).not.toHaveBeenCalled();+ for (const callback of Object.values(native)) expect(callback).not.toHaveBeenCalled();+ });++ it.each(["not_requested", "read_error"] as const)("enters shutdown before checkpointing for %s", async (mode) => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ if (mode === "read_error") hotRestart.read.mockRejectedValueOnce(new Error("unreadable intent"));+ const recovery = createHeartbeatRecovery(database.db, deps);+ expect(await recovery.prepareHotRestartShutdown("SIGTERM")).toEqual({ mode, skipDrain: false, activeRunIds: [] });+ expect(native.closeIdle).toHaveBeenCalledAfter(deps.enterShutdown);+ expect(hotRestart.read).toHaveBeenCalledAfter(native.closeIdle);+ expect(database.access).not.toHaveBeenCalled();+ });++ it("does not adopt or query runs without a restart intent", async () => {+ const database = guardedDatabase();+ expect(await createHeartbeatRecovery(database.db, callbacks(database.db)).reconcileHotRestartAdoption()).toEqual({+ mode: "not_requested", adoptedRunIds: [], finalizedWhileDownRunIds: [], lostRunIds: [], skippedRunIds: [],+ });+ expect(database.access).not.toHaveBeenCalled();+ });++ it("does not broaden an empty selective shutdown drain to every run", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ expect(await createHeartbeatRecovery(database.db, deps).drainRunningRunsForShutdown("SIGINT", new Date(), [])).toEqual({+ interrupted: 0, interruptedRunIds: [], retryRunIds: [], restartSuspendedRunIds: [],+ });+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled();+ expect(deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();+ });++ it.each(["running", "failed"] as const)("keeps a %s ownership hold when a status race wins", async (status) => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { id: "run", runtimeMode: "native", status } as Run;+ const winner = { ...run, status: "cancelled" };+ deps.setRunStatusFromLive.mockResolvedValue({ updated: false, run: winner });+ expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "live_process_identifier" })).toBe(winner);+ expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", status, [status], expect.objectContaining({ errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE }));+ expect(deps.appendRunEvent).not.toHaveBeenCalled();+ expect(database.access).not.toHaveBeenCalled();+ });++ it("persists an authentication ownership hold before its event without terminating or retrying", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { id: "run", runtimeMode: "native", status: "running" } as Run;+ const blocked = { ...run, nativePhase: "terminal_failure", errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE };+ deps.setRunStatusFromLive.mockResolvedValue({ updated: true, run: blocked });+ expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "adopted_runner_authentication_timeout" })).toBe(blocked);+ expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", "running", ["running"], expect.objectContaining({ nativePhase: "terminal_failure" }));+ expect(deps.appendRunEvent).toHaveBeenCalledAfter(deps.setRunStatusFromLive);+ expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled();+ expect(deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();+ expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();+ expect(database.access).not.toHaveBeenCalled();+ });+});++const support = await getEmbeddedPostgresTestSupport();+const describePostgres = support.supported ? describe : describe.skip;+if (!support.supported) console.warn(`Skipping recovery module database tests: ${support.reason}`);++describePostgres("heartbeat recovery module ownership", () => {+ let db: Db;+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-recovery-module-");+ db = createDb(database.connectionString);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => { await database?.cleanup(); });++ async function seedRun() {+ const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), issueId = randomUUID();+ await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `T${companyId.slice(0, 6)}` });+ await db.insert(agents).values({ id: agentId, companyId, name: "Recovery", role: "engineer", adapterType: "process" });+ await db.insert(issues).values({ id: issueId, companyId, title: "Recover this run", status: "in_progress", assigneeAgentId: agentId });+ return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];+ }++ it("uses the shared live-execution set after construction across two factories", async () => {+ const run = await seedRun();+ const deps = callbacks(db);+ const first = createHeartbeatRecovery(db, deps);+ const second = createHeartbeatRecovery(db, deps);+ deps.activeRunExecutions.add(run.id);+ expect(await first.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });+ expect(await second.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");+ await Promise.all([...deps.activeRunExecutionPromises]);+ await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ });++ it("settles persisted native results before claiming restart authority and tracks the exact execution", async () => {+ const deps = callbacks(db);+ const claim: NativeRestartRecoveryClaim = { kind: "bootstrap_incomplete", runId: randomUUID(), leaseOwner: "claimed-owner", controllerGeneration: 3, providerAttempt: 2, restartKind: "hard", recoveryRequestId: null };+ native.claims.mockResolvedValue([claim]);+ let finish!: () => void;+ const execution = new Promise<void>(resolve => { finish = resolve; });+ deps.executeRun.mockReturnValue(execution);+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();+ expect(result.claims).toEqual([claim]);+ expect(native.claims).toHaveBeenCalledAfter(native.finalizations);+ expect(deps.executeRun).toHaveBeenCalledWith(claim.runId, { nativeLeaseOwner: claim.leaseOwner, nativeRestartRecovery: claim });+ expect(deps.activeRunExecutionPromises.size).toBeGreaterThan(0);+ finish();+ await Promise.all([...deps.activeRunExecutionPromises]);+ await vi.waitFor(() => expect(deps.activeRunExecutionPromises.size).toBe(0));+ });++ it("does not claim or execute recovery when persisted finalization fails", async () => {+ const deps = callbacks(db);+ native.finalizations.mockRejectedValueOnce(new Error("result settlement failed"));+ await expect(createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart()).rejects.toThrow("result settlement failed");+ expect(native.claims).not.toHaveBeenCalled();+ expect(deps.executeRun).not.toHaveBeenCalled();+ expect(native.cleanups).not.toHaveBeenCalled();+ });++ it("keeps blocked and ambiguous ownership out of execution", async () => {+ const deps = callbacks(db);+ const dispositions: NativeRestartRecoveryDisposition[] = [+ { kind: "blocked", runId: randomUUID(), reason: "conflicting_owner" },+ { kind: "awaiting_evidence", runId: randomUUID(), reason: "missing_stop_evidence" },+ ];+ native.claims.mockResolvedValue(dispositions);+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();+ expect(result.blockedRunIds).toEqual([dispositions[0].runId]);+ expect(result.awaitingEvidenceRunIds).toEqual([dispositions[1].runId]);+ expect(result.claims).toEqual([]);+ expect(deps.executeRun).not.toHaveBeenCalled();+ await Promise.all([...deps.activeRunExecutionPromises]);+ });++ it("rearms the persisted native retry deadline through the service callback", async () => {+ const run = await seedRun();+ const nextAttemptAt = new Date(Date.now() + 60_000);+ await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status: "failed" }).where(eq(heartbeatRuns.id, run.id));+ await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt });+ const deps = callbacks(db);+ const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();+ expect(result.scheduledRetryRunIds).toEqual([run.id]);+ expect(deps.scheduleNativeSessionResumeDispatch).toHaveBeenCalledWith(run.id, nextAttemptAt);+ expect(deps.executeRun).not.toHaveBeenCalled();+ await Promise.all([...deps.activeRunExecutionPromises]);+ await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));+ await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ });++ it("retains shutdown ownership until background native cleanup physically settles", async () => {+ const deps = callbacks(db);+ let settle!: () => void;+ native.drainMaintenance.mockReturnValueOnce(new Promise<void>(resolve => { settle = resolve; }));+ await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();+ await vi.waitFor(() => expect(native.drainMaintenance).toHaveBeenCalled());+ expect(deps.activeRunExecutionPromises.size).toBe(1);+ settle();+ await Promise.all([...deps.activeRunExecutionPromises]);+ await vi.waitFor(() => expect(deps.activeRunExecutionPromises.size).toBe(0));+ });+});diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.tsnew file mode 100644index 0000000000..4798251a1f--- /dev/null+++ b/server/src/services/heartbeat/recovery.ts@@ -0,0 +1,2423 @@+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";+import { CHAT_COMPLETION_WAKE_REASON } from "../chat-completion-delivery.js";+import {+ hasStopOnlyCleanup,+ settleStopOnlyCleanup,+} from "../sandbox-stop-and-retain.js";+import { recordNativeLocalProcessStop } from "../native-local-process-stop.js";+import {+ legacyControllerBootId,+ hasLiveLegacyController,+ revokeExpiredLegacyController,+} from "../legacy-controller-lease.js";+import { remoteTerminationReceipt } from "../remote-execution-termination.js";+import { runUsedConversationAdapter } from "../conversation-continuation.js";+import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";+import { randomUUID } from "node:crypto";+import {+ and,+ asc,+ eq,+ gt,+ inArray,+ isNull,+ lte,+ ne,+ or,+ sql,+} from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import {+ agents,+ environmentLeases,+ heartbeatRuns,+ issues,+ nativeRunFinalizations,+} from "@paperclipai/db";+import { logger } from "../../middleware/logger.js";+import {+ claimNativeRestartRecoveries,+ closeIdleWarmNativeSessionsForRestart,+ currentNativeControllerIdentity,+ dispatchNativeSessionResumptions,+ detachNativeSessionsForRestart,+ isNativeSessionId,+ readNativeWorkspaceSyncReference,+ type NativeRestartRecoveryClaim,+ reconcileNativeFinalizations,+ reconcileRetainedNativeSessionCleanup,+ reconcileRetainedNativeSessionCleanups,+} from "../native-runtime/index.js";+import { drainRetainedRunnerdMaintenanceOperations } from "../../vendor/paperclip-runner/index.js";+import { runningProcesses } from "../../adapters/index.js";+import { parseObject } from "../../adapters/utils.js";+import {+ isNativeRunnerOwnershipHeld,+ NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,+} from "../native-runtime/native-runner-ownership.js";+import { performance } from "node:perf_hooks";+import { buildProcessLossDiagnostic } from "../process-loss-diagnostics.js";+import { logActivity } from "../activity-log.js";+import { releaseRuntimeServicesForRun } from "../workspace-runtime.js";+import { isProcessGroupAlive } from "../local-service-supervisor.js";+import {+ findMissingHotRestartSnapshotRunIds,+ readHotRestartIntent,+ removeHotRestartIntent,+ shouldHonorHotRestartIntentForProcess,+ writeHotRestartReport,+ writeHotRestartShutdownSnapshot,+ type HotRestartIntentRun,+ type HotRestartReportRun,+} from "../hot-restart.js";+import { serverVersion } from "../../version.js";++import type { environmentService } from "../environments.js";+import type { environmentRuntimeService, ProviderResourceDisposition } from "../environment-runtime.js";+import type { agentInstructionWorkingCopyService } from "../agent-instruction-working-copies.js";+import type { createHeartbeatRunState } from "./run-state.js";+import type { HeartbeatRetryDependencies, createHeartbeatRetries } from "./retries.js";++type HeartbeatRun = typeof heartbeatRuns.$inferSelect;+type HeartbeatRetries = ReturnType<typeof createHeartbeatRetries>;++/** Recovery owns reconciliation; the service retains execution and lifecycle effects. */+export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDependencies,+ "getAgent" | "appendRunEvent" | "setRunStatusIfRunning" |+ "setWakeupStatus" | "releaseIssueExecutionAndPromote" | "finalizeAgentStatus"> {+ enterShutdown: () => void;+ getRun: (runId: string, options?: Parameters<ReturnType<typeof createHeartbeatRunState>["getRun"]>[1]) => Promise<+ Awaited<ReturnType<ReturnType<typeof createHeartbeatRunState>["getRun"]>> | null+ >;+ environmentRuntime: ReturnType<typeof environmentRuntimeService>;+ environmentsSvc: Pick<ReturnType<typeof environmentService>, "getById" | "getLeaseById" | "releaseLease">;+ instructionCopies: Pick<ReturnType<typeof agentInstructionWorkingCopyService>, "recoverStopped" | "recoverCaptured">;+ runtimeEnv: Record<string, string | undefined>;+ // Preserve the module-wide sets shared by scheduler and route service instances.+ activeRunExecutions: Set<string>;+ activeRunExecutionPromises: Set<Promise<void>>;+ scheduleBoundedRetryForRun: HeartbeatRetries["scheduleBoundedRetryForRun"];+ scheduleInteractionContinuationInfrastructureRetryIfEligible: HeartbeatRetries["scheduleInteractionContinuationInfrastructureRetryIfEligible"];+ timerClaimWasFirstHeartbeat: HeartbeatRetries["timerClaimWasFirstHeartbeat"];+ executeRun: (runId: string, options?: {+ nativeLeaseOwner?: string;+ nativeRestartRecovery?: NativeRestartRecoveryClaim;+ }) => Promise<void>;+ scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date) => void;+ cancelHeartbeatNativeRun: (input: {+ db: Db; runId: string; reason: string; runtimeMode: string | null;+ }) => Promise<unknown>;+ terminateHeartbeatRunProcess: (input: {+ pid: number | null | undefined;+ processGroupId: number | null | undefined;+ graceMs?: number;+ }) => Promise<void>;+ setRunStatus: (runId: string, status: string, patch?: Partial<typeof heartbeatRuns.$inferInsert>) => Promise<HeartbeatRun | null>;+ setRunStatusFromLive: (runId: string, status: string, fromStatuses: string[], patch?: Partial<typeof heartbeatRuns.$inferInsert>) => Promise<{+ updated: boolean;+ run: HeartbeatRun | null;+ }>;+ mergeRunStopMetadataForAgent: (+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",+ options?: {+ resultJson?: Record<string, unknown> | null;+ conversationContinuationEligible?: boolean;+ errorCode?: string | null;+ errorMessage?: string | null;+ },+ ) => Record<string, unknown> | null;+ classifyAndPersistRunLiveness: (run: HeartbeatRun, resultJson?: Record<string, unknown> | null) => Promise<HeartbeatRun | null>;+ releaseEnvironmentLeasesForRun: (input: {+ runId: string;+ companyId: string;+ agentId: string;+ status: string | null | undefined;+ failureReason?: string | null;+ providerResourceDisposition?: ProviderResourceDisposition;+ }) => Promise<void>;+ acknowledgeRemoteStop: (runId: string, companyId: string) => Promise<void>;+ resumeRemoteStopComments: (run: HeartbeatRun) => Promise<void>;+ dispatchPendingNativeStatusWakeups: () => Promise<unknown>;+ cancelRunInternal: (runId: string, reason?: string, options?: { resultJson?: Record<string, unknown> }) => Promise<unknown>;+ startNextQueuedRunForAgent: (agentId: string) => Promise<unknown>;+}++export const DETACHED_PROCESS_ERROR_CODE = "process_detached";++const NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE =+ "Native execution ownership could not be verified; automatic recovery is blocked";++// The reaper sweeps at most this many pending_cleanup leases per tick.+const PENDING_CLEANUP_SWEEP_PAGE_SIZE = 20;++const pendingCleanupAttemptsInFlight = new Set<string>();++// Escalate and slow cleanup after this many attempts; never abandon a live lease.+const PENDING_CLEANUP_SWEEP_ATTEMPT_CAP = 5;++// The reaper stores its retry state under these keys in the lease metadata.+const PENDING_CLEANUP_ATTEMPTS_METADATA_KEY = "pendingCleanupRetryAttempts";++const PENDING_CLEANUP_CAP_WARNED_METADATA_KEY = "pendingCleanupRetryCapWarned";++// The reaper sweeps at most this many orphaned active leases per tick.+const ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE = 20;++// A provider or plugin destroy rejection can carry a bearer credential, a+// signed URL, or provider response detail in its name, code, message, cause, or+// stack. The exception fields cross the server boundary, so they are not a+// trusted enum. The pending_cleanup sweep logs never read the exception. Each+// catch site logs a constant, locally generated `errorKind` instead.+const PENDING_CLEANUP_RETRY_ERROR_KIND = "destroy_failed";++const PENDING_CLEANUP_SWEEP_ERROR_KIND = "sweep_failed";++const ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND = "orphaned_active_lease_sweep_failed";++// Read the stored retry attempt count as a safe value, directly in SQL. A+// provider can write a malformed value under the attempts key. The type guard+// makes any non-number value read as zero. The reader computes as numeric and+// never casts to int, so a finite number outside the 32-bit range (for example+// 1e300) never throws. The reader clamps a negative value to zero and a positive+// value to the attempt cap. One malformed lease therefore never aborts the page+// sweep. This matches the TypeScript reader `readPendingCleanupRetryAttempts`,+// which clamps to the same range. The claim predicate compares the two readers,+// so both must yield the same value for every input.+function pendingCleanupAttemptsSql() {+ return sql`+ case+ when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}) = 'number'+ then least(+ greatest(+ floor((${environmentLeases.metadata} ->> ${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY})::numeric),+ 0+ ),+ ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}+ )+ else 0+ end`;+}++function pendingCleanupRetryDueSql(explicitRetry = false) {+ // Ownership renewal and retry cooldown are separate clocks. A late renewal+ // cannot change when a completed attempt may retry. Older claims used the+ // retry field for both clocks, so retain that fallback until they settle.+ const deadline = sql`case+ when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'+ and ${environmentLeases.metadata} ? 'pendingCleanupLeaseExpiresAtMs'+ then ${environmentLeases.metadata}->'pendingCleanupLeaseExpiresAtMs'+ else ${environmentLeases.metadata}->'pendingCleanupRetryAfterMs' end`;+ return sql`case+ when ${explicitRetry} and coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true' then true+ when jsonb_typeof(${deadline}) = 'number'+ then (${deadline})::numeric <= ${Date.now()}+ or (${deadline})::numeric > ${Date.now() + 30 * 60_000 + 1_000}+ else true end`;+}++// Choose the `jsonb_set` target root. A provider can write a scalar or array+// metadata root. `jsonb_set` fails on a non-object root, so the reader uses the+// stored metadata only when its root is an object. A NULL, scalar, or array root+// reads as an empty object. `jsonb_typeof(NULL)` is NULL, so the else branch also+// covers a NULL root.+function pendingCleanupMetadataObjectSql() {+ return sql`case when jsonb_typeof(${environmentLeases.metadata}) = 'object' then ${environmentLeases.metadata} else '{}'::jsonb end`;+}++// Read the stored cap-warned flag as a safe boolean, directly in SQL. A+// malformed value reads as false, so the boolean cast never throws.+function pendingCleanupCapWarnedSql() {+ return sql`coalesce(+ case+ when jsonb_typeof(${environmentLeases.metadata} -> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}) = 'boolean'+ then (${environmentLeases.metadata} ->> ${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY})::boolean+ else false+ end,+ false+ )`;+}++export const CANCELLABLE_HEARTBEAT_RUN_STATUSES = [+ "queued",+ "running",+ "scheduled_retry",+] as const;++const NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY = "nativeQuestionCancellation";++export const HEARTBEAT_RUN_TERMINAL_STATUSES = [+ "succeeded",+ "interrupted",+ "failed",+ "cancelled",+ "timed_out",+] as const;++const SESSIONED_LOCAL_ADAPTERS = new Set([+ "claude_local",+ "codex_local",+ "cursor",+ "gemini_local",+ "hermes_local",+ "kimi_local",+ "opencode_local",+ "pi_local",+]);++function isTrackedLocalChildProcessAdapter(adapterType: string) {+ return SESSIONED_LOCAL_ADAPTERS.has(adapterType);+}++// A positive liveness check means some process currently owns the PID.+// On Linux, PIDs can be recycled, so this is a best-effort signal rather+// than proof that the original child is still alive.+export function isProcessAlive(pid: number | null | undefined) {+ if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0)+ return false;+ try {+ process.kill(pid, 0);+ return true;+ } catch (error) {+ const code = (error as NodeJS.ErrnoException | undefined)?.code;+ if (code === "EPERM") return true;+ if (code === "ESRCH") return false;+ return false;+ }+}++function buildProcessLossMessage(+ run: {+ processPid: number | null;+ processGroupId: number | null;+ },+ options?: { descendantOnly?: boolean },+) {+ if (options?.descendantOnly && run.processGroupId) {+ return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`;+ }+ if (run.processPid) {+ return `Process lost -- child pid ${run.processPid} is no longer running`;+ }+ if (run.processGroupId) {+ return `Process lost -- process group ${run.processGroupId} is no longer running`;+ }+ return "Process lost -- server may have restarted";+}++function readHotRestartAdoptionMetadata(+ resultJson: Record<string, unknown> | null | undefined,+) {+ const result = parseObject(resultJson);+ const hotRestart = parseObject(result.hotRestart);+ if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string")+ return null;+ return hotRestart;+}++function mergeHotRestartAdoptionResultJson(+ resultJson: Record<string, unknown> | null | undefined,+ input: {+ adoptedAt: Date;+ previousServerPid: number;+ newServerPid: number;+ previousServerVersion: string | null;+ newServerVersion: string;+ processPid: number | null;+ processGroupId: number | null;+ },+) {+ const result = parseObject(resultJson);+ const existing = parseObject(result.hotRestart);+ return {+ ...result,+ hotRestart: {+ ...existing,+ adopted: true,+ adoptedAt: input.adoptedAt.toISOString(),+ previousServerPid: input.previousServerPid,+ newServerPid: input.newServerPid,+ previousServerVersion: input.previousServerVersion,+ newServerVersion: input.newServerVersion,+ processPid: input.processPid,+ processGroupId: input.processGroupId,+ },+ };+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++/** Restart recovery shares the service lifecycle callbacks and execution ownership. */+export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryDependencies) {+ const {+ enterShutdown,+ scheduleBoundedRetryForRun,+ appendRunEvent,+ environmentRuntime,+ scheduleNativeSessionResumeDispatch,+ getRun,+ executeRun,+ activeRunExecutionPromises,+ cancelHeartbeatNativeRun,+ terminateHeartbeatRunProcess,+ setRunStatusIfRunning,+ mergeRunStopMetadataForAgent,+ setWakeupStatus,+ classifyAndPersistRunLiveness,+ releaseEnvironmentLeasesForRun,+ releaseIssueExecutionAndPromote,+ finalizeAgentStatus,+ timerClaimWasFirstHeartbeat,+ environmentsSvc,+ acknowledgeRemoteStop,+ resumeRemoteStopComments,+ setRunStatusFromLive,+ instructionCopies,+ dispatchPendingNativeStatusWakeups,+ runtimeEnv,+ cancelRunInternal,+ activeRunExecutions,+ setRunStatus,+ getAgent,+ scheduleInteractionContinuationInfrastructureRetryIfEligible,+ startNextQueuedRunForAgent,+ } = dependencies;++ async function enqueueProcessLossRetry(+ run: typeof heartbeatRuns.$inferSelect,+ agent: typeof agents.$inferSelect,+ now: Date,+ ) {+ // Completion deliveries own their durable bounded retry and reply identity.+ // A second process-loss retry would compete for the same outbox input.+ if (run.contextSnapshot?.wakeReason === CHAT_COMPLETION_WAKE_REASON &&+ Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds)) return null;+ // Native sessions have their own fenced same-run controller. Legacy+ // bootstrap recovery shares the durable delay and incident counter with+ // transient retries; process loss must not open a second retry budget.+ if (run.runtimeMode === "native" || await legacyExecutionNeedsReconciliationWithEvidence(db, run))+ return null;+ const scheduled = await scheduleBoundedRetryForRun(run, agent, { now });+ return scheduled.outcome === "scheduled" ? scheduled.run : null;+ }++ function toHotRestartIntentRun(input: {+ run: typeof heartbeatRuns.$inferSelect;+ adapterType: string;+ }): HotRestartIntentRun {+ const context = parseObject(input.run.contextSnapshot);+ return {+ runId: input.run.id,+ companyId: input.run.companyId,+ agentId: input.run.agentId,+ adapterType: input.adapterType,+ status: input.run.status,+ processPid: input.run.processPid ?? null,+ processGroupId: input.run.processGroupId ?? null,+ issueId: readNonEmptyString(context.issueId),+ runtimeMode: input.run.runtimeMode,+ nativeSessionId: input.run.nativeSessionId,+ runnerInstanceId: input.run.runnerInstanceId,+ processStartedAt: input.run.processStartedAt?.toISOString() ?? null,+ };+ }++ function isServerStdioBoundHotRestartRun(input: {+ run: typeof heartbeatRuns.$inferSelect;+ adapterType: string;+ adapterConfig: unknown;+ }) {+ const context = parseObject(input.run.contextSnapshot);+ if (+ context.processTopology === "server_stdio" ||+ context.executionEngine === "acp"+ ) {+ return true;+ }+ if (+ context.processTopology === "detached" ||+ context.executionEngine === "cli"+ ) {+ return false;+ }+ if (+ !["claude_local", "codex_local", "gemini_local"].includes(+ input.adapterType,+ )+ ) {+ return false;+ }+ return (+ readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli"+ );+ }++ async function prepareHotRestartShutdown(+ signal: "SIGINT" | "SIGTERM",+ now = new Date(),+ ) {+ enterShutdown();+ const idleSessions = await closeIdleWarmNativeSessionsForRestart();+ if (idleSessions.failed > 0) {+ logger.warn({ idleSessions }, "idle native sessions could not checkpoint before controller shutdown");+ }+ let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;+ try {+ intent = await readHotRestartIntent();+ } catch (err) {+ logger.warn(+ { err },+ "failed to read hot-restart intent; falling back to normal shutdown drain",+ );+ return {+ mode: "read_error" as const,+ skipDrain: false as const,+ activeRunIds: [] as string[],+ };+ }++ if (!intent)+ return {+ mode: "not_requested" as const,+ skipDrain: false as const,+ activeRunIds: [] as string[],+ };+ if (intent.drainRequired)+ return {+ mode: "drain_required" as const,+ skipDrain: false as const,+ activeRunIds: [] as string[],+ };+ if (!shouldHonorHotRestartIntentForProcess(intent)) {+ logger.warn(+ { expectedPid: intent.previousServerPid, currentPid: process.pid },+ "hot-restart intent targets a different server pid; falling back to normal shutdown drain",+ );+ return {+ mode: "pid_mismatch" as const,+ skipDrain: false as const,+ activeRunIds: [] as string[],+ };+ }++ const activeRuns = await db+ .select({+ run: heartbeatRuns,+ adapterType: agents.adapterType,+ adapterConfig: agents.adapterConfig,+ })+ .from(heartbeatRuns)+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))+ .where(eq(heartbeatRuns.status, "running"));+ const snapshotRuns = activeRuns.map(toHotRestartIntentRun);+ const intentWithVersion = {+ ...intent,+ previousServerVersion: intent.previousServerVersion ?? serverVersion,+ };++ const serverStdioRuns = activeRuns.filter(isServerStdioBoundHotRestartRun);+ if (serverStdioRuns.length > 0) {+ const activeServerStdioRunIds = serverStdioRuns.map(({ run }) => run.id);+ await writeHotRestartShutdownSnapshot({+ intent: intentWithVersion,+ signal,+ activeRuns: snapshotRuns,+ drainReason: "active_acp_run",+ drainRunIds: activeServerStdioRunIds,+ capturedAt: now,+ });++ logger.warn(+ {+ signal,+ previousServerPid: intent.previousServerPid,+ activeRunIds: snapshotRuns.map((run) => run.runId),+ activeServerStdioRunIds,+ drainReason: "active_acp_run",+ },+ "server-stdio agent run prevents hot-restart adoption; using graceful drain and retry",+ );++ return {+ mode: "acp_drain_required" as const,+ skipDrain: false as const,+ activeRunIds: snapshotRuns.map((run) => run.runId),+ activeAcpRunIds: activeServerStdioRunIds,+ drainRunIds: activeServerStdioRunIds,+ drainReason: "active_acp_run" as const,+ };+ }++ await writeHotRestartShutdownSnapshot({+ intent: intentWithVersion,+ signal,+ activeRuns: snapshotRuns,+ capturedAt: now,+ });++ for (const { run } of activeRuns) {+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: "info",+ message:+ "Hot restart requested; leaving child process alive for startup adoption",+ payload: {+ signal,+ previousServerPid: intent.previousServerPid,+ previousServerVersion: intentWithVersion.previousServerVersion,+ processPid: run.processPid ?? null,+ processGroupId: run.processGroupId ?? null,+ },+ });+ }++ const nativeRunIds = activeRuns+ .filter(+ ({ run, adapterType }) =>+ adapterType === "paperclip_runner" &&+ isNativeSessionId(run.nativeSessionId),+ )+ .map(({ run }) => run.id);+ const detachedNativeSessions =+ await detachNativeSessionsForRestart(nativeRunIds);++ logger.info(+ {+ signal,+ previousServerPid: intent.previousServerPid,+ activeRunIds: snapshotRuns.map((run) => run.runId),+ detachedNativeSessions,+ },+ "hot-restart shutdown snapshot captured; skipping graceful run drain",+ );++ return {+ mode: "hot_restart" as const,+ skipDrain: true as const,+ activeRunIds: snapshotRuns.map((run) => run.runId),+ };+ }++ async function reconcileHotRestartAdoption(now = new Date()) {+ let intent: Awaited<ReturnType<typeof readHotRestartIntent>>;+ try {+ intent = await readHotRestartIntent();+ } catch (err) {+ logger.warn(+ { err },+ "failed to read hot-restart intent on startup; skipping adoption",+ );+ return {+ mode: "read_error" as const,+ adoptedRunIds: [] as string[],+ finalizedWhileDownRunIds: [] as string[],+ lostRunIds: [] as string[],+ skippedRunIds: [] as string[],+ };+ }+ if (!intent) {+ return {+ mode: "not_requested" as const,+ adoptedRunIds: [] as string[],+ finalizedWhileDownRunIds: [] as string[],+ lostRunIds: [] as string[],+ skippedRunIds: [] as string[],+ };+ }++ if (!intent.shutdownSnapshot) {+ const log = intent.drainRequired+ ? logger.info.bind(logger)+ : logger.warn.bind(logger);+ log(+ {+ previousServerPid: intent.previousServerPid,+ preflightActiveRunIds: intent.preflightActiveRunIds,+ drainReason: intent.drainReason ?? null,+ },+ intent.drainRequired+ ? "drain-required restart intent has no adoption snapshot"+ : "hot-restart intent present but shutdown snapshot is missing; no runs can be adopted",+ );+ }+ const candidates = intent.shutdownSnapshot?.activeRuns ?? [];+ const missingSnapshotRunIds = findMissingHotRestartSnapshotRunIds(intent);+ const reconciliationRunIds = [+ ...new Set([+ ...candidates.map((run) => run.runId),+ ...missingSnapshotRunIds,+ ]),+ ];+ const currentRows =+ reconciliationRunIds.length > 0+ ? await db+ .select({+ run: heartbeatRuns,+ adapterType: agents.adapterType,+ })+ .from(heartbeatRuns)+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))+ .where(inArray(heartbeatRuns.id, reconciliationRunIds))+ : [];+ const currentByRunId = new Map(currentRows.map((row) => [row.run.id, row]));++ const reportRuns: HotRestartReportRun[] = [];+ const adoptedRunIds: string[] = [];+ const finalizedWhileDownRunIds: string[] = [];+ const lostRunIds: string[] = [];+ const skippedRunIds: string[] = [];++ const classify = (+ candidate: HotRestartIntentRun,+ classification: HotRestartReportRun["classification"],+ reason: string,+ patch?: Partial<HotRestartIntentRun>,+ ) => {+ const run = {+ ...candidate,+ ...patch,+ classification,+ reason,+ } satisfies HotRestartReportRun;+ reportRuns.push(run);+ if (classification === "adopted") adoptedRunIds.push(candidate.runId);+ else if (classification === "finalized_while_down")+ finalizedWhileDownRunIds.push(candidate.runId);+ else if (classification === "lost") lostRunIds.push(candidate.runId);+ else skippedRunIds.push(candidate.runId);+ };++ for (const runId of missingSnapshotRunIds) {+ const current = currentByRunId.get(runId);+ if (!current) {+ finalizedWhileDownRunIds.push(runId);+ continue;+ }++ const candidate = toHotRestartIntentRun(current);+ if (current.run.status !== "running") {+ classify(+ candidate,+ "finalized_while_down",+ `run_status_${current.run.status}`,+ );+ } else {+ classify(candidate, "lost", "missing_shutdown_snapshot");+ }+ }++ if (lostRunIds.length > 0) {+ logger.error(+ { previousServerPid: intent.previousServerPid, lostRunIds },+ "hot-restart shutdown snapshot omitted live preflight runs; reporting them as lost",+ );+ }++ for (const candidate of candidates) {+ const current = currentByRunId.get(candidate.runId);+ if (!current) {+ classify(candidate, "finalized_while_down", "run_row_missing");+ continue;+ }++ const { run, adapterType } = current;+ const patch = {+ adapterType,+ status: run.status,+ processPid: run.processPid ?? candidate.processPid,+ processGroupId: run.processGroupId ?? candidate.processGroupId,+ };++ if (run.status !== "running") {+ classify(+ candidate,+ "finalized_while_down",+ `run_status_${run.status}`,+ patch,+ );+ continue;+ }++ const hasSelectiveAcpDrain =+ intent.drainReason === "active_acp_run" &&+ (intent.drainRunIds?.length ?? 0) > 0;+ if (+ hasSelectiveAcpDrain &&+ intent.drainRunIds?.includes(candidate.runId)+ ) {+ // A selective ACP drain is expected to persist a terminal row before+ // the new server starts. If the process was terminated but that write+ // failed, surface the run as lost instead of hiding it as an expected+ // drain skip.+ classify(candidate, "lost", "selective_drain_not_finalized", patch);+ continue;+ }+ if (intent.drainRequired && !hasSelectiveAcpDrain) {+ classify(candidate, "skipped", "drain_required", patch);+ continue;+ }++ if (run.runtimeMode === "native" && adapterType === "paperclip_runner") {+ classify(candidate, "skipped", "native_restart_recovery_owned", patch);+ continue;+ }++ if (!isTrackedLocalChildProcessAdapter(adapterType)) {+ classify(+ candidate,+ "skipped",+ "adapter_not_local_child_process",+ patch,+ );+ continue;+ }++ const processPid = run.processPid ?? candidate.processPid;+ const processGroupId = run.processGroupId ?? candidate.processGroupId;+ const processPidAlive = isProcessAlive(processPid);+ const processGroupAlive = isProcessGroupAlive(processGroupId);+ if (!processPid && !processGroupId) {+ classify(candidate, "lost", "missing_process_metadata", patch);+ continue;+ }+ if (!processPidAlive && !processGroupAlive) {+ classify(candidate, "lost", "process_not_alive", patch);+ continue;+ }++ const resultJson = mergeHotRestartAdoptionResultJson(+ parseObject(run.resultJson),+ {+ adoptedAt: now,+ previousServerPid: intent.previousServerPid,+ newServerPid: process.pid,+ previousServerVersion: intent.previousServerVersion,+ newServerVersion: serverVersion,+ processPid,+ processGroupId,+ },+ );+ const updated = await db+ .update(heartbeatRuns)+ .set({+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(resultJson, true),+ error:+ run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error,+ errorCode:+ run.errorCode === DETACHED_PROCESS_ERROR_CODE+ ? null+ : run.errorCode,+ updatedAt: now,+ })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.status, "running"),+ ),+ )+ .returning()+ .then((rows) => rows[0] ?? null);++ if (!updated) {+ const latest = await db+ .select({ status: heartbeatRuns.status })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, run.id))+ .then((rows) => rows[0] ?? null);+ if (latest && latest.status !== "running") {+ classify(+ candidate,+ "finalized_while_down",+ `run_status_${latest.status}`,+ patch,+ );+ } else {+ classify(candidate, "lost", "adoption_update_not_applied", patch);+ }+ continue;+ }++ await appendRunEvent(updated, {+ eventType: "lifecycle",+ stream: "system",+ level: "info",+ message: "Adopted live child process after hot restart",+ payload: {+ previousServerPid: intent.previousServerPid,+ newServerPid: process.pid,+ previousServerVersion: intent.previousServerVersion,+ newServerVersion: serverVersion,+ processPid,+ processGroupId,+ },+ });+ classify(+ candidate,+ "adopted",+ processPidAlive ? "process_pid_alive" : "process_group_alive",+ patch,+ );+ }++ const report = await writeHotRestartReport({+ version: 1,+ requestedAt: intent.requestedAt,+ completedAt: now.toISOString(),+ drainRequired: intent.drainRequired,+ drainReason:+ intent.drainReason ?? (intent.drainRequired ? "requested" : null),+ previousServerPid: intent.previousServerPid,+ newServerPid: process.pid,+ previousServerVersion: intent.previousServerVersion,+ newServerVersion: serverVersion,+ adoptedRunIds,+ finalizedWhileDownRunIds,+ lostRunIds,+ skippedRunIds,+ runs: reportRuns,+ });+ await removeHotRestartIntent(undefined, intent);++ logger.info(+ {+ previousServerPid: report.previousServerPid,+ newServerPid: report.newServerPid,+ adoptedRunIds,+ finalizedWhileDownRunIds,+ lostRunIds,+ missingSnapshotRunIds,+ skippedRunIds,+ },+ "hot-restart adoption report written",+ );++ return {+ mode: "reported" as const,+ adoptedRunIds,+ finalizedWhileDownRunIds,+ lostRunIds,+ skippedRunIds,+ };+ }++ async function recoverNativeRunsAfterRestart(now = new Date()) {+ // A result committed before the old controller stopped outranks process+ // recovery. Finish its durable workspace/status suffix before deciding+ // whether any provider authority needs to be reopened.+ await reconcileNativeFinalizations(db, undefined, {+ environmentRuntime,+ onWorkspaceSettled: settleRecoveredNativeWorkspace,+ });+ scheduleRetainedNativeSessionCleanup();+ const intent = await readHotRestartIntent().catch((error) => {+ logger.warn(+ { err: error },+ "failed to read hot-restart intent before native startup recovery",+ );+ return null;+ });+ const restartKind = intent ? ("hot" as const) : ("hard" as const);+ const previousStartedAt = intent?.previousServerStartedAt+ ? new Date(intent.previousServerStartedAt)+ : null;+ const scheduledNativeRetries = await db+ .select({+ runId: nativeRunFinalizations.runId,+ nextAttemptAt: nativeRunFinalizations.nextAttemptAt,+ })+ .from(nativeRunFinalizations)+ .innerJoin(+ heartbeatRuns,+ eq(heartbeatRuns.id, nativeRunFinalizations.runId),+ )+ .where(+ and(+ eq(heartbeatRuns.runtimeMode, "native"),+ inArray(heartbeatRuns.status, ["running", "failed"]),+ isNull(nativeRunFinalizations.resultId),+ eq(nativeRunFinalizations.phase, "retryable_failure"),+ gt(nativeRunFinalizations.nextAttemptAt, now),+ ),+ );+ for (const scheduled of scheduledNativeRetries) {+ if (scheduled.nextAttemptAt) {+ scheduleNativeSessionResumeDispatch(+ scheduled.runId,+ scheduled.nextAttemptAt,+ );+ }+ }+ const dispositions = await claimNativeRestartRecoveries({+ db,+ restartKind,+ recoveryRequestId: intent?.recoveryRequestId ?? null,+ coordinatedPreviousController: intent+ ? {+ pid: intent.previousServerPid,+ processStartedAt:+ previousStartedAt && !Number.isNaN(previousStartedAt.getTime())+ ? previousStartedAt+ : null,+ }+ : null,+ now,+ });++ const claims = dispositions.filter(+ (disposition): disposition is NativeRestartRecoveryClaim =>+ disposition.kind === "reattach_existing_runner" ||+ disposition.kind === "reattach_remote_runner" ||+ disposition.kind === "resume_dead_runner" ||+ disposition.kind === "bootstrap_incomplete",+ );+ for (const disposition of dispositions) {+ const run = await getRun(disposition.runId);+ if (run) {+ const isClaim =+ disposition.kind === "reattach_existing_runner" ||+ disposition.kind === "reattach_remote_runner" ||+ disposition.kind === "resume_dead_runner" ||+ disposition.kind === "bootstrap_incomplete";+ await appendRunEvent(run, {+ eventType: "native.recovery.transition",+ stream: "system",+ level: disposition.kind === "blocked" ? "warn" : "info",+ message:+ disposition.kind === "reattach_existing_runner" ||+ disposition.kind === "reattach_remote_runner"+ ? "Recovering the existing native runner process after server restart"+ : disposition.kind === "resume_dead_runner"+ ? "Resuming the durable native provider session after runner process loss"+ : disposition.kind === "bootstrap_incomplete"+ ? "Restarting an incomplete native runner bootstrap on the same heartbeat run"+ : disposition.kind === "awaiting_evidence"+ ? "Native restart recovery is waiting for safe ownership evidence"+ : disposition.kind === "already_finalized"+ ? "Native restart recovery found an already-finalized result"+ : "Native restart recovery blocked ambiguous or conflicting ownership",+ payload: {+ restartKind: isClaim ? disposition.restartKind : restartKind,+ recoveryRequestId: isClaim+ ? disposition.recoveryRequestId+ : (intent?.recoveryRequestId ?? null),+ runnerDisposition: disposition.kind,+ ...(isClaim+ ? {+ controllerGeneration: disposition.controllerGeneration,+ providerAttempt: disposition.providerAttempt,+ }+ : { reason: disposition.reason }),+ ...(disposition.kind === "reattach_existing_runner"+ ? {+ processPid: disposition.process.pid,+ processGroupId: disposition.process.processGroupId,+ processStartedAt: disposition.process.startedAt,+ }+ : {}),+ },+ });+ }+ }+ for (const claim of claims) {+ const execution = executeRun(claim.runId, {+ nativeLeaseOwner: claim.leaseOwner,+ nativeRestartRecovery: claim,+ }).catch((error) => {+ logger.error(+ { err: error, runId: claim.runId, disposition: claim.kind },+ "native restart recovery execution failed",+ );+ });+ activeRunExecutionPromises.add(execution);+ void execution.finally(() =>+ activeRunExecutionPromises.delete(execution),+ );+ }++ return {+ restartKind,+ claims,+ dispositions,+ scheduledRetryRunIds: scheduledNativeRetries.map((entry) => entry.runId),+ awaitingEvidenceRunIds: dispositions+ .filter((entry) => entry.kind === "awaiting_evidence")+ .map((entry) => entry.runId),+ blockedRunIds: dispositions+ .filter((entry) => entry.kind === "blocked")+ .map((entry) => entry.runId),+ };+ }++ async function drainRunningRunsForShutdown(+ signal: "SIGINT" | "SIGTERM",+ now = new Date(),+ runIds: readonly string[] | null = null,+ ) {+ const selectedRunIds = runIds ? [...new Set(runIds)] : null;+ if (selectedRunIds?.length === 0) {+ return {+ interrupted: 0,+ interruptedRunIds: [],+ retryRunIds: [],+ restartSuspendedRunIds: [],+ };+ }+ const activeRuns = await db+ .select({+ run: heartbeatRuns,+ agent: agents,+ })+ .from(heartbeatRuns)+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))+ .where(+ selectedRunIds+ ? and(+ eq(heartbeatRuns.status, "running"),+ inArray(heartbeatRuns.id, selectedRunIds),+ )+ : eq(heartbeatRuns.status, "running"),+ );++ const interruptedRunIds: string[] = [];+ const retryRunIds: string[] = [];+ const restartSuspendedRunIds: string[] = [];++ for (const { run, agent } of activeRuns) {+ // Shutdown owns only this boot's legacy executions. Expired foreign+ // owners belong to the reaper, not another container's drain.+ if (run.runtimeMode === "legacy" && run.controllerBootId &&+ run.controllerBootId !== legacyControllerBootId) continue;+ if (isNativeRunnerOwnershipHeld(run)) continue;+ if (+ run.runtimeMode === "native" &&+ agent.adapterType === "paperclip_runner"+ ) {+ // A graceful shutdown relinquishes controller authority just like a+ // hot restart. Leaving the old event consumer attached lets its+ // finalizer interrupt/suspend Claude while the next server is adopting+ // the same turn.+ await detachNativeSessionsForRestart([run.id]);+ const recoveryHistoryEntry = JSON.stringify({+ at: now.toISOString(),+ restartKind: "graceful",+ disposition: "restart_suspended",+ reason: signal,+ processPid: run.processPid,+ processStartedAt: run.processStartedAt?.toISOString() ?? null,+ });+ await db+ .update(nativeRunFinalizations)+ .set({+ recoveryState: "awaiting_runner_reattach",+ recoveryHistory: sql`(+ select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb)+ from jsonb_array_elements(+ coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)+ || jsonb_build_array(${recoveryHistoryEntry}::jsonb)+ ) with ordinality as history(item, ordinal)+ where ordinal > greatest(+ jsonb_array_length(+ coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb)+ || jsonb_build_array(${recoveryHistoryEntry}::jsonb)+ ) - 20,+ 0+ )+ )`,+ updatedAt: now,+ })+ .where(+ and(+ eq(nativeRunFinalizations.runId, run.id),+ isNull(nativeRunFinalizations.resultId),+ ),+ );+ await appendRunEvent(run, {+ eventType: "native.recovery.transition",+ stream: "system",+ level: "info",+ message:+ "Server shutdown suspended native controller ownership without cancelling provider work",+ payload: {+ restartKind: "graceful",+ signal,+ runnerDisposition: "awaiting_runner_reattach",+ processPid: run.processPid,+ processGroupId: run.processGroupId,+ processStartedAt: run.processStartedAt?.toISOString() ?? null,+ retryRunCreated: false,+ },+ });+ restartSuspendedRunIds.push(run.id);+ continue;+ }+ const message = `Interrupted by graceful server shutdown (${signal})`;+ const running = runningProcesses.get(run.id);+ try {+ if (run.runtimeMode === "native") {+ await cancelHeartbeatNativeRun({+ db,+ runId: run.id,+ reason: message,+ runtimeMode: run.runtimeMode,+ });+ }+ if (running) {+ await terminateHeartbeatRunProcess({+ pid: running.child.pid,+ processGroupId: running.processGroupId,+ graceMs: Math.max(1, running.graceSec) * 1000,+ });+ }+ } finally {+ runningProcesses.delete(run.id);+ }++ const persistedCancellationResult =+ run.runtimeMode === "native"+ ? await getRun(run.id).then((current) =>+ parseObject(current?.resultJson),+ )+ : parseObject(run.resultJson);++ const interruptedStatus = await setRunStatusIfRunning(+ run.id,+ "interrupted",+ {+ finishedAt: now,+ error: message,+ errorCode: "server_shutdown_interrupted",+ signal,+ resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {+ conversationContinuationEligible: await runUsedConversationAdapter(db, run),+ resultJson: persistedCancellationResult,+ errorCode: "server_shutdown_interrupted",+ errorMessage: message,+ }),+ },+ );+ if (!interruptedStatus.updated || !interruptedStatus.run) continue;+ let interrupted = interruptedStatus.run;+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {+ finishedAt: now,+ error: null,+ });+ interrupted =+ (await classifyAndPersistRunLiveness(+ interrupted,+ parseObject(interrupted.resultJson),+ )) ?? interrupted;++ await releaseEnvironmentLeasesForRun({+ runId: interrupted.id,+ companyId: interrupted.companyId,+ agentId: interrupted.agentId,+ status: interrupted.status,+ failureReason: interrupted.error ?? undefined,+ });++ const retry = await enqueueProcessLossRetry(interrupted, agent, now);+ if (!retry) {+ await releaseIssueExecutionAndPromote(interrupted);+ } else {+ retryRunIds.push(retry.id);+ }++ await appendRunEvent(interrupted, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message,+ payload: {+ signal,+ ...(run.processPid ? { processPid: run.processPid } : {}),+ ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),+ ...(retry ? { retryRunId: retry.id } : {}),+ },+ });++ await finalizeAgentStatus(run.agentId, "interrupted", message, {+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ });+ interruptedRunIds.push(interrupted.id);+ }++ if (interruptedRunIds.length > 0) {+ logger.warn(+ {+ signal,+ interrupted: interruptedRunIds.length,+ interruptedRunIds,+ retryRunIds,+ },+ "interrupted running heartbeat runs for graceful shutdown",+ );+ }++ return {+ interrupted: interruptedRunIds.length,+ interruptedRunIds,+ retryRunIds,+ restartSuspendedRunIds,+ };+ }++ // Clamp the stored attempt count to the range [0, cap]. The SQL reader+ // `pendingCleanupAttemptsSql` clamps to the same range, so both readers yield+ // the same value for every input. The claim predicate compares the two values,+ // so this alignment lets the claim match for a malformed lease.+ function readPendingCleanupRetryAttempts(+ metadata: Record<string, unknown>,+ ): number {+ const value = metadata[PENDING_CLEANUP_ATTEMPTS_METADATA_KEY];+ if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)+ return 0;+ return Math.min(Math.floor(value), PENDING_CLEANUP_SWEEP_ATTEMPT_CAP);+ }++ // Atomically claim one retry attempt on a pending_cleanup lease. The update+ // only matches when the lease is still pending_cleanup and its stored attempt+ // count still equals `expectedAttempts`. Two concurrent sweeps read the same+ // count, but Postgres serializes the two updates on the row and only the first+ // matches the guard. The loser gets zero rows and skips the lease. The persisted in-flight deadline also prevents a later tick+ // from starting another attempt while this one is still running.+ // Returns the attempt identity only for the sweep that won the claim.+ //+ // The update patches cleanup ownership fields without writing a copied+ // metadata object, so a concurrent write to an unrelated metadata key+ // survives. The guard reads the stored count through the safe SQL reader, so a+ // malformed value never throws.+ async function claimPendingCleanupRetryAttempt(+ leaseId: string,+ expectedAttempts: number,+ manualAttempt?: { previousId: unknown },+ ): Promise<string | null> {+ const now = new Date();+ const attemptId = randomUUID();+ const claimed = await db+ .update(environmentLeases)+ .set({+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_ATTEMPTS_METADATA_KEY}], to_jsonb(${expectedAttempts + 1}::int), true)+ || ${JSON.stringify({ ...(manualAttempt ? { pendingCleanupManualAttemptId: randomUUID() } : {}),+ pendingCleanupAttemptId: attemptId, pendingCleanupInFlight: true,+ pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000,+ })}::jsonb`,+ lastUsedAt: now,+ updatedAt: now,+ })+ .where(+ and(+ eq(environmentLeases.id, leaseId),+ eq(environmentLeases.status, "pending_cleanup"),+ sql`${pendingCleanupAttemptsSql()} = ${expectedAttempts}`,+ manualAttempt ? sql`coalesce(${environmentLeases.metadata}->'pendingCleanupManualAttemptId', 'null'::jsonb) is not distinct from ${JSON.stringify(manualAttempt.previousId ?? null)}::jsonb` : undefined,+ pendingCleanupRetryDueSql(Boolean(manualAttempt)),+ ),+ )+ .returning({ id: environmentLeases.id });+ return claimed.length > 0 ? attemptId : null;+ }++ // Atomically claim the one-time cap warning for a lease. The update only+ // matches when the lease is still pending_cleanup, its stored attempt count is+ // at or above the cap, and it has not yet carried the warned flag. Two+ // concurrent sweeps that both reach the cap race here, but only one update+ // sets the flag and returns a row. The loser skips the warning. This keeps the+ // warning to one log line per lease.+ //+ // The status and cap predicates are the last line of defense. They stop a warn+ // flag write to a lease that left pending_cleanup or dropped below the cap+ // between the read and this claim. The update writes only the warned key with+ // `jsonb_set`, so a concurrent write to an unrelated metadata key survives.+ async function claimPendingCleanupCapWarning(+ leaseId: string,+ ): Promise<boolean> {+ const now = new Date();+ const claimed = await db+ .update(environmentLeases)+ .set({+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, array[${PENDING_CLEANUP_CAP_WARNED_METADATA_KEY}], to_jsonb(true), true)`,+ updatedAt: now,+ })+ .where(+ and(+ eq(environmentLeases.id, leaseId),+ eq(environmentLeases.status, "pending_cleanup"),+ sql`${pendingCleanupAttemptsSql()} >= ${PENDING_CLEANUP_SWEEP_ATTEMPT_CAP}`,+ sql`${pendingCleanupCapWarnedSql()} = false`,+ ),+ )+ .returning({ id: environmentLeases.id });+ return claimed.length > 0;+ }++ // Defer a pending_cleanup lease whose provider plugin is not ready this tick.+ // The sweep reads one page of the oldest rows, ordered by `updatedAt`. A lease+ // that the sweep only skips keeps its old `updatedAt`, so it stays the oldest+ // and refills the page on every tick. That starves a newer lease whose+ // provider is ready. The defer bumps `updatedAt` to now, so the unavailable+ // lease moves to the back of the queue and a ready lease takes its page slot.+ // The defer never writes the attempt count, so a long provider outage never+ // consumes a finite retry. The status guard keeps the write on a lease that is+ // still pending_cleanup.+ async function deferPendingCleanupLease(leaseId: string): Promise<void> {+ const now = new Date();+ await db+ .update(environmentLeases)+ .set({ updatedAt: now })+ .where(+ and(+ eq(environmentLeases.id, leaseId),+ eq(environmentLeases.status, "pending_cleanup"),+ ),+ );+ }++ // Move a guarded orphan candidate to the back of the sweep order. The+ // shared-resource guard below can skip a row on every tick as long as the+ // other owner stays live, retained, or pending cleanup. The select orders+ // by `updatedAt` ascending and takes only the oldest page, so an untouched+ // skipped row keeps refilling that same page and blocks every row behind+ // it. The bump pushes the row past the fixed-size page, so the next tick+ // reaches the rows behind it. It costs the row one extra backoff wait,+ // which is safe because the guard means a physical sandbox still exists.+ async function deferOrphanedActiveLease(leaseId: string): Promise<void> {+ const now = new Date();+ await db+ .update(environmentLeases)+ .set({ updatedAt: now })+ .where(+ and(+ eq(environmentLeases.id, leaseId),+ eq(environmentLeases.status, "active"),+ ),+ );+ }++ // An active lease is reachable only while its heartbeat run keeps the+ // running status. The reaper writes the run status and the lease release as+ // two separate statements, so a restart between them can leave a terminal+ // run and an active lease. `releaseRunLeases` also skips a lease whose+ // environment row is gone, so that lease stays active too. No later query+ // finds either lease, because every production query selects an active+ // lease by its environment or by its heartbeat run id, never by age. This+ // sweep finds both stranded classes and moves each lease to pending_cleanup,+ // so the existing pending_cleanup sweep tears the sandbox down from the+ // data already on the lease row.+ async function sweepOrphanedActiveLeases(opts: {+ backoffMs: number;+ }): Promise<{ recovered: number }> {+ const cutoff = new Date(Date.now() - opts.backoffMs);++ const rows = await db+ .select({ lease: environmentLeases })+ .from(environmentLeases)+ .leftJoin(+ heartbeatRuns,+ eq(environmentLeases.heartbeatRunId, heartbeatRuns.id),+ )+ .where(+ and(+ eq(environmentLeases.status, "active"),+ or(+ isNull(environmentLeases.heartbeatRunId),+ inArray(heartbeatRuns.status, [+ ...HEARTBEAT_RUN_TERMINAL_STATUSES,+ ]),+ ),+ lte(environmentLeases.updatedAt, cutoff),+ ),+ )+ .orderBy(asc(environmentLeases.updatedAt))+ .limit(ORPHANED_ACTIVE_LEASE_SWEEP_PAGE_SIZE);++ let recovered = 0;+ for (const { lease } of rows) {+ // A provider resource id names one physical sandbox. A different lease+ // row can still hold that same resource in a live status, so this sweep+ // must not tear down a sandbox that a different lease still owns.+ if (lease.provider && lease.providerLeaseId) {+ const [otherOwner] = await db+ .select({ id: environmentLeases.id })+ .from(environmentLeases)+ .where(+ and(+ ne(environmentLeases.id, lease.id),+ eq(environmentLeases.provider, lease.provider),+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId),+ inArray(environmentLeases.status, [+ "active",+ "retained",+ "pending_cleanup",+ ]),+ ),+ )+ .limit(1);+ if (otherOwner) {+ // Defer this row so the fixed-size page reaches the rows behind+ // it next tick, instead of re-selecting the same guarded rows+ // forever.+ await deferOrphanedActiveLease(lease.id);+ continue;+ }+ }++ // Keep the row's existing updatedAt value. The select above already+ // proved the row is older than the backoff cutoff, so the+ // pending_cleanup sweep can accept the same row in this same tick. A+ // fresh timestamp here would push the row inside that sweep's own+ // backoff window and delay the teardown by one full tick.+ const flipped = await db+ .update(environmentLeases)+ .set({+ status: "pending_cleanup",+ failureReason: "orphaned_active_lease_recovered",+ })+ .where(+ and(+ eq(environmentLeases.id, lease.id),+ eq(environmentLeases.status, "active"),+ ),+ )+ .returning({ id: environmentLeases.id });+ if (flipped.length > 0) recovered += 1;+ }++ return { recovered };+ }++ // Retry the leases stranded in "pending_cleanup". A failed destroy leaves a+ // lease in that state forever without this sweep. The reaper tick runs the+ // sweep. The backoff equals the reaper staleness threshold, so a lease waits+ // for that period between attempts. The sweep reads and writes the attempt+ // count in the lease metadata. It warns once when a lease reaches the attempt+ // threshold, then continues with slower retries. Live attempts renew their+ // cleanup claim; after controller loss, exact-resource destruction may repeat.+ async function sweepPendingCleanupLeases(opts?: {+ backoffMs?: number;+ /** One cleanup attempt per explicit user Retry, for this failed run only.+ * A later user Retry may bypass the cooldown after a provider failure,+ * but cannot take over an in-flight cleanup attempt.+ */+ explicitRetry?: { companyId: string; runId: string; actorId: string; reason?: "retry_failed_run" | "queued_comment_interrupt" };+ }): Promise<{+ swept: number;+ destroyed: number;+ capped: number;+ }> {+ const backoffMs = opts?.backoffMs ?? 0;+ const now = new Date();+ const cutoff = new Date(now.getTime() - backoffMs);++ // Flush the in-process orphan-cleanup buffer first. A failed acquire buffers+ // an orphan there when every synchronous pending-cleanup write failed after a+ // failed teardown. The flush re-inserts each buffered record, so a durable+ // `pending_cleanup` row lands once the database recovers. The flush runs+ // before the read below, so this same tick tears down a freshly-landed row.+ try {+ const flushed = opts?.explicitRetry ? null : await environmentRuntime.flushDeferredOrphanCleanups?.();+ if (flushed && (flushed.recovered > 0 || flushed.pending > 0)) {+ logger.info(+ { recovered: flushed.recovered, pending: flushed.pending },+ "flushed the in-process orphan sandbox cleanup buffer to the database",+ );+ }+ } catch {+ // A flush failure never stops the sweep. The buffer keeps the orphan for a+ // later tick, and the database rows below still need this sweep. The caught+ // exception never enters the log, because a write error can carry a+ // credential in its message, code, cause, or stack.+ logger.warn(+ "orphan sandbox cleanup buffer flush failed; the sweep continues",+ );+ }++ const rows = await db+ .select()+ .from(environmentLeases)+ .where(+ and(+ eq(environmentLeases.status, "pending_cleanup"),+ opts?.explicitRetry ? eq(environmentLeases.companyId, opts.explicitRetry.companyId) : undefined,+ opts?.explicitRetry ? eq(environmentLeases.heartbeatRunId, opts.explicitRetry.runId) : undefined,+ pendingCleanupRetryDueSql(Boolean(opts?.explicitRetry)),+ backoffMs > 0 ? lte(environmentLeases.updatedAt, cutoff) : undefined,+ ),+ )+ .orderBy(asc(environmentLeases.updatedAt))+ .limit(PENDING_CLEANUP_SWEEP_PAGE_SIZE);++ let destroyed = 0;+ let capped = 0;+ for (const row of rows) {+ if (pendingCleanupAttemptsInFlight.has(row.id)) continue;+ const metadata = { ...(row.metadata ?? {}) } as Record<string, unknown>;+ const attempts = readPendingCleanupRetryAttempts(metadata);++ const environment = row.environmentId+ ? await environmentsSvc.getById(row.environmentId)+ : null;+ const lease = await environmentsSvc.getLeaseById(row.id);+ if (!lease) continue;++ // An orphan ephemeral lease keeps its provider, its provider lease id, and+ // its sandbox config in the lease row. A failed acquire records it, and its+ // environment row may be gone or foreign-bound. A reuse_by_environment lease+ // whose environment a delete removed keeps the same recorded data, because+ // the schema sets the environment reference to null on delete and preserves+ // the row. Both leases tear down from the recorded lease data through+ // `retryPendingSandboxTeardown`, which never reads the environment row. So+ // the sweep uses that path whenever the lease is an orphan ephemeral lease+ // or its environment row is gone. A reuse_by_environment lease whose+ // environment still exists tears down through `destroyRunLease`. That+ // path uses the provider and configuration recorded on the lease first;+ // the environment is lifecycle context and only a legacy fallback.+ const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral";+ const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease);++ // Do not consume a finite cleanup attempt while the provider plugin is+ // briefly unavailable. A plugin worker restart, a plugin reload, or a+ // plugin reinstall makes the provider unavailable for a short window. The+ // plugin can be missing or not ready in that window. A teardown then throws,+ // and the atomic claim below would count that throw against the cap, so a+ // long restart or reload could exhaust the retries and strand a live+ // sandbox. So probe the provider first, and defer the lease this tick when+ // the provider is not ready. The sweep preserves the pending_cleanup row,+ // and a later sweep retries after the provider recovers. The probe reports+ // ready only for a permanent condition (a missing provider string, a+ // built-in provider, or no worker manager), so a genuine teardown failure+ // still runs, throws, and counts toward the cap. A runtime with no probe+ // method treats the lease as ready, so the sweep keeps its earlier+ // behavior.+ const workerReady = environmentRuntime.isPendingCleanupWorkerReady+ ? await environmentRuntime.isPendingCleanupWorkerReady({+ environment,+ lease,+ })+ : true;+ if (!workerReady) {+ // Move the unavailable lease to the back of the sweep queue. Otherwise+ // the oldest unavailable rows refill the page on every tick and starve a+ // newer lease that has a ready provider. The defer bumps `updatedAt`+ // only, so it consumes no finite retry attempt.+ await deferPendingCleanupLease(row.id);+ continue;+ }++ // Atomically claim the attempt before the retry. Only the winning sweep+ // increments the count and tears the sandbox down, so an overlapping sweep+ // cannot start another attempt while this one holds the cleanup lease.+ // The deadline survives a server restart; the counter saturates at the+ // escalation threshold while attempt identities remain unique.+ const claimed = await claimPendingCleanupRetryAttempt(row.id, attempts,+ opts?.explicitRetry ? { previousId: metadata.pendingCleanupManualAttemptId } : undefined);+ if (!claimed) continue;+ pendingCleanupAttemptsInFlight.add(row.id);+ lease.metadata = { ...lease.metadata, pendingCleanupAttemptId: claimed };+ let activeRenewal: Promise<void> | null = null;+ const renewal = setInterval(() => {+ if (activeRenewal) return;+ activeRenewal = db.update(environmentLeases).set({+ metadata: sql`jsonb_set(${pendingCleanupMetadataObjectSql()}, '{pendingCleanupLeaseExpiresAtMs}', to_jsonb(${Date.now() + 15 * 60_000}::bigint))`,+ }).where(and(eq(environmentLeases.id, row.id), eq(environmentLeases.status, "pending_cleanup"),+ sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`,+ sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`))+ .then(() => {})+ .catch(() => logger.warn({ leaseId: row.id }, "cleanup ownership renewal failed"))+ .finally(() => { activeRenewal = null; });+ }, 30_000);+ renewal.unref();++ try {+ if (opts?.explicitRetry) await logActivity(db, {+ companyId: row.companyId, actorType: "user", actorId: opts.explicitRetry.actorId,+ action: "environment_lease.cleanup_retried", entityType: "environment_lease", entityId: row.id,+ runId: opts.explicitRetry.runId, details: { attempt: attempts + 1, reason: opts.explicitRetry.reason ?? "retry_failed_run" },+ });+ if (useRecordedTeardown) {+ // Tear the sandbox down from the recorded provider config and the+ // cleanup-authorized secret versions. Preserve any provider receipt;+ // a completed retry must grant the same evidence as initial cleanup.+ const receipt = await environmentRuntime.retryPendingSandboxTeardown({+ environment,+ lease,+ });+ const released = hasStopOnlyCleanup(lease)+ ? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt })+ : await environmentsSvc.releaseLease(lease.id, "expired", {+ expectedPendingCleanupAttemptId: claimed,+ cleanupStatus: "success",+ failureReason: "pending_cleanup_retry",+ remoteExecutionTermination: remoteTerminationReceipt(lease, receipt),+ });+ if (released) destroyed += 1;+ } else if (environment) {+ const result = await environmentRuntime.destroyRunLease({+ environment,+ lease,+ failureReason: "pending_cleanup_retry",+ });+ if (result && result.status !== "pending_cleanup") {+ destroyed += 1;+ }+ }+ } catch {+ // The recorded-data teardown throws on failure, so revert the lease to+ // pending_cleanup for a later sweep. The claimed attempt still counts+ // for backoff, so requests stay bounded. The `destroyRunLease`+ // path reverts the lease itself, so this revert only runs for the+ // recorded-data teardown path.+ if (useRecordedTeardown) {+ await environmentsSvc.releaseLease(lease.id, "pending_cleanup", {+ expectedPendingCleanupAttemptId: claimed,+ cleanupStatus: "failed",+ failureReason: "pending_cleanup_retry",+ });+ }+ // Log a constant errorKind only. The exception can carry a credential in+ // its name, code, message, cause, or stack, so the sweep never reads it.+ logger.warn(+ {+ errorKind: PENDING_CLEANUP_RETRY_ERROR_KIND,+ leaseId: row.id,+ environmentId: row.environmentId,+ attempts: attempts + 1,+ },+ "pending_cleanup lease retry failed",+ );+ } finally {+ clearInterval(renewal);+ // A hung renewal must not retain process-local cleanup ownership.+ // Late renewals are attempt-fenced and never write the retry cooldown.+ pendingCleanupAttemptsInFlight.delete(row.id);+ }+ if (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP) {+ capped += 1;+ // Warn once, then continue automatic cleanup with backoff. The atomic claim+ // keeps the warning to one log line even when two sweeps overlap.+ if (metadata[PENDING_CLEANUP_CAP_WARNED_METADATA_KEY] !== true) {+ const warned = await claimPendingCleanupCapWarning(row.id);+ if (warned) {+ logger.warn(+ { leaseId: row.id, environmentId: row.environmentId, attempts },+ "environment lease needs operator attention; automatic cleanup continues with backoff",+ );+ }+ }+ }+ // Persist the cooldown independently of process memory. A crash before+ // this write leaves the bounded in-flight lease for a later sweep.+ await db.update(environmentLeases).set({+ metadata: sql`${pendingCleanupMetadataObjectSql()} || ${JSON.stringify({+ pendingCleanupInFlight: false,+ pendingCleanupRetryAfterMs: Date.now() + (attempts + 1 >= PENDING_CLEANUP_SWEEP_ATTEMPT_CAP+ ? 30 * 60_000 : Math.min(30 * 60_000, Math.max(30_000, backoffMs))),+ })}::jsonb`,+ }).where(and(eq(environmentLeases.id, lease.id),+ sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${claimed}`));+ if (lease.heartbeatRunId) {+ // Delivery failure must not revert successful provider cleanup. A new+ // message can still use the persisted receipt on its next admission.+ await (async () => {+ await acknowledgeRemoteStop(lease.heartbeatRunId!, lease.companyId);+ const run = await getRun(lease.heartbeatRunId!);+ if (run) await resumeRemoteStopComments(run);+ })().catch(() => logger.warn({ leaseId: lease.id }, "could not reconsider messages after cleanup retry"));+ }+ }++ return { swept: rows.length, destroyed, capped };+ }++ async function markNativeOwnershipUnverified(+ run: typeof heartbeatRuns.$inferSelect,+ evidence: {+ reason:+ | "live_process_identifier"+ | "observed_owner_unverified"+ | "adopted_runner_authentication_timeout"+ | "native_chat_workspace_scope_mismatch";+ processPidAlive?: boolean;+ processGroupAlive?: boolean;+ },+ ) {+ const durableOwnershipHold =+ evidence.reason === "adopted_runner_authentication_timeout" ||+ evidence.reason === "native_chat_workspace_scope_mismatch";+ if (+ run.errorCode === NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE &&+ run.error === NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE &&+ (!durableOwnershipHold || isNativeRunnerOwnershipHeld(run))+ )+ return run;+ const blockedStatus = run.status === "failed" ? "failed" : "running";+ const blockedWrite = await setRunStatusFromLive(+ run.id,+ blockedStatus,+ [blockedStatus],+ {+ error: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,+ errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,+ ...(durableOwnershipHold+ ? {+ nativePhase: "terminal_failure",+ nativePhaseUpdatedAt: new Date(),+ }+ : {}),+ },+ );+ if (!blockedWrite.updated || !blockedWrite.run) {+ return blockedWrite.run ?? run;+ }+ const blocked = blockedWrite.run;+ await appendRunEvent(blocked, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE,+ payload: {+ reason: evidence.reason,+ ...(evidence.processPidAlive === true ? { processPidAlive: true } : {}),+ ...(evidence.processGroupAlive === true+ ? { processGroupAlive: true }+ : {}),+ },+ });+ return blocked;+ }++ async function settleRecoveredNativeWorkspace(input: {+ runId: string;+ companyId: string;+ agentId: string;+ succeeded: boolean;+ }) {+ const settledRun = await getRun(input.runId);+ const workspaceSyncReference = readNativeWorkspaceSyncReference(+ parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,+ );+ await releaseEnvironmentLeasesForRun({+ runId: input.runId,+ companyId: input.companyId,+ agentId: input.agentId,+ status: settledRun?.status,+ failureReason: settledRun?.error ?? undefined,+ providerResourceDisposition: input.succeeded && (!parseObject(settledRun?.resultJson).workspaceExportRetry || workspaceSyncReference?.resourceDisposition === "destroy")+ ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")+ : "stop_and_retain",+ });+ await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);+ await finalizeAgentStatus(+ input.agentId,+ input.succeeded ? "succeeded" : "failed",+ input.succeeded+ ? null+ : (settledRun?.error ?? "native_workspace_sync_out_failed"),+ {+ wasFirstHeartbeat: settledRun+ ? timerClaimWasFirstHeartbeat(settledRun)+ : undefined,+ },+ ).catch(() => undefined);+ }++ function scheduleRetainedNativeSessionCleanup() {+ // The per-database sweep joins startup and periodic callers. One bounded+ // control-only repair must not hold up unrelated provider ingress or the+ // entire orphan reaper, but shutdown must still await its physical owner.+ const cleanup = reconcileRetainedNativeSessionCleanups(db, {+ cleanup: (input) => reconcileRetainedNativeSessionCleanup(db, input),+ onError: (error, runId) => {+ logger.warn(+ { err: error, runId },+ "retained native session cleanup failed",+ );+ },+ })+ .then(() => undefined)+ .catch((error) => {+ logger.warn({ err: error }, "retained native cleanup discovery failed");+ })+ // The bounded maintenance attempt may fail before an already-started+ // database callback settles. Keep shutdown ownership until the original+ // operations finish; their timeout cannot authorize closing the database.+ .finally(() => drainRetainedRunnerdMaintenanceOperations());+ activeRunExecutionPromises.add(cleanup);+ void cleanup.finally(() => activeRunExecutionPromises.delete(cleanup));+ }++ async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {+ const staleThresholdMs = opts?.staleThresholdMs ?? 0;+ const now = new Date();+ // Recovery never launches a provider or infers stopped ownership from+ // terminal status. Uncaptured local copies require durable stop evidence.+ await instructionCopies.recoverStopped().catch(error => {+ logger.warn({ err: error }, "failed to recover stopped instruction copies");+ });+ await instructionCopies.recoverCaptured().catch(error => {+ logger.warn({ err: error }, "failed to retry captured instruction revisions");+ });++ // Complete persisted native results before generic orphan recovery. The+ // reconciler reads the durable workspace barrier and persisted runtime+ // mode, never the current feature flag.+ await reconcileNativeFinalizations(db, undefined, {+ environmentRuntime,+ onWorkspaceSettled: settleRecoveredNativeWorkspace,+ }).catch((error) => {+ logger.warn(+ { err: error },+ "failed to reconcile persisted native finalizations before orphan reaping",+ );+ });+ scheduleRetainedNativeSessionCleanup();+ await dispatchPendingNativeStatusWakeups().catch((error) => {+ logger.warn(+ { err: error },+ "failed to dispatch persisted native status wake intents before orphan reaping",+ );+ });++ // A retryable native run can retain process identifiers from the failed+ // attempt. Inspect them before the recovery claim: a live identifier is+ // unowned and blocks recovery, while identifiers that are all dead can be+ // cleared with a compare-and-set so the explicit retryable failure becomes+ // claimable in this same sweep.+ const retryableNativeProcesses = await db+ .select({ run: heartbeatRuns })+ .from(heartbeatRuns)+ .innerJoin(+ nativeRunFinalizations,+ eq(nativeRunFinalizations.runId, heartbeatRuns.id),+ )+ .where(+ and(+ inArray(heartbeatRuns.status, ["running", "failed"]),+ eq(heartbeatRuns.runtimeMode, "native"),+ eq(nativeRunFinalizations.phase, "retryable_failure"),+ isNull(nativeRunFinalizations.resultId),+ ),+ );+ const claimableNativeRunIds = new Set<string>();+ for (const { run } of retryableNativeProcesses) {+ if (isNativeRunnerOwnershipHeld(run)) continue;+ if (!run.processPid && !run.processGroupId) {+ claimableNativeRunIds.add(run.id);+ continue;+ }+ const processPidAlive =+ !!run.processPid && isProcessAlive(run.processPid);+ const processGroupAlive =+ !!run.processGroupId && isProcessGroupAlive(run.processGroupId);+ if (processPidAlive || processGroupAlive) {+ await markNativeOwnershipUnverified(run, {+ reason: "live_process_identifier",+ processPidAlive,+ processGroupAlive,+ });+ continue;+ }+ const cleared = await db.transaction(async tx => {+ const cleared = await tx+ .update(heartbeatRuns)+ .set({+ processPid: null,+ processGroupId: null,+ processStartedAt: null,+ updatedAt: now,+ })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.runtimeMode, "native"),+ run.processPid === null+ ? isNull(heartbeatRuns.processPid)+ : eq(heartbeatRuns.processPid, run.processPid),+ run.processGroupId === null+ ? isNull(heartbeatRuns.processGroupId)+ : eq(heartbeatRuns.processGroupId, run.processGroupId),+ run.processStartedAt === null+ ? isNull(heartbeatRuns.processStartedAt)+ : eq(heartbeatRuns.processStartedAt, run.processStartedAt),+ ),+ )+ .returning({ id: heartbeatRuns.id })+ .then((rows) => rows[0] ?? null);+ if (cleared) await recordNativeLocalProcessStop(tx as unknown as Db, run);+ return cleared;+ });+ if (cleared) claimableNativeRunIds.add(cleared.id);+ }++ // An explicit result-less retryable failure resumes on the original run.+ // The database lease is claimed before dispatch so concurrent service+ // instances cannot open competing recoveries; executeRun receives the exact+ // claimed owner. Expired `observed` ownership never enters this set.+ const nativeResumeClaims =+ claimableNativeRunIds.size === 0+ ? []+ : await dispatchNativeSessionResumptions({+ db,+ runnerInstanceId:+ runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat",+ now,+ runIds: [...claimableNativeRunIds],+ dispatch: (claim) => {+ const execution = executeRun(claim.runId, {+ nativeLeaseOwner: claim.leaseOwner,+ }).catch((error) => {+ logger.error(+ { err: error, runId: claim.runId },+ "persisted native session resume failed",+ );+ });+ activeRunExecutionPromises.add(execution);+ void execution.finally(() =>+ activeRunExecutionPromises.delete(execution),+ );+ },+ }).catch((error) => {+ logger.warn(+ { err: error },+ "failed to claim persisted native session resumptions",+ );+ return [];+ });+ const resumedRunIds = new Set(+ nativeResumeClaims.map((claim) => claim.runId),+ );++ // A terminal issue transition writes this intent in the same transaction+ // that closes the question's task, even when its card is retained. Consume+ // it before generic orphan recovery so a restart preserves cancellation.+ const cancellationRequests = await db+ .select({+ id: heartbeatRuns.id,+ contextSnapshot: heartbeatRuns.contextSnapshot,+ })+ .from(heartbeatRuns)+ .where(+ and(+ inArray(heartbeatRuns.status, [+ ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,+ ]),+ sql`${heartbeatRuns.contextSnapshot} -> ${NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY} is not null`,+ ),+ );+ for (const request of cancellationRequests) {+ const marker = parseObject(+ parseObject(request.contextSnapshot)[+ NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY+ ],+ );+ const issueId = readNonEmptyString(marker.issueId);+ const issueStatus = readNonEmptyString(marker.issueStatus);+ const interactionId = readNonEmptyString(marker.interactionId);+ const kind = readNonEmptyString(marker.kind);+ const reason =+ kind === "interaction_withdrawn"+ ? "Question withdrawn while waiting for operator input"+ : kind === "interaction_cancelled"+ ? "Cancelled while waiting for operator input"+ : "Task closed while waiting for operator input";+ try {+ await cancelRunInternal(request.id, reason, {+ resultJson: {+ ...(kind === "interaction_withdrawn" && interactionId+ ? { withdrawnInteractionId: interactionId }+ : {}),+ ...(kind === "interaction_cancelled" && interactionId+ ? { cancelledInteractionId: interactionId }+ : {}),+ ...((!kind || kind === "issue_terminal") && issueStatus+ ? { cancelledByIssueStatus: issueStatus }+ : {}),+ ...(issueId ? { cancelledIssueId: issueId } : {}),+ },+ });+ } catch (err) {+ // Keep the marker intact for the next startup/periodic sweep.+ logger.warn(+ { err, runId: request.id },+ "native question cancellation recovery attempt failed",+ );+ }+ }++ // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)+ const activeRuns = await db+ .select({+ run: heartbeatRuns,+ adapterType: agents.adapterType,+ adapterConfig: agents.adapterConfig,+ nativeCoordinatorPhase: nativeRunFinalizations.phase,+ nativeRecoveryState: nativeRunFinalizations.recoveryState,+ nativeControllerBootId: nativeRunFinalizations.controllerBootId,+ nativeControllerPid: nativeRunFinalizations.controllerPid,+ nativeControllerProcessStartedAt:+ nativeRunFinalizations.controllerProcessStartedAt,+ nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,+ })+ .from(heartbeatRuns)+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))+ .leftJoin(+ nativeRunFinalizations,+ eq(nativeRunFinalizations.runId, heartbeatRuns.id),+ )+ .where(eq(heartbeatRuns.status, "running"));++ const monitorIssueIds = [+ ...new Set(+ activeRuns.flatMap(({ run }) => {+ const runContext = parseObject(run.contextSnapshot);+ if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due")+ return [];+ const issueId = readNonEmptyString(runContext.issueId);+ return issueId ? [issueId] : [];+ }),+ ),+ ];+ const monitorIssues =+ monitorIssueIds.length > 0+ ? await db+ .select({+ id: issues.id,+ companyId: issues.companyId,+ monitorNextCheckAt: issues.monitorNextCheckAt,+ })+ .from(issues)+ .where(inArray(issues.id, monitorIssueIds))+ : [];+ const monitorNextCheckAtByIssue = new Map(+ monitorIssues.map((issue) => [+ `${issue.companyId}:${issue.id}`,+ issue.monitorNextCheckAt,+ ]),+ );++ const reaped: string[] = [];+ const currentNativeController = await currentNativeControllerIdentity();++ for (const {+ run,+ adapterType,+ adapterConfig,+ nativeCoordinatorPhase,+ nativeRecoveryState,+ nativeControllerBootId,+ nativeControllerPid,+ nativeControllerProcessStartedAt,+ nativeControllerLeaseExpiresAt,+ } of activeRuns) {+ // Authentication timeout requires an explicit ownership resolution, not+ // repeated reattachment or a process-gone guess on subsequent sweeps.+ if (isNativeRunnerOwnershipHeld(run)) continue;+ const nativeRun = run.runtimeMode === "native";+ const nativeProcessPidAlive =+ nativeRun && !!run.processPid && isProcessAlive(run.processPid);+ const nativeProcessGroupAlive =+ nativeRun &&+ !!run.processGroupId &&+ isProcessGroupAlive(run.processGroupId);+ const coordinatorOwnedByCurrentController =+ nativeRun &&+ nativeControllerBootId === currentNativeController.bootId &&+ nativeControllerPid === currentNativeController.pid &&+ nativeControllerProcessStartedAt?.getTime() ===+ currentNativeController.processStartedAt.getTime() &&+ !!nativeControllerLeaseExpiresAt &&+ nativeControllerLeaseExpiresAt.getTime() > now.getTime();+ const locallyTracked =+ runningProcesses.has(run.id) ||+ activeRunExecutions.has(run.id) ||+ coordinatorOwnedByCurrentController;+ if (+ nativeRun &&+ ([+ "awaiting_evidence",+ "awaiting_runner_reattach",+ "resuming_session",+ "bootstrap_incomplete",+ ].includes(nativeRecoveryState ?? "") ||+ nativeCoordinatorPhase === "retryable_failure")+ ) {+ continue;+ }+ const observedOwnerUnverified =+ nativeRun &&+ (nativeCoordinatorPhase === "observed" ||+ (nativeCoordinatorPhase === null &&+ run.nativePhase === "observed")) &&+ !resumedRunIds.has(run.id) &&+ !locallyTracked;+ // Persisted numeric process identifiers prove only that some process is+ // alive, not that Paperclip still owns it. Likewise an observed native+ // coordinator without a live in-process execution has no durable proof+ // that its prior provider owner stopped. Keep both cases running but+ // blocked: never signal, finalize, or retry them automatically. This gate+ // intentionally precedes resumedRunIds so a claim cannot bypass the+ // ownership check.+ if (+ !locallyTracked &&+ (nativeProcessPidAlive ||+ nativeProcessGroupAlive ||+ observedOwnerUnverified)+ ) {+ await markNativeOwnershipUnverified(run, {+ reason:+ nativeProcessPidAlive || nativeProcessGroupAlive+ ? "live_process_identifier"+ : "observed_owner_unverified",+ processPidAlive: nativeProcessPidAlive,+ processGroupAlive: nativeProcessGroupAlive,+ });+ continue;+ }+ if (resumedRunIds.has(run.id)) continue;+ if (locallyTracked) continue;+ if (await hasLiveLegacyController(db, run)) continue;++ // Apply staleness threshold to avoid false positives+ if (staleThresholdMs > 0) {+ const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;+ if (now.getTime() - refTime < staleThresholdMs) continue;+ }++ const currentAdapterTracksLocalChild =+ isTrackedLocalChildProcessAdapter(adapterType);+ const tracksLegacyLocalChild =+ run.runtimeMode !== "native" && currentAdapterTracksLocalChild;+ // Native runner processes also persist child metadata, but they must not+ // inherit legacy retry or termination authority. Use their PID/group only+ // for a read-only liveness check so a lost in-memory handle cannot cause+ // overlapping provider/tool execution while that child is still alive.+ const checksPersistedChildLiveness =+ currentAdapterTracksLocalChild || run.runtimeMode === "native";+ const processPidAlive =+ checksPersistedChildLiveness &&+ run.processPid &&+ isProcessAlive(run.processPid);+ const processGroupAlive =+ checksPersistedChildLiveness &&+ run.processGroupId &&+ isProcessGroupAlive(run.processGroupId);+ if (+ (processPidAlive || processGroupAlive) &&+ readHotRestartAdoptionMetadata(parseObject(run.resultJson))+ ) {+ continue;+ }+ if (processPidAlive || processGroupAlive) {+ if (run.errorCode !== DETACHED_PROCESS_ERROR_CODE) {+ const detachedMessage = processPidAlive+ ? `Lost in-memory process handle, but child pid ${run.processPid} is still alive`+ : `Lost in-memory process handle, but persisted process group ${run.processGroupId} is still alive`;+ const detachedRun = await setRunStatus(run.id, "running", {+ error: detachedMessage,+ errorCode: DETACHED_PROCESS_ERROR_CODE,+ });+ if (detachedRun) {+ await appendRunEvent(detachedRun, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: detachedMessage,+ payload: {+ processPid: run.processPid ?? null,+ processGroupId: run.processGroupId ?? null,+ ownedProcessHandle: false,+ },+ });+ }+ }+ continue;+ }++ const runContext = parseObject(run.contextSnapshot);+ const monitorIssueId = readNonEmptyString(runContext.issueId);+ const monitorNextCheckAt = monitorIssueId+ ? monitorNextCheckAtByIssue.get(`${run.companyId}:${monitorIssueId}`)+ : undefined;+ const monitorDispatchLostWithoutFutureWake =+ readNonEmptyString(runContext.wakeReason) === "issue_monitor_due" &&+ monitorNextCheckAt !== undefined &&+ (!monitorNextCheckAt || monitorNextCheckAt.getTime() <= now.getTime());+ const shouldRetry =+ (run.processLossRetryCount ?? 0) < 1 &&+ ((tracksLegacyLocalChild &&+ (!!run.processPid || !!run.processGroupId)) ||+ monitorDispatchLostWithoutFutureWake);+ if (!(await revokeExpiredLegacyController(db, run))) continue;+ const baseMessage = buildProcessLossMessage(run);+ const processLossDiagnostic = buildProcessLossDiagnostic({+ run,+ nowMs: Date.now(),+ observerStartedAtMs: performance.timeOrigin,+ checksPersistedChildLiveness,+ retryEligible: shouldRetry,+ });+ const conversationContinuationEligible = await runUsedConversationAdapter(db, run);++ const failureWrite = await setRunStatusFromLive(+ run.id,+ "failed",+ ["running"],+ {+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,+ errorCode: "process_lost",+ finishedAt: now,+ resultJson: (() => {+ const result = mergeRunStopMetadataForAgent(+ { adapterType, adapterConfig },+ "failed",+ {+ conversationContinuationEligible,+ resultJson: { ...parseObject(run.resultJson), processLossDiagnostic },+ errorCode: "process_lost",+ errorMessage: shouldRetry+ ? `${baseMessage}; retrying once`+ : baseMessage,+ },+ );+ return result;+ })(),+ },+ );+ if (!failureWrite.updated || !failureWrite.run) continue;+ let finalizedRun: typeof heartbeatRuns.$inferSelect | null =+ failureWrite.run;+ await setWakeupStatus(run.wakeupRequestId, "failed", {+ finishedAt: now,+ error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage,+ });+ if (!finalizedRun) finalizedRun = await getRun(run.id);+ if (!finalizedRun) continue;+ finalizedRun =+ (await classifyAndPersistRunLiveness(+ finalizedRun,+ parseObject(finalizedRun.resultJson),+ )) ?? finalizedRun;+ await releaseEnvironmentLeasesForRun({+ runId: finalizedRun.id,+ companyId: finalizedRun.companyId,+ agentId: finalizedRun.agentId,+ status: finalizedRun.status,+ failureReason: finalizedRun.error ?? undefined,+ });++ let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;+ const retryAgent = await getAgent(run.agentId);+ if (shouldRetry) {+ if (retryAgent) {+ retriedRun = await enqueueProcessLossRetry(+ finalizedRun,+ retryAgent,+ now,+ );+ }+ } else if (retryAgent) {+ const scheduled =+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(+ finalizedRun,+ retryAgent,+ );+ retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null;+ }++ if (!retriedRun) {+ await releaseIssueExecutionAndPromote(finalizedRun);+ }++ await appendRunEvent(finalizedRun, {+ eventType: "lifecycle",+ stream: "system",+ level: "error",+ message: shouldRetry+ ? `${baseMessage}; queued retry ${retriedRun?.id ?? ""}`.trim()+ : baseMessage,+ payload: {+ ...(run.processPid ? { processPid: run.processPid } : {}),+ ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),+ ...(retriedRun ? { retryRunId: retriedRun.id } : {}),+ },+ });++ await finalizeAgentStatus(run.agentId, "failed", baseMessage, {+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ });+ await startNextQueuedRunForAgent(run.agentId);+ runningProcesses.delete(run.id);+ reaped.push(run.id);+ }++ if (reaped.length > 0) {+ logger.warn(+ { reapedCount: reaped.length, runIds: reaped },+ "reaped orphaned heartbeat runs",+ );+ }++ // Recover an active lease whose run already ended before the same-tick+ // pending_cleanup sweep, so this tick can stop the recovered sandbox.+ // Isolate the sweep so its failure never hides the reaper result.+ try {+ const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({+ backoffMs: staleThresholdMs,+ });+ if (orphanedActiveLeaseSweep.recovered > 0) {+ logger.warn(+ { recovered: orphanedActiveLeaseSweep.recovered },+ "recovered orphaned active environment leases",+ );+ }+ } catch {+ // Log a constant errorKind only. The exception can carry a credential in+ // its name, code, message, cause, or stack, so the sweep never reads it.+ logger.error(+ { errorKind: ORPHANED_ACTIVE_LEASE_SWEEP_ERROR_KIND },+ "orphaned active environment lease sweep failed",+ );+ }++ // Retry stranded pending_cleanup leases on the same tick. Isolate the sweep+ // so its failure never hides the reaper result. The backoff equals the+ // reaper staleness threshold.+ try {+ const sweep = await sweepPendingCleanupLeases({+ backoffMs: staleThresholdMs,+ });+ if (sweep.destroyed > 0 || sweep.capped > 0) {+ logger.warn(+ {+ destroyed: sweep.destroyed,+ capped: sweep.capped,+ swept: sweep.swept,+ },+ "swept pending_cleanup environment leases",+ );+ }+ } catch {+ // Log a constant errorKind only. The exception can carry a credential in+ // its name, code, message, cause, or stack, so the sweep never reads it.+ logger.error(+ { errorKind: PENDING_CLEANUP_SWEEP_ERROR_KIND },+ "pending_cleanup lease sweep failed",+ );+ }++ return { reaped: reaped.length, runIds: reaped };+ }++ return {+ sweepPendingCleanupLeases,+ markNativeOwnershipUnverified,+ prepareHotRestartShutdown,+ reconcileHotRestartAdoption,+ recoverNativeRunsAfterRestart,+ reapOrphanedRuns,+ sweepOrphanedActiveLeases,+ drainRunningRunsForShutdown,+ };+}