The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(heartbeat): extract run cancellation and cleanup (#15685)
Dotta0ca4b7c4c061
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex f6fcad7747..35fcb16bc1 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1290,6 +1290,16 @@ reporting and forwarding callbacks preserve construction order for the otherheartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`.Keep lifecycle policy changes separate from this extraction and adapter execution.+Run cancellation and cleanup are in `server/src/services/heartbeat/run-control.ts`.+It owns Stop, agent and budget cancellation, environment lease release, and+resumption of saved comments after execution stops. `createHeartbeatRunControl`+binds the database, lifecycle and admission callbacks, and shared executor+barriers without starting work. The service retains the process-wide execution+and cancellation maps so scheduler and route instances observe the same owners.+Forwarding callbacks preserve construction order for budget, retry, recovery,+and queue services. Existing public helpers remain re-exported by `heartbeat.ts`.+Keep cancellation policy changes separate from this extraction and execution.+## Wake Context DeliveryBuilt-in adapters deliver wake context through the run prompt, including structureddiff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex be77bf2f40..616cdcdeab 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,3 +1,15 @@+import {+ cancelHeartbeatNativeRun,+ terminateHeartbeatRunProcess,+ providerResourceDispositionForTerminalRun,+ createHeartbeatRunControl,+ type CancelRunOptions,+} from "./heartbeat/run-control.js";+export {+ leaseReleaseStatusForRunStatus,+ providerResourceDispositionForTerminalRun,+ cancelHeartbeatNativeRun,+} from "./heartbeat/run-control.js";import {NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,isHeartbeatRunTerminalStatus,@@ -25,7 +37,6 @@ export {} from "./heartbeat/queue.js";import {isProcessAlive,- CANCELLABLE_HEARTBEAT_RUN_STATUSES,createHeartbeatRecovery,} from "./heartbeat/recovery.js";import {@@ -247,10 +258,8 @@ import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/asimport { externalObjectService } from "./external-objects.js";import { resolvePaperclipInstanceRoot } from "../home-paths.js";import { dotRunnerBroker } from "./dot-runner-broker.js";-import { isAiAuthenticationBlocked } from "./ai-auth-failure.js";+import {- nativeRetryCancellationCommitCondition,- claimCancellationRequest,startupCancellationFence,} from "./native-runtime/native-cancellation-request.js";import {@@ -262,7 +271,10 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./ageimport type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js";-import { cancellationResultJson, canContinueCancelledRun, readRunCancellation, requestedRunCancellation } from "./run-cancellation.js";+import {+ cancellationResultJson,+ readRunCancellation,+} from "./run-cancellation.js";import {withNativeWorkspaceFinalizationOwnership,NativeWorkspaceFinalizationBusyError,@@ -278,11 +290,8 @@ import {hasWorkspaceRestoreFailure,} from "@paperclipai/shared";-import { settleSlackConversation } from "./slack-conversation-lifecycle.js";import { toolActionDeliveryService } from "./tool-action-delivery.js";--import { readQueuedInteractionResponse } from "./queued-interaction-response.js";import {isConversation,prepareConversationTurn,@@ -293,29 +302,18 @@ import { getConversationConfirmationContext } from "./conversation-confirmation-import {recordNativeLocalProcessStop,} from "./native-local-process-stop.js";-import {- isAcknowledgedNativeStop,- acknowledgedNativeStopExecutionHasStopped,-} from "./acknowledged-native-stop.js";+import {legacyControllerBootId,renewLegacyControllerLease,watchLegacyControllerLease,} from "./legacy-controller-lease.js";-import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";+import {- hasRemoteTerminationReceipt,remoteExecutionHasStopped,- stoppedRemoteCleanupScopes,} from "./remote-execution-termination.js";import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";-import {- admitExplicitNativeContinuation,- undeliveredLegacyUserCommentIds,-} from "./explicit-native-continuation.js";-import {- isCancelledNativeStartup,-} from "./cancelled-native-startup.js";+import { connectionIntentService } from "./connection-intents.js";import {prepareManagedAiRuntime,@@ -327,12 +325,11 @@ import {import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared";import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js";import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js";-import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js";import {- CONVERSATION_CONTINUATION_POLICY,+ getExecutionBlocker,+} from "./execution-blocker.js";+import {claimedAdapterType,- runUsedConversationAdapter,- hasConversationContinuationPolicy,} from "./conversation-continuation.js";import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";@@ -345,10 +342,8 @@ import {} from "./legacy-execution-recovery.js";import {adapterExecutionControls,- captureAdapterStopOwnership,createAdapterExecutionControl,registerAdapterExecutionControl,- waitForAdapterStop,} from "./adapter-execution-control.js";import { executionFailureRetryCount } from "./execution-recovery-attempt.js";import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";@@ -386,14 +381,11 @@ import {eq,exists,getTableColumns,- gt,gte,inArray,isNull,- isNotNull,lt,lte,- ne,notInArray,or,sql,@@ -404,7 +396,6 @@ import {ISSUE_DISPOSITION_REPAIR_RETRY_REASON,PROVIDER_QUOTA_MONITOR_SERVICE_NAME,isEnvironmentDriverSupportedForAdapter,- type EnvironmentLeaseStatus,type ExecutionWorkspace,type IssueExecutionMonitorClearReason,type IssueExecutionMonitorPolicy,@@ -419,11 +410,9 @@ import {chatActions,companies,completionContracts,- environmentLeases,heartbeatRunEvents,heartbeatRuns,issueComments,- issueRecoveryActions,issues,nativeRunFinalizations,projects,@@ -449,10 +438,7 @@ export { scrubGitCredentialText };import { publishLiveEvent } from "./live-events.js";import {- queuedCommentIdsFromWakePayload,queuedCommentIdsFromRunContext,- withQueuedCommentIdsInWakePayload,- withQueuedCommentIdsInRunContext,} from "./issue-queued-comment-queue.js";import { documentService } from "./documents.js";import { getTaskPlanContext } from "./task-plan-context.js";@@ -463,10 +449,8 @@ import {buildNativeExecutionInput,buildNativeExecutionWithCheckpoint,buildNativeRuntimeContext,- cancelNativeSession,claimWarmNativeInstructionCopy,nativeSessionWorkspaceScope,- closeWarmNativeSessionsForEnvironment,reserveWarmNativeInstructionDirectory,dispatchNativeSessionResumptions,ensureNativeCompletionContract,@@ -560,7 +544,9 @@ import { reportRunFailure } from "./run-failure-report.js";import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";import { companySkillService } from "./company-skills.js";-import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";+import {+ budgetService,+} from "./budgets.js";import {secretService,} from "./secrets.js";@@ -647,7 +633,6 @@ import {} from "./workspace-operations.js";import {isProcessGroupAlive,- terminateLocalService,} from "./local-service-supervisor.js";import {isRuntimeOwnedGitBranch,@@ -709,7 +694,6 @@ import {} from "../modules/run-dispatch/index.js";import {createWakeQueue,- WakeQueueApplicationError,type IssueSnapshot as WakeQueueIssueSnapshot,type PostCommitEffect as WakeQueuePostCommitEffect,type ReleaseRecoveryBlockedNoticeKind,@@ -1018,21 +1002,6 @@ export function getTaskDrainStatus(): {};}-export function leaseReleaseStatusForRunStatus(- status: string | null | undefined,-): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {- if (status === "cancelled") return "expired";- return status === "failed" || status === "timed_out" ? "failed" : "released";-}--export function providerResourceDispositionForTerminalRun(- desired: ProviderResourceDisposition | undefined,- status: string | null | undefined,-): ProviderResourceDisposition | undefined {- if (desired !== "keep_running") return desired;- return status === "succeeded" ? desired : "stop_and_retain";-}-export interface NativeSandboxLifecycle {runnerProcess: "per_turn" | "warm";sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";@@ -1283,33 +1252,6 @@ function isSameTaskScope(left: string | null, right: string | null) {return (left ?? null) === (right ?? null);}-async function terminateHeartbeatRunProcess(input: {- pid: number | null | undefined;- processGroupId: number | null | undefined;- graceMs?: number;- signal?: NodeJS.Signals;-}) {- const pid = input.pid ?? null;- const processGroupId = input.processGroupId ?? null;- if (typeof pid !== "number" && typeof processGroupId !== "number") return;-- await terminateLocalService(- {- pid:- typeof pid === "number" && Number.isInteger(pid) && pid > 0- ? pid- : (processGroupId ?? 0),- processGroupId:- typeof processGroupId === "number" &&- Number.isInteger(processGroupId) &&- processGroupId > 0- ? processGroupId- : null,- },- { forceAfterMs: input.graceMs, signal: input.signal },- );-}-export type HeartbeatEnvironmentRuntime = ReturnType<typeof environmentRuntimeService>;@@ -1351,38 +1293,6 @@ export interface HeartbeatServiceOptions {}) => Promise<void>;}-export async function cancelHeartbeatNativeRun(input: {- db: Db;- runId: string;- reason: string;- runtimeMode: string | null;- cancellationRequestId?: string;- cancel?: (- runId: string,- reason: string,- options: { db: Db; scope: "run"; cancellationRequestId?: string },- ) => Promise<{ decision: unknown | null; auditId: string | null }>;-}) {- if (input.runtimeMode !== "native") {- return { decision: null, auditId: null };- }- const cancellation = input.cancel- ? await input.cancel(input.runId, input.reason, {- db: input.db,- scope: "run",- ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),- })- : await cancelNativeSession(input.runId, input.reason, {- db: input.db,- scope: "run",- ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),- });- if (!cancellation.decision || !cancellation.auditId) {- throw new Error("native_cancellation_outcome_not_audited");- }- return cancellation;-}-class NativeSessionResumeScheduledError extends Error {constructor(readonly original: unknown) {super("Native session recovery has been scheduled for the same run.");@@ -2061,6 +1971,74 @@ export function heartbeatService(return runLifecycle.publishRunLifecyclePluginEvent(...args);}+ const runControl = createHeartbeatRunControl(db, {+ getRun,+ options,+ envOrchestrator,+ activeRunExecutions,+ enqueueWakeup,+ getAgent,+ sweepPendingCleanupLeases,+ getSchedulingSuppression,+ wakeQueue,+ applyWakeQueuePostCommitEffects,+ processRunCancellationSettlements,+ mergeRunStopMetadataForAgent,+ setRunStatusFromLive,+ failedProcessRunCancellations,+ setWakeupStatus,+ appendRunEvent,+ finalizeAgentStatus,+ timerClaimWasFirstHeartbeat,+ startNextQueuedRunForAgent,+ setRunStatus,+ });++ const {+ cancelInvocationsForAgentsInternal,+ } = runControl;++ // Earlier factories capture these callbacks before run-control construction.+ function cancelBudgetScopeWork(...args: Parameters<typeof runControl.cancelBudgetScopeWork>) {+ return runControl.cancelBudgetScopeWork(...args);+ }++ function releaseIssueExecutionAndPromote(...args: Parameters<typeof runControl.releaseIssueExecutionAndPromote>) {+ return runControl.releaseIssueExecutionAndPromote(...args);+ }++ function releaseEnvironmentLeasesForRun(...args: Parameters<typeof runControl.releaseEnvironmentLeasesForRun>) {+ return runControl.releaseEnvironmentLeasesForRun(...args);+ }++ function acknowledgeRemoteStop(...args: Parameters<typeof runControl.acknowledgeRemoteStop>) {+ return runControl.acknowledgeRemoteStop(...args);+ }++ function resumeRemoteStopComments(...args: Parameters<typeof runControl.resumeRemoteStopComments>) {+ return runControl.resumeRemoteStopComments(...args);+ }++ function cancelRunInternal(...args: Parameters<typeof runControl.cancelRunInternal>) {+ return runControl.cancelRunInternal(...args);+ }++ function resumeExecutionWaitComments(...args: Parameters<typeof runControl.resumeExecutionWaitComments>) {+ return runControl.resumeExecutionWaitComments(...args);+ }++ function resumeQueuedCommentInterrupt(...args: Parameters<typeof runControl.resumeQueuedCommentInterrupt>) {+ return runControl.resumeQueuedCommentInterrupt(...args);+ }++ function resumeSavedLegacyComments(...args: Parameters<typeof runControl.resumeSavedLegacyComments>) {+ return runControl.resumeSavedLegacyComments(...args);+ }++ function cancelActiveForAgentInternal(...args: Parameters<typeof runControl.cancelActiveForAgentInternal>) {+ return runControl.cancelActiveForAgentInternal(...args);+ }+// Applies the post-commit effects a wake-queue release returns, exactly as// the original release function did before its writes moved into that// module: publish + dispatch a promoted or recovery run, log a reopened@@ -2123,422 +2101,6 @@ export function heartbeatService(}const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });- async function releaseEnvironmentLeasesForRun(input: {- runId: string;- companyId: string;- agentId: string;- status: string | null | undefined;- failureReason?: string | null;- providerResourceDisposition?: ProviderResourceDisposition;- nativeLifecycleTelemetry?: {- provider: string;- harness: string;- lifecycleMode: "per_turn" | "warm";- sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";- };- }) {- const leaseOwnerRun = await getRun(input.runId);- if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;- // Recovery can finish workspace copy-back outside the executor's finally.- // Successful copy-back does not earn warm retention for a failed turn.- const status = leaseOwnerRun?.status ?? input.status;- const providerResourceDisposition = providerResourceDispositionForTerminalRun(- input.providerResourceDisposition,- status,- );- if (providerResourceDisposition === "destroy") {- const closeResult = await (- options.closeWarmNativeSessionsForRun ??- (async ({ runId, reason }) => {- const environmentIds = await db- .selectDistinct({ environmentId: environmentLeases.environmentId })- .from(environmentLeases)- .where(- and(- eq(environmentLeases.heartbeatRunId, runId),- eq(environmentLeases.status, "active"),- ),- )- .then((rows) =>- rows.flatMap((row) =>- typeof row.environmentId === "string" &&- row.environmentId.length > 0- ? [row.environmentId]- : [],- ),- );- const aggregate = { closed: 0, busy: 0, failed: 0 };- for (const environmentId of environmentIds) {- const result = await closeWarmNativeSessionsForEnvironment({- environmentId,- reason,- });- aggregate.closed += result.closed;- aggregate.busy += result.busy;- aggregate.failed += result.failed;- }- return aggregate;- })- )({- runId: input.runId,- reason: "terminal heartbeat run destroyed its environment lease",- }).catch((err) => {- logger.warn(- { err, runId: input.runId },- "failed to close warm native sessions before environment lease destruction",- );- return { closed: 0, busy: 0, failed: 1 };- });- // A failed remote checkpoint cannot veto destruction of the isolated- // sandbox after the run stopped. Provider destruction supplies the exit- // proof; it does not turn the interrupted checkpoint into a success.- const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&- ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)- ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(- eq(environmentLeases.companyId, input.companyId),- eq(environmentLeases.heartbeatRunId, input.runId),- ))- : [];- const canDestroyRemote = remoteLeases.length > 0 &&- remoteLeases.every(lease => lease.provider && lease.provider !== "local");- if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {- logger.warn(- { runId: input.runId, warmNativeSessions: closeResult },- "deferred environment lease destruction until warm native sessions close",- );- return;- }- }- const releaseResult = await envOrchestrator- .releaseForRun({- heartbeatRunId: input.runId,- companyId: input.companyId,- agentId: input.agentId,- status: leaseReleaseStatusForRunStatus(status),- failureReason: input.failureReason ?? undefined,- providerResourceDisposition,- nativeLifecycleTelemetry: input.nativeLifecycleTelemetry,- })- .catch((err) => {- logger.warn(- { err, runId: input.runId },- "failed to release environment leases for heartbeat run",- );- return null;- });- for (const releaseError of releaseResult?.errors ?? []) {- logger.warn(- {- err: releaseError.error,- leaseId: releaseError.leaseId,- runId: input.runId,- },- "failed to release environment lease for heartbeat run",- );- }- await acknowledgeRemoteStop(input.runId, input.companyId);- }-- async function acknowledgeRemoteStop(runId: string, companyId: string) {- // The provider receipt arrives after adapter settlement. A remote ACP child- // has no host PID, so only this target-aware boundary can acknowledge Stop.- const stopped = await getRun(runId);- if (stopped?.runtimeMode === "native") {- const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId);- for (const remoteCleanupScope of scopes ?? []) {- completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });- }- }- if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" &&- parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&- await runUsedConversationAdapter(db, stopped) &&- await remoteExecutionHasStopped(db, companyId, runId)) {- await db.update(heartbeatRuns).set({- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({- executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),- state: "acknowledged", acknowledgedAt: new Date().toISOString(),- proof: "provider_termination_receipt" },- conversationContinuation: CONVERSATION_CONTINUATION_POLICY,- })}::jsonb`,- updatedAt: new Date(),- }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),- eq(heartbeatRuns.status, "cancelled")));- }- }-- async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) {- if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;- const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt- ? await db.select().from(nativeRunFinalizations).where(and(- eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),- )) : [];- const cancelledPreparation = run.runtimeMode === "legacy" &&- await isCancelledNativeStartup(db, run, preparationCoordinator);- if (run.runtimeMode !== "native" && !cancelledPreparation &&- parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true &&- !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return;- const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);- if (!issueId) return;- const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null;- const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase,- leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and(- eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),- )) : [];- // Stop ends one response. Saved user input can enter ordinary admission- // once its executor settles; it does not need a manufactured crash incident.- // The ordinary path still enforces task holds, ownership, and native session- // cleanup before starting a provider.- const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) &&- !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner &&- ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") &&- await acknowledgedNativeStopExecutionHasStopped(db, currentRun) &&- !(await getExecutionBlocker(db, run.companyId, issueId));- const legacyContinuation = run.runtimeMode === "legacy" &&- hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) &&- !(await getExecutionBlocker(db, run.companyId, issueId));- if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return;- const pending = await db.select().from(agentWakeupRequests).where(and(- eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),- eq(agentWakeupRequests.status, "deferred_issue_execution"),- eq(agentWakeupRequests.requestedByActorType, "user"),- requestId ? eq(agentWakeupRequests.id, requestId) : undefined,- sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,- )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50);- for (const wake of pending) {- if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue;- let payload = parseObject(wake.payload);- if (payload.queuedCommentInterrupt) {- await resumeQueuedCommentInterrupt(wake.companyId, wake.id);- continue;- }- let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);- let commentId = deriveCommentId(context, payload);- let requestedByActorId = wake.requestedByActorId;- const reason = readNonEmptyString(context.wakeReason) ?? wake.reason;- if (run.runtimeMode === "native" || cancelledPreparation) {- const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId,- queuedCommentIdsFromWakePayload(payload));- if (!ids.length) continue;- payload = withQueuedCommentIdsInWakePayload(payload, ids);- context = withQueuedCommentIdsInRunContext(context, ids);- commentId = ids.at(-1)!;- // A coalesced queue can contain several authors. Its saved comments,- // not the outer wake's first author, authorize the remaining input.- const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and(- eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId),- ));- requestedByActorId = author?.id ?? null;- }- if (legacyContinuation || stoppedNativeContinuation) {- if (!commentId || !run.finishedAt || !requestedByActorId ||- !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue;- const [comment] = await db.select().from(issueComments).where(and(- eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId),- sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"),- eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt),- isNull(issueComments.createdByRunId),- stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt),- ));- if (!comment?.body.trim()) continue;- } else {- let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." };- const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId,- agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId,- reason, commentId, successorRunId: randomUUID(), dryRun: true,- ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),- onBlocked: (reason, message) => { wait = { reason, message }; },- });- if (!admitted) {- await db.update(agentWakeupRequests).set({- payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',- coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,- updatedAt: new Date(),- }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),- eq(agentWakeupRequests.status, "deferred_issue_execution")));- continue;- }- }- // Re-enter ordinary admission with the original user's authority. It- // atomically adopts the deferred comments and still applies every gate.- await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"],- reason, payload, contextSnapshot: context,- requestedByActorType: "user", requestedByActorId,- ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),- idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id);- break;- }- }-- async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) {- return resumeSavedLegacyComments(companyId, queueId, true, opts);- }-- async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) {- const [wake] = await db.select().from(agentWakeupRequests).where(and(- eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),- eq(agentWakeupRequests.status, "deferred_issue_execution"),- ));- if (!wake) return;- const payload = parseObject(wake.payload);- let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId);- let commentIds = queuedCommentIdsFromWakePayload(payload);- const issueId = readNonEmptyString(payload.issueId);- if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return;- const response = await readQueuedInteractionResponse(db, companyId, issueId, payload);- if (!commentIds.length && !response) return;- // Resolved cards are immutable input. Only an explicit Interrupt click can- // authorize continuation across a stopped execution; ordinary completion- // uses normal deferred-wake promotion.- if (response && !interrupted) return;- if (!interrupted) {- commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds);- if (!commentIds.length) return;- // The queue itself may have begun as a system wake. The saved human- // comment, not that wake's origin or mutable caller payload, is authority.- const [comment] = await db.select().from(issueComments).where(and(- eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),- eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"),- isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),- )).orderBy(desc(issueComments.createdAt)).limit(1);- if (!comment?.body.trim() || !comment.authorUserId) return;- actorId = comment.authorUserId;- }- if (!actorId) return;- const agent = await getAgent(wake.agentId);- if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return;- const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(- eq(heartbeatRuns.companyId, companyId),- eq(heartbeatRuns.agentId, wake.agentId),- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,- inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),- )).limit(1);- if (active) return;- if (interrupted && opts?.retryCleanup) {- // Only the HTTP click grants an extra cleanup attempt. Periodic retries- // reuse the intent to deliver, never a fresh provider teardown budget.- const sourceRun = await db.transaction(async tx => {- const [task] = await tx.select().from(issues).where(and(- eq(issues.companyId, companyId), eq(issues.id, issueId),- )).for("update");- const [current] = await tx.select().from(agentWakeupRequests).where(and(- eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),- eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"),- sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,- sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`,- ));- if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) ||- !current || (!queuedCommentIdsFromWakePayload(current.payload).length &&- !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null;- const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(- eq(heartbeatRuns.companyId, companyId),- eq(heartbeatRuns.agentId, wake.agentId),- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,- inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),- )).limit(1);- if (successor) return null;- const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId);- const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and(- eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId),- eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"),- inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]),- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,- )).then(rows => rows[0]) : null;- if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null;- // Older ephemeral leases recorded successful cleanup without a provider- // receipt. Re-verify them through the recorded teardown path; a timestamp- // alone never certifies termination. Retained/reusable resources stay put.- const historical = await tx.select().from(environmentLeases).where(and(- eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id),- eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt),- inArray(environmentLeases.status, ["released", "expired", "failed"]),- )).for("update");- for (const lease of historical) {- if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue;- // Provider resource IDs identify physical sandboxes. A lease in any- // company can still own this resource; never destroy it on behalf of- // this company. This existence-only guard exposes no foreign data.- const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(- ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),- eq(environmentLeases.providerLeaseId, lease.providerLeaseId),- or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])),- )).limit(1);- if (otherOwner) continue;- await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() })- .where(eq(environmentLeases.id, lease.id));- }- return run;- });- if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: {- companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt",- } });- }- const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds);- delete deliveryPayload.queuedCommentInterrupt;- await enqueueWakeup(wake.agentId, {- source: "on_demand", triggerDetail: "manual", reason: "issue_commented",- payload: deliveryPayload, contextSnapshot: response- ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId,- responsibleUserId: actorId }- : withQueuedCommentIdsInRunContext({- issueId, triggeredBy: "board", actorId, responsibleUserId: actorId,- }, commentIds),- requestedByActorType: "user", requestedByActorId: actorId,- ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }),- issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] },- idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`,- }, queueId);- }-- async function resumeExecutionWaitComments() {- if ((await getSchedulingSuppression()).suppressed) return;- const waits = await db.select({ wake: agentWakeupRequests })- .from(agentWakeupRequests)- .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),- sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,- eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))- .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))- .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(- eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),- executionBlockerPredicate(),- ))), eq(agentWakeupRequests.status, "deferred_issue_execution"),- eq(agentWakeupRequests.requestedByActorType, "user"),- sql`${agentWakeupRequests.payload}->'executionWait' is not null`,- sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`,- lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),- notInArray(issues.status, ["done", "cancelled"])))- .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);- const seen = new Set<string>();- for (const { wake } of waits) {- const issueId = String(wake.payload?.issueId);- if (seen.has(issueId)) continue;- seen.add(issueId);- // Advance the cursor even for invalid evidence so one damaged task cannot- // starve later requests in this bounded scan. Preserve concurrent edits.- const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and(- eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId),- eq(agentWakeupRequests.status, "deferred_issue_execution"),- lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),- )).returning({ id: agentWakeupRequests.id });- if (!claimed) continue;- // Match normal admission's deterministic current blocker selection. An- // arbitrary historical action must not choose the retry's source run.- const blocker = await getExecutionBlocker(db, wake.companyId, issueId);- const sourceId = blocker?.runId;- if (!sourceId || !isUuidLike(sourceId)) continue;- const run = await getRun(sourceId, { includeExecutionEvidence: true });- if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;- if (canContinueCancelledRun(run)) {- await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {- logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");- });- continue;- }- await resumeRemoteStopComments(run, wake.id).catch(err => {- logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");- });- }- }-async function getAgentInvokability(agent: typeof agents.$inferSelect | null | undefined,) {@@ -10618,683 +10180,6 @@ export function heartbeatService(}}- async function releaseIssueExecutionAndPromote(- run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">,- options: { suppressImmediateRecovery?: boolean } = {},- ) {- try {- const source = await getRun(run.id);- const { postCommitEffects } = await wakeQueue.releaseIssueExecution({- companyId: run.companyId,- runId: run.id,- now: new Date(),- // A durable authentication card owns recovery. This covers the review path,- // while continuation classification blocks periodic generic retries.- suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source),- });- await applyWakeQueuePostCommitEffects(postCommitEffects);- const completed = await getRun(run.id);- const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId)- ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId;- if (completed?.status === "succeeded" && issueId) {- await settleSlackConversation(db, run.companyId, issueId).catch((err) => {- logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation");- });- }- } catch (error) {- if (- error instanceof WakeQueueApplicationError &&- error.code === "responsible_user_unresolved"- ) {- // Every other `responsible_user_unresolved` HttpError in this file- // carries `code` inside its own `details`; match that shape here too.- throw new HttpError(422, error.message, { code: error.code, ...error.details });- }- throw error;- }- }-- async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) {- const runIssueId = sql<- string | null- >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`;- const effectiveProjectId = sql<- string | null- >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;-- const rows = await db- .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })- .from(heartbeatRuns)- .leftJoin(- issues,- and(- eq(issues.companyId, companyId),- sql`${issues.id}::text = ${runIssueId}`,- ),- )- .where(- and(- eq(heartbeatRuns.companyId, companyId),- createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined,- inArray(heartbeatRuns.status, [- ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,- ]),- sql`${effectiveProjectId} = ${projectId}`,- ),- );-- return rows.map((row) => row.id);- }-- async function listProjectScopedWakeupIds(- companyId: string,- projectId: string,- database: Db = db,- ) {- const wakeIssueId = sql<- string | null- >`${agentWakeupRequests.payload} ->> 'issueId'`;- const effectiveProjectId = sql<- string | null- >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;-- const rows = await database- .selectDistinctOn([agentWakeupRequests.id], {- id: agentWakeupRequests.id,- })- .from(agentWakeupRequests)- .leftJoin(- issues,- and(- eq(issues.companyId, companyId),- sql`${issues.id}::text = ${wakeIssueId}`,- ),- )- .where(- and(- eq(agentWakeupRequests.companyId, companyId),- inArray(agentWakeupRequests.status, [- "queued",- "deferred_issue_execution",- ]),- sql`${agentWakeupRequests.runId} is null`,- sql`${effectiveProjectId} = ${projectId}`,- ),- );-- return rows.map((row) => row.id);- }-- async function cancelPendingWakeupsForBudgetScope(- scope: BudgetEnforcementScope,- ) {- return withCurrentBudgetEnforcement(db, scope, async (tx) => {- const now = new Date();- let wakeupIds: string[] = [];-- if (scope.scopeType === "company") {- wakeupIds = await tx- .select({ id: agentWakeupRequests.id })- .from(agentWakeupRequests)- .where(- and(- eq(agentWakeupRequests.companyId, scope.companyId),- inArray(agentWakeupRequests.status, [- "queued",- "deferred_issue_execution",- ]),- sql`${agentWakeupRequests.runId} is null`,- ),- )- .then((rows) => rows.map((row) => row.id));- } else if (scope.scopeType === "agent") {- wakeupIds = await tx- .select({ id: agentWakeupRequests.id })- .from(agentWakeupRequests)- .where(- and(- eq(agentWakeupRequests.companyId, scope.companyId),- eq(agentWakeupRequests.agentId, scope.scopeId),- inArray(agentWakeupRequests.status, [- "queued",- "deferred_issue_execution",- ]),- sql`${agentWakeupRequests.runId} is null`,- ),- )- .then((rows) => rows.map((row) => row.id));- } else {- wakeupIds = await listProjectScopedWakeupIds(- scope.companyId,- scope.scopeId,- tx,- );- }-- if (wakeupIds.length === 0) return 0;-- await tx- .update(agentWakeupRequests)- .set({- status: "cancelled",- finishedAt: now,- error: "Cancelled due to budget pause",- updatedAt: now,- })- .where(and(- inArray(agentWakeupRequests.id, wakeupIds),- inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),- isNull(agentWakeupRequests.runId),- scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined,- ));-- return wakeupIds.length;- });- }-- type CancelRunOptions = {- budgetEnforcement?: BudgetEnforcementScope;- /** Optional board request identity, atomically reserved for native Stop. */- cancellationRequestId?: string;- cancellationRequestedByUserId?: string | null;- errorCode?: string;- resultJson?: Record<string, unknown>;- eventMessage?: string;- eventPayload?: Record<string, unknown>;- /** Per-call graceful process shutdown window, bounded to a safe range. */- terminationGraceMs?: number;- /** Caller is immediately scheduling an explicit successor path. */- suppressImmediateRecovery?: boolean;- };-- function cancellationTerminationGraceMs(- configuredGraceSec: number,- requestedGraceMs: number | undefined,- ) {- const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000;- if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) {- return configuredGraceMs;- }- return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs)));- }-- async function cancelRunInternal(- runId: string,- reason = "Cancelled by control plane",- options: CancelRunOptions = {},- ) {- let run = await getRun(runId);- if (!run) throw notFound("Heartbeat run not found");- if (options.cancellationRequestId) {- run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null);- }- // The caller claim checked retry eligibility under both durable row locks.- // This is only a cancellation candidate: dispatch rechecks the coordinator- // under lock, and the final CAS requires its own unchanged acknowledged- // retry fence. Re-reading only retryable_failure here would strand replay.- const pendingNativeRetry =- run.runtimeMode === "native" && run.status === "failed" && (- Boolean(options.cancellationRequestId) || await db- .select({ runId: nativeRunFinalizations.runId })- .from(nativeRunFinalizations)- .where(- and(- eq(nativeRunFinalizations.runId, run.id),- eq(nativeRunFinalizations.companyId, run.companyId),- eq(nativeRunFinalizations.phase, "retryable_failure"),- ),- )- .then((rows) => rows.length > 0)- );- if (- !pendingNativeRetry &&- !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(- run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number],- )- )- return run;- const agent = await getAgent(run.agentId);- const errorCode = options.errorCode ?? "cancelled";- const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason);-- const pendingProcessCancellation = processRunCancellationSettlements.get(- run.id,- );- if (pendingProcessCancellation) {- await pendingProcessCancellation.settled;- if (pendingProcessCancellation.failed)- throw pendingProcessCancellation.error;- return getRun(run.id);- }- const running = runningProcesses.get(run.id);- const stopOwnership =- run.runtimeMode !== "native"- ? captureAdapterStopOwnership(run.id)- : undefined;- const control = stopOwnership?.control;- // Capture the existing adapter owner before waiting on the run lock. Then- // atomically fence preparation and refresh the selected runtime, so Stop- // cannot miss a native handoff that won after its first read.- // Established legacy processes must still be stopped if the database is- // unavailable. Only native or not-yet-dispatched preparation needs this- // additional durable fence before its existing cancellation path.- if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) {- const fence = async (tx: Db) => {- const [fenced] = await tx.update(heartbeatRuns).set({- // Record handoff intent before native cancellation can finalize and release- // the run. Only its audited stop acknowledgement suppresses recovery.- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||- ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true- ? { reassignmentStopRequested: true } : {})}::jsonb ||- ${JSON.stringify({ cancellation })}::jsonb ||- jsonb_build_object('startupCancellation', jsonb_build_object(- 'requestedAt', ${new Date().toISOString()}::text,- 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'- and ${heartbeatRuns.runtimeModeResolvedAt} is null- and ${heartbeatRuns.executionStage} = 'preparing'- and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)- ))`,- }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,- pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],- ))).returning();- return fenced ?? null;- };- let fenced: typeof run | null;- try {- fenced = options.budgetEnforcement- ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence)- : await fence(db);- } catch (error) {- stopOwnership?.release();- throw error;- }- if (!fenced) {- stopOwnership?.release();- return getRun(runId);- }- run = fenced;- }- const resultJson = { ...(agent- ? {- ...mergeRunStopMetadataForAgent(agent, "cancelled", {- resultJson: parseObject(run.resultJson),- errorCode,- errorMessage: reason,- }),- ...(options.resultJson ?? {}),- }- : options.resultJson), cancellation };-- try {- let releaseProcessCancellation: (() => void) | undefined;- const processCancellationSettlement =- run.runtimeMode !== "native" &&- !control &&- running- ? {- settled: new Promise<void>((resolve) => {- releaseProcessCancellation = resolve;- }),- failed: false,- error: undefined as unknown,- }- : undefined;- if (processCancellationSettlement) {- // No await between joining an existing owner above and registering ours.- processRunCancellationSettlements.set(- run.id,- processCancellationSettlement,- );- }- const cancellation = await (async () => {- try {- if (control) {- await db- .update(heartbeatRuns)- .set({- error: reason,- errorCode,- // Stop may have read the row before adapter settlement saved- // a required-copyback receipt. Merge into current database JSON- // and leave those server-owned fields to the restore recorder.- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({- ...resultJson,- ...(!running ? { executionCancellation: {- state: "requested", requestedAt: new Date().toISOString(),- } } : {}),- }, true),- updatedAt: new Date(),- })- .where(- and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.status, "running"),- ),- );- control.controller.abort(new Error(reason));- }- let terminationSettled = false;- try {- await cancelHeartbeatNativeRun({- db,- runId: run.id,- reason,- runtimeMode: run.runtimeMode,- ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),- });- if (running) {- await terminateHeartbeatRunProcess({- pid: running.child.pid,- processGroupId: running.processGroupId,- // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM- // can leave commands in their separate process groups alive.- signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined,- graceMs: cancellationTerminationGraceMs(- running.graceSec,- options.terminationGraceMs,- ),- });- }- terminationSettled = true;- } finally {- if (- (!processCancellationSettlement || terminationSettled) &&- runningProcesses.get(run.id) === running- ) {- runningProcesses.delete(run.id);- }- }-- if (control) {- await waitForAdapterStop(control.settled, undefined, {- runId: run.id,- adapterType: agent?.adapterType,- runtimeMode: run.runtimeMode,- abortRequested: control.controller.signal.aborted,- }, control);- const stopped = await getRun(run.id);- if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) {- if (- parseObject(stopped.resultJson?.executionCancellation).state !==- "acknowledged"- ) {- throw conflict(- "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.",- );- }- // The owned adapter already finalized this run and its lifecycle.- // Do not replay the process cancellation side effects below.- return { run: stopped, updated: false };- }- }-- const finishedAt = new Date();- const persistedCancellationResult =- run.runtimeMode === "native"- ? await getRun(run.id).then((current) =>- parseObject(current?.resultJson),- )- : {};- return await setRunStatusFromLive(- run.id,- "cancelled",- pendingNativeRetry- ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"]- : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],- {- finishedAt,- error: reason,- errorCode,- ...(resultJson ||- Object.keys(persistedCancellationResult).length > 0- ? {- resultJson: {- ...persistedCancellationResult,- ...(resultJson ?? {}),- // A scheduler placeholder has no process to acknowledge.- // Preserve its normal release policy instead of treating- // it as an operator stop of provider work.- ...(processCancellationSettlement && agent && running && (- (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) ||- (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0)- )- ? mergeRunStopMetadataForAgent(agent, "cancelled", {- resultJson: {- ...resultJson,- executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() },- },- errorCode, errorMessage: reason,- })- : {}),- // The native cancellation helper may have advanced a durable- // pending intent to its acknowledged state after `run` was- // first read. Never let that stale snapshot overwrite the- // authoritative post-dispatch acknowledgement.- ...(Object.hasOwn(- persistedCancellationResult,- "nativeCancellation",- )- ? {- nativeCancellation:- persistedCancellationResult.nativeCancellation,- }- : {}),- },- }- : {}),- },- undefined,- pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined,- );- } catch (error) {- if (processCancellationSettlement) {- processCancellationSettlement.failed = true;- processCancellationSettlement.error = error;- if (activeRunExecutions.has(run.id)) {- failedProcessRunCancellations.set(- run.id,- processCancellationSettlement,- );- }- }- throw error;- } finally {- if (processCancellationSettlement) {- if (- processRunCancellationSettlements.get(run.id) ===- processCancellationSettlement- ) {- processRunCancellationSettlements.delete(run.id);- }- // Always settle waiters, including termination and DB errors. A- // failed Stop remains an error to its caller, never a cancellation- // receipt; the signal-bearing executor may then record failure.- releaseProcessCancellation?.();- }- }- })();- const cancelled = cancellation.run;-- if (cancellation.updated && cancelled) {- await setWakeupStatus(run.wakeupRequestId, "cancelled", {- finishedAt: cancelled.finishedAt ?? new Date(),- error: reason,- });- await appendRunEvent(cancelled, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message: options.eventMessage ?? "run cancelled",- payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) },- });- await releaseIssueExecutionAndPromote(cancelled, {- suppressImmediateRecovery: options.suppressImmediateRecovery,- });- await finalizeAgentStatus(run.agentId, "cancelled", undefined, {- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),- });- await startNextQueuedRunForAgent(run.agentId);- }- return cancelled;- } finally {- stopOwnership?.release();- }- }-- async function cancelActiveForAgentInternal(- agentId: string,- reason = "Cancelled due to agent pause",- errorCode = "cancelled",- ) {- const agent = await getAgent(agentId);- const runs = await db- .select()- .from(heartbeatRuns)- .where(- and(- eq(heartbeatRuns.agentId, agentId),- inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),- ),- );-- for (const run of runs) {- const stopOwnership =- run.runtimeMode !== "native"- ? captureAdapterStopOwnership(run.id)- : undefined;- try {- if (stopOwnership?.control) {- await cancelRunInternal(run.id, reason, { errorCode });- continue;- }- if (run.runtimeMode === "native") {- await db.update(heartbeatRuns).set({ resultJson:- sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`,- }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status)));- await cancelHeartbeatNativeRun({- db,- runId: run.id,- reason,- runtimeMode: run.runtimeMode,- });- }- const persistedCancellationResult =- run.runtimeMode === "native"- ? await getRun(run.id).then((current) =>- parseObject(current?.resultJson),- )- : parseObject(run.resultJson);- await setRunStatus(run.id, "cancelled", {- finishedAt: new Date(),- error: reason,- errorCode,- resultJson: {- ...persistedCancellationResult,- ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", {- resultJson: persistedCancellationResult, errorCode, errorMessage: reason,- }) : {}),- cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason),- },- });-- await setWakeupStatus(run.wakeupRequestId, "cancelled", {- finishedAt: new Date(),- error: reason,- });-- const running = runningProcesses.get(run.id);- if (running) {- await terminateHeartbeatRunProcess({- pid: running.child.pid,- processGroupId: running.processGroupId,- graceMs: Math.max(1, running.graceSec) * 1000,- });- }- runningProcesses.delete(run.id);- await releaseIssueExecutionAndPromote(run);- } finally {- stopOwnership?.release();- }- }-- return runs.length;- }-- async function cancelPendingWakeupsForAgentsInternal(- agentIds: string[],- reason: string,- ) {- const uniqueAgentIds = [...new Set(agentIds)].filter(- (agentId) => agentId.length > 0,- );- if (uniqueAgentIds.length === 0) return 0;-- const now = new Date();- const wakeupIds = await db- .select({ id: agentWakeupRequests.id })- .from(agentWakeupRequests)- .where(- and(- inArray(agentWakeupRequests.agentId, uniqueAgentIds),- inArray(agentWakeupRequests.status, [- "queued",- "deferred_issue_execution",- ]),- sql`${agentWakeupRequests.runId} is null`,- ),- )- .then((rows) => rows.map((row) => row.id));-- if (wakeupIds.length === 0) return 0;-- await db- .update(agentWakeupRequests)- .set({- status: "cancelled",- finishedAt: now,- error: reason,- updatedAt: now,- })- .where(inArray(agentWakeupRequests.id, wakeupIds));-- return wakeupIds.length;- }-- async function cancelInvocationsForAgentsInternal(- agentIds: string[],- reason: string,- ) {- const uniqueAgentIds = [...new Set(agentIds)].filter(- (agentId) => agentId.length > 0,- );- let runsCancelled = 0;- for (const agentId of uniqueAgentIds) {- runsCancelled += await cancelActiveForAgentInternal(agentId, reason);- }- const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal(- uniqueAgentIds,- reason,- );- return {- agentIds: uniqueAgentIds,- runsCancelled,- wakeupsCancelled,- };- }-- async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) {- const runIds = scope.scopeType === "project"- ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore)- : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(- eq(heartbeatRuns.companyId, scope.companyId),- scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined,- scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined,- inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),- )).then((rows) => rows.map((row) => row.id));- for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope });- await cancelPendingWakeupsForBudgetScope(scope);- }-return {waitForRunExecutionDrain: async (runId: string,diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.tsnew file mode 100644index 0000000000..7af48aeee5--- /dev/null+++ b/server/src/services/heartbeat/run-control.test.ts@@ -0,0 +1,310 @@+import { randomUUID } from "node:crypto";+import type { Db } from "@paperclipai/db";+import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects } from "@paperclipai/db";+import { eq, sql } from "drizzle-orm";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import { runningProcesses } from "../../adapters/index.js";+import { adapterExecutionControls } from "../adapter-execution-control.js";+import { terminateLocalService } from "../local-service-supervisor.js";+import { createHeartbeatRunState } from "./run-state.js";+import { createHeartbeatRunPreparation } from "./run-preparation.js";+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";+import { createHeartbeatRunControl, type HeartbeatRunControlDependencies } from "./run-control.js";++vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));+vi.mock("../local-service-supervisor.js", async (importOriginal) => ({+ ...await importOriginal<typeof import("../local-service-supervisor.js")>(),+ terminateLocalService: vi.fn(async () => undefined),+}));++type Run = typeof heartbeatRuns.$inferSelect;+function callbacks(db: Db) {+ const state = createHeartbeatRunState(db);+ const lifecycleDependencies = {+ ...state,+ ...createHeartbeatRunPreparation(db),+ issuesSvc: {+ addComment: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["addComment"]>(),+ listReviewAttention: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["listReviewAttention"]>(),+ },+ recovery: { escalateStrandedAssignedIssue: vi.fn<HeartbeatLifecycleDependencies["recovery"]["escalateStrandedAssignedIssue"]>() },+ companySkills: { completeTestRunForIssue: vi.fn<HeartbeatLifecycleDependencies["companySkills"]["completeTestRunForIssue"]>() },+ budgets: { getInvocationBlock: vi.fn(async () => null) },+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },+ enqueueWakeup: vi.fn<HeartbeatLifecycleDependencies["enqueueWakeup"]>(),+ getCurrentUserRedactionOptions: vi.fn(async () => ({})),+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),+ emitTerminalAgentTaskRun: vi.fn<HeartbeatLifecycleDependencies["emitTerminalAgentTaskRun"]>(),+ budgetHooks: {},+ } satisfies HeartbeatLifecycleDependencies;+ const lifecycle = createHeartbeatLifecycle(db, lifecycleDependencies);+ return {+ ...state,+ ...lifecycle,+ getRun: vi.fn(state.getRun),+ getAgent: vi.fn(state.getAgent),+ setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive),+ setRunStatus: vi.fn(lifecycle.setRunStatus),+ setWakeupStatus: vi.fn(lifecycle.setWakeupStatus),+ appendRunEvent: vi.fn(lifecycle.appendRunEvent),+ finalizeAgentStatus: vi.fn<HeartbeatRunControlDependencies["finalizeAgentStatus"]>(),+ options: { closeWarmNativeSessionsForRun: vi.fn(async () => ({ closed: 1, busy: 0, failed: 0 })) },+ envOrchestrator: { releaseForRun: vi.fn<HeartbeatRunControlDependencies["envOrchestrator"]["releaseForRun"]>(async () => ({ released: [], errors: [] })) },+ enqueueWakeup: lifecycleDependencies.enqueueWakeup,+ startNextQueuedRunForAgent: vi.fn<HeartbeatRunControlDependencies["startNextQueuedRunForAgent"]>(),+ sweepPendingCleanupLeases: vi.fn<HeartbeatRunControlDependencies["sweepPendingCleanupLeases"]>(),+ timerClaimWasFirstHeartbeat: vi.fn<HeartbeatRunControlDependencies["timerClaimWasFirstHeartbeat"]>(() => undefined),+ wakeQueue: { releaseIssueExecution: vi.fn<HeartbeatRunControlDependencies["wakeQueue"]["releaseIssueExecution"]>(async () => ({ outcome: { kind: "released" }, postCommitEffects: [] })) },+ applyWakeQueuePostCommitEffects: vi.fn(async () => undefined),+ getSchedulingSuppression: vi.fn(async () => ({ suppressed: false })),+ activeRunExecutions: new Set<string>(),+ processRunCancellationSettlements: new Map<string, { settled: Promise<void>; failed: boolean; error?: unknown }>(),+ failedProcessRunCancellations: new Map<string, { settled: Promise<void>; failed: boolean; error?: unknown }>(),+ } satisfies HeartbeatRunControlDependencies;+}+function guardedDatabase() {+ const access = vi.fn(() => { throw new Error("Unexpected run-control database access"); });+ return { db: new Proxy({}, { get: access }) as Db, access };+}++describe("heartbeat run-control module boundary", () => {+ afterEach(() => adapterExecutionControls.clear());++ it("constructs without database access, cancellation, or cleanup", () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ createHeartbeatRunControl(database.db, deps);+ expect(database.access).not.toHaveBeenCalled();+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();+ expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();+ });++ it("keeps an already completed run out of cancellation effects", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { id: "finished", status: "succeeded", runtimeMode: "legacy" } as Run;+ deps.getRun.mockResolvedValue(run);+ expect(await createHeartbeatRunControl(database.db, deps).cancelRunInternal(run.id)).toBe(run);+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.getAgent).not.toHaveBeenCalled();+ expect(deps.appendRunEvent).not.toHaveBeenCalled();+ });++ it("keeps a suppressed saved-input scan out of the database and wake admission", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true });+ await createHeartbeatRunControl(database.db, deps).resumeExecutionWaitComments();+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();+ });++ it("ignores an empty cancellation batch", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ expect(await createHeartbeatRunControl(database.db, deps).cancelInvocationsForAgentsInternal(["", ""], "pause")).toEqual({ agentIds: [], runsCancelled: 0, wakeupsCancelled: 0 });+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.getAgent).not.toHaveBeenCalled();+ });++ it("waits for an owned adapter before admitting saved comments", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { id: "owned", status: "cancelled", runtimeMode: "legacy" } as Run;+ adapterExecutionControls.set(run.id, {} as never);+ await createHeartbeatRunControl(database.db, deps).resumeRemoteStopComments(run);+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();+ });++ it("retains leases while native execution ownership is unverified", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ deps.getRun.mockResolvedValue({ runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run);+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "held", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" });+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();+ });++ it("uses the persisted terminal outcome when choosing warm resource retention", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run);+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "failed", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" });+ expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ heartbeatRunId: "failed", status: "failed", providerResourceDisposition: "stop_and_retain" }));+ expect(database.access).not.toHaveBeenCalled();+ });++ it("defers destruction when a warm session remains busy", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run);+ deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 });+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" });+ expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" });+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();+ expect(database.access).not.toHaveBeenCalled();+ });+});++const support = await getEmbeddedPostgresTestSupport();+if (!support.supported) console.warn(`Skipping run-control database tests: ${support.reason}`);+describe.skipIf(!support.supported)("heartbeat run-control database wiring", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: Db;+ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-run-control-module");+ db = createDb(database.connectionString);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => {+ await db?.$client.end();+ await database?.cleanup();+ });+ beforeEach(async () => {+ vi.mocked(terminateLocalService).mockReset().mockResolvedValue(undefined);+ await db.execute(sql`truncate table companies restart identity cascade`);+ });+ afterEach(() => runningProcesses.clear());++ async function fixture() {+ const [company] = await db.insert(companies).values({ name: "Run control", issuePrefix: "RC" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Controlled agent", adapterType: "codex_local" }).returning();+ return { company, agent };+ }+ async function liveRun(companyId: string, agentId: string) {+ const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "legacy", runtimeModeResolvedAt: new Date() }).returning();+ runningProcesses.set(run.id, { child: { pid: 424242 } as never, graceSec: 4, processGroupId: 424242 });+ return run;+ }++ it("reports a missing run without entering cancellation", async () => {+ const deps = callbacks(db);+ await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found");+ expect(deps.getAgent).not.toHaveBeenCalled();+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();+ expect(deps.appendRunEvent).not.toHaveBeenCalled();+ });++ it("joins concurrent Stop requests across factories through the shared process barrier", async () => {+ const { company, agent } = await fixture();+ const run = await liveRun(company.id, agent.id);+ const deps = callbacks(db);+ const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get");+ let releaseTermination!: () => void;+ const terminating = new Promise<void>(resolve => { releaseTermination = resolve; });+ vi.mocked(terminateLocalService).mockReturnValue(terminating);+ const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop", { terminationGraceMs: 1 });+ await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true));+ const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop again");+ await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2));+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();+ expect(terminateLocalService).toHaveBeenCalledTimes(1);+ releaseTermination();+ const results = await Promise.all([first, second]);+ expect(results.map(result => result?.status)).toEqual(["cancelled", "cancelled"]);+ expect(deps.setRunStatusFromLive).toHaveBeenCalledTimes(1);+ expect(deps.appendRunEvent).toHaveBeenCalledTimes(1);+ expect(deps.finalizeAgentStatus).toHaveBeenCalledTimes(1);+ expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledTimes(1);+ expect(deps.processRunCancellationSettlements.size).toBe(0);+ expect(runningProcesses.has(run.id)).toBe(false);+ expect(terminateLocalService).toHaveBeenCalledWith({ pid: 424242, processGroupId: 424242 }, { forceAfterMs: 100, signal: "SIGINT" });+ });++ it("settles every joined Stop caller with the same termination failure", async () => {+ const { company, agent } = await fixture();+ const run = await liveRun(company.id, agent.id);+ const deps = callbacks(db);+ const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get");+ deps.activeRunExecutions.add(run.id);+ let rejectTermination!: (error: Error) => void;+ vi.mocked(terminateLocalService).mockReturnValue(new Promise<void>((_resolve, reject) => { rejectTermination = reject; }));+ const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id);+ await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true));+ const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id);+ const results = Promise.allSettled([first, second]);+ await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2));+ const failure = new Error("provider did not stop");+ rejectTermination(failure);+ expect(await results).toEqual([{ status: "rejected", reason: failure }, { status: "rejected", reason: failure }]);+ expect(terminateLocalService).toHaveBeenCalledTimes(1);+ expect(deps.processRunCancellationSettlements.size).toBe(0);+ expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure);+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();+ expect((await deps.getRun(run.id))?.status).toBe("running");+ });++ it("retains failed Stop evidence and the owned process until a later request succeeds", async () => {+ const { company, agent } = await fixture();+ const run = await liveRun(company.id, agent.id);+ const deps = callbacks(db);+ deps.activeRunExecutions.add(run.id);+ const failure = new Error("termination failed");+ vi.mocked(terminateLocalService).mockRejectedValueOnce(failure);+ await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).rejects.toBe(failure);+ expect(deps.failedProcessRunCancellations.get(run.id)).toMatchObject({ failed: true, error: failure });+ expect(deps.processRunCancellationSettlements.size).toBe(0);+ expect(runningProcesses.has(run.id)).toBe(true);+ expect((await deps.getRun(run.id))?.status).toBe("running");+ expect(deps.finalizeAgentStatus).not.toHaveBeenCalled();+ expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" });+ expect(terminateLocalService).toHaveBeenCalledTimes(2);+ // The executor still owns removal of this evidence after it observes Stop.+ expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure);+ });++ it("deduplicates agent batches and cancels only unclaimed pending wakes", async () => {+ const { company, agent } = await fixture();+ const [otherAgent] = await db.insert(agents).values({ companyId: company.id, name: "Other agent" }).returning();+ const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "succeeded" }).returning();+ const wakes = await db.insert(agentWakeupRequests).values([+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" },+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "deferred_issue_execution" },+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" },+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", runId: finished.id },+ { companyId: company.id, agentId: otherAgent.id, source: "assignment", status: "queued" },+ ]).returning();+ const deps = callbacks(db);+ expect(await createHeartbeatRunControl(db, deps).cancelInvocationsForAgentsInternal([agent.id, "", agent.id], "pause")).toEqual({ agentIds: [agent.id], runsCancelled: 0, wakeupsCancelled: 2 });+ const persisted = await db.select().from(agentWakeupRequests);+ expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "cancelled", "claimed", "queued", "queued"]);+ expect(deps.getAgent).toHaveBeenCalledTimes(1);+ });++ it("rechecks stale budget enforcement before cancelling runs or queued wakes", async () => {+ const { company, agent } = await fixture();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", runtimeMode: "legacy" }).returning();+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }).returning();+ const deps = callbacks(db);+ await createHeartbeatRunControl(db, deps).cancelBudgetScopeWork({ companyId: company.id, scopeType: "agent", scopeId: agent.id, enforcement: { policyId: randomUUID(), version: 1 } });+ expect((await deps.getRun(run.id))?.status).toBe("queued");+ expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued");+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();+ expect(deps.finalizeAgentStatus).not.toHaveBeenCalled();+ expect(terminateLocalService).not.toHaveBeenCalled();+ });++ it("limits project budget cancellation by company, effective project, and admission cutoff", async () => {+ const { company, agent } = await fixture();+ const [otherCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning();+ const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Other company agent" }).returning();+ const projectRows = await db.insert(projects).values([+ { companyId: company.id, name: "Target" }, { companyId: company.id, name: "Override" },+ ]).returning();+ const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: projectRows[0].id, title: "Bound task", issueNumber: 1, identifier: "RC-1" }).returning();+ const old = new Date("2026-01-01T00:00:00Z");+ const cutoff = new Date("2026-01-02T00:00:00Z");+ const wakes = await db.insert(agentWakeupRequests).values([+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id } },+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id, projectId: projectRows[1].id } },+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: cutoff, payload: { projectId: projectRows[0].id } },+ { companyId: otherCompany.id, agentId: otherAgent.id, source: "assignment", status: "queued", createdAt: old, payload: { projectId: projectRows[0].id } },+ ]).returning();+ await createHeartbeatRunControl(db, callbacks(db)).cancelBudgetScopeWork({ companyId: company.id, scopeType: "project", scopeId: projectRows[0].id, createdBefore: cutoff });+ const persisted = await db.select().from(agentWakeupRequests);+ expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "queued", "queued", "queued"]);+ });+});diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.tsnew file mode 100644index 0000000000..dd3f9d8dc3--- /dev/null+++ b/server/src/services/heartbeat/run-control.ts@@ -0,0 +1,1360 @@+import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";+import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js";+import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js";+import {+ type WakeupOptions,+} from "./run-preparation.js";+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";+import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";+import {+ nativeRetryCancellationCommitCondition,+ claimCancellationRequest,+} from "../native-runtime/native-cancellation-request.js";+import {+ canContinueCancelledRun,+ readRunCancellation,+ requestedRunCancellation,+} from "../run-cancellation.js";+import { settleSlackConversation } from "../slack-conversation-lifecycle.js";+import { readQueuedInteractionResponse } from "../queued-interaction-response.js";+import {+ isAcknowledgedNativeStop,+ acknowledgedNativeStopExecutionHasStopped,+} from "../acknowledged-native-stop.js";+import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js";+import {+ hasRemoteTerminationReceipt,+ remoteExecutionHasStopped,+ stoppedRemoteCleanupScopes,+} from "../remote-execution-termination.js";+import {+ admitExplicitNativeContinuation,+ undeliveredLegacyUserCommentIds,+} from "../explicit-native-continuation.js";+import { isCancelledNativeStartup } from "../cancelled-native-startup.js";+import {+ executionBlockerPredicate,+ getExecutionBlocker,+} from "../execution-blocker.js";+import {+ CONVERSATION_CONTINUATION_POLICY,+ runUsedConversationAdapter,+ hasConversationContinuationPolicy,+} from "../conversation-continuation.js";+import {+ adapterExecutionControls,+ captureAdapterStopOwnership,+ waitForAdapterStop,+} from "../adapter-execution-control.js";+import { randomUUID } from "node:crypto";+import {+ and,+ asc,+ desc,+ eq,+ exists,+ gt,+ inArray,+ isNull,+ isNotNull,+ lt,+ lte,+ ne,+ notInArray,+ or,+ sql,+} from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import {+ type EnvironmentLeaseStatus,+} from "@paperclipai/shared";+import {+ agentWakeupRequests,+ companies,+ environmentLeases,+ heartbeatRuns,+ issueComments,+ issueRecoveryActions,+ issues,+ nativeRunFinalizations,+} from "@paperclipai/db";+import {+ conflict,+ HttpError,+ notFound,+} from "../../errors.js";+import { logger } from "../../middleware/logger.js";+import {+ queuedCommentIdsFromWakePayload,+ withQueuedCommentIdsInWakePayload,+ withQueuedCommentIdsInRunContext,+} from "../issue-queued-comment-queue.js";+import {+ cancelNativeSession,+ closeWarmNativeSessionsForEnvironment,+} from "../native-runtime/index.js";+import { runningProcesses } from "../../adapters/index.js";+import { parseObject } from "../../adapters/utils.js";+import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js";+import {+ withCurrentBudgetEnforcement,+ type BudgetEnforcementScope,+} from "../budgets.js";+import { terminateLocalService } from "../local-service-supervisor.js";+import { deriveCommentId } from "../../modules/run-dispatch/index.js";+import { WakeQueueApplicationError } from "../../modules/wake-queue/index.js";+import { isUuidLike } from "@paperclipai/shared";+import {+ type ProviderResourceDisposition,+} from "../environment-runtime.js";++import type { createHeartbeatRunState } from "./run-state.js";+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";+import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js";+import type { createHeartbeatQueue } from "./queue.js";+import type { createHeartbeatRecovery } from "./recovery.js";+import type { createWakeQueue, PostCommitEffect as WakeQueuePostCommitEffect } from "../../modules/wake-queue/index.js";+import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";++type ProcessRunCancellationSettlement = {+ settled: Promise<void>;+ failed: boolean;+ error?: unknown;+};++/** Cancellation shares executor ownership and delegates lifecycle and admission effects. */+export interface HeartbeatRunControlDependencies extends Pick<ReturnType<typeof createHeartbeatRunState>,+ "getRun" | "getAgent">,+ Pick<ReturnType<typeof createHeartbeatLifecycle>,+ "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">,+ Pick<HeartbeatRetryDependencies, "finalizeAgentStatus"> {+ options: {+ closeWarmNativeSessionsForRun?: (input: {+ runId: string;+ reason: string;+ }) => Promise<{ closed: number; busy: number; failed: number }>;+ };+ envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "releaseForRun">;+ enqueueWakeup: ReturnType<typeof createHeartbeatQueue>["enqueueWakeup"];+ startNextQueuedRunForAgent: ReturnType<typeof createHeartbeatQueue>["startNextQueuedRunForAgent"];+ sweepPendingCleanupLeases: ReturnType<typeof createHeartbeatRecovery>["sweepPendingCleanupLeases"];+ timerClaimWasFirstHeartbeat: ReturnType<typeof createHeartbeatRetries>["timerClaimWasFirstHeartbeat"];+ wakeQueue: Pick<ReturnType<typeof createWakeQueue>, "releaseIssueExecution">;+ applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise<void>;+ getSchedulingSuppression: () => Promise<{ suppressed: boolean }>;+ // Scheduler and route service instances must retain the same in-process barriers.+ activeRunExecutions: Set<string>;+ processRunCancellationSettlements: Map<string, ProcessRunCancellationSettlement>;+ failedProcessRunCancellations: Map<string, ProcessRunCancellationSettlement>;+}++export function leaseReleaseStatusForRunStatus(+ status: string | null | undefined,+): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {+ if (status === "cancelled") return "expired";+ return status === "failed" || status === "timed_out" ? "failed" : "released";+}++export function providerResourceDispositionForTerminalRun(+ desired: ProviderResourceDisposition | undefined,+ status: string | null | undefined,+): ProviderResourceDisposition | undefined {+ if (desired !== "keep_running") return desired;+ return status === "succeeded" ? desired : "stop_and_retain";+}++export async function terminateHeartbeatRunProcess(input: {+ pid: number | null | undefined;+ processGroupId: number | null | undefined;+ graceMs?: number;+ signal?: NodeJS.Signals;+}) {+ const pid = input.pid ?? null;+ const processGroupId = input.processGroupId ?? null;+ if (typeof pid !== "number" && typeof processGroupId !== "number") return;++ await terminateLocalService(+ {+ pid:+ typeof pid === "number" && Number.isInteger(pid) && pid > 0+ ? pid+ : (processGroupId ?? 0),+ processGroupId:+ typeof processGroupId === "number" &&+ Number.isInteger(processGroupId) &&+ processGroupId > 0+ ? processGroupId+ : null,+ },+ { forceAfterMs: input.graceMs, signal: input.signal },+ );+}++export async function cancelHeartbeatNativeRun(input: {+ db: Db;+ runId: string;+ reason: string;+ runtimeMode: string | null;+ cancellationRequestId?: string;+ cancel?: (+ runId: string,+ reason: string,+ options: { db: Db; scope: "run"; cancellationRequestId?: string },+ ) => Promise<{ decision: unknown | null; auditId: string | null }>;+}) {+ if (input.runtimeMode !== "native") {+ return { decision: null, auditId: null };+ }+ const cancellation = input.cancel+ ? await input.cancel(input.runId, input.reason, {+ db: input.db,+ scope: "run",+ ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),+ })+ : await cancelNativeSession(input.runId, input.reason, {+ db: input.db,+ scope: "run",+ ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),+ });+ if (!cancellation.decision || !cancellation.auditId) {+ throw new Error("native_cancellation_outcome_not_audited");+ }+ return cancellation;+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++export type CancelRunOptions = {+ budgetEnforcement?: BudgetEnforcementScope;+ /** Optional board request identity, atomically reserved for native Stop. */+ cancellationRequestId?: string;+ cancellationRequestedByUserId?: string | null;+ errorCode?: string;+ resultJson?: Record<string, unknown>;+ eventMessage?: string;+ eventPayload?: Record<string, unknown>;+ /** Per-call graceful process shutdown window, bounded to a safe range. */+ terminationGraceMs?: number;+ /** Caller is immediately scheduling an explicit successor path. */+ suppressImmediateRecovery?: boolean;+ };++/** Run control binds cancellation, resource release, and saved-input resumption. */+export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunControlDependencies) {+ const {+ getRun,+ options,+ envOrchestrator,+ activeRunExecutions,+ enqueueWakeup,+ getAgent,+ sweepPendingCleanupLeases,+ getSchedulingSuppression,+ wakeQueue,+ applyWakeQueuePostCommitEffects,+ processRunCancellationSettlements,+ mergeRunStopMetadataForAgent,+ setRunStatusFromLive,+ failedProcessRunCancellations,+ setWakeupStatus,+ appendRunEvent,+ finalizeAgentStatus,+ timerClaimWasFirstHeartbeat,+ startNextQueuedRunForAgent,+ setRunStatus,+ } = dependencies;++ async function releaseEnvironmentLeasesForRun(input: {+ runId: string;+ companyId: string;+ agentId: string;+ status: string | null | undefined;+ failureReason?: string | null;+ providerResourceDisposition?: ProviderResourceDisposition;+ nativeLifecycleTelemetry?: {+ provider: string;+ harness: string;+ lifecycleMode: "per_turn" | "warm";+ sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";+ };+ }) {+ const leaseOwnerRun = await getRun(input.runId);+ if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;+ // Recovery can finish workspace copy-back outside the executor's finally.+ // Successful copy-back does not earn warm retention for a failed turn.+ const status = leaseOwnerRun?.status ?? input.status;+ const providerResourceDisposition = providerResourceDispositionForTerminalRun(+ input.providerResourceDisposition,+ status,+ );+ if (providerResourceDisposition === "destroy") {+ const closeResult = await (+ options.closeWarmNativeSessionsForRun ??+ (async ({ runId, reason }) => {+ const environmentIds = await db+ .selectDistinct({ environmentId: environmentLeases.environmentId })+ .from(environmentLeases)+ .where(+ and(+ eq(environmentLeases.heartbeatRunId, runId),+ eq(environmentLeases.status, "active"),+ ),+ )+ .then((rows) =>+ rows.flatMap((row) =>+ typeof row.environmentId === "string" &&+ row.environmentId.length > 0+ ? [row.environmentId]+ : [],+ ),+ );+ const aggregate = { closed: 0, busy: 0, failed: 0 };+ for (const environmentId of environmentIds) {+ const result = await closeWarmNativeSessionsForEnvironment({+ environmentId,+ reason,+ });+ aggregate.closed += result.closed;+ aggregate.busy += result.busy;+ aggregate.failed += result.failed;+ }+ return aggregate;+ })+ )({+ runId: input.runId,+ reason: "terminal heartbeat run destroyed its environment lease",+ }).catch((err) => {+ logger.warn(+ { err, runId: input.runId },+ "failed to close warm native sessions before environment lease destruction",+ );+ return { closed: 0, busy: 0, failed: 1 };+ });+ // A failed remote checkpoint cannot veto destruction of the isolated+ // sandbox after the run stopped. Provider destruction supplies the exit+ // proof; it does not turn the interrupted checkpoint into a success.+ const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&+ ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)+ ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(+ eq(environmentLeases.companyId, input.companyId),+ eq(environmentLeases.heartbeatRunId, input.runId),+ ))+ : [];+ const canDestroyRemote = remoteLeases.length > 0 &&+ remoteLeases.every(lease => lease.provider && lease.provider !== "local");+ if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {+ logger.warn(+ { runId: input.runId, warmNativeSessions: closeResult },+ "deferred environment lease destruction until warm native sessions close",+ );+ return;+ }+ }+ const releaseResult = await envOrchestrator+ .releaseForRun({+ heartbeatRunId: input.runId,+ companyId: input.companyId,+ agentId: input.agentId,+ status: leaseReleaseStatusForRunStatus(status),+ failureReason: input.failureReason ?? undefined,+ providerResourceDisposition,+ nativeLifecycleTelemetry: input.nativeLifecycleTelemetry,+ })+ .catch((err) => {+ logger.warn(+ { err, runId: input.runId },+ "failed to release environment leases for heartbeat run",+ );+ return null;+ });+ for (const releaseError of releaseResult?.errors ?? []) {+ logger.warn(+ {+ err: releaseError.error,+ leaseId: releaseError.leaseId,+ runId: input.runId,+ },+ "failed to release environment lease for heartbeat run",+ );+ }+ await acknowledgeRemoteStop(input.runId, input.companyId);+ }++ async function acknowledgeRemoteStop(runId: string, companyId: string) {+ // The provider receipt arrives after adapter settlement. A remote ACP child+ // has no host PID, so only this target-aware boundary can acknowledge Stop.+ const stopped = await getRun(runId);+ if (stopped?.runtimeMode === "native") {+ const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId);+ for (const remoteCleanupScope of scopes ?? []) {+ completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });+ }+ }+ if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" &&+ parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&+ await runUsedConversationAdapter(db, stopped) &&+ await remoteExecutionHasStopped(db, companyId, runId)) {+ await db.update(heartbeatRuns).set({+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({+ executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),+ state: "acknowledged", acknowledgedAt: new Date().toISOString(),+ proof: "provider_termination_receipt" },+ conversationContinuation: CONVERSATION_CONTINUATION_POLICY,+ })}::jsonb`,+ updatedAt: new Date(),+ }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),+ eq(heartbeatRuns.status, "cancelled")));+ }+ }++ async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) {+ if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;+ const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt+ ? await db.select().from(nativeRunFinalizations).where(and(+ eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),+ )) : [];+ const cancelledPreparation = run.runtimeMode === "legacy" &&+ await isCancelledNativeStartup(db, run, preparationCoordinator);+ if (run.runtimeMode !== "native" && !cancelledPreparation &&+ parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true &&+ !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return;+ const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);+ if (!issueId) return;+ const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null;+ const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase,+ leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and(+ eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),+ )) : [];+ // Stop ends one response. Saved user input can enter ordinary admission+ // once its executor settles; it does not need a manufactured crash incident.+ // The ordinary path still enforces task holds, ownership, and native session+ // cleanup before starting a provider.+ const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) &&+ !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner &&+ ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") &&+ await acknowledgedNativeStopExecutionHasStopped(db, currentRun) &&+ !(await getExecutionBlocker(db, run.companyId, issueId));+ const legacyContinuation = run.runtimeMode === "legacy" &&+ hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) &&+ !(await getExecutionBlocker(db, run.companyId, issueId));+ if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return;+ const pending = await db.select().from(agentWakeupRequests).where(and(+ eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),+ eq(agentWakeupRequests.status, "deferred_issue_execution"),+ eq(agentWakeupRequests.requestedByActorType, "user"),+ requestId ? eq(agentWakeupRequests.id, requestId) : undefined,+ sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,+ )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50);+ for (const wake of pending) {+ if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue;+ let payload = parseObject(wake.payload);+ if (payload.queuedCommentInterrupt) {+ await resumeQueuedCommentInterrupt(wake.companyId, wake.id);+ continue;+ }+ let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);+ let commentId = deriveCommentId(context, payload);+ let requestedByActorId = wake.requestedByActorId;+ const reason = readNonEmptyString(context.wakeReason) ?? wake.reason;+ if (run.runtimeMode === "native" || cancelledPreparation) {+ const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId,+ queuedCommentIdsFromWakePayload(payload));+ if (!ids.length) continue;+ payload = withQueuedCommentIdsInWakePayload(payload, ids);+ context = withQueuedCommentIdsInRunContext(context, ids);+ commentId = ids.at(-1)!;+ // A coalesced queue can contain several authors. Its saved comments,+ // not the outer wake's first author, authorize the remaining input.+ const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and(+ eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId),+ ));+ requestedByActorId = author?.id ?? null;+ }+ if (legacyContinuation || stoppedNativeContinuation) {+ if (!commentId || !run.finishedAt || !requestedByActorId ||+ !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue;+ const [comment] = await db.select().from(issueComments).where(and(+ eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId),+ sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"),+ eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt),+ isNull(issueComments.createdByRunId),+ stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt),+ ));+ if (!comment?.body.trim()) continue;+ } else {+ let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." };+ const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId,+ agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId,+ reason, commentId, successorRunId: randomUUID(), dryRun: true,+ ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),+ onBlocked: (reason, message) => { wait = { reason, message }; },+ });+ if (!admitted) {+ await db.update(agentWakeupRequests).set({+ payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',+ coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,+ updatedAt: new Date(),+ }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),+ eq(agentWakeupRequests.status, "deferred_issue_execution")));+ continue;+ }+ }+ // Re-enter ordinary admission with the original user's authority. It+ // atomically adopts the deferred comments and still applies every gate.+ await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"],+ reason, payload, contextSnapshot: context,+ requestedByActorType: "user", requestedByActorId,+ ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),+ idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id);+ break;+ }+ }++ async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) {+ return resumeSavedLegacyComments(companyId, queueId, true, opts);+ }++ async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) {+ const [wake] = await db.select().from(agentWakeupRequests).where(and(+ eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),+ eq(agentWakeupRequests.status, "deferred_issue_execution"),+ ));+ if (!wake) return;+ const payload = parseObject(wake.payload);+ let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId);+ let commentIds = queuedCommentIdsFromWakePayload(payload);+ const issueId = readNonEmptyString(payload.issueId);+ if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return;+ const response = await readQueuedInteractionResponse(db, companyId, issueId, payload);+ if (!commentIds.length && !response) return;+ // Resolved cards are immutable input. Only an explicit Interrupt click can+ // authorize continuation across a stopped execution; ordinary completion+ // uses normal deferred-wake promotion.+ if (response && !interrupted) return;+ if (!interrupted) {+ commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds);+ if (!commentIds.length) return;+ // The queue itself may have begun as a system wake. The saved human+ // comment, not that wake's origin or mutable caller payload, is authority.+ const [comment] = await db.select().from(issueComments).where(and(+ eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),+ eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"),+ isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),+ )).orderBy(desc(issueComments.createdAt)).limit(1);+ if (!comment?.body.trim() || !comment.authorUserId) return;+ actorId = comment.authorUserId;+ }+ if (!actorId) return;+ const agent = await getAgent(wake.agentId);+ if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return;+ const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(+ eq(heartbeatRuns.companyId, companyId),+ eq(heartbeatRuns.agentId, wake.agentId),+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,+ inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),+ )).limit(1);+ if (active) return;+ if (interrupted && opts?.retryCleanup) {+ // Only the HTTP click grants an extra cleanup attempt. Periodic retries+ // reuse the intent to deliver, never a fresh provider teardown budget.+ const sourceRun = await db.transaction(async tx => {+ const [task] = await tx.select().from(issues).where(and(+ eq(issues.companyId, companyId), eq(issues.id, issueId),+ )).for("update");+ const [current] = await tx.select().from(agentWakeupRequests).where(and(+ eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),+ eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"),+ sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,+ sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`,+ ));+ if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) ||+ !current || (!queuedCommentIdsFromWakePayload(current.payload).length &&+ !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null;+ const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(+ eq(heartbeatRuns.companyId, companyId),+ eq(heartbeatRuns.agentId, wake.agentId),+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,+ inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),+ )).limit(1);+ if (successor) return null;+ const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId);+ const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and(+ eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId),+ eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"),+ inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]),+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,+ )).then(rows => rows[0]) : null;+ if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null;+ // Older ephemeral leases recorded successful cleanup without a provider+ // receipt. Re-verify them through the recorded teardown path; a timestamp+ // alone never certifies termination. Retained/reusable resources stay put.+ const historical = await tx.select().from(environmentLeases).where(and(+ eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id),+ eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt),+ inArray(environmentLeases.status, ["released", "expired", "failed"]),+ )).for("update");+ for (const lease of historical) {+ if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue;+ // Provider resource IDs identify physical sandboxes. A lease in any+ // company can still own this resource; never destroy it on behalf of+ // this company. This existence-only guard exposes no foreign data.+ const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(+ ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId),+ or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])),+ )).limit(1);+ if (otherOwner) continue;+ await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() })+ .where(eq(environmentLeases.id, lease.id));+ }+ return run;+ });+ if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: {+ companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt",+ } });+ }+ const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds);+ delete deliveryPayload.queuedCommentInterrupt;+ await enqueueWakeup(wake.agentId, {+ source: "on_demand", triggerDetail: "manual", reason: "issue_commented",+ payload: deliveryPayload, contextSnapshot: response+ ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId,+ responsibleUserId: actorId }+ : withQueuedCommentIdsInRunContext({+ issueId, triggeredBy: "board", actorId, responsibleUserId: actorId,+ }, commentIds),+ requestedByActorType: "user", requestedByActorId: actorId,+ ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }),+ issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] },+ idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`,+ }, queueId);+ }++ async function resumeExecutionWaitComments() {+ if ((await getSchedulingSuppression()).suppressed) return;+ const waits = await db.select({ wake: agentWakeupRequests })+ .from(agentWakeupRequests)+ .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),+ sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,+ eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))+ .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))+ .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(+ eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),+ executionBlockerPredicate(),+ ))), eq(agentWakeupRequests.status, "deferred_issue_execution"),+ eq(agentWakeupRequests.requestedByActorType, "user"),+ sql`${agentWakeupRequests.payload}->'executionWait' is not null`,+ sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`,+ lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),+ notInArray(issues.status, ["done", "cancelled"])))+ .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);+ const seen = new Set<string>();+ for (const { wake } of waits) {+ const issueId = String(wake.payload?.issueId);+ if (seen.has(issueId)) continue;+ seen.add(issueId);+ // Advance the cursor even for invalid evidence so one damaged task cannot+ // starve later requests in this bounded scan. Preserve concurrent edits.+ const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and(+ eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId),+ eq(agentWakeupRequests.status, "deferred_issue_execution"),+ lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),+ )).returning({ id: agentWakeupRequests.id });+ if (!claimed) continue;+ // Match normal admission's deterministic current blocker selection. An+ // arbitrary historical action must not choose the retry's source run.+ const blocker = await getExecutionBlocker(db, wake.companyId, issueId);+ const sourceId = blocker?.runId;+ if (!sourceId || !isUuidLike(sourceId)) continue;+ const run = await getRun(sourceId, { includeExecutionEvidence: true });+ if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;+ if (canContinueCancelledRun(run)) {+ await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {+ logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");+ });+ continue;+ }+ await resumeRemoteStopComments(run, wake.id).catch(err => {+ logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");+ });+ }+ }++ async function releaseIssueExecutionAndPromote(+ run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">,+ options: { suppressImmediateRecovery?: boolean } = {},+ ) {+ try {+ const source = await getRun(run.id);+ const { postCommitEffects } = await wakeQueue.releaseIssueExecution({+ companyId: run.companyId,+ runId: run.id,+ now: new Date(),+ // A durable authentication card owns recovery. This covers the review path,+ // while continuation classification blocks periodic generic retries.+ suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source),+ });+ await applyWakeQueuePostCommitEffects(postCommitEffects);+ const completed = await getRun(run.id);+ const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId)+ ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId;+ if (completed?.status === "succeeded" && issueId) {+ await settleSlackConversation(db, run.companyId, issueId).catch((err) => {+ logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation");+ });+ }+ } catch (error) {+ if (+ error instanceof WakeQueueApplicationError &&+ error.code === "responsible_user_unresolved"+ ) {+ // Every other `responsible_user_unresolved` HttpError in this file+ // carries `code` inside its own `details`; match that shape here too.+ throw new HttpError(422, error.message, { code: error.code, ...error.details });+ }+ throw error;+ }+ }++ async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) {+ const runIssueId = sql<+ string | null+ >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`;+ const effectiveProjectId = sql<+ string | null+ >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;++ const rows = await db+ .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })+ .from(heartbeatRuns)+ .leftJoin(+ issues,+ and(+ eq(issues.companyId, companyId),+ sql`${issues.id}::text = ${runIssueId}`,+ ),+ )+ .where(+ and(+ eq(heartbeatRuns.companyId, companyId),+ createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined,+ inArray(heartbeatRuns.status, [+ ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,+ ]),+ sql`${effectiveProjectId} = ${projectId}`,+ ),+ );++ return rows.map((row) => row.id);+ }++ async function listProjectScopedWakeupIds(+ companyId: string,+ projectId: string,+ database: Db = db,+ ) {+ const wakeIssueId = sql<+ string | null+ >`${agentWakeupRequests.payload} ->> 'issueId'`;+ const effectiveProjectId = sql<+ string | null+ >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;++ const rows = await database+ .selectDistinctOn([agentWakeupRequests.id], {+ id: agentWakeupRequests.id,+ })+ .from(agentWakeupRequests)+ .leftJoin(+ issues,+ and(+ eq(issues.companyId, companyId),+ sql`${issues.id}::text = ${wakeIssueId}`,+ ),+ )+ .where(+ and(+ eq(agentWakeupRequests.companyId, companyId),+ inArray(agentWakeupRequests.status, [+ "queued",+ "deferred_issue_execution",+ ]),+ sql`${agentWakeupRequests.runId} is null`,+ sql`${effectiveProjectId} = ${projectId}`,+ ),+ );++ return rows.map((row) => row.id);+ }++ async function cancelPendingWakeupsForBudgetScope(+ scope: BudgetEnforcementScope,+ ) {+ return withCurrentBudgetEnforcement(db, scope, async (tx) => {+ const now = new Date();+ let wakeupIds: string[] = [];++ if (scope.scopeType === "company") {+ wakeupIds = await tx+ .select({ id: agentWakeupRequests.id })+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, scope.companyId),+ inArray(agentWakeupRequests.status, [+ "queued",+ "deferred_issue_execution",+ ]),+ sql`${agentWakeupRequests.runId} is null`,+ ),+ )+ .then((rows) => rows.map((row) => row.id));+ } else if (scope.scopeType === "agent") {+ wakeupIds = await tx+ .select({ id: agentWakeupRequests.id })+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, scope.companyId),+ eq(agentWakeupRequests.agentId, scope.scopeId),+ inArray(agentWakeupRequests.status, [+ "queued",+ "deferred_issue_execution",+ ]),+ sql`${agentWakeupRequests.runId} is null`,+ ),+ )+ .then((rows) => rows.map((row) => row.id));+ } else {+ wakeupIds = await listProjectScopedWakeupIds(+ scope.companyId,+ scope.scopeId,+ tx,+ );+ }++ if (wakeupIds.length === 0) return 0;++ await tx+ .update(agentWakeupRequests)+ .set({+ status: "cancelled",+ finishedAt: now,+ error: "Cancelled due to budget pause",+ updatedAt: now,+ })+ .where(and(+ inArray(agentWakeupRequests.id, wakeupIds),+ inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),+ isNull(agentWakeupRequests.runId),+ scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined,+ ));++ return wakeupIds.length;+ });+ }++ function cancellationTerminationGraceMs(+ configuredGraceSec: number,+ requestedGraceMs: number | undefined,+ ) {+ const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000;+ if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) {+ return configuredGraceMs;+ }+ return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs)));+ }++ async function cancelRunInternal(+ runId: string,+ reason = "Cancelled by control plane",+ options: CancelRunOptions = {},+ ) {+ let run = await getRun(runId);+ if (!run) throw notFound("Heartbeat run not found");+ if (options.cancellationRequestId) {+ run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null);+ }+ // The caller claim checked retry eligibility under both durable row locks.+ // This is only a cancellation candidate: dispatch rechecks the coordinator+ // under lock, and the final CAS requires its own unchanged acknowledged+ // retry fence. Re-reading only retryable_failure here would strand replay.+ const pendingNativeRetry =+ run.runtimeMode === "native" && run.status === "failed" && (+ Boolean(options.cancellationRequestId) || await db+ .select({ runId: nativeRunFinalizations.runId })+ .from(nativeRunFinalizations)+ .where(+ and(+ eq(nativeRunFinalizations.runId, run.id),+ eq(nativeRunFinalizations.companyId, run.companyId),+ eq(nativeRunFinalizations.phase, "retryable_failure"),+ ),+ )+ .then((rows) => rows.length > 0)+ );+ if (+ !pendingNativeRetry &&+ !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(+ run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number],+ )+ )+ return run;+ const agent = await getAgent(run.agentId);+ const errorCode = options.errorCode ?? "cancelled";+ const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason);++ const pendingProcessCancellation = processRunCancellationSettlements.get(+ run.id,+ );+ if (pendingProcessCancellation) {+ await pendingProcessCancellation.settled;+ if (pendingProcessCancellation.failed)+ throw pendingProcessCancellation.error;+ return getRun(run.id);+ }+ const running = runningProcesses.get(run.id);+ const stopOwnership =+ run.runtimeMode !== "native"+ ? captureAdapterStopOwnership(run.id)+ : undefined;+ const control = stopOwnership?.control;+ // Capture the existing adapter owner before waiting on the run lock. Then+ // atomically fence preparation and refresh the selected runtime, so Stop+ // cannot miss a native handoff that won after its first read.+ // Established legacy processes must still be stopped if the database is+ // unavailable. Only native or not-yet-dispatched preparation needs this+ // additional durable fence before its existing cancellation path.+ if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) {+ const fence = async (tx: Db) => {+ const [fenced] = await tx.update(heartbeatRuns).set({+ // Record handoff intent before native cancellation can finalize and release+ // the run. Only its audited stop acknowledgement suppresses recovery.+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||+ ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true+ ? { reassignmentStopRequested: true } : {})}::jsonb ||+ ${JSON.stringify({ cancellation })}::jsonb ||+ jsonb_build_object('startupCancellation', jsonb_build_object(+ 'requestedAt', ${new Date().toISOString()}::text,+ 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'+ and ${heartbeatRuns.runtimeModeResolvedAt} is null+ and ${heartbeatRuns.executionStage} = 'preparing'+ and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)+ ))`,+ }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,+ pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],+ ))).returning();+ return fenced ?? null;+ };+ let fenced: typeof run | null;+ try {+ fenced = options.budgetEnforcement+ ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence)+ : await fence(db);+ } catch (error) {+ stopOwnership?.release();+ throw error;+ }+ if (!fenced) {+ stopOwnership?.release();+ return getRun(runId);+ }+ run = fenced;+ }+ const resultJson = { ...(agent+ ? {+ ...mergeRunStopMetadataForAgent(agent, "cancelled", {+ resultJson: parseObject(run.resultJson),+ errorCode,+ errorMessage: reason,+ }),+ ...(options.resultJson ?? {}),+ }+ : options.resultJson), cancellation };++ try {+ let releaseProcessCancellation: (() => void) | undefined;+ const processCancellationSettlement =+ run.runtimeMode !== "native" &&+ !control &&+ running+ ? {+ settled: new Promise<void>((resolve) => {+ releaseProcessCancellation = resolve;+ }),+ failed: false,+ error: undefined as unknown,+ }+ : undefined;+ if (processCancellationSettlement) {+ // No await between joining an existing owner above and registering ours.+ processRunCancellationSettlements.set(+ run.id,+ processCancellationSettlement,+ );+ }+ const cancellation = await (async () => {+ try {+ if (control) {+ await db+ .update(heartbeatRuns)+ .set({+ error: reason,+ errorCode,+ // Stop may have read the row before adapter settlement saved+ // a required-copyback receipt. Merge into current database JSON+ // and leave those server-owned fields to the restore recorder.+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({+ ...resultJson,+ ...(!running ? { executionCancellation: {+ state: "requested", requestedAt: new Date().toISOString(),+ } } : {}),+ }, true),+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.status, "running"),+ ),+ );+ control.controller.abort(new Error(reason));+ }+ let terminationSettled = false;+ try {+ await cancelHeartbeatNativeRun({+ db,+ runId: run.id,+ reason,+ runtimeMode: run.runtimeMode,+ ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),+ });+ if (running) {+ await terminateHeartbeatRunProcess({+ pid: running.child.pid,+ processGroupId: running.processGroupId,+ // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM+ // can leave commands in their separate process groups alive.+ signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined,+ graceMs: cancellationTerminationGraceMs(+ running.graceSec,+ options.terminationGraceMs,+ ),+ });+ }+ terminationSettled = true;+ } finally {+ if (+ (!processCancellationSettlement || terminationSettled) &&+ runningProcesses.get(run.id) === running+ ) {+ runningProcesses.delete(run.id);+ }+ }++ if (control) {+ await waitForAdapterStop(control.settled, undefined, {+ runId: run.id,+ adapterType: agent?.adapterType,+ runtimeMode: run.runtimeMode,+ abortRequested: control.controller.signal.aborted,+ }, control);+ const stopped = await getRun(run.id);+ if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) {+ if (+ parseObject(stopped.resultJson?.executionCancellation).state !==+ "acknowledged"+ ) {+ throw conflict(+ "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.",+ );+ }+ // The owned adapter already finalized this run and its lifecycle.+ // Do not replay the process cancellation side effects below.+ return { run: stopped, updated: false };+ }+ }++ const finishedAt = new Date();+ const persistedCancellationResult =+ run.runtimeMode === "native"+ ? await getRun(run.id).then((current) =>+ parseObject(current?.resultJson),+ )+ : {};+ return await setRunStatusFromLive(+ run.id,+ "cancelled",+ pendingNativeRetry+ ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"]+ : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],+ {+ finishedAt,+ error: reason,+ errorCode,+ ...(resultJson ||+ Object.keys(persistedCancellationResult).length > 0+ ? {+ resultJson: {+ ...persistedCancellationResult,+ ...(resultJson ?? {}),+ // A scheduler placeholder has no process to acknowledge.+ // Preserve its normal release policy instead of treating+ // it as an operator stop of provider work.+ ...(processCancellationSettlement && agent && running && (+ (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) ||+ (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0)+ )+ ? mergeRunStopMetadataForAgent(agent, "cancelled", {+ resultJson: {+ ...resultJson,+ executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() },+ },+ errorCode, errorMessage: reason,+ })+ : {}),+ // The native cancellation helper may have advanced a durable+ // pending intent to its acknowledged state after `run` was+ // first read. Never let that stale snapshot overwrite the+ // authoritative post-dispatch acknowledgement.+ ...(Object.hasOwn(+ persistedCancellationResult,+ "nativeCancellation",+ )+ ? {+ nativeCancellation:+ persistedCancellationResult.nativeCancellation,+ }+ : {}),+ },+ }+ : {}),+ },+ undefined,+ pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined,+ );+ } catch (error) {+ if (processCancellationSettlement) {+ processCancellationSettlement.failed = true;+ processCancellationSettlement.error = error;+ if (activeRunExecutions.has(run.id)) {+ failedProcessRunCancellations.set(+ run.id,+ processCancellationSettlement,+ );+ }+ }+ throw error;+ } finally {+ if (processCancellationSettlement) {+ if (+ processRunCancellationSettlements.get(run.id) ===+ processCancellationSettlement+ ) {+ processRunCancellationSettlements.delete(run.id);+ }+ // Always settle waiters, including termination and DB errors. A+ // failed Stop remains an error to its caller, never a cancellation+ // receipt; the signal-bearing executor may then record failure.+ releaseProcessCancellation?.();+ }+ }+ })();+ const cancelled = cancellation.run;++ if (cancellation.updated && cancelled) {+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {+ finishedAt: cancelled.finishedAt ?? new Date(),+ error: reason,+ });+ await appendRunEvent(cancelled, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: options.eventMessage ?? "run cancelled",+ payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) },+ });+ await releaseIssueExecutionAndPromote(cancelled, {+ suppressImmediateRecovery: options.suppressImmediateRecovery,+ });+ await finalizeAgentStatus(run.agentId, "cancelled", undefined, {+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ });+ await startNextQueuedRunForAgent(run.agentId);+ }+ return cancelled;+ } finally {+ stopOwnership?.release();+ }+ }++ async function cancelActiveForAgentInternal(+ agentId: string,+ reason = "Cancelled due to agent pause",+ errorCode = "cancelled",+ ) {+ const agent = await getAgent(agentId);+ const runs = await db+ .select()+ .from(heartbeatRuns)+ .where(+ and(+ eq(heartbeatRuns.agentId, agentId),+ inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),+ ),+ );++ for (const run of runs) {+ const stopOwnership =+ run.runtimeMode !== "native"+ ? captureAdapterStopOwnership(run.id)+ : undefined;+ try {+ if (stopOwnership?.control) {+ await cancelRunInternal(run.id, reason, { errorCode });+ continue;+ }+ if (run.runtimeMode === "native") {+ await db.update(heartbeatRuns).set({ resultJson:+ sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`,+ }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status)));+ await cancelHeartbeatNativeRun({+ db,+ runId: run.id,+ reason,+ runtimeMode: run.runtimeMode,+ });+ }+ const persistedCancellationResult =+ run.runtimeMode === "native"+ ? await getRun(run.id).then((current) =>+ parseObject(current?.resultJson),+ )+ : parseObject(run.resultJson);+ await setRunStatus(run.id, "cancelled", {+ finishedAt: new Date(),+ error: reason,+ errorCode,+ resultJson: {+ ...persistedCancellationResult,+ ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", {+ resultJson: persistedCancellationResult, errorCode, errorMessage: reason,+ }) : {}),+ cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason),+ },+ });++ await setWakeupStatus(run.wakeupRequestId, "cancelled", {+ finishedAt: new Date(),+ error: reason,+ });++ const running = runningProcesses.get(run.id);+ if (running) {+ await terminateHeartbeatRunProcess({+ pid: running.child.pid,+ processGroupId: running.processGroupId,+ graceMs: Math.max(1, running.graceSec) * 1000,+ });+ }+ runningProcesses.delete(run.id);+ await releaseIssueExecutionAndPromote(run);+ } finally {+ stopOwnership?.release();+ }+ }++ return runs.length;+ }++ async function cancelPendingWakeupsForAgentsInternal(+ agentIds: string[],+ reason: string,+ ) {+ const uniqueAgentIds = [...new Set(agentIds)].filter(+ (agentId) => agentId.length > 0,+ );+ if (uniqueAgentIds.length === 0) return 0;++ const now = new Date();+ const wakeupIds = await db+ .select({ id: agentWakeupRequests.id })+ .from(agentWakeupRequests)+ .where(+ and(+ inArray(agentWakeupRequests.agentId, uniqueAgentIds),+ inArray(agentWakeupRequests.status, [+ "queued",+ "deferred_issue_execution",+ ]),+ sql`${agentWakeupRequests.runId} is null`,+ ),+ )+ .then((rows) => rows.map((row) => row.id));++ if (wakeupIds.length === 0) return 0;++ await db+ .update(agentWakeupRequests)+ .set({+ status: "cancelled",+ finishedAt: now,+ error: reason,+ updatedAt: now,+ })+ .where(inArray(agentWakeupRequests.id, wakeupIds));++ return wakeupIds.length;+ }++ async function cancelInvocationsForAgentsInternal(+ agentIds: string[],+ reason: string,+ ) {+ const uniqueAgentIds = [...new Set(agentIds)].filter(+ (agentId) => agentId.length > 0,+ );+ let runsCancelled = 0;+ for (const agentId of uniqueAgentIds) {+ runsCancelled += await cancelActiveForAgentInternal(agentId, reason);+ }+ const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal(+ uniqueAgentIds,+ reason,+ );+ return {+ agentIds: uniqueAgentIds,+ runsCancelled,+ wakeupsCancelled,+ };+ }++ async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) {+ const runIds = scope.scopeType === "project"+ ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore)+ : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(+ eq(heartbeatRuns.companyId, scope.companyId),+ scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined,+ scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined,+ inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),+ )).then((rows) => rows.map((row) => row.id));+ for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope });+ await cancelPendingWakeupsForBudgetScope(scope);+ }++ return {+ cancelBudgetScopeWork,+ releaseIssueExecutionAndPromote,+ releaseEnvironmentLeasesForRun,+ acknowledgeRemoteStop,+ resumeRemoteStopComments,+ cancelRunInternal,+ resumeExecutionWaitComments,+ resumeQueuedCommentInterrupt,+ resumeSavedLegacyComments,+ cancelActiveForAgentInternal,+ cancelInvocationsForAgentsInternal,+ };+}