Skip to content

The open-source app everyone uses to manage agents at work

refactor(heartbeat): extract run cancellation and cleanup (#15685)

Dotta0ca4b7c4c061
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index f6fcad7747..35fcb16bc1 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -1290,6 +1290,16 @@ reporting and forwarding callbacks preserve construction order for the other
heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`.
Keep lifecycle policy changes separate from this extraction and adapter execution.
+Run cancellation and cleanup are in `server/src/services/heartbeat/run-control.ts`.
+It owns Stop, agent and budget cancellation, environment lease release, and
+resumption of saved comments after execution stops. `createHeartbeatRunControl`
+binds the database, lifecycle and admission callbacks, and shared executor
+barriers without starting work. The service retains the process-wide execution
+and cancellation maps so scheduler and route instances observe the same owners.
+Forwarding callbacks preserve construction order for budget, retry, recovery,
+and queue services. Existing public helpers remain re-exported by `heartbeat.ts`.
+Keep cancellation policy changes separate from this extraction and execution.
+
## Wake Context Delivery
Built-in adapters deliver wake context through the run prompt, including structured
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index be77bf2f40..616cdcdeab 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -1,3 +1,15 @@
+import {
+ cancelHeartbeatNativeRun,
+ terminateHeartbeatRunProcess,
+ providerResourceDispositionForTerminalRun,
+ createHeartbeatRunControl,
+ type CancelRunOptions,
+} from "./heartbeat/run-control.js";
+export {
+ leaseReleaseStatusForRunStatus,
+ providerResourceDispositionForTerminalRun,
+ cancelHeartbeatNativeRun,
+} from "./heartbeat/run-control.js";
import {
NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
isHeartbeatRunTerminalStatus,
@@ -25,7 +37,6 @@ export {
} from "./heartbeat/queue.js";
import {
isProcessAlive,
- CANCELLABLE_HEARTBEAT_RUN_STATUSES,
createHeartbeatRecovery,
} from "./heartbeat/recovery.js";
import {
@@ -247,10 +258,8 @@ import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/as
import { externalObjectService } from "./external-objects.js";
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
import { dotRunnerBroker } from "./dot-runner-broker.js";
-import { isAiAuthenticationBlocked } from "./ai-auth-failure.js";
+
import {
- nativeRetryCancellationCommitCondition,
- claimCancellationRequest,
startupCancellationFence,
} from "./native-runtime/native-cancellation-request.js";
import {
@@ -262,7 +271,10 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./age
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js";
-import { cancellationResultJson, canContinueCancelledRun, readRunCancellation, requestedRunCancellation } from "./run-cancellation.js";
+import {
+ cancellationResultJson,
+ readRunCancellation,
+} from "./run-cancellation.js";
import {
withNativeWorkspaceFinalizationOwnership,
NativeWorkspaceFinalizationBusyError,
@@ -278,11 +290,8 @@ import {
hasWorkspaceRestoreFailure,
} from "@paperclipai/shared";
-import { settleSlackConversation } from "./slack-conversation-lifecycle.js";
import { toolActionDeliveryService } from "./tool-action-delivery.js";
-
-import { readQueuedInteractionResponse } from "./queued-interaction-response.js";
import {
isConversation,
prepareConversationTurn,
@@ -293,29 +302,18 @@ import { getConversationConfirmationContext } from "./conversation-confirmation-
import {
recordNativeLocalProcessStop,
} from "./native-local-process-stop.js";
-import {
- isAcknowledgedNativeStop,
- acknowledgedNativeStopExecutionHasStopped,
-} from "./acknowledged-native-stop.js";
+
import {
legacyControllerBootId,
renewLegacyControllerLease,
watchLegacyControllerLease,
} from "./legacy-controller-lease.js";
-import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
+
import {
- hasRemoteTerminationReceipt,
remoteExecutionHasStopped,
- stoppedRemoteCleanupScopes,
} from "./remote-execution-termination.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
-import {
- admitExplicitNativeContinuation,
- undeliveredLegacyUserCommentIds,
-} from "./explicit-native-continuation.js";
-import {
- isCancelledNativeStartup,
-} from "./cancelled-native-startup.js";
+
import { connectionIntentService } from "./connection-intents.js";
import {
prepareManagedAiRuntime,
@@ -327,12 +325,11 @@ import {
import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared";
import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js";
import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js";
-import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js";
import {
- CONVERSATION_CONTINUATION_POLICY,
+ getExecutionBlocker,
+} from "./execution-blocker.js";
+import {
claimedAdapterType,
- runUsedConversationAdapter,
- hasConversationContinuationPolicy,
} from "./conversation-continuation.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
@@ -345,10 +342,8 @@ import {
} from "./legacy-execution-recovery.js";
import {
adapterExecutionControls,
- captureAdapterStopOwnership,
createAdapterExecutionControl,
registerAdapterExecutionControl,
- waitForAdapterStop,
} from "./adapter-execution-control.js";
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";
@@ -386,14 +381,11 @@ import {
eq,
exists,
getTableColumns,
- gt,
gte,
inArray,
isNull,
- isNotNull,
lt,
lte,
- ne,
notInArray,
or,
sql,
@@ -404,7 +396,6 @@ import {
ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
isEnvironmentDriverSupportedForAdapter,
- type EnvironmentLeaseStatus,
type ExecutionWorkspace,
type IssueExecutionMonitorClearReason,
type IssueExecutionMonitorPolicy,
@@ -419,11 +410,9 @@ import {
chatActions,
companies,
completionContracts,
- environmentLeases,
heartbeatRunEvents,
heartbeatRuns,
issueComments,
- issueRecoveryActions,
issues,
nativeRunFinalizations,
projects,
@@ -449,10 +438,7 @@ export { scrubGitCredentialText };
import { publishLiveEvent } from "./live-events.js";
import {
- queuedCommentIdsFromWakePayload,
queuedCommentIdsFromRunContext,
- withQueuedCommentIdsInWakePayload,
- withQueuedCommentIdsInRunContext,
} from "./issue-queued-comment-queue.js";
import { documentService } from "./documents.js";
import { getTaskPlanContext } from "./task-plan-context.js";
@@ -463,10 +449,8 @@ import {
buildNativeExecutionInput,
buildNativeExecutionWithCheckpoint,
buildNativeRuntimeContext,
- cancelNativeSession,
claimWarmNativeInstructionCopy,
nativeSessionWorkspaceScope,
- closeWarmNativeSessionsForEnvironment,
reserveWarmNativeInstructionDirectory,
dispatchNativeSessionResumptions,
ensureNativeCompletionContract,
@@ -560,7 +544,9 @@ import { reportRunFailure } from "./run-failure-report.js";
import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";
import { companySkillService } from "./company-skills.js";
-import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js";
+import {
+ budgetService,
+} from "./budgets.js";
import {
secretService,
} from "./secrets.js";
@@ -647,7 +633,6 @@ import {
} from "./workspace-operations.js";
import {
isProcessGroupAlive,
- terminateLocalService,
} from "./local-service-supervisor.js";
import {
isRuntimeOwnedGitBranch,
@@ -709,7 +694,6 @@ import {
} from "../modules/run-dispatch/index.js";
import {
createWakeQueue,
- WakeQueueApplicationError,
type IssueSnapshot as WakeQueueIssueSnapshot,
type PostCommitEffect as WakeQueuePostCommitEffect,
type ReleaseRecoveryBlockedNoticeKind,
@@ -1018,21 +1002,6 @@ export function getTaskDrainStatus(): {
};
}
-export function leaseReleaseStatusForRunStatus(
- status: string | null | undefined,
-): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {
- if (status === "cancelled") return "expired";
- return status === "failed" || status === "timed_out" ? "failed" : "released";
-}
-
-export function providerResourceDispositionForTerminalRun(
- desired: ProviderResourceDisposition | undefined,
- status: string | null | undefined,
-): ProviderResourceDisposition | undefined {
- if (desired !== "keep_running") return desired;
- return status === "succeeded" ? desired : "stop_and_retain";
-}
-
export interface NativeSandboxLifecycle {
runnerProcess: "per_turn" | "warm";
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
@@ -1283,33 +1252,6 @@ function isSameTaskScope(left: string | null, right: string | null) {
return (left ?? null) === (right ?? null);
}
-async function terminateHeartbeatRunProcess(input: {
- pid: number | null | undefined;
- processGroupId: number | null | undefined;
- graceMs?: number;
- signal?: NodeJS.Signals;
-}) {
- const pid = input.pid ?? null;
- const processGroupId = input.processGroupId ?? null;
- if (typeof pid !== "number" && typeof processGroupId !== "number") return;
-
- await terminateLocalService(
- {
- pid:
- typeof pid === "number" && Number.isInteger(pid) && pid > 0
- ? pid
- : (processGroupId ?? 0),
- processGroupId:
- typeof processGroupId === "number" &&
- Number.isInteger(processGroupId) &&
- processGroupId > 0
- ? processGroupId
- : null,
- },
- { forceAfterMs: input.graceMs, signal: input.signal },
- );
-}
-
export type HeartbeatEnvironmentRuntime = ReturnType<
typeof environmentRuntimeService
>;
@@ -1351,38 +1293,6 @@ export interface HeartbeatServiceOptions {
}) => Promise<void>;
}
-export async function cancelHeartbeatNativeRun(input: {
- db: Db;
- runId: string;
- reason: string;
- runtimeMode: string | null;
- cancellationRequestId?: string;
- cancel?: (
- runId: string,
- reason: string,
- options: { db: Db; scope: "run"; cancellationRequestId?: string },
- ) => Promise<{ decision: unknown | null; auditId: string | null }>;
-}) {
- if (input.runtimeMode !== "native") {
- return { decision: null, auditId: null };
- }
- const cancellation = input.cancel
- ? await input.cancel(input.runId, input.reason, {
- db: input.db,
- scope: "run",
- ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),
- })
- : await cancelNativeSession(input.runId, input.reason, {
- db: input.db,
- scope: "run",
- ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),
- });
- if (!cancellation.decision || !cancellation.auditId) {
- throw new Error("native_cancellation_outcome_not_audited");
- }
- return cancellation;
-}
-
class NativeSessionResumeScheduledError extends Error {
constructor(readonly original: unknown) {
super("Native session recovery has been scheduled for the same run.");
@@ -2061,6 +1971,74 @@ export function heartbeatService(
return runLifecycle.publishRunLifecyclePluginEvent(...args);
}
+ const runControl = createHeartbeatRunControl(db, {
+ getRun,
+ options,
+ envOrchestrator,
+ activeRunExecutions,
+ enqueueWakeup,
+ getAgent,
+ sweepPendingCleanupLeases,
+ getSchedulingSuppression,
+ wakeQueue,
+ applyWakeQueuePostCommitEffects,
+ processRunCancellationSettlements,
+ mergeRunStopMetadataForAgent,
+ setRunStatusFromLive,
+ failedProcessRunCancellations,
+ setWakeupStatus,
+ appendRunEvent,
+ finalizeAgentStatus,
+ timerClaimWasFirstHeartbeat,
+ startNextQueuedRunForAgent,
+ setRunStatus,
+ });
+
+ const {
+ cancelInvocationsForAgentsInternal,
+ } = runControl;
+
+ // Earlier factories capture these callbacks before run-control construction.
+ function cancelBudgetScopeWork(...args: Parameters<typeof runControl.cancelBudgetScopeWork>) {
+ return runControl.cancelBudgetScopeWork(...args);
+ }
+
+ function releaseIssueExecutionAndPromote(...args: Parameters<typeof runControl.releaseIssueExecutionAndPromote>) {
+ return runControl.releaseIssueExecutionAndPromote(...args);
+ }
+
+ function releaseEnvironmentLeasesForRun(...args: Parameters<typeof runControl.releaseEnvironmentLeasesForRun>) {
+ return runControl.releaseEnvironmentLeasesForRun(...args);
+ }
+
+ function acknowledgeRemoteStop(...args: Parameters<typeof runControl.acknowledgeRemoteStop>) {
+ return runControl.acknowledgeRemoteStop(...args);
+ }
+
+ function resumeRemoteStopComments(...args: Parameters<typeof runControl.resumeRemoteStopComments>) {
+ return runControl.resumeRemoteStopComments(...args);
+ }
+
+ function cancelRunInternal(...args: Parameters<typeof runControl.cancelRunInternal>) {
+ return runControl.cancelRunInternal(...args);
+ }
+
+ function resumeExecutionWaitComments(...args: Parameters<typeof runControl.resumeExecutionWaitComments>) {
+ return runControl.resumeExecutionWaitComments(...args);
+ }
+
+ function resumeQueuedCommentInterrupt(...args: Parameters<typeof runControl.resumeQueuedCommentInterrupt>) {
+ return runControl.resumeQueuedCommentInterrupt(...args);
+ }
+
+ function resumeSavedLegacyComments(...args: Parameters<typeof runControl.resumeSavedLegacyComments>) {
+ return runControl.resumeSavedLegacyComments(...args);
+ }
+
+ function cancelActiveForAgentInternal(...args: Parameters<typeof runControl.cancelActiveForAgentInternal>) {
+ return runControl.cancelActiveForAgentInternal(...args);
+ }
+
// Applies the post-commit effects a wake-queue release returns, exactly as
// the original release function did before its writes moved into that
// module: publish + dispatch a promoted or recovery run, log a reopened
@@ -2123,422 +2101,6 @@ export function heartbeatService(
}
const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });
- async function releaseEnvironmentLeasesForRun(input: {
- runId: string;
- companyId: string;
- agentId: string;
- status: string | null | undefined;
- failureReason?: string | null;
- providerResourceDisposition?: ProviderResourceDisposition;
- nativeLifecycleTelemetry?: {
- provider: string;
- harness: string;
- lifecycleMode: "per_turn" | "warm";
- sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
- };
- }) {
- const leaseOwnerRun = await getRun(input.runId);
- if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;
- // Recovery can finish workspace copy-back outside the executor's finally.
- // Successful copy-back does not earn warm retention for a failed turn.
- const status = leaseOwnerRun?.status ?? input.status;
- const providerResourceDisposition = providerResourceDispositionForTerminalRun(
- input.providerResourceDisposition,
- status,
- );
- if (providerResourceDisposition === "destroy") {
- const closeResult = await (
- options.closeWarmNativeSessionsForRun ??
- (async ({ runId, reason }) => {
- const environmentIds = await db
- .selectDistinct({ environmentId: environmentLeases.environmentId })
- .from(environmentLeases)
- .where(
- and(
- eq(environmentLeases.heartbeatRunId, runId),
- eq(environmentLeases.status, "active"),
- ),
- )
- .then((rows) =>
- rows.flatMap((row) =>
- typeof row.environmentId === "string" &&
- row.environmentId.length > 0
- ? [row.environmentId]
- : [],
- ),
- );
- const aggregate = { closed: 0, busy: 0, failed: 0 };
- for (const environmentId of environmentIds) {
- const result = await closeWarmNativeSessionsForEnvironment({
- environmentId,
- reason,
- });
- aggregate.closed += result.closed;
- aggregate.busy += result.busy;
- aggregate.failed += result.failed;
- }
- return aggregate;
- })
- )({
- runId: input.runId,
- reason: "terminal heartbeat run destroyed its environment lease",
- }).catch((err) => {
- logger.warn(
- { err, runId: input.runId },
- "failed to close warm native sessions before environment lease destruction",
- );
- return { closed: 0, busy: 0, failed: 1 };
- });
- // A failed remote checkpoint cannot veto destruction of the isolated
- // sandbox after the run stopped. Provider destruction supplies the exit
- // proof; it does not turn the interrupted checkpoint into a success.
- const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&
- ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)
- ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
- eq(environmentLeases.companyId, input.companyId),
- eq(environmentLeases.heartbeatRunId, input.runId),
- ))
- : [];
- const canDestroyRemote = remoteLeases.length > 0 &&
- remoteLeases.every(lease => lease.provider && lease.provider !== "local");
- if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {
- logger.warn(
- { runId: input.runId, warmNativeSessions: closeResult },
- "deferred environment lease destruction until warm native sessions close",
- );
- return;
- }
- }
- const releaseResult = await envOrchestrator
- .releaseForRun({
- heartbeatRunId: input.runId,
- companyId: input.companyId,
- agentId: input.agentId,
- status: leaseReleaseStatusForRunStatus(status),
- failureReason: input.failureReason ?? undefined,
- providerResourceDisposition,
- nativeLifecycleTelemetry: input.nativeLifecycleTelemetry,
- })
- .catch((err) => {
- logger.warn(
- { err, runId: input.runId },
- "failed to release environment leases for heartbeat run",
- );
- return null;
- });
- for (const releaseError of releaseResult?.errors ?? []) {
- logger.warn(
- {
- err: releaseError.error,
- leaseId: releaseError.leaseId,
- runId: input.runId,
- },
- "failed to release environment lease for heartbeat run",
- );
- }
- await acknowledgeRemoteStop(input.runId, input.companyId);
- }
-
- async function acknowledgeRemoteStop(runId: string, companyId: string) {
- // The provider receipt arrives after adapter settlement. A remote ACP child
- // has no host PID, so only this target-aware boundary can acknowledge Stop.
- const stopped = await getRun(runId);
- if (stopped?.runtimeMode === "native") {
- const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId);
- for (const remoteCleanupScope of scopes ?? []) {
- completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });
- }
- }
- if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" &&
- parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&
- await runUsedConversationAdapter(db, stopped) &&
- await remoteExecutionHasStopped(db, companyId, runId)) {
- await db.update(heartbeatRuns).set({
- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({
- executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),
- state: "acknowledged", acknowledgedAt: new Date().toISOString(),
- proof: "provider_termination_receipt" },
- conversationContinuation: CONVERSATION_CONTINUATION_POLICY,
- })}::jsonb`,
- updatedAt: new Date(),
- }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),
- eq(heartbeatRuns.status, "cancelled")));
- }
- }
-
- async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) {
- if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;
- const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt
- ? await db.select().from(nativeRunFinalizations).where(and(
- eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),
- )) : [];
- const cancelledPreparation = run.runtimeMode === "legacy" &&
- await isCancelledNativeStartup(db, run, preparationCoordinator);
- if (run.runtimeMode !== "native" && !cancelledPreparation &&
- parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true &&
- !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return;
- const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);
- if (!issueId) return;
- const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null;
- const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase,
- leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and(
- eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),
- )) : [];
- // Stop ends one response. Saved user input can enter ordinary admission
- // once its executor settles; it does not need a manufactured crash incident.
- // The ordinary path still enforces task holds, ownership, and native session
- // cleanup before starting a provider.
- const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) &&
- !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner &&
- ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") &&
- await acknowledgedNativeStopExecutionHasStopped(db, currentRun) &&
- !(await getExecutionBlocker(db, run.companyId, issueId));
- const legacyContinuation = run.runtimeMode === "legacy" &&
- hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) &&
- !(await getExecutionBlocker(db, run.companyId, issueId));
- if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return;
- const pending = await db.select().from(agentWakeupRequests).where(and(
- eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- eq(agentWakeupRequests.requestedByActorType, "user"),
- requestId ? eq(agentWakeupRequests.id, requestId) : undefined,
- sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
- )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50);
- for (const wake of pending) {
- if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue;
- let payload = parseObject(wake.payload);
- if (payload.queuedCommentInterrupt) {
- await resumeQueuedCommentInterrupt(wake.companyId, wake.id);
- continue;
- }
- let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
- let commentId = deriveCommentId(context, payload);
- let requestedByActorId = wake.requestedByActorId;
- const reason = readNonEmptyString(context.wakeReason) ?? wake.reason;
- if (run.runtimeMode === "native" || cancelledPreparation) {
- const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId,
- queuedCommentIdsFromWakePayload(payload));
- if (!ids.length) continue;
- payload = withQueuedCommentIdsInWakePayload(payload, ids);
- context = withQueuedCommentIdsInRunContext(context, ids);
- commentId = ids.at(-1)!;
- // A coalesced queue can contain several authors. Its saved comments,
- // not the outer wake's first author, authorize the remaining input.
- const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and(
- eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId),
- ));
- requestedByActorId = author?.id ?? null;
- }
- if (legacyContinuation || stoppedNativeContinuation) {
- if (!commentId || !run.finishedAt || !requestedByActorId ||
- !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue;
- const [comment] = await db.select().from(issueComments).where(and(
- eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId),
- sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"),
- eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt),
- isNull(issueComments.createdByRunId),
- stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt),
- ));
- if (!comment?.body.trim()) continue;
- } else {
- let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." };
- const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId,
- agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId,
- reason, commentId, successorRunId: randomUUID(), dryRun: true,
- ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),
- onBlocked: (reason, message) => { wait = { reason, message }; },
- });
- if (!admitted) {
- await db.update(agentWakeupRequests).set({
- payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
- coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
- updatedAt: new Date(),
- }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
- eq(agentWakeupRequests.status, "deferred_issue_execution")));
- continue;
- }
- }
- // Re-enter ordinary admission with the original user's authority. It
- // atomically adopts the deferred comments and still applies every gate.
- await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"],
- reason, payload, contextSnapshot: context,
- requestedByActorType: "user", requestedByActorId,
- ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),
- idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id);
- break;
- }
- }
-
- async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) {
- return resumeSavedLegacyComments(companyId, queueId, true, opts);
- }
-
- async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) {
- const [wake] = await db.select().from(agentWakeupRequests).where(and(
- eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- ));
- if (!wake) return;
- const payload = parseObject(wake.payload);
- let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId);
- let commentIds = queuedCommentIdsFromWakePayload(payload);
- const issueId = readNonEmptyString(payload.issueId);
- if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return;
- const response = await readQueuedInteractionResponse(db, companyId, issueId, payload);
- if (!commentIds.length && !response) return;
- // Resolved cards are immutable input. Only an explicit Interrupt click can
- // authorize continuation across a stopped execution; ordinary completion
- // uses normal deferred-wake promotion.
- if (response && !interrupted) return;
- if (!interrupted) {
- commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds);
- if (!commentIds.length) return;
- // The queue itself may have begun as a system wake. The saved human
- // comment, not that wake's origin or mutable caller payload, is authority.
- const [comment] = await db.select().from(issueComments).where(and(
- eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),
- eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"),
- isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),
- )).orderBy(desc(issueComments.createdAt)).limit(1);
- if (!comment?.body.trim() || !comment.authorUserId) return;
- actorId = comment.authorUserId;
- }
- if (!actorId) return;
- const agent = await getAgent(wake.agentId);
- if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return;
- const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
- eq(heartbeatRuns.companyId, companyId),
- eq(heartbeatRuns.agentId, wake.agentId),
- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
- inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),
- )).limit(1);
- if (active) return;
- if (interrupted && opts?.retryCleanup) {
- // Only the HTTP click grants an extra cleanup attempt. Periodic retries
- // reuse the intent to deliver, never a fresh provider teardown budget.
- const sourceRun = await db.transaction(async tx => {
- const [task] = await tx.select().from(issues).where(and(
- eq(issues.companyId, companyId), eq(issues.id, issueId),
- )).for("update");
- const [current] = await tx.select().from(agentWakeupRequests).where(and(
- eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"),
- sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
- sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`,
- ));
- if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) ||
- !current || (!queuedCommentIdsFromWakePayload(current.payload).length &&
- !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null;
- const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
- eq(heartbeatRuns.companyId, companyId),
- eq(heartbeatRuns.agentId, wake.agentId),
- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
- inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),
- )).limit(1);
- if (successor) return null;
- const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId);
- const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and(
- eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId),
- eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"),
- inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]),
- sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
- )).then(rows => rows[0]) : null;
- if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null;
- // Older ephemeral leases recorded successful cleanup without a provider
- // receipt. Re-verify them through the recorded teardown path; a timestamp
- // alone never certifies termination. Retained/reusable resources stay put.
- const historical = await tx.select().from(environmentLeases).where(and(
- eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id),
- eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt),
- inArray(environmentLeases.status, ["released", "expired", "failed"]),
- )).for("update");
- for (const lease of historical) {
- if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue;
- // Provider resource IDs identify physical sandboxes. A lease in any
- // company can still own this resource; never destroy it on behalf of
- // this company. This existence-only guard exposes no foreign data.
- const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
- ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),
- eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
- or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])),
- )).limit(1);
- if (otherOwner) continue;
- await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() })
- .where(eq(environmentLeases.id, lease.id));
- }
- return run;
- });
- if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: {
- companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt",
- } });
- }
- const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds);
- delete deliveryPayload.queuedCommentInterrupt;
- await enqueueWakeup(wake.agentId, {
- source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
- payload: deliveryPayload, contextSnapshot: response
- ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId,
- responsibleUserId: actorId }
- : withQueuedCommentIdsInRunContext({
- issueId, triggeredBy: "board", actorId, responsibleUserId: actorId,
- }, commentIds),
- requestedByActorType: "user", requestedByActorId: actorId,
- ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }),
- issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] },
- idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`,
- }, queueId);
- }
-
- async function resumeExecutionWaitComments() {
- if ((await getSchedulingSuppression()).suppressed) return;
- const waits = await db.select({ wake: agentWakeupRequests })
- .from(agentWakeupRequests)
- .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
- sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
- eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
- .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
- .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
- eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
- executionBlockerPredicate(),
- ))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
- eq(agentWakeupRequests.requestedByActorType, "user"),
- sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
- sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`,
- lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
- notInArray(issues.status, ["done", "cancelled"])))
- .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
- const seen = new Set<string>();
- for (const { wake } of waits) {
- const issueId = String(wake.payload?.issueId);
- if (seen.has(issueId)) continue;
- seen.add(issueId);
- // Advance the cursor even for invalid evidence so one damaged task cannot
- // starve later requests in this bounded scan. Preserve concurrent edits.
- const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and(
- eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
- )).returning({ id: agentWakeupRequests.id });
- if (!claimed) continue;
- // Match normal admission's deterministic current blocker selection. An
- // arbitrary historical action must not choose the retry's source run.
- const blocker = await getExecutionBlocker(db, wake.companyId, issueId);
- const sourceId = blocker?.runId;
- if (!sourceId || !isUuidLike(sourceId)) continue;
- const run = await getRun(sourceId, { includeExecutionEvidence: true });
- if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
- if (canContinueCancelledRun(run)) {
- await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
- logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
- });
- continue;
- }
- await resumeRemoteStopComments(run, wake.id).catch(err => {
- logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
- });
- }
- }
-
async function getAgentInvokability(
agent: typeof agents.$inferSelect | null | undefined,
) {
@@ -10618,683 +10180,6 @@ export function heartbeatService(
}
}
- async function releaseIssueExecutionAndPromote(
- run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">,
- options: { suppressImmediateRecovery?: boolean } = {},
- ) {
- try {
- const source = await getRun(run.id);
- const { postCommitEffects } = await wakeQueue.releaseIssueExecution({
- companyId: run.companyId,
- runId: run.id,
- now: new Date(),
- // A durable authentication card owns recovery. This covers the review path,
- // while continuation classification blocks periodic generic retries.
- suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source),
- });
- await applyWakeQueuePostCommitEffects(postCommitEffects);
- const completed = await getRun(run.id);
- const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId)
- ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId;
- if (completed?.status === "succeeded" && issueId) {
- await settleSlackConversation(db, run.companyId, issueId).catch((err) => {
- logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation");
- });
- }
- } catch (error) {
- if (
- error instanceof WakeQueueApplicationError &&
- error.code === "responsible_user_unresolved"
- ) {
- // Every other `responsible_user_unresolved` HttpError in this file
- // carries `code` inside its own `details`; match that shape here too.
- throw new HttpError(422, error.message, { code: error.code, ...error.details });
- }
- throw error;
- }
- }
-
- async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) {
- const runIssueId = sql<
- string | null
- >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
- const effectiveProjectId = sql<
- string | null
- >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
-
- const rows = await db
- .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
- .from(heartbeatRuns)
- .leftJoin(
- issues,
- and(
- eq(issues.companyId, companyId),
- sql`${issues.id}::text = ${runIssueId}`,
- ),
- )
- .where(
- and(
- eq(heartbeatRuns.companyId, companyId),
- createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined,
- inArray(heartbeatRuns.status, [
- ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
- ]),
- sql`${effectiveProjectId} = ${projectId}`,
- ),
- );
-
- return rows.map((row) => row.id);
- }
-
- async function listProjectScopedWakeupIds(
- companyId: string,
- projectId: string,
- database: Db = db,
- ) {
- const wakeIssueId = sql<
- string | null
- >`${agentWakeupRequests.payload} ->> 'issueId'`;
- const effectiveProjectId = sql<
- string | null
- >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
-
- const rows = await database
- .selectDistinctOn([agentWakeupRequests.id], {
- id: agentWakeupRequests.id,
- })
- .from(agentWakeupRequests)
- .leftJoin(
- issues,
- and(
- eq(issues.companyId, companyId),
- sql`${issues.id}::text = ${wakeIssueId}`,
- ),
- )
- .where(
- and(
- eq(agentWakeupRequests.companyId, companyId),
- inArray(agentWakeupRequests.status, [
- "queued",
- "deferred_issue_execution",
- ]),
- sql`${agentWakeupRequests.runId} is null`,
- sql`${effectiveProjectId} = ${projectId}`,
- ),
- );
-
- return rows.map((row) => row.id);
- }
-
- async function cancelPendingWakeupsForBudgetScope(
- scope: BudgetEnforcementScope,
- ) {
- return withCurrentBudgetEnforcement(db, scope, async (tx) => {
- const now = new Date();
- let wakeupIds: string[] = [];
-
- if (scope.scopeType === "company") {
- wakeupIds = await tx
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, scope.companyId),
- inArray(agentWakeupRequests.status, [
- "queued",
- "deferred_issue_execution",
- ]),
- sql`${agentWakeupRequests.runId} is null`,
- ),
- )
- .then((rows) => rows.map((row) => row.id));
- } else if (scope.scopeType === "agent") {
- wakeupIds = await tx
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, scope.companyId),
- eq(agentWakeupRequests.agentId, scope.scopeId),
- inArray(agentWakeupRequests.status, [
- "queued",
- "deferred_issue_execution",
- ]),
- sql`${agentWakeupRequests.runId} is null`,
- ),
- )
- .then((rows) => rows.map((row) => row.id));
- } else {
- wakeupIds = await listProjectScopedWakeupIds(
- scope.companyId,
- scope.scopeId,
- tx,
- );
- }
-
- if (wakeupIds.length === 0) return 0;
-
- await tx
- .update(agentWakeupRequests)
- .set({
- status: "cancelled",
- finishedAt: now,
- error: "Cancelled due to budget pause",
- updatedAt: now,
- })
- .where(and(
- inArray(agentWakeupRequests.id, wakeupIds),
- inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),
- isNull(agentWakeupRequests.runId),
- scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined,
- ));
-
- return wakeupIds.length;
- });
- }
-
- type CancelRunOptions = {
- budgetEnforcement?: BudgetEnforcementScope;
- /** Optional board request identity, atomically reserved for native Stop. */
- cancellationRequestId?: string;
- cancellationRequestedByUserId?: string | null;
- errorCode?: string;
- resultJson?: Record<string, unknown>;
- eventMessage?: string;
- eventPayload?: Record<string, unknown>;
- /** Per-call graceful process shutdown window, bounded to a safe range. */
- terminationGraceMs?: number;
- /** Caller is immediately scheduling an explicit successor path. */
- suppressImmediateRecovery?: boolean;
- };
-
- function cancellationTerminationGraceMs(
- configuredGraceSec: number,
- requestedGraceMs: number | undefined,
- ) {
- const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000;
- if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) {
- return configuredGraceMs;
- }
- return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs)));
- }
-
- async function cancelRunInternal(
- runId: string,
- reason = "Cancelled by control plane",
- options: CancelRunOptions = {},
- ) {
- let run = await getRun(runId);
- if (!run) throw notFound("Heartbeat run not found");
- if (options.cancellationRequestId) {
- run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null);
- }
- // The caller claim checked retry eligibility under both durable row locks.
- // This is only a cancellation candidate: dispatch rechecks the coordinator
- // under lock, and the final CAS requires its own unchanged acknowledged
- // retry fence. Re-reading only retryable_failure here would strand replay.
- const pendingNativeRetry =
- run.runtimeMode === "native" && run.status === "failed" && (
- Boolean(options.cancellationRequestId) || await db
- .select({ runId: nativeRunFinalizations.runId })
- .from(nativeRunFinalizations)
- .where(
- and(
- eq(nativeRunFinalizations.runId, run.id),
- eq(nativeRunFinalizations.companyId, run.companyId),
- eq(nativeRunFinalizations.phase, "retryable_failure"),
- ),
- )
- .then((rows) => rows.length > 0)
- );
- if (
- !pendingNativeRetry &&
- !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(
- run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number],
- )
- )
- return run;
- const agent = await getAgent(run.agentId);
- const errorCode = options.errorCode ?? "cancelled";
- const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason);
-
- const pendingProcessCancellation = processRunCancellationSettlements.get(
- run.id,
- );
- if (pendingProcessCancellation) {
- await pendingProcessCancellation.settled;
- if (pendingProcessCancellation.failed)
- throw pendingProcessCancellation.error;
- return getRun(run.id);
- }
- const running = runningProcesses.get(run.id);
- const stopOwnership =
- run.runtimeMode !== "native"
- ? captureAdapterStopOwnership(run.id)
- : undefined;
- const control = stopOwnership?.control;
- // Capture the existing adapter owner before waiting on the run lock. Then
- // atomically fence preparation and refresh the selected runtime, so Stop
- // cannot miss a native handoff that won after its first read.
- // Established legacy processes must still be stopped if the database is
- // unavailable. Only native or not-yet-dispatched preparation needs this
- // additional durable fence before its existing cancellation path.
- if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) {
- const fence = async (tx: Db) => {
- const [fenced] = await tx.update(heartbeatRuns).set({
- // Record handoff intent before native cancellation can finalize and release
- // the run. Only its audited stop acknowledgement suppresses recovery.
- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
- ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true
- ? { reassignmentStopRequested: true } : {})}::jsonb ||
- ${JSON.stringify({ cancellation })}::jsonb ||
- jsonb_build_object('startupCancellation', jsonb_build_object(
- 'requestedAt', ${new Date().toISOString()}::text,
- 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'
- and ${heartbeatRuns.runtimeModeResolvedAt} is null
- and ${heartbeatRuns.executionStage} = 'preparing'
- and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)
- ))`,
- }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,
- pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
- ))).returning();
- return fenced ?? null;
- };
- let fenced: typeof run | null;
- try {
- fenced = options.budgetEnforcement
- ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence)
- : await fence(db);
- } catch (error) {
- stopOwnership?.release();
- throw error;
- }
- if (!fenced) {
- stopOwnership?.release();
- return getRun(runId);
- }
- run = fenced;
- }
- const resultJson = { ...(agent
- ? {
- ...mergeRunStopMetadataForAgent(agent, "cancelled", {
- resultJson: parseObject(run.resultJson),
- errorCode,
- errorMessage: reason,
- }),
- ...(options.resultJson ?? {}),
- }
- : options.resultJson), cancellation };
-
- try {
- let releaseProcessCancellation: (() => void) | undefined;
- const processCancellationSettlement =
- run.runtimeMode !== "native" &&
- !control &&
- running
- ? {
- settled: new Promise<void>((resolve) => {
- releaseProcessCancellation = resolve;
- }),
- failed: false,
- error: undefined as unknown,
- }
- : undefined;
- if (processCancellationSettlement) {
- // No await between joining an existing owner above and registering ours.
- processRunCancellationSettlements.set(
- run.id,
- processCancellationSettlement,
- );
- }
- const cancellation = await (async () => {
- try {
- if (control) {
- await db
- .update(heartbeatRuns)
- .set({
- error: reason,
- errorCode,
- // Stop may have read the row before adapter settlement saved
- // a required-copyback receipt. Merge into current database JSON
- // and leave those server-owned fields to the restore recorder.
- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({
- ...resultJson,
- ...(!running ? { executionCancellation: {
- state: "requested", requestedAt: new Date().toISOString(),
- } } : {}),
- }, true),
- updatedAt: new Date(),
- })
- .where(
- and(
- eq(heartbeatRuns.id, run.id),
- eq(heartbeatRuns.status, "running"),
- ),
- );
- control.controller.abort(new Error(reason));
- }
- let terminationSettled = false;
- try {
- await cancelHeartbeatNativeRun({
- db,
- runId: run.id,
- reason,
- runtimeMode: run.runtimeMode,
- ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),
- });
- if (running) {
- await terminateHeartbeatRunProcess({
- pid: running.child.pid,
- processGroupId: running.processGroupId,
- // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM
- // can leave commands in their separate process groups alive.
- signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined,
- graceMs: cancellationTerminationGraceMs(
- running.graceSec,
- options.terminationGraceMs,
- ),
- });
- }
- terminationSettled = true;
- } finally {
- if (
- (!processCancellationSettlement || terminationSettled) &&
- runningProcesses.get(run.id) === running
- ) {
- runningProcesses.delete(run.id);
- }
- }
-
- if (control) {
- await waitForAdapterStop(control.settled, undefined, {
- runId: run.id,
- adapterType: agent?.adapterType,
- runtimeMode: run.runtimeMode,
- abortRequested: control.controller.signal.aborted,
- }, control);
- const stopped = await getRun(run.id);
- if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) {
- if (
- parseObject(stopped.resultJson?.executionCancellation).state !==
- "acknowledged"
- ) {
- throw conflict(
- "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.",
- );
- }
- // The owned adapter already finalized this run and its lifecycle.
- // Do not replay the process cancellation side effects below.
- return { run: stopped, updated: false };
- }
- }
-
- const finishedAt = new Date();
- const persistedCancellationResult =
- run.runtimeMode === "native"
- ? await getRun(run.id).then((current) =>
- parseObject(current?.resultJson),
- )
- : {};
- return await setRunStatusFromLive(
- run.id,
- "cancelled",
- pendingNativeRetry
- ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"]
- : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
- {
- finishedAt,
- error: reason,
- errorCode,
- ...(resultJson ||
- Object.keys(persistedCancellationResult).length > 0
- ? {
- resultJson: {
- ...persistedCancellationResult,
- ...(resultJson ?? {}),
- // A scheduler placeholder has no process to acknowledge.
- // Preserve its normal release policy instead of treating
- // it as an operator stop of provider work.
- ...(processCancellationSettlement && agent && running && (
- (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) ||
- (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0)
- )
- ? mergeRunStopMetadataForAgent(agent, "cancelled", {
- resultJson: {
- ...resultJson,
- executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() },
- },
- errorCode, errorMessage: reason,
- })
- : {}),
- // The native cancellation helper may have advanced a durable
- // pending intent to its acknowledged state after `run` was
- // first read. Never let that stale snapshot overwrite the
- // authoritative post-dispatch acknowledgement.
- ...(Object.hasOwn(
- persistedCancellationResult,
- "nativeCancellation",
- )
- ? {
- nativeCancellation:
- persistedCancellationResult.nativeCancellation,
- }
- : {}),
- },
- }
- : {}),
- },
- undefined,
- pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined,
- );
- } catch (error) {
- if (processCancellationSettlement) {
- processCancellationSettlement.failed = true;
- processCancellationSettlement.error = error;
- if (activeRunExecutions.has(run.id)) {
- failedProcessRunCancellations.set(
- run.id,
- processCancellationSettlement,
- );
- }
- }
- throw error;
- } finally {
- if (processCancellationSettlement) {
- if (
- processRunCancellationSettlements.get(run.id) ===
- processCancellationSettlement
- ) {
- processRunCancellationSettlements.delete(run.id);
- }
- // Always settle waiters, including termination and DB errors. A
- // failed Stop remains an error to its caller, never a cancellation
- // receipt; the signal-bearing executor may then record failure.
- releaseProcessCancellation?.();
- }
- }
- })();
- const cancelled = cancellation.run;
-
- if (cancellation.updated && cancelled) {
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
- finishedAt: cancelled.finishedAt ?? new Date(),
- error: reason,
- });
- await appendRunEvent(cancelled, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message: options.eventMessage ?? "run cancelled",
- payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) },
- });
- await releaseIssueExecutionAndPromote(cancelled, {
- suppressImmediateRecovery: options.suppressImmediateRecovery,
- });
- await finalizeAgentStatus(run.agentId, "cancelled", undefined, {
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
- });
- await startNextQueuedRunForAgent(run.agentId);
- }
- return cancelled;
- } finally {
- stopOwnership?.release();
- }
- }
-
- async function cancelActiveForAgentInternal(
- agentId: string,
- reason = "Cancelled due to agent pause",
- errorCode = "cancelled",
- ) {
- const agent = await getAgent(agentId);
- const runs = await db
- .select()
- .from(heartbeatRuns)
- .where(
- and(
- eq(heartbeatRuns.agentId, agentId),
- inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),
- ),
- );
-
- for (const run of runs) {
- const stopOwnership =
- run.runtimeMode !== "native"
- ? captureAdapterStopOwnership(run.id)
- : undefined;
- try {
- if (stopOwnership?.control) {
- await cancelRunInternal(run.id, reason, { errorCode });
- continue;
- }
- if (run.runtimeMode === "native") {
- await db.update(heartbeatRuns).set({ resultJson:
- sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`,
- }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status)));
- await cancelHeartbeatNativeRun({
- db,
- runId: run.id,
- reason,
- runtimeMode: run.runtimeMode,
- });
- }
- const persistedCancellationResult =
- run.runtimeMode === "native"
- ? await getRun(run.id).then((current) =>
- parseObject(current?.resultJson),
- )
- : parseObject(run.resultJson);
- await setRunStatus(run.id, "cancelled", {
- finishedAt: new Date(),
- error: reason,
- errorCode,
- resultJson: {
- ...persistedCancellationResult,
- ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", {
- resultJson: persistedCancellationResult, errorCode, errorMessage: reason,
- }) : {}),
- cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason),
- },
- });
-
- await setWakeupStatus(run.wakeupRequestId, "cancelled", {
- finishedAt: new Date(),
- error: reason,
- });
-
- const running = runningProcesses.get(run.id);
- if (running) {
- await terminateHeartbeatRunProcess({
- pid: running.child.pid,
- processGroupId: running.processGroupId,
- graceMs: Math.max(1, running.graceSec) * 1000,
- });
- }
- runningProcesses.delete(run.id);
- await releaseIssueExecutionAndPromote(run);
- } finally {
- stopOwnership?.release();
- }
- }
-
- return runs.length;
- }
-
- async function cancelPendingWakeupsForAgentsInternal(
- agentIds: string[],
- reason: string,
- ) {
- const uniqueAgentIds = [...new Set(agentIds)].filter(
- (agentId) => agentId.length > 0,
- );
- if (uniqueAgentIds.length === 0) return 0;
-
- const now = new Date();
- const wakeupIds = await db
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(
- and(
- inArray(agentWakeupRequests.agentId, uniqueAgentIds),
- inArray(agentWakeupRequests.status, [
- "queued",
- "deferred_issue_execution",
- ]),
- sql`${agentWakeupRequests.runId} is null`,
- ),
- )
- .then((rows) => rows.map((row) => row.id));
-
- if (wakeupIds.length === 0) return 0;
-
- await db
- .update(agentWakeupRequests)
- .set({
- status: "cancelled",
- finishedAt: now,
- error: reason,
- updatedAt: now,
- })
- .where(inArray(agentWakeupRequests.id, wakeupIds));
-
- return wakeupIds.length;
- }
-
- async function cancelInvocationsForAgentsInternal(
- agentIds: string[],
- reason: string,
- ) {
- const uniqueAgentIds = [...new Set(agentIds)].filter(
- (agentId) => agentId.length > 0,
- );
- let runsCancelled = 0;
- for (const agentId of uniqueAgentIds) {
- runsCancelled += await cancelActiveForAgentInternal(agentId, reason);
- }
- const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal(
- uniqueAgentIds,
- reason,
- );
- return {
- agentIds: uniqueAgentIds,
- runsCancelled,
- wakeupsCancelled,
- };
- }
-
- async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) {
- const runIds = scope.scopeType === "project"
- ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore)
- : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
- eq(heartbeatRuns.companyId, scope.companyId),
- scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined,
- scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined,
- inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),
- )).then((rows) => rows.map((row) => row.id));
- for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope });
- await cancelPendingWakeupsForBudgetScope(scope);
- }
-
return {
waitForRunExecutionDrain: async (
runId: string,
diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts
new file mode 100644
index 0000000000..7af48aeee5
--- /dev/null
+++ b/server/src/services/heartbeat/run-control.test.ts
@@ -0,0 +1,310 @@
+import { randomUUID } from "node:crypto";
+import type { Db } from "@paperclipai/db";
+import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import { runningProcesses } from "../../adapters/index.js";
+import { adapterExecutionControls } from "../adapter-execution-control.js";
+import { terminateLocalService } from "../local-service-supervisor.js";
+import { createHeartbeatRunState } from "./run-state.js";
+import { createHeartbeatRunPreparation } from "./run-preparation.js";
+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";
+import { createHeartbeatRunControl, type HeartbeatRunControlDependencies } from "./run-control.js";
+
+vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
+vi.mock("../local-service-supervisor.js", async (importOriginal) => ({
+ ...await importOriginal<typeof import("../local-service-supervisor.js")>(),
+ terminateLocalService: vi.fn(async () => undefined),
+}));
+
+type Run = typeof heartbeatRuns.$inferSelect;
+function callbacks(db: Db) {
+ const state = createHeartbeatRunState(db);
+ const lifecycleDependencies = {
+ ...state,
+ ...createHeartbeatRunPreparation(db),
+ issuesSvc: {
+ addComment: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["addComment"]>(),
+ listReviewAttention: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["listReviewAttention"]>(),
+ },
+ recovery: { escalateStrandedAssignedIssue: vi.fn<HeartbeatLifecycleDependencies["recovery"]["escalateStrandedAssignedIssue"]>() },
+ companySkills: { completeTestRunForIssue: vi.fn<HeartbeatLifecycleDependencies["companySkills"]["completeTestRunForIssue"]>() },
+ budgets: { getInvocationBlock: vi.fn(async () => null) },
+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
+ enqueueWakeup: vi.fn<HeartbeatLifecycleDependencies["enqueueWakeup"]>(),
+ getCurrentUserRedactionOptions: vi.fn(async () => ({})),
+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),
+ emitTerminalAgentTaskRun: vi.fn<HeartbeatLifecycleDependencies["emitTerminalAgentTaskRun"]>(),
+ budgetHooks: {},
+ } satisfies HeartbeatLifecycleDependencies;
+ const lifecycle = createHeartbeatLifecycle(db, lifecycleDependencies);
+ return {
+ ...state,
+ ...lifecycle,
+ getRun: vi.fn(state.getRun),
+ getAgent: vi.fn(state.getAgent),
+ setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive),
+ setRunStatus: vi.fn(lifecycle.setRunStatus),
+ setWakeupStatus: vi.fn(lifecycle.setWakeupStatus),
+ appendRunEvent: vi.fn(lifecycle.appendRunEvent),
+ finalizeAgentStatus: vi.fn<HeartbeatRunControlDependencies["finalizeAgentStatus"]>(),
+ options: { closeWarmNativeSessionsForRun: vi.fn(async () => ({ closed: 1, busy: 0, failed: 0 })) },
+ envOrchestrator: { releaseForRun: vi.fn<HeartbeatRunControlDependencies["envOrchestrator"]["releaseForRun"]>(async () => ({ released: [], errors: [] })) },
+ enqueueWakeup: lifecycleDependencies.enqueueWakeup,
+ startNextQueuedRunForAgent: vi.fn<HeartbeatRunControlDependencies["startNextQueuedRunForAgent"]>(),
+ sweepPendingCleanupLeases: vi.fn<HeartbeatRunControlDependencies["sweepPendingCleanupLeases"]>(),
+ timerClaimWasFirstHeartbeat: vi.fn<HeartbeatRunControlDependencies["timerClaimWasFirstHeartbeat"]>(() => undefined),
+ wakeQueue: { releaseIssueExecution: vi.fn<HeartbeatRunControlDependencies["wakeQueue"]["releaseIssueExecution"]>(async () => ({ outcome: { kind: "released" }, postCommitEffects: [] })) },
+ applyWakeQueuePostCommitEffects: vi.fn(async () => undefined),
+ getSchedulingSuppression: vi.fn(async () => ({ suppressed: false })),
+ activeRunExecutions: new Set<string>(),
+ processRunCancellationSettlements: new Map<string, { settled: Promise<void>; failed: boolean; error?: unknown }>(),
+ failedProcessRunCancellations: new Map<string, { settled: Promise<void>; failed: boolean; error?: unknown }>(),
+ } satisfies HeartbeatRunControlDependencies;
+}
+function guardedDatabase() {
+ const access = vi.fn(() => { throw new Error("Unexpected run-control database access"); });
+ return { db: new Proxy({}, { get: access }) as Db, access };
+}
+
+describe("heartbeat run-control module boundary", () => {
+ afterEach(() => adapterExecutionControls.clear());
+
+ it("constructs without database access, cancellation, or cleanup", () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ createHeartbeatRunControl(database.db, deps);
+ expect(database.access).not.toHaveBeenCalled();
+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();
+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
+ expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();
+ });
+
+ it("keeps an already completed run out of cancellation effects", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { id: "finished", status: "succeeded", runtimeMode: "legacy" } as Run;
+ deps.getRun.mockResolvedValue(run);
+ expect(await createHeartbeatRunControl(database.db, deps).cancelRunInternal(run.id)).toBe(run);
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.getAgent).not.toHaveBeenCalled();
+ expect(deps.appendRunEvent).not.toHaveBeenCalled();
+ });
+
+ it("keeps a suppressed saved-input scan out of the database and wake admission", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true });
+ await createHeartbeatRunControl(database.db, deps).resumeExecutionWaitComments();
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();
+ });
+
+ it("ignores an empty cancellation batch", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ expect(await createHeartbeatRunControl(database.db, deps).cancelInvocationsForAgentsInternal(["", ""], "pause")).toEqual({ agentIds: [], runsCancelled: 0, wakeupsCancelled: 0 });
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.getAgent).not.toHaveBeenCalled();
+ });
+
+ it("waits for an owned adapter before admitting saved comments", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { id: "owned", status: "cancelled", runtimeMode: "legacy" } as Run;
+ adapterExecutionControls.set(run.id, {} as never);
+ await createHeartbeatRunControl(database.db, deps).resumeRemoteStopComments(run);
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();
+ });
+
+ it("retains leases while native execution ownership is unverified", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ deps.getRun.mockResolvedValue({ runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run);
+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "held", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" });
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled();
+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
+ });
+
+ it("uses the persisted terminal outcome when choosing warm resource retention", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run);
+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "failed", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" });
+ expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ heartbeatRunId: "failed", status: "failed", providerResourceDisposition: "stop_and_retain" }));
+ expect(database.access).not.toHaveBeenCalled();
+ });
+
+ it("defers destruction when a warm session remains busy", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run);
+ deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 });
+ await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" });
+ expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" });
+ expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled();
+ expect(database.access).not.toHaveBeenCalled();
+ });
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+if (!support.supported) console.warn(`Skipping run-control database tests: ${support.reason}`);
+describe.skipIf(!support.supported)("heartbeat run-control database wiring", () => {
+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+ let db: Db;
+ beforeAll(async () => {
+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-run-control-module");
+ db = createDb(database.connectionString);
+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+ afterAll(async () => {
+ await db?.$client.end();
+ await database?.cleanup();
+ });
+ beforeEach(async () => {
+ vi.mocked(terminateLocalService).mockReset().mockResolvedValue(undefined);
+ await db.execute(sql`truncate table companies restart identity cascade`);
+ });
+ afterEach(() => runningProcesses.clear());
+
+ async function fixture() {
+ const [company] = await db.insert(companies).values({ name: "Run control", issuePrefix: "RC" }).returning();
+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Controlled agent", adapterType: "codex_local" }).returning();
+ return { company, agent };
+ }
+ async function liveRun(companyId: string, agentId: string) {
+ const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "legacy", runtimeModeResolvedAt: new Date() }).returning();
+ runningProcesses.set(run.id, { child: { pid: 424242 } as never, graceSec: 4, processGroupId: 424242 });
+ return run;
+ }
+
+ it("reports a missing run without entering cancellation", async () => {
+ const deps = callbacks(db);
+ await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found");
+ expect(deps.getAgent).not.toHaveBeenCalled();
+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
+ expect(deps.appendRunEvent).not.toHaveBeenCalled();
+ });
+
+ it("joins concurrent Stop requests across factories through the shared process barrier", async () => {
+ const { company, agent } = await fixture();
+ const run = await liveRun(company.id, agent.id);
+ const deps = callbacks(db);
+ const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get");
+ let releaseTermination!: () => void;
+ const terminating = new Promise<void>(resolve => { releaseTermination = resolve; });
+ vi.mocked(terminateLocalService).mockReturnValue(terminating);
+ const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop", { terminationGraceMs: 1 });
+ await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true));
+ const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop again");
+ await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2));
+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
+ expect(terminateLocalService).toHaveBeenCalledTimes(1);
+ releaseTermination();
+ const results = await Promise.all([first, second]);
+ expect(results.map(result => result?.status)).toEqual(["cancelled", "cancelled"]);
+ expect(deps.setRunStatusFromLive).toHaveBeenCalledTimes(1);
+ expect(deps.appendRunEvent).toHaveBeenCalledTimes(1);
+ expect(deps.finalizeAgentStatus).toHaveBeenCalledTimes(1);
+ expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledTimes(1);
+ expect(deps.processRunCancellationSettlements.size).toBe(0);
+ expect(runningProcesses.has(run.id)).toBe(false);
+ expect(terminateLocalService).toHaveBeenCalledWith({ pid: 424242, processGroupId: 424242 }, { forceAfterMs: 100, signal: "SIGINT" });
+ });
+
+ it("settles every joined Stop caller with the same termination failure", async () => {
+ const { company, agent } = await fixture();
+ const run = await liveRun(company.id, agent.id);
+ const deps = callbacks(db);
+ const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get");
+ deps.activeRunExecutions.add(run.id);
+ let rejectTermination!: (error: Error) => void;
+ vi.mocked(terminateLocalService).mockReturnValue(new Promise<void>((_resolve, reject) => { rejectTermination = reject; }));
+ const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id);
+ await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true));
+ const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id);
+ const results = Promise.allSettled([first, second]);
+ await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2));
+ const failure = new Error("provider did not stop");
+ rejectTermination(failure);
+ expect(await results).toEqual([{ status: "rejected", reason: failure }, { status: "rejected", reason: failure }]);
+ expect(terminateLocalService).toHaveBeenCalledTimes(1);
+ expect(deps.processRunCancellationSettlements.size).toBe(0);
+ expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure);
+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
+ expect((await deps.getRun(run.id))?.status).toBe("running");
+ });
+
+ it("retains failed Stop evidence and the owned process until a later request succeeds", async () => {
+ const { company, agent } = await fixture();
+ const run = await liveRun(company.id, agent.id);
+ const deps = callbacks(db);
+ deps.activeRunExecutions.add(run.id);
+ const failure = new Error("termination failed");
+ vi.mocked(terminateLocalService).mockRejectedValueOnce(failure);
+ await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).rejects.toBe(failure);
+ expect(deps.failedProcessRunCancellations.get(run.id)).toMatchObject({ failed: true, error: failure });
+ expect(deps.processRunCancellationSettlements.size).toBe(0);
+ expect(runningProcesses.has(run.id)).toBe(true);
+ expect((await deps.getRun(run.id))?.status).toBe("running");
+ expect(deps.finalizeAgentStatus).not.toHaveBeenCalled();
+ expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" });
+ expect(terminateLocalService).toHaveBeenCalledTimes(2);
+ // The executor still owns removal of this evidence after it observes Stop.
+ expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure);
+ });
+
+ it("deduplicates agent batches and cancels only unclaimed pending wakes", async () => {
+ const { company, agent } = await fixture();
+ const [otherAgent] = await db.insert(agents).values({ companyId: company.id, name: "Other agent" }).returning();
+ const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "succeeded" }).returning();
+ const wakes = await db.insert(agentWakeupRequests).values([
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" },
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "deferred_issue_execution" },
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" },
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", runId: finished.id },
+ { companyId: company.id, agentId: otherAgent.id, source: "assignment", status: "queued" },
+ ]).returning();
+ const deps = callbacks(db);
+ expect(await createHeartbeatRunControl(db, deps).cancelInvocationsForAgentsInternal([agent.id, "", agent.id], "pause")).toEqual({ agentIds: [agent.id], runsCancelled: 0, wakeupsCancelled: 2 });
+ const persisted = await db.select().from(agentWakeupRequests);
+ expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "cancelled", "claimed", "queued", "queued"]);
+ expect(deps.getAgent).toHaveBeenCalledTimes(1);
+ });
+
+ it("rechecks stale budget enforcement before cancelling runs or queued wakes", async () => {
+ const { company, agent } = await fixture();
+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", runtimeMode: "legacy" }).returning();
+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }).returning();
+ const deps = callbacks(db);
+ await createHeartbeatRunControl(db, deps).cancelBudgetScopeWork({ companyId: company.id, scopeType: "agent", scopeId: agent.id, enforcement: { policyId: randomUUID(), version: 1 } });
+ expect((await deps.getRun(run.id))?.status).toBe("queued");
+ expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued");
+ expect(deps.setRunStatusFromLive).not.toHaveBeenCalled();
+ expect(deps.finalizeAgentStatus).not.toHaveBeenCalled();
+ expect(terminateLocalService).not.toHaveBeenCalled();
+ });
+
+ it("limits project budget cancellation by company, effective project, and admission cutoff", async () => {
+ const { company, agent } = await fixture();
+ const [otherCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning();
+ const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Other company agent" }).returning();
+ const projectRows = await db.insert(projects).values([
+ { companyId: company.id, name: "Target" }, { companyId: company.id, name: "Override" },
+ ]).returning();
+ const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: projectRows[0].id, title: "Bound task", issueNumber: 1, identifier: "RC-1" }).returning();
+ const old = new Date("2026-01-01T00:00:00Z");
+ const cutoff = new Date("2026-01-02T00:00:00Z");
+ const wakes = await db.insert(agentWakeupRequests).values([
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id } },
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id, projectId: projectRows[1].id } },
+ { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: cutoff, payload: { projectId: projectRows[0].id } },
+ { companyId: otherCompany.id, agentId: otherAgent.id, source: "assignment", status: "queued", createdAt: old, payload: { projectId: projectRows[0].id } },
+ ]).returning();
+ await createHeartbeatRunControl(db, callbacks(db)).cancelBudgetScopeWork({ companyId: company.id, scopeType: "project", scopeId: projectRows[0].id, createdBefore: cutoff });
+ const persisted = await db.select().from(agentWakeupRequests);
+ expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "queued", "queued", "queued"]);
+ });
+});
diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts
new file mode 100644
index 0000000000..dd3f9d8dc3
--- /dev/null
+++ b/server/src/services/heartbeat/run-control.ts
@@ -0,0 +1,1360 @@
+import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";
+import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js";
+import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js";
+import {
+ type WakeupOptions,
+} from "./run-preparation.js";
+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
+import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";
+import {
+ nativeRetryCancellationCommitCondition,
+ claimCancellationRequest,
+} from "../native-runtime/native-cancellation-request.js";
+import {
+ canContinueCancelledRun,
+ readRunCancellation,
+ requestedRunCancellation,
+} from "../run-cancellation.js";
+import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
+import { readQueuedInteractionResponse } from "../queued-interaction-response.js";
+import {
+ isAcknowledgedNativeStop,
+ acknowledgedNativeStopExecutionHasStopped,
+} from "../acknowledged-native-stop.js";
+import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js";
+import {
+ hasRemoteTerminationReceipt,
+ remoteExecutionHasStopped,
+ stoppedRemoteCleanupScopes,
+} from "../remote-execution-termination.js";
+import {
+ admitExplicitNativeContinuation,
+ undeliveredLegacyUserCommentIds,
+} from "../explicit-native-continuation.js";
+import { isCancelledNativeStartup } from "../cancelled-native-startup.js";
+import {
+ executionBlockerPredicate,
+ getExecutionBlocker,
+} from "../execution-blocker.js";
+import {
+ CONVERSATION_CONTINUATION_POLICY,
+ runUsedConversationAdapter,
+ hasConversationContinuationPolicy,
+} from "../conversation-continuation.js";
+import {
+ adapterExecutionControls,
+ captureAdapterStopOwnership,
+ waitForAdapterStop,
+} from "../adapter-execution-control.js";
+import { randomUUID } from "node:crypto";
+import {
+ and,
+ asc,
+ desc,
+ eq,
+ exists,
+ gt,
+ inArray,
+ isNull,
+ isNotNull,
+ lt,
+ lte,
+ ne,
+ notInArray,
+ or,
+ sql,
+} from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import {
+ type EnvironmentLeaseStatus,
+} from "@paperclipai/shared";
+import {
+ agentWakeupRequests,
+ companies,
+ environmentLeases,
+ heartbeatRuns,
+ issueComments,
+ issueRecoveryActions,
+ issues,
+ nativeRunFinalizations,
+} from "@paperclipai/db";
+import {
+ conflict,
+ HttpError,
+ notFound,
+} from "../../errors.js";
+import { logger } from "../../middleware/logger.js";
+import {
+ queuedCommentIdsFromWakePayload,
+ withQueuedCommentIdsInWakePayload,
+ withQueuedCommentIdsInRunContext,
+} from "../issue-queued-comment-queue.js";
+import {
+ cancelNativeSession,
+ closeWarmNativeSessionsForEnvironment,
+} from "../native-runtime/index.js";
+import { runningProcesses } from "../../adapters/index.js";
+import { parseObject } from "../../adapters/utils.js";
+import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js";
+import {
+ withCurrentBudgetEnforcement,
+ type BudgetEnforcementScope,
+} from "../budgets.js";
+import { terminateLocalService } from "../local-service-supervisor.js";
+import { deriveCommentId } from "../../modules/run-dispatch/index.js";
+import { WakeQueueApplicationError } from "../../modules/wake-queue/index.js";
+import { isUuidLike } from "@paperclipai/shared";
+import {
+ type ProviderResourceDisposition,
+} from "../environment-runtime.js";
+
+import type { createHeartbeatRunState } from "./run-state.js";
+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
+import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js";
+import type { createHeartbeatQueue } from "./queue.js";
+import type { createHeartbeatRecovery } from "./recovery.js";
+import type { createWakeQueue, PostCommitEffect as WakeQueuePostCommitEffect } from "../../modules/wake-queue/index.js";
+import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";
+
+type ProcessRunCancellationSettlement = {
+ settled: Promise<void>;
+ failed: boolean;
+ error?: unknown;
+};
+
+/** Cancellation shares executor ownership and delegates lifecycle and admission effects. */
+export interface HeartbeatRunControlDependencies extends Pick<ReturnType<typeof createHeartbeatRunState>,
+ "getRun" | "getAgent">,
+ Pick<ReturnType<typeof createHeartbeatLifecycle>,
+ "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">,
+ Pick<HeartbeatRetryDependencies, "finalizeAgentStatus"> {
+ options: {
+ closeWarmNativeSessionsForRun?: (input: {
+ runId: string;
+ reason: string;
+ }) => Promise<{ closed: number; busy: number; failed: number }>;
+ };
+ envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "releaseForRun">;
+ enqueueWakeup: ReturnType<typeof createHeartbeatQueue>["enqueueWakeup"];
+ startNextQueuedRunForAgent: ReturnType<typeof createHeartbeatQueue>["startNextQueuedRunForAgent"];
+ sweepPendingCleanupLeases: ReturnType<typeof createHeartbeatRecovery>["sweepPendingCleanupLeases"];
+ timerClaimWasFirstHeartbeat: ReturnType<typeof createHeartbeatRetries>["timerClaimWasFirstHeartbeat"];
+ wakeQueue: Pick<ReturnType<typeof createWakeQueue>, "releaseIssueExecution">;
+ applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise<void>;
+ getSchedulingSuppression: () => Promise<{ suppressed: boolean }>;
+ // Scheduler and route service instances must retain the same in-process barriers.
+ activeRunExecutions: Set<string>;
+ processRunCancellationSettlements: Map<string, ProcessRunCancellationSettlement>;
+ failedProcessRunCancellations: Map<string, ProcessRunCancellationSettlement>;
+}
+
+export function leaseReleaseStatusForRunStatus(
+ status: string | null | undefined,
+): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {
+ if (status === "cancelled") return "expired";
+ return status === "failed" || status === "timed_out" ? "failed" : "released";
+}
+
+export function providerResourceDispositionForTerminalRun(
+ desired: ProviderResourceDisposition | undefined,
+ status: string | null | undefined,
+): ProviderResourceDisposition | undefined {
+ if (desired !== "keep_running") return desired;
+ return status === "succeeded" ? desired : "stop_and_retain";
+}
+
+export async function terminateHeartbeatRunProcess(input: {
+ pid: number | null | undefined;
+ processGroupId: number | null | undefined;
+ graceMs?: number;
+ signal?: NodeJS.Signals;
+}) {
+ const pid = input.pid ?? null;
+ const processGroupId = input.processGroupId ?? null;
+ if (typeof pid !== "number" && typeof processGroupId !== "number") return;
+
+ await terminateLocalService(
+ {
+ pid:
+ typeof pid === "number" && Number.isInteger(pid) && pid > 0
+ ? pid
+ : (processGroupId ?? 0),
+ processGroupId:
+ typeof processGroupId === "number" &&
+ Number.isInteger(processGroupId) &&
+ processGroupId > 0
+ ? processGroupId
+ : null,
+ },
+ { forceAfterMs: input.graceMs, signal: input.signal },
+ );
+}
+
+export async function cancelHeartbeatNativeRun(input: {
+ db: Db;
+ runId: string;
+ reason: string;
+ runtimeMode: string | null;
+ cancellationRequestId?: string;
+ cancel?: (
+ runId: string,
+ reason: string,
+ options: { db: Db; scope: "run"; cancellationRequestId?: string },
+ ) => Promise<{ decision: unknown | null; auditId: string | null }>;
+}) {
+ if (input.runtimeMode !== "native") {
+ return { decision: null, auditId: null };
+ }
+ const cancellation = input.cancel
+ ? await input.cancel(input.runId, input.reason, {
+ db: input.db,
+ scope: "run",
+ ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),
+ })
+ : await cancelNativeSession(input.runId, input.reason, {
+ db: input.db,
+ scope: "run",
+ ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}),
+ });
+ if (!cancellation.decision || !cancellation.auditId) {
+ throw new Error("native_cancellation_outcome_not_audited");
+ }
+ return cancellation;
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+export type CancelRunOptions = {
+ budgetEnforcement?: BudgetEnforcementScope;
+ /** Optional board request identity, atomically reserved for native Stop. */
+ cancellationRequestId?: string;
+ cancellationRequestedByUserId?: string | null;
+ errorCode?: string;
+ resultJson?: Record<string, unknown>;
+ eventMessage?: string;
+ eventPayload?: Record<string, unknown>;
+ /** Per-call graceful process shutdown window, bounded to a safe range. */
+ terminationGraceMs?: number;
+ /** Caller is immediately scheduling an explicit successor path. */
+ suppressImmediateRecovery?: boolean;
+ };
+
+/** Run control binds cancellation, resource release, and saved-input resumption. */
+export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunControlDependencies) {
+ const {
+ getRun,
+ options,
+ envOrchestrator,
+ activeRunExecutions,
+ enqueueWakeup,
+ getAgent,
+ sweepPendingCleanupLeases,
+ getSchedulingSuppression,
+ wakeQueue,
+ applyWakeQueuePostCommitEffects,
+ processRunCancellationSettlements,
+ mergeRunStopMetadataForAgent,
+ setRunStatusFromLive,
+ failedProcessRunCancellations,
+ setWakeupStatus,
+ appendRunEvent,
+ finalizeAgentStatus,
+ timerClaimWasFirstHeartbeat,
+ startNextQueuedRunForAgent,
+ setRunStatus,
+ } = dependencies;
+
+ async function releaseEnvironmentLeasesForRun(input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ status: string | null | undefined;
+ failureReason?: string | null;
+ providerResourceDisposition?: ProviderResourceDisposition;
+ nativeLifecycleTelemetry?: {
+ provider: string;
+ harness: string;
+ lifecycleMode: "per_turn" | "warm";
+ sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
+ };
+ }) {
+ const leaseOwnerRun = await getRun(input.runId);
+ if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;
+ // Recovery can finish workspace copy-back outside the executor's finally.
+ // Successful copy-back does not earn warm retention for a failed turn.
+ const status = leaseOwnerRun?.status ?? input.status;
+ const providerResourceDisposition = providerResourceDispositionForTerminalRun(
+ input.providerResourceDisposition,
+ status,
+ );
+ if (providerResourceDisposition === "destroy") {
+ const closeResult = await (
+ options.closeWarmNativeSessionsForRun ??
+ (async ({ runId, reason }) => {
+ const environmentIds = await db
+ .selectDistinct({ environmentId: environmentLeases.environmentId })
+ .from(environmentLeases)
+ .where(
+ and(
+ eq(environmentLeases.heartbeatRunId, runId),
+ eq(environmentLeases.status, "active"),
+ ),
+ )
+ .then((rows) =>
+ rows.flatMap((row) =>
+ typeof row.environmentId === "string" &&
+ row.environmentId.length > 0
+ ? [row.environmentId]
+ : [],
+ ),
+ );
+ const aggregate = { closed: 0, busy: 0, failed: 0 };
+ for (const environmentId of environmentIds) {
+ const result = await closeWarmNativeSessionsForEnvironment({
+ environmentId,
+ reason,
+ });
+ aggregate.closed += result.closed;
+ aggregate.busy += result.busy;
+ aggregate.failed += result.failed;
+ }
+ return aggregate;
+ })
+ )({
+ runId: input.runId,
+ reason: "terminal heartbeat run destroyed its environment lease",
+ }).catch((err) => {
+ logger.warn(
+ { err, runId: input.runId },
+ "failed to close warm native sessions before environment lease destruction",
+ );
+ return { closed: 0, busy: 0, failed: 1 };
+ });
+ // A failed remote checkpoint cannot veto destruction of the isolated
+ // sandbox after the run stopped. Provider destruction supplies the exit
+ // proof; it does not turn the interrupted checkpoint into a success.
+ const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&
+ ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)
+ ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
+ eq(environmentLeases.companyId, input.companyId),
+ eq(environmentLeases.heartbeatRunId, input.runId),
+ ))
+ : [];
+ const canDestroyRemote = remoteLeases.length > 0 &&
+ remoteLeases.every(lease => lease.provider && lease.provider !== "local");
+ if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {
+ logger.warn(
+ { runId: input.runId, warmNativeSessions: closeResult },
+ "deferred environment lease destruction until warm native sessions close",
+ );
+ return;
+ }
+ }
+ const releaseResult = await envOrchestrator
+ .releaseForRun({
+ heartbeatRunId: input.runId,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ status: leaseReleaseStatusForRunStatus(status),
+ failureReason: input.failureReason ?? undefined,
+ providerResourceDisposition,
+ nativeLifecycleTelemetry: input.nativeLifecycleTelemetry,
+ })
+ .catch((err) => {
+ logger.warn(
+ { err, runId: input.runId },
+ "failed to release environment leases for heartbeat run",
+ );
+ return null;
+ });
+ for (const releaseError of releaseResult?.errors ?? []) {
+ logger.warn(
+ {
+ err: releaseError.error,
+ leaseId: releaseError.leaseId,
+ runId: input.runId,
+ },
+ "failed to release environment lease for heartbeat run",
+ );
+ }
+ await acknowledgeRemoteStop(input.runId, input.companyId);
+ }
+
+ async function acknowledgeRemoteStop(runId: string, companyId: string) {
+ // The provider receipt arrives after adapter settlement. A remote ACP child
+ // has no host PID, so only this target-aware boundary can acknowledge Stop.
+ const stopped = await getRun(runId);
+ if (stopped?.runtimeMode === "native") {
+ const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId);
+ for (const remoteCleanupScope of scopes ?? []) {
+ completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });
+ }
+ }
+ if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" &&
+ parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&
+ await runUsedConversationAdapter(db, stopped) &&
+ await remoteExecutionHasStopped(db, companyId, runId)) {
+ await db.update(heartbeatRuns).set({
+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({
+ executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),
+ state: "acknowledged", acknowledgedAt: new Date().toISOString(),
+ proof: "provider_termination_receipt" },
+ conversationContinuation: CONVERSATION_CONTINUATION_POLICY,
+ })}::jsonb`,
+ updatedAt: new Date(),
+ }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),
+ eq(heartbeatRuns.status, "cancelled")));
+ }
+ }
+
+ async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) {
+ if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;
+ const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt
+ ? await db.select().from(nativeRunFinalizations).where(and(
+ eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),
+ )) : [];
+ const cancelledPreparation = run.runtimeMode === "legacy" &&
+ await isCancelledNativeStartup(db, run, preparationCoordinator);
+ if (run.runtimeMode !== "native" && !cancelledPreparation &&
+ parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true &&
+ !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return;
+ const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);
+ if (!issueId) return;
+ const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null;
+ const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase,
+ leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and(
+ eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),
+ )) : [];
+ // Stop ends one response. Saved user input can enter ordinary admission
+ // once its executor settles; it does not need a manufactured crash incident.
+ // The ordinary path still enforces task holds, ownership, and native session
+ // cleanup before starting a provider.
+ const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) &&
+ !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner &&
+ ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") &&
+ await acknowledgedNativeStopExecutionHasStopped(db, currentRun) &&
+ !(await getExecutionBlocker(db, run.companyId, issueId));
+ const legacyContinuation = run.runtimeMode === "legacy" &&
+ hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) &&
+ !(await getExecutionBlocker(db, run.companyId, issueId));
+ if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return;
+ const pending = await db.select().from(agentWakeupRequests).where(and(
+ eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ eq(agentWakeupRequests.requestedByActorType, "user"),
+ requestId ? eq(agentWakeupRequests.id, requestId) : undefined,
+ sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
+ )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50);
+ for (const wake of pending) {
+ if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue;
+ let payload = parseObject(wake.payload);
+ if (payload.queuedCommentInterrupt) {
+ await resumeQueuedCommentInterrupt(wake.companyId, wake.id);
+ continue;
+ }
+ let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
+ let commentId = deriveCommentId(context, payload);
+ let requestedByActorId = wake.requestedByActorId;
+ const reason = readNonEmptyString(context.wakeReason) ?? wake.reason;
+ if (run.runtimeMode === "native" || cancelledPreparation) {
+ const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId,
+ queuedCommentIdsFromWakePayload(payload));
+ if (!ids.length) continue;
+ payload = withQueuedCommentIdsInWakePayload(payload, ids);
+ context = withQueuedCommentIdsInRunContext(context, ids);
+ commentId = ids.at(-1)!;
+ // A coalesced queue can contain several authors. Its saved comments,
+ // not the outer wake's first author, authorize the remaining input.
+ const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and(
+ eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId),
+ ));
+ requestedByActorId = author?.id ?? null;
+ }
+ if (legacyContinuation || stoppedNativeContinuation) {
+ if (!commentId || !run.finishedAt || !requestedByActorId ||
+ !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue;
+ const [comment] = await db.select().from(issueComments).where(and(
+ eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId),
+ sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"),
+ eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt),
+ isNull(issueComments.createdByRunId),
+ stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt),
+ ));
+ if (!comment?.body.trim()) continue;
+ } else {
+ let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." };
+ const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId,
+ agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId,
+ reason, commentId, successorRunId: randomUUID(), dryRun: true,
+ ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),
+ onBlocked: (reason, message) => { wait = { reason, message }; },
+ });
+ if (!admitted) {
+ await db.update(agentWakeupRequests).set({
+ payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
+ coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
+ updatedAt: new Date(),
+ }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution")));
+ continue;
+ }
+ }
+ // Re-enter ordinary admission with the original user's authority. It
+ // atomically adopts the deferred comments and still applies every gate.
+ await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"],
+ reason, payload, contextSnapshot: context,
+ requestedByActorType: "user", requestedByActorId,
+ ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}),
+ idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id);
+ break;
+ }
+ }
+
+ async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) {
+ return resumeSavedLegacyComments(companyId, queueId, true, opts);
+ }
+
+ async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) {
+ const [wake] = await db.select().from(agentWakeupRequests).where(and(
+ eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ ));
+ if (!wake) return;
+ const payload = parseObject(wake.payload);
+ let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId);
+ let commentIds = queuedCommentIdsFromWakePayload(payload);
+ const issueId = readNonEmptyString(payload.issueId);
+ if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return;
+ const response = await readQueuedInteractionResponse(db, companyId, issueId, payload);
+ if (!commentIds.length && !response) return;
+ // Resolved cards are immutable input. Only an explicit Interrupt click can
+ // authorize continuation across a stopped execution; ordinary completion
+ // uses normal deferred-wake promotion.
+ if (response && !interrupted) return;
+ if (!interrupted) {
+ commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds);
+ if (!commentIds.length) return;
+ // The queue itself may have begun as a system wake. The saved human
+ // comment, not that wake's origin or mutable caller payload, is authority.
+ const [comment] = await db.select().from(issueComments).where(and(
+ eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),
+ eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"),
+ isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),
+ )).orderBy(desc(issueComments.createdAt)).limit(1);
+ if (!comment?.body.trim() || !comment.authorUserId) return;
+ actorId = comment.authorUserId;
+ }
+ if (!actorId) return;
+ const agent = await getAgent(wake.agentId);
+ if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return;
+ const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
+ eq(heartbeatRuns.companyId, companyId),
+ eq(heartbeatRuns.agentId, wake.agentId),
+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
+ inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),
+ )).limit(1);
+ if (active) return;
+ if (interrupted && opts?.retryCleanup) {
+ // Only the HTTP click grants an extra cleanup attempt. Periodic retries
+ // reuse the intent to deliver, never a fresh provider teardown budget.
+ const sourceRun = await db.transaction(async tx => {
+ const [task] = await tx.select().from(issues).where(and(
+ eq(issues.companyId, companyId), eq(issues.id, issueId),
+ )).for("update");
+ const [current] = await tx.select().from(agentWakeupRequests).where(and(
+ eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
+ sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`,
+ ));
+ if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) ||
+ !current || (!queuedCommentIdsFromWakePayload(current.payload).length &&
+ !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null;
+ const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
+ eq(heartbeatRuns.companyId, companyId),
+ eq(heartbeatRuns.agentId, wake.agentId),
+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
+ inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]),
+ )).limit(1);
+ if (successor) return null;
+ const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId);
+ const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and(
+ eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId),
+ eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"),
+ inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]),
+ sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
+ )).then(rows => rows[0]) : null;
+ if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null;
+ // Older ephemeral leases recorded successful cleanup without a provider
+ // receipt. Re-verify them through the recorded teardown path; a timestamp
+ // alone never certifies termination. Retained/reusable resources stay put.
+ const historical = await tx.select().from(environmentLeases).where(and(
+ eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id),
+ eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt),
+ inArray(environmentLeases.status, ["released", "expired", "failed"]),
+ )).for("update");
+ for (const lease of historical) {
+ if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue;
+ // Provider resource IDs identify physical sandboxes. A lease in any
+ // company can still own this resource; never destroy it on behalf of
+ // this company. This existence-only guard exposes no foreign data.
+ const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
+ ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),
+ eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
+ or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])),
+ )).limit(1);
+ if (otherOwner) continue;
+ await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() })
+ .where(eq(environmentLeases.id, lease.id));
+ }
+ return run;
+ });
+ if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: {
+ companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt",
+ } });
+ }
+ const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds);
+ delete deliveryPayload.queuedCommentInterrupt;
+ await enqueueWakeup(wake.agentId, {
+ source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
+ payload: deliveryPayload, contextSnapshot: response
+ ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId,
+ responsibleUserId: actorId }
+ : withQueuedCommentIdsInRunContext({
+ issueId, triggeredBy: "board", actorId, responsibleUserId: actorId,
+ }, commentIds),
+ requestedByActorType: "user", requestedByActorId: actorId,
+ ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }),
+ issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] },
+ idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`,
+ }, queueId);
+ }
+
+ async function resumeExecutionWaitComments() {
+ if ((await getSchedulingSuppression()).suppressed) return;
+ const waits = await db.select({ wake: agentWakeupRequests })
+ .from(agentWakeupRequests)
+ .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
+ sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
+ eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
+ .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
+ .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
+ eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
+ executionBlockerPredicate(),
+ ))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ eq(agentWakeupRequests.requestedByActorType, "user"),
+ sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
+ sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`,
+ lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
+ notInArray(issues.status, ["done", "cancelled"])))
+ .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
+ const seen = new Set<string>();
+ for (const { wake } of waits) {
+ const issueId = String(wake.payload?.issueId);
+ if (seen.has(issueId)) continue;
+ seen.add(issueId);
+ // Advance the cursor even for invalid evidence so one damaged task cannot
+ // starve later requests in this bounded scan. Preserve concurrent edits.
+ const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and(
+ eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
+ )).returning({ id: agentWakeupRequests.id });
+ if (!claimed) continue;
+ // Match normal admission's deterministic current blocker selection. An
+ // arbitrary historical action must not choose the retry's source run.
+ const blocker = await getExecutionBlocker(db, wake.companyId, issueId);
+ const sourceId = blocker?.runId;
+ if (!sourceId || !isUuidLike(sourceId)) continue;
+ const run = await getRun(sourceId, { includeExecutionEvidence: true });
+ if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
+ if (canContinueCancelledRun(run)) {
+ await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
+ logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
+ });
+ continue;
+ }
+ await resumeRemoteStopComments(run, wake.id).catch(err => {
+ logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
+ });
+ }
+ }
+
+ async function releaseIssueExecutionAndPromote(
+ run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId">,
+ options: { suppressImmediateRecovery?: boolean } = {},
+ ) {
+ try {
+ const source = await getRun(run.id);
+ const { postCommitEffects } = await wakeQueue.releaseIssueExecution({
+ companyId: run.companyId,
+ runId: run.id,
+ now: new Date(),
+ // A durable authentication card owns recovery. This covers the review path,
+ // while continuation classification blocks periodic generic retries.
+ suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source),
+ });
+ await applyWakeQueuePostCommitEffects(postCommitEffects);
+ const completed = await getRun(run.id);
+ const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId)
+ ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId;
+ if (completed?.status === "succeeded" && issueId) {
+ await settleSlackConversation(db, run.companyId, issueId).catch((err) => {
+ logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation");
+ });
+ }
+ } catch (error) {
+ if (
+ error instanceof WakeQueueApplicationError &&
+ error.code === "responsible_user_unresolved"
+ ) {
+ // Every other `responsible_user_unresolved` HttpError in this file
+ // carries `code` inside its own `details`; match that shape here too.
+ throw new HttpError(422, error.message, { code: error.code, ...error.details });
+ }
+ throw error;
+ }
+ }
+
+ async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) {
+ const runIssueId = sql<
+ string | null
+ >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`;
+ const effectiveProjectId = sql<
+ string | null
+ >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`;
+
+ const rows = await db
+ .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id })
+ .from(heartbeatRuns)
+ .leftJoin(
+ issues,
+ and(
+ eq(issues.companyId, companyId),
+ sql`${issues.id}::text = ${runIssueId}`,
+ ),
+ )
+ .where(
+ and(
+ eq(heartbeatRuns.companyId, companyId),
+ createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined,
+ inArray(heartbeatRuns.status, [
+ ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
+ ]),
+ sql`${effectiveProjectId} = ${projectId}`,
+ ),
+ );
+
+ return rows.map((row) => row.id);
+ }
+
+ async function listProjectScopedWakeupIds(
+ companyId: string,
+ projectId: string,
+ database: Db = db,
+ ) {
+ const wakeIssueId = sql<
+ string | null
+ >`${agentWakeupRequests.payload} ->> 'issueId'`;
+ const effectiveProjectId = sql<
+ string | null
+ >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`;
+
+ const rows = await database
+ .selectDistinctOn([agentWakeupRequests.id], {
+ id: agentWakeupRequests.id,
+ })
+ .from(agentWakeupRequests)
+ .leftJoin(
+ issues,
+ and(
+ eq(issues.companyId, companyId),
+ sql`${issues.id}::text = ${wakeIssueId}`,
+ ),
+ )
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ inArray(agentWakeupRequests.status, [
+ "queued",
+ "deferred_issue_execution",
+ ]),
+ sql`${agentWakeupRequests.runId} is null`,
+ sql`${effectiveProjectId} = ${projectId}`,
+ ),
+ );
+
+ return rows.map((row) => row.id);
+ }
+
+ async function cancelPendingWakeupsForBudgetScope(
+ scope: BudgetEnforcementScope,
+ ) {
+ return withCurrentBudgetEnforcement(db, scope, async (tx) => {
+ const now = new Date();
+ let wakeupIds: string[] = [];
+
+ if (scope.scopeType === "company") {
+ wakeupIds = await tx
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, scope.companyId),
+ inArray(agentWakeupRequests.status, [
+ "queued",
+ "deferred_issue_execution",
+ ]),
+ sql`${agentWakeupRequests.runId} is null`,
+ ),
+ )
+ .then((rows) => rows.map((row) => row.id));
+ } else if (scope.scopeType === "agent") {
+ wakeupIds = await tx
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, scope.companyId),
+ eq(agentWakeupRequests.agentId, scope.scopeId),
+ inArray(agentWakeupRequests.status, [
+ "queued",
+ "deferred_issue_execution",
+ ]),
+ sql`${agentWakeupRequests.runId} is null`,
+ ),
+ )
+ .then((rows) => rows.map((row) => row.id));
+ } else {
+ wakeupIds = await listProjectScopedWakeupIds(
+ scope.companyId,
+ scope.scopeId,
+ tx,
+ );
+ }
+
+ if (wakeupIds.length === 0) return 0;
+
+ await tx
+ .update(agentWakeupRequests)
+ .set({
+ status: "cancelled",
+ finishedAt: now,
+ error: "Cancelled due to budget pause",
+ updatedAt: now,
+ })
+ .where(and(
+ inArray(agentWakeupRequests.id, wakeupIds),
+ inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),
+ isNull(agentWakeupRequests.runId),
+ scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined,
+ ));
+
+ return wakeupIds.length;
+ });
+ }
+
+ function cancellationTerminationGraceMs(
+ configuredGraceSec: number,
+ requestedGraceMs: number | undefined,
+ ) {
+ const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000;
+ if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) {
+ return configuredGraceMs;
+ }
+ return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs)));
+ }
+
+ async function cancelRunInternal(
+ runId: string,
+ reason = "Cancelled by control plane",
+ options: CancelRunOptions = {},
+ ) {
+ let run = await getRun(runId);
+ if (!run) throw notFound("Heartbeat run not found");
+ if (options.cancellationRequestId) {
+ run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null);
+ }
+ // The caller claim checked retry eligibility under both durable row locks.
+ // This is only a cancellation candidate: dispatch rechecks the coordinator
+ // under lock, and the final CAS requires its own unchanged acknowledged
+ // retry fence. Re-reading only retryable_failure here would strand replay.
+ const pendingNativeRetry =
+ run.runtimeMode === "native" && run.status === "failed" && (
+ Boolean(options.cancellationRequestId) || await db
+ .select({ runId: nativeRunFinalizations.runId })
+ .from(nativeRunFinalizations)
+ .where(
+ and(
+ eq(nativeRunFinalizations.runId, run.id),
+ eq(nativeRunFinalizations.companyId, run.companyId),
+ eq(nativeRunFinalizations.phase, "retryable_failure"),
+ ),
+ )
+ .then((rows) => rows.length > 0)
+ );
+ if (
+ !pendingNativeRetry &&
+ !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(
+ run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number],
+ )
+ )
+ return run;
+ const agent = await getAgent(run.agentId);
+ const errorCode = options.errorCode ?? "cancelled";
+ const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason);
+
+ const pendingProcessCancellation = processRunCancellationSettlements.get(
+ run.id,
+ );
+ if (pendingProcessCancellation) {
+ await pendingProcessCancellation.settled;
+ if (pendingProcessCancellation.failed)
+ throw pendingProcessCancellation.error;
+ return getRun(run.id);
+ }
+ const running = runningProcesses.get(run.id);
+ const stopOwnership =
+ run.runtimeMode !== "native"
+ ? captureAdapterStopOwnership(run.id)
+ : undefined;
+ const control = stopOwnership?.control;
+ // Capture the existing adapter owner before waiting on the run lock. Then
+ // atomically fence preparation and refresh the selected runtime, so Stop
+ // cannot miss a native handoff that won after its first read.
+ // Established legacy processes must still be stopped if the database is
+ // unavailable. Only native or not-yet-dispatched preparation needs this
+ // additional durable fence before its existing cancellation path.
+ if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) {
+ const fence = async (tx: Db) => {
+ const [fenced] = await tx.update(heartbeatRuns).set({
+ // Record handoff intent before native cancellation can finalize and release
+ // the run. Only its audited stop acknowledgement suppresses recovery.
+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
+ ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true
+ ? { reassignmentStopRequested: true } : {})}::jsonb ||
+ ${JSON.stringify({ cancellation })}::jsonb ||
+ jsonb_build_object('startupCancellation', jsonb_build_object(
+ 'requestedAt', ${new Date().toISOString()}::text,
+ 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'
+ and ${heartbeatRuns.runtimeModeResolvedAt} is null
+ and ${heartbeatRuns.executionStage} = 'preparing'
+ and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false)
+ ))`,
+ }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status,
+ pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
+ ))).returning();
+ return fenced ?? null;
+ };
+ let fenced: typeof run | null;
+ try {
+ fenced = options.budgetEnforcement
+ ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence)
+ : await fence(db);
+ } catch (error) {
+ stopOwnership?.release();
+ throw error;
+ }
+ if (!fenced) {
+ stopOwnership?.release();
+ return getRun(runId);
+ }
+ run = fenced;
+ }
+ const resultJson = { ...(agent
+ ? {
+ ...mergeRunStopMetadataForAgent(agent, "cancelled", {
+ resultJson: parseObject(run.resultJson),
+ errorCode,
+ errorMessage: reason,
+ }),
+ ...(options.resultJson ?? {}),
+ }
+ : options.resultJson), cancellation };
+
+ try {
+ let releaseProcessCancellation: (() => void) | undefined;
+ const processCancellationSettlement =
+ run.runtimeMode !== "native" &&
+ !control &&
+ running
+ ? {
+ settled: new Promise<void>((resolve) => {
+ releaseProcessCancellation = resolve;
+ }),
+ failed: false,
+ error: undefined as unknown,
+ }
+ : undefined;
+ if (processCancellationSettlement) {
+ // No await between joining an existing owner above and registering ours.
+ processRunCancellationSettlements.set(
+ run.id,
+ processCancellationSettlement,
+ );
+ }
+ const cancellation = await (async () => {
+ try {
+ if (control) {
+ await db
+ .update(heartbeatRuns)
+ .set({
+ error: reason,
+ errorCode,
+ // Stop may have read the row before adapter settlement saved
+ // a required-copyback receipt. Merge into current database JSON
+ // and leave those server-owned fields to the restore recorder.
+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({
+ ...resultJson,
+ ...(!running ? { executionCancellation: {
+ state: "requested", requestedAt: new Date().toISOString(),
+ } } : {}),
+ }, true),
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, run.id),
+ eq(heartbeatRuns.status, "running"),
+ ),
+ );
+ control.controller.abort(new Error(reason));
+ }
+ let terminationSettled = false;
+ try {
+ await cancelHeartbeatNativeRun({
+ db,
+ runId: run.id,
+ reason,
+ runtimeMode: run.runtimeMode,
+ ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}),
+ });
+ if (running) {
+ await terminateHeartbeatRunProcess({
+ pid: running.child.pid,
+ processGroupId: running.processGroupId,
+ // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM
+ // can leave commands in their separate process groups alive.
+ signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined,
+ graceMs: cancellationTerminationGraceMs(
+ running.graceSec,
+ options.terminationGraceMs,
+ ),
+ });
+ }
+ terminationSettled = true;
+ } finally {
+ if (
+ (!processCancellationSettlement || terminationSettled) &&
+ runningProcesses.get(run.id) === running
+ ) {
+ runningProcesses.delete(run.id);
+ }
+ }
+
+ if (control) {
+ await waitForAdapterStop(control.settled, undefined, {
+ runId: run.id,
+ adapterType: agent?.adapterType,
+ runtimeMode: run.runtimeMode,
+ abortRequested: control.controller.signal.aborted,
+ }, control);
+ const stopped = await getRun(run.id);
+ if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) {
+ if (
+ parseObject(stopped.resultJson?.executionCancellation).state !==
+ "acknowledged"
+ ) {
+ throw conflict(
+ "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.",
+ );
+ }
+ // The owned adapter already finalized this run and its lifecycle.
+ // Do not replay the process cancellation side effects below.
+ return { run: stopped, updated: false };
+ }
+ }
+
+ const finishedAt = new Date();
+ const persistedCancellationResult =
+ run.runtimeMode === "native"
+ ? await getRun(run.id).then((current) =>
+ parseObject(current?.resultJson),
+ )
+ : {};
+ return await setRunStatusFromLive(
+ run.id,
+ "cancelled",
+ pendingNativeRetry
+ ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"]
+ : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES],
+ {
+ finishedAt,
+ error: reason,
+ errorCode,
+ ...(resultJson ||
+ Object.keys(persistedCancellationResult).length > 0
+ ? {
+ resultJson: {
+ ...persistedCancellationResult,
+ ...(resultJson ?? {}),
+ // A scheduler placeholder has no process to acknowledge.
+ // Preserve its normal release policy instead of treating
+ // it as an operator stop of provider work.
+ ...(processCancellationSettlement && agent && running && (
+ (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) ||
+ (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0)
+ )
+ ? mergeRunStopMetadataForAgent(agent, "cancelled", {
+ resultJson: {
+ ...resultJson,
+ executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() },
+ },
+ errorCode, errorMessage: reason,
+ })
+ : {}),
+ // The native cancellation helper may have advanced a durable
+ // pending intent to its acknowledged state after `run` was
+ // first read. Never let that stale snapshot overwrite the
+ // authoritative post-dispatch acknowledgement.
+ ...(Object.hasOwn(
+ persistedCancellationResult,
+ "nativeCancellation",
+ )
+ ? {
+ nativeCancellation:
+ persistedCancellationResult.nativeCancellation,
+ }
+ : {}),
+ },
+ }
+ : {}),
+ },
+ undefined,
+ pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined,
+ );
+ } catch (error) {
+ if (processCancellationSettlement) {
+ processCancellationSettlement.failed = true;
+ processCancellationSettlement.error = error;
+ if (activeRunExecutions.has(run.id)) {
+ failedProcessRunCancellations.set(
+ run.id,
+ processCancellationSettlement,
+ );
+ }
+ }
+ throw error;
+ } finally {
+ if (processCancellationSettlement) {
+ if (
+ processRunCancellationSettlements.get(run.id) ===
+ processCancellationSettlement
+ ) {
+ processRunCancellationSettlements.delete(run.id);
+ }
+ // Always settle waiters, including termination and DB errors. A
+ // failed Stop remains an error to its caller, never a cancellation
+ // receipt; the signal-bearing executor may then record failure.
+ releaseProcessCancellation?.();
+ }
+ }
+ })();
+ const cancelled = cancellation.run;
+
+ if (cancellation.updated && cancelled) {
+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {
+ finishedAt: cancelled.finishedAt ?? new Date(),
+ error: reason,
+ });
+ await appendRunEvent(cancelled, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: options.eventMessage ?? "run cancelled",
+ payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) },
+ });
+ await releaseIssueExecutionAndPromote(cancelled, {
+ suppressImmediateRecovery: options.suppressImmediateRecovery,
+ });
+ await finalizeAgentStatus(run.agentId, "cancelled", undefined, {
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ });
+ await startNextQueuedRunForAgent(run.agentId);
+ }
+ return cancelled;
+ } finally {
+ stopOwnership?.release();
+ }
+ }
+
+ async function cancelActiveForAgentInternal(
+ agentId: string,
+ reason = "Cancelled due to agent pause",
+ errorCode = "cancelled",
+ ) {
+ const agent = await getAgent(agentId);
+ const runs = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(
+ and(
+ eq(heartbeatRuns.agentId, agentId),
+ inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),
+ ),
+ );
+
+ for (const run of runs) {
+ const stopOwnership =
+ run.runtimeMode !== "native"
+ ? captureAdapterStopOwnership(run.id)
+ : undefined;
+ try {
+ if (stopOwnership?.control) {
+ await cancelRunInternal(run.id, reason, { errorCode });
+ continue;
+ }
+ if (run.runtimeMode === "native") {
+ await db.update(heartbeatRuns).set({ resultJson:
+ sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`,
+ }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status)));
+ await cancelHeartbeatNativeRun({
+ db,
+ runId: run.id,
+ reason,
+ runtimeMode: run.runtimeMode,
+ });
+ }
+ const persistedCancellationResult =
+ run.runtimeMode === "native"
+ ? await getRun(run.id).then((current) =>
+ parseObject(current?.resultJson),
+ )
+ : parseObject(run.resultJson);
+ await setRunStatus(run.id, "cancelled", {
+ finishedAt: new Date(),
+ error: reason,
+ errorCode,
+ resultJson: {
+ ...persistedCancellationResult,
+ ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", {
+ resultJson: persistedCancellationResult, errorCode, errorMessage: reason,
+ }) : {}),
+ cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason),
+ },
+ });
+
+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {
+ finishedAt: new Date(),
+ error: reason,
+ });
+
+ const running = runningProcesses.get(run.id);
+ if (running) {
+ await terminateHeartbeatRunProcess({
+ pid: running.child.pid,
+ processGroupId: running.processGroupId,
+ graceMs: Math.max(1, running.graceSec) * 1000,
+ });
+ }
+ runningProcesses.delete(run.id);
+ await releaseIssueExecutionAndPromote(run);
+ } finally {
+ stopOwnership?.release();
+ }
+ }
+
+ return runs.length;
+ }
+
+ async function cancelPendingWakeupsForAgentsInternal(
+ agentIds: string[],
+ reason: string,
+ ) {
+ const uniqueAgentIds = [...new Set(agentIds)].filter(
+ (agentId) => agentId.length > 0,
+ );
+ if (uniqueAgentIds.length === 0) return 0;
+
+ const now = new Date();
+ const wakeupIds = await db
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ inArray(agentWakeupRequests.agentId, uniqueAgentIds),
+ inArray(agentWakeupRequests.status, [
+ "queued",
+ "deferred_issue_execution",
+ ]),
+ sql`${agentWakeupRequests.runId} is null`,
+ ),
+ )
+ .then((rows) => rows.map((row) => row.id));
+
+ if (wakeupIds.length === 0) return 0;
+
+ await db
+ .update(agentWakeupRequests)
+ .set({
+ status: "cancelled",
+ finishedAt: now,
+ error: reason,
+ updatedAt: now,
+ })
+ .where(inArray(agentWakeupRequests.id, wakeupIds));
+
+ return wakeupIds.length;
+ }
+
+ async function cancelInvocationsForAgentsInternal(
+ agentIds: string[],
+ reason: string,
+ ) {
+ const uniqueAgentIds = [...new Set(agentIds)].filter(
+ (agentId) => agentId.length > 0,
+ );
+ let runsCancelled = 0;
+ for (const agentId of uniqueAgentIds) {
+ runsCancelled += await cancelActiveForAgentInternal(agentId, reason);
+ }
+ const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal(
+ uniqueAgentIds,
+ reason,
+ );
+ return {
+ agentIds: uniqueAgentIds,
+ runsCancelled,
+ wakeupsCancelled,
+ };
+ }
+
+ async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) {
+ const runIds = scope.scopeType === "project"
+ ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore)
+ : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
+ eq(heartbeatRuns.companyId, scope.companyId),
+ scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined,
+ scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined,
+ inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]),
+ )).then((rows) => rows.map((row) => row.id));
+ for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope });
+ await cancelPendingWakeupsForBudgetScope(scope);
+ }
+
+ return {
+ cancelBudgetScopeWork,
+ releaseIssueExecutionAndPromote,
+ releaseEnvironmentLeasesForRun,
+ acknowledgeRemoteStop,
+ resumeRemoteStopComments,
+ cancelRunInternal,
+ resumeExecutionWaitComments,
+ resumeQueuedCommentInterrupt,
+ resumeSavedLegacyComments,
+ cancelActiveForAgentInternal,
+ cancelInvocationsForAgentsInternal,
+ };
+}