The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(heartbeat): extract prepared runtime execution (#15803)
Dotta1fb3f9746e6b
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 1e9be2453b..f58ef608b0 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1358,8 +1358,16 @@ Native selection retains the row lock that serializes it with cancellation andcontroller lease renewal. The executor receives an explicit selection result;lifecycle and ownership callbacks update its teardown state immediately, includingwhen preparation later throws. Public sandbox lifecycle and model fallback helpers-remain re-exported by `heartbeat.ts`. Provider dispatch and final cleanup stay in-the executor.+remain re-exported by `heartbeat.ts`.++Runtime execution is in `server/src/services/heartbeat/runtime-execution.ts`.+`executeHeartbeatRuntime` dispatches a prepared native session or legacy adapter,+captures usage, finalizes workspace operations, and releases gateway tokens and+instruction copies. The executor supplies the atomic dispatch gate and retains+admission, terminal run status, retry handling, and outer lease cleanup. An explicit+dispatch result preserves early cancellation returns; ownership callbacks update+teardown state before errors propagate. Instruction getters retain updates made+during provider execution. Keep policy changes separate from this extraction.## Wake Context Deliverydiff --git a/doc/composer-stop.md b/doc/composer-stop.mdindex 5579d55f10..bd0165a19c 100644--- a/doc/composer-stop.md+++ b/doc/composer-stop.md@@ -159,7 +159,9 @@ pnpm exec playwright test --config tests/e2e/playwright-composer-stop.config.tsThe suite boots a disposable local-trusted instance on port 3199 (override with`PAPERCLIP_E2E_PORT`). It never attaches to an existing server. Native coverageis explicitly skipped without the fixture; it must not use a logged-in provider-as a fallback. Test companies are archived during cleanup.+as a fallback. The acceptance server pins its Codex command resolver to the fixture+and uses an empty credential home, because an installed Codex dependency takes+precedence over `PATH`. Test companies are archived during cleanup.For each runner, the journey starts a parent, child, and unrelated task, plus aterminal child. It sends while running and verifies the durable queue, clicksdiff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 10cbb2b81f..ae86b02f87 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,3 +1,4 @@+import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js";import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js";import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";export {@@ -9,7 +10,6 @@ export {import {createHeartbeatScheduling,formatIssueIdentifierLink,- parseNativeSessionGoalControl,} from "./heartbeat/scheduling.js";import { agentExecutionsHaveStopped } from "./agent-execution-stop.js";import {@@ -120,12 +120,6 @@ import {applyRunScopedMentionedSkillKeys,MANAGED_GITHUB_TOKEN_KEYS,configuredPaperclipApiBaseUrl,- buildPaperclipRuntimeMcpServers,- createAdapterRuntimeToolAccess,- paperclipApiBaseUrl,- createAdapterRuntimeMcpAccess,- createManagedMcpRunConfig,- revokeHeartbeatRunGatewayTokens,createHeartbeatRunPreparation,} from "./heartbeat/run-preparation.js";export {@@ -168,9 +162,6 @@ import {buildRunWorkspaceHints,buildReferencedProjectRunObservability,assertGitSensitiveAdapterWorkspaceValid,- isWorkspaceValidationFailure,- fingerprintFinalizeWorkspaceBranchValidation,- attachPaperclipSessionMetadataToSessionParams,type EffectiveRunSessionConfigMetadata,createHeartbeatWorkspaceResolver,} from "./heartbeat/workspaces.js";@@ -245,19 +236,14 @@ export {import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";-import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";-import {- configuredEnvironmentProjection,-} from "../vendor/paperclip-runner/index.js";import { decisionModelService } from "./decision-models.js";import { TASK_QUESTION_GUIDANCE } from "./issue-question-context.js";import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js";-import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils";import { prepareConnectionInstructionDelivery } from "./connection-instructions.js";import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js";-import { externalObjectService } from "./external-objects.js";+import { resolvePaperclipInstanceRoot } from "../home-paths.js";import {@@ -270,22 +256,14 @@ import {import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./agent-directory-working-copies.js";import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";-import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js";import {- withNativeWorkspaceFinalizationOwnership,NativeWorkspaceFinalizationBusyError,NativeWorkspaceFinalizationOwnershipLostError,- type NativeWorkspaceFinalizationOwnership,} from "./native-runtime/native-workspace-finalization-ownership.js";import { reserveRunBudget } from "./budget-reservations.js";import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js";-import { createRunUsageRecorder } from "./usage-receipts.js";-import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";-import {- hasWorkspaceRestoreFailure,-} from "@paperclipai/shared";import { toolActionDeliveryService } from "./tool-action-delivery.js";import { liveVoiceExecutionGuidance } from "./voice/voice-execution-guidance.js";@@ -307,9 +285,6 @@ import {watchLegacyControllerLease,} from "./legacy-controller-lease.js";-import {- remoteExecutionHasStopped,-} from "./remote-execution-termination.js";import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";import { connectionIntentService } from "./connection-intents.js";@@ -336,7 +311,6 @@ import {import {adapterExecutionControls,createAdapterExecutionControl,- registerAdapterExecutionControl,} from "./adapter-execution-control.js";import { executionFailureRetryCount } from "./execution-recovery-attempt.js";import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js";@@ -357,9 +331,7 @@ import {startAdapterExecutionTargetPaperclipBridge,} from "@paperclipai/adapter-utils/execution-target";import { agentInstructionWorkingCopyService, collectStoppedInstructionCopyWithRetries, instructionWorkingCopyGuidance } from "./agent-instruction-working-copies.js";-import {- resolveManagedOpenAiBilling,-} from "@paperclipai/adapter-utils";+import fs from "node:fs/promises";import path from "node:path";import { execFile as execFileCallback } from "node:child_process";@@ -398,11 +370,9 @@ import {nativeRunFinalizations,projects,projectWorkspaces,- workspaceOperations,} from "@paperclipai/db";-import { conflict, HttpError, notFound } from "../errors.js";+import { HttpError, notFound } from "../errors.js";import {- getStartupTraceContext,getStartupTracer,} from "../instrumentation.js";import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js";@@ -425,27 +395,19 @@ import {import { getTaskPlanContext } from "./task-plan-context.js";import {- buildNativeProviderEnvironment,claimWarmNativeInstructionCopy,nativeSessionWorkspaceScope,reserveWarmNativeInstructionDirectory,dispatchNativeSessionResumptions,- executePaperclipNativeSession,- finalizeNativeRun,- isRunnerIngressAuthorized,- materializeLegacyQuestionResponseWakeProjection,NativeCancellationPendingRecoveryError,NativeControllerDetachedForRestartError,prepareNativeWorkspaceSync,readNativeWorkspaceSyncReference,- recordNativeFinalizationFailure,type NativeRestartRecoveryClaim,resolveHeartbeatNativeRuntimeMode,} from "./native-runtime/index.js";import {- buildNativeHeartbeatPreparationSpans,- buildNativeWakeIngressSpan,recordFailedSkillPreparation,type NativeRunHistoricalSpan,} from "./native-runtime/native-run-trace.js";@@ -459,19 +421,16 @@ import { createNativeSessionHandoffLoader } from "./native-runtime/native-sessioimport { getRunLogStore, type RunLogHandle } from "./run-log-store.js";import {providerTraceStore,- PROVIDER_TRACE_MAX_BYTES,} from "./provider-trace-store.js";import { getServerAdapter, runningProcesses } from "../adapters/index.js";import type {- AdapterExecutionResult,AdapterInvocationMeta,AdapterRuntimeEvent,} from "../adapters/index.js";-import { createLocalAgentJwt } from "../agent-auth-jwt.js";+import { createRuntimeToolsToken } from "../runtime-tools-token.js";import {parseObject,- asNumber,} from "../adapters/utils.js";import {@@ -523,11 +482,8 @@ import {buildWorkspaceReadyMetadata,buildWorkspaceReadyPresentation,cleanupExecutionWorkspaceArtifacts,- ensureGitWorktreeBranchCoherent,ensurePersistedExecutionWorkspaceAvailable,ensureRuntimeServicesForRun,- formatManagedGitWorktreeBranchInspection,- inspectManagedGitWorktreeBranch,persistAdapterManagedRuntimeServices,realizeExecutionWorkspace,releaseRuntimeServicesForRun,@@ -542,10 +498,7 @@ import {} from "./workspace-instance-cleanup.js";import { issueService } from "./issues.js";import {- blockRunnerGoalRecovery,failRunnerGoalAction,- isRunnerGoalActionCompleted,- settleLiveRunnerGoalBeforeInterrupt,} from "./runner-goals.js";import {@@ -961,25 +914,6 @@ export interface HeartbeatServiceOptions {}) => Promise<void>;}-class NativeSessionResumeScheduledError extends Error {- constructor(readonly original: unknown) {- super("Native session recovery has been scheduled for the same run.");- this.name = "NativeSessionResumeScheduledError";- }-}--class NativeWorkspaceFinalizeScheduledError extends Error {- constructor(- readonly original: unknown,- readonly terminalFailure: boolean,- readonly reasonCode:- "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",- ) {- super("Native workspace finalization recovery has been scheduled.");- this.name = "NativeWorkspaceFinalizeScheduledError";- }-}-type WorkspaceReadyCommentWriter = {addComment: (issueId: string,@@ -5656,918 +5590,106 @@ export function heartbeatService(if (!runtimeSelection.selected) return;const { nativeExecution, nativeRunnerInstanceId, getNativeFreshSessionHandoff } = runtimeSelection;nativeWorkspaceSync = runtimeSelection.nativeWorkspaceSync;- const localAgentJwtScope =- issueRef?.workMode === "skill_test"- ? { kind: "skill_test" as const, issueId: issueRef.id }- : { kind: "standard" as const };- const authToken =- nativeRuntimeResolution.kind === "legacy" &&- adapter.supportsLocalAgentJwt- ? createLocalAgentJwt(- agent.id,- agent.companyId,- agent.adapterType,- run.id,- run.responsibleUserId,- localAgentJwtScope,- )- : null;- if (- nativeRuntimeResolution.kind === "legacy" &&- adapter.supportsLocalAgentJwt &&- !authToken- ) {- logger.warn(- {- companyId: agent.companyId,- agentId: agent.id,- runId: run.id,- adapterType: agent.adapterType,- },- "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY",- );- }- let adapterFinalizeOutcome: "succeeded" | "failed" | null = null;- const inspectFinalizeWorkspaceBranch = async () => {- const workspaceRecord = persistedExecutionWorkspace?.id- ? await executionWorkspacesSvc.getById(- persistedExecutionWorkspace.id,- )- : persistedExecutionWorkspace;- if (workspaceRecord?.strategyType !== "git_worktree") return null;-- const worktreePath =- readNonEmptyString(workspaceRecord.providerRef) ??- readNonEmptyString(workspaceRecord.cwd) ??- readNonEmptyString(executionWorkspace.worktreePath) ??- readNonEmptyString(executionWorkspace.cwd);- const expectedBranchName =- readNonEmptyString(workspaceRecord.branchName) ??- readNonEmptyString(executionWorkspace.branchName);- if (!worktreePath || !expectedBranchName) return null;-- const inspection = await inspectManagedGitWorktreeBranch({- worktreePath,- expectedBranchName,- });- return { workspaceRecord, inspection };- };- const recordWorkspaceFinalize = async (- status: "succeeded" | "failed",- metadata?: Record<string, unknown>,- ) => {- if (adapterFinalizeOutcome) return;- let finalizeBranchMetadata: Record<string, unknown> | null = null;- let finalizeBranchRepairMetadata: Record<string, unknown> | null =- null;- if (status === "succeeded") {- const branchInspection = await inspectFinalizeWorkspaceBranch();- if (branchInspection) {- let inspection = branchInspection.inspection;- const initialManagedGitWorktreeBranch =- formatManagedGitWorktreeBranchInspection(inspection);- if (- !inspection.valid &&- inspection.reasonCode === "branch_mismatch" &&- inspection.repoRoot- ) {- let repairedExpectedBranchName = inspection.expectedBranchName;- try {- const coherence = await ensureGitWorktreeBranchCoherent({- db,- repoRoot: inspection.repoRoot,- worktreePath: inspection.worktreePath,- expectedBranchName: inspection.expectedBranchName,- actualBranchName: inspection.actualBranchName,- sourceIssue: issueRef- ? {- id: issueRef.id,- identifier: issueRef.identifier,- title: issueRef.title,- workMode: issueRef.workMode,- }- : null,- executionWorkspaceId: branchInspection.workspaceRecord.id,- heartbeatRunId: run.id,- enableWorkspaceBranchReconcileForward:- resolvedInstanceSettings.experimental- .enableWorkspaceBranchReconcileForward,- enableWorkspaceDirtyQuarantineRepair:- resolvedInstanceSettings.experimental- .enableWorkspaceDirtyQuarantineRepair,- persistForwardReconcile: false,- reconcileOperationPhase: "workspace_finalize",- recorder: workspaceOperationRecorder,- });- if (- coherence.branchName &&- coherence.branchName !==- branchInspection.workspaceRecord.branchName- ) {- repairedExpectedBranchName = coherence.branchName;- executionWorkspace.branchName = coherence.branchName;- executionWorkspace.warnings.push(...coherence.warnings);- }- } catch (repairErr) {- const workspaceValidationFailure =- isWorkspaceValidationFailure(repairErr) ? repairErr : null;- finalizeBranchMetadata = {- executionWorkspaceId: branchInspection.workspaceRecord.id,- ...initialManagedGitWorktreeBranch,- };- finalizeBranchRepairMetadata = {- attempted: true,- succeeded: false,- initial: initialManagedGitWorktreeBranch,- reason:- repairErr instanceof Error- ? repairErr.message- : String(repairErr),- };- await workspaceOperationRecorder.recordOperation({- phase: "workspace_finalize",- cwd: executionWorkspace.cwd,- metadata: {- adapterType: agent.adapterType,- executionTargetKind: executionTarget?.kind ?? "local",- ...metadata,- managedGitWorktreeBranch: finalizeBranchMetadata,- managedGitWorktreeBranchRepair:- finalizeBranchRepairMetadata,- ...(workspaceValidationFailure?.resultJson- ? {- workspaceValidation:- workspaceValidationFailure.resultJson- .workspaceValidation ??- workspaceValidationFailure.resultJson,- }- : {}),- },- run: async () => ({- status: "failed",- stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`,- }),- });- adapterFinalizeOutcome = "failed";- throw repairErr;- }-- const repairedInspection =- await inspectManagedGitWorktreeBranch({- worktreePath: inspection.worktreePath,- expectedBranchName: repairedExpectedBranchName,- repoRoot: inspection.repoRoot,- });- finalizeBranchRepairMetadata = {- attempted: true,- succeeded: repairedInspection.valid,- initial: initialManagedGitWorktreeBranch,- repaired:- formatManagedGitWorktreeBranchInspection(- repairedInspection,- ),- };- inspection = repairedInspection;- }-- const managedGitWorktreeBranch =- formatManagedGitWorktreeBranchInspection(inspection);- finalizeBranchMetadata = {- executionWorkspaceId: branchInspection.workspaceRecord.id,- ...managedGitWorktreeBranch,- };- if (!inspection.valid) {- const workspaceValidationFingerprint =- fingerprintFinalizeWorkspaceBranchValidation({- issueId: issueRef?.id ?? null,- executionWorkspaceId: branchInspection.workspaceRecord.id,- inspection: managedGitWorktreeBranch,- });- await workspaceOperationRecorder.recordOperation({- phase: "workspace_finalize",- cwd: executionWorkspace.cwd,- metadata: {- adapterType: agent.adapterType,- executionTargetKind: executionTarget?.kind ?? "local",- ...metadata,- managedGitWorktreeBranch: finalizeBranchMetadata,- ...(finalizeBranchRepairMetadata- ? {- managedGitWorktreeBranchRepair:- finalizeBranchRepairMetadata,- }- : {}),- },- run: async () => ({- status: "failed",- stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`,- }),- });- adapterFinalizeOutcome = "failed";- throw new WorkspaceValidationFailure(- `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`,- {- workspaceValidation: {- reason: "git_worktree_branch_incoherence",- fingerprint: workspaceValidationFingerprint,- adapterType: agent.adapterType,- issueId: issueRef?.id ?? null,- issueIdentifier: issueRef?.identifier ?? null,- persistedExecutionWorkspaceId:- branchInspection.workspaceRecord.id,- executionWorkspaceCwd: executionWorkspace.cwd,- managedGitWorktreeBranch: finalizeBranchMetadata,- },- },- );- }- }- }- await workspaceOperationRecorder.recordOperation({- phase: "workspace_finalize",- cwd: executionWorkspace.cwd,- metadata: {- adapterType: agent.adapterType,- executionTargetKind: executionTarget?.kind ?? "local",- ...metadata,- ...(finalizeBranchMetadata- ? { managedGitWorktreeBranch: finalizeBranchMetadata }- : {}),- ...(finalizeBranchRepairMetadata- ? {- managedGitWorktreeBranchRepair:- finalizeBranchRepairMetadata,- }- : {}),- },- run: async () => ({ status }),- });- // Only mark the outcome after the row landed, so a transient write- // failure on the succeeded path can still be recovered by recording- // finalize=failed from the catch path below.- adapterFinalizeOutcome = status;- };-- const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType });- persistUsageCaptureFailure = usageRecorder.persistFailure;- let adapterResult: AdapterExecutionResult;- const runGoalControlRequestId = readNonEmptyString(- context.goalControlRequestId,- );- try {- if (nativeRuntimeResolution.kind === "native") {- if (!nativeExecution || !nativeRunnerInstanceId)- throw new Error("native_runtime_selection_not_persisted");- const expectedNativeMcpDigest =- "runtimeContext" in nativeExecution &&- nativeExecution.runtimeContext.mcp.bindingId- ? nativeExecution.runtimeContext.mcp.digest- : null;- const nativeMcpServers = await buildPaperclipRuntimeMcpServers({- db,- agent,- runId: run.id,- expectedAssignmentDigest: expectedNativeMcpDigest,- });- if ("runtimeContext" in nativeExecution) {- if (nativeMcpServers.length > 1)- throw new Error(- "native MCP realization must produce one aggregate gateway",- );- const server = nativeMcpServers[0] ?? null;- const digest = server?.connectionId.startsWith("assignment:")- ? server.connectionId.slice("assignment:".length)- : null;- if (digest && digest !== expectedNativeMcpDigest) {- throw new Error("native MCP assignment digest mismatch");- }- }- const nativeMcpServer = nativeMcpServers[0] ?? null;- let sessionGoalControl = parseNativeSessionGoalControl(- context.runnerGoalControl,- );- if (runGoalControlRequestId && !sessionGoalControl) {- throw new Error("session_goal_control_payload_invalid");- }- // A hard restart replays the heartbeat context, not a new user- // action. Do not repeat a completed create/replace/edit (which- // could reactivate or clear a goal that finished while detached).- const completedGoalControl =- sessionGoalControl !== null &&- taskKey !== null &&- (await isRunnerGoalActionCompleted(- db,- {- companyId: agent.companyId,- agentId: agent.id,- issueId: taskKey,- },- sessionGoalControl.requestId,- ));- if (completedGoalControl) sessionGoalControl = null;- const nativeDispatchAtMs = Date.now();- const runCreatedAtMs = run.createdAt.getTime();- const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();- const wakeComments = Array.isArray(- parseObject(context.paperclipWake).comments,- )- ? (parseObject(context.paperclipWake).comments as unknown[])- : [];- const wakeIngressSpan = buildNativeWakeIngressSpan({- runCreatedAtMs,- wakeComments,- attestedQuestionResponseAtMs,- });- if (wakeIngressSpan)- nativeRunnerPreparationSpans.unshift(wakeIngressSpan);- nativeRunnerPreparationSpans.push(- ...buildNativeHeartbeatPreparationSpans({- runCreatedAtMs,- runStartedAtMs,- attemptStartedAtMs,- environmentAcquireStartedAtMs,- environmentRealizeEndedAtMs,- nativeDispatchAtMs,- }),- );- const guardedDispatch =- await dispatchResolvedInteractionContinuationWithAtomicGate(- (markDispatchStarted) => {- return executePaperclipNativeSession({- db,- execution: nativeExecution,- getFreshSessionHandoff: getNativeFreshSessionHandoff,- refreshTools: context.refreshTools === true,- conversationMode: isConversation(issueContext),- turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000,- runnerInstanceId: nativeRunnerInstanceId,- leaseOwner: runOptions.nativeLeaseOwner,- restartRecovery: runOptions.nativeRestartRecovery,- backend:- options.nativeSessionBackendFactory?.(nativeExecution),- useRunnerd: agent.adapterType === "paperclip_runner",- adapterType: agent.adapterType,- sessionGoalControl,- resumeSessionGoalHeartbeat:- context.resumeSessionGoalHeartbeat === true ||- completedGoalControl,- onGoalCheckpoint: async (snapshot) => {- if (!taskKey) return;- const params =- attachPaperclipSessionMetadataToSessionParams(- {- ...runtimeSessionParamsForAdapter,- sessionId: snapshot.identity.sessionId,- cwd: executionWorkspace.cwd,- },- configuredModel,- sessionConfigMetadata,- )!;- const displayId =- snapshot.providerSessionId ?? snapshot.sessionId;- await upsertTaskSession({- companyId: agent.companyId,- agentId: agent.id,- adapterType: agent.adapterType,- taskKey,- sessionParamsJson: params,- sessionDisplayId: displayId,- lastRunId: run.id,- lastError: null,- });- goalCheckpointSession.current = { params, displayId };- },- onLog,- onEvent: onAdapterEvent,- instructionWorkingCopy: nativeInstructionWorkingCopy(),-- onUsage: async receipt => { await usageRecorder.capture(receipt); },- preparationSpans: nativeRunnerPreparationSpans,- // Bootstrap with executable/home discovery while keeping- // configured provider values and the server-selected- // workspace boundary authoritative.- managedGitHub: !useHostGitHub && githubSelection.configured,- billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined,- managedAiCredentialIdentity: managedAiRuntime?.identity,- managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record<string, unknown>).CODEX_HOME) : undefined,- dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined,- runnerEnvironment: {- ...configuredEnvironmentProjection(configuredTaskEnvironment),- ...buildNativeProviderEnvironment(- adapterEnv,- process.env,- executionWorkspace.cwd,- ),- ...buildAgentIdentityEnv(agentIdentity),- ...(instructionCopy && isAgentDirectoryCopy(instructionCopy) ? { AGENT_HOME: instructionCopy.executionRoot } : {}),- ...(nativeMcpServer- ? {- PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name,- PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url,- PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token,- }- : {}),- ...(providerTraceCapture- ? {- PAPERCLIP_PROVIDER_TRACE_PATH:- providerTraceCapture.path,- PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(- PROVIDER_TRACE_MAX_BYTES,- ),- }- : {}),- },- runnerExecutionTarget: executionTarget,- runnerIngressAuthorized: isRunnerIngressAuthorized(- nativeRuntimeResolution,- ),- runnerPublicUrl:- runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null,- runnerCaBundlePath:- runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() ||- null,- runnerRemoteBinaryPath:- runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() ||- null,- runnerRemoteCodexPath:- runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() ||- null,- runnerRemoteCodexNpmSpec:- runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() ||- null,- runnerRemoteProviderPackPath:- runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() ||- null,- stopTaskForReassignment: async (target) => {- await settleLiveRunnerGoalBeforeInterrupt(db, target);- if (!target.runId) return;- const prior = await getRun(target.runId);- if (!prior || prior.companyId !== target.companyId || prior.agentId !== target.agentId) {- throw conflict("Reassignment run binding changed");- }- const stopped = await cancelRunInternal(target.runId, "Cancelled for task reassignment", {- errorCode: "issue_reassigned", suppressImmediateRecovery: true,- resultJson: { reassignmentStopConfirmed: true },- });- if (stopped && ["running", "queued", "scheduled_retry"].includes(stopped.status)) {- throw conflict("The previous run did not stop; reassignment was not applied");- }- },- enqueueWakeup,- syncIssueExternalObjects: externalObjectService(db, {- pluginWorkerManager: options.pluginWorkerManager,- enabled: async () => (await instanceSettings.getExperimental()).enableExternalObjects === true,- }).syncIssueSafely,- onSpawn: async (meta) => {- markDispatchStarted();- await persistRunProcessMetadata(run.id, { ...meta, targetKind: executionTarget?.kind ?? "local" });- },- });- },- );- if (!guardedDispatch.dispatched) return;- nativeDispatchStarted = true;- adapterResult = await guardedDispatch.resultPromise;- } else {- const interactionId = readNonEmptyString(context.interactionId);- const legacyQuestionResponse =- issueRef &&- interactionId &&- readNonEmptyString(context.interactionKind) ===- "ask_user_questions" &&- readNonEmptyString(context.interactionStatus) === "answered"- ? await materializeLegacyQuestionResponseWakeProjection({- db,- companyId: agent.companyId,- issueId: issueRef.id,- runId: run.id,- agentId: agent.id,- interactionId,- })- : null;- // Do not write the answer projection back to `context`: legacy- // adapters need it in their prompt, but the authoritative answers- // remain on the interaction instead of being duplicated in the- // heartbeat run snapshot.- const adapterContext: Record<string, unknown> = {- ...context,- ...(legacyQuestionResponse- ? {- [PAPERCLIP_WAKE_PAYLOAD_KEY]: {- ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]),- questionResponse: legacyQuestionResponse,- },- }- : {}),- };- const runtimeTools = createAdapterRuntimeToolAccess({- agentId: agent.id,- companyId: agent.companyId,- runId: run.id,- responsibleUserId: run.responsibleUserId,- });- if (!runtimeTools) {- logger.warn(- {- companyId: agent.companyId,- agentId: agent.id,- runId: run.id,- },- "runtime connection tools could not be delivered",- );- }- const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({- db,- agent,- runId: run.id,- });- const runtimeToolDelivery =- adapter.runtimeToolDelivery ?? "invocation_context";- if (runtimeTools && runtimeToolDelivery === "native_mcp") {- runtimeMcpServers.unshift({- name: "Paperclip connections",- url: runtimeTools.mcpEndpoint,- token: runtimeTools.bearerToken,- connectionId: "paperclip-runtime-tools",- });- }- if (authToken && configuredPaperclipApiBaseUrl() && issueRef) {- runtimeMcpServers.unshift({ name: "Paperclip projects", url: `${paperclipApiBaseUrl()}/api/mcp/project-tools`,- token: authToken, connectionId: "paperclip-project-tools" });- }- const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers);- if (runtimeTools && runtimeToolDelivery === "invocation_context") {- adapterContext.paperclipRuntimeTools = runtimeTools;- }- const managedMcpConfig = await createManagedMcpRunConfig({- db,- agent,- runId: run.id,- config: runtimeConfig,- projectId: issueRef?.projectId ?? null,- issueId: issueRef?.id ?? null,- });- if (managedMcpConfig) {- adapterContext.paperclipManagedMcp = managedMcpConfig;- }- const guardedDispatch =- await dispatchResolvedInteractionContinuationWithAtomicGate(- (markDispatchStarted) => {- legacyAdapterEntered = true;- return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({- getFreshSessionHandoff,- agentIdentity,- runId: run.id,- agent,- runtime: runtimeForAdapter,- config: runtimeConfig,- context: adapterContext,- executionContinuation: executionContinuation ?? null,- runtimeCommandSpec:- adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null,- executionTarget,- executionTransport: remoteExecution- ? {- remoteExecution: remoteExecution as unknown as Record<- string,- unknown- >,- }- : undefined,- runtimeMcp,- runtimeTools,- onLog,- onMeta: onAdapterMeta,- onEvent: onAdapterEvent,- onUsage: async receipt => { await usageRecorder.capture(receipt); },- onExecutionPhase: executionControl.phases.enter,- startupTraceContext: getStartupTraceContext(),- onRuntimeProgress: async (progress) => {- await recordCurrentHeartbeatRunRuntimeProgress(- run,- progress,- issueId,- );- },- onProviderStopped: collectStoppedInstructions,- onDispatch: markDispatchStarted,- signal: executionControl.controller.signal,- ...(executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" ? {- stopRemoteStartup: async () => {- // Scope comes from the running host invocation, never agent- // config. Keep adapter ownership until setup has unwound.- if (!executionControl.controller.signal.aborted) {- throw new Error("Remote startup stop requires a cancelled run");- }- const release = await envOrchestrator.releaseForRun({- heartbeatRunId: run.id,- companyId: agent.companyId,- agentId: agent.id,- status: "released",- providerResourceDisposition: "stop_and_retain",- cancelActiveWork: true,- });- if (release.errors.length || !await remoteExecutionHasStopped(db, agent.companyId, run.id)) {- throw new Error("Could not verify remote startup stopped");- }- },- } : {}),- onCancellationReady: async () => {- await registerAdapterExecutionControl(run.id, executionControl);- const current = await getRun(run.id);- if (!current || isHeartbeatRunTerminalStatus(current.status)) {- executionControl.controller.abort(new Error("Run stopped before provider startup"));- }- },- onSpawn: async (meta) => {- markDispatchStarted();- await persistRunProcessMetadata(run.id, {- pid: meta.pid,- processGroupId:- "processGroupId" in meta &&- typeof meta.processGroupId === "number"- ? meta.processGroupId- : null,- startedAt: meta.startedAt,- });- },- authToken: authToken ?? undefined,- }));- },- );- if (!guardedDispatch.dispatched) return;- adapterResult = await guardedDispatch.resultPromise;- }- adapterResult = identityRedactor.redact(adapterResult);- if (run.runtimeMode === "legacy" && hasWorkspaceRestoreFailure(adapterResult.resultJson)- && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") {- requiredWorkspaceRestoreEvidence = {- workspaceRestoreFailure: adapterResult.resultJson!.workspaceRestoreFailure,- ...(adapterResult.resultJson?.workspaceRestoreDiagnostic ? { workspaceRestoreDiagnostic: adapterResult.resultJson.workspaceRestoreDiagnostic } : {}),- };- // Retention is the fallback even if recording this receipt fails.- providerResourceDispositionForRun = "stop_and_retain";- await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);- }- for (const stream of ["stdout", "stderr"] as const) {- const tail = identityRedactor.finish(stream);- if (tail) await appendIdentityRedactedLog(stream, tail);- }- if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };-- if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {- const captured = await usageRecorder.complete(adapterResult);- adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };- } else {- // Stop may already own the terminal result. Preserve its metadata- // while durably recording the proof needed to release admission.- await db.update(heartbeatRuns).set({ costAccountingPending: true,- usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`,- }).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));- }- adapterResult = applyWorkspaceRestoreFailure(adapterResult);- // A returned result can include a failed restore. Keep the workspace- // barrier closed until required files have been restored.- // If recording the barrier itself fails, propagate as a run failure- // rather than silently leaving dependents stranded behind a missing- // finalize row.- const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => {- try {- if (nativeWorkspaceSync) {- const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(- eq(workspaceOperations.companyId, run.companyId),- eq(workspaceOperations.heartbeatRunId, run.id),- eq(workspaceOperations.phase, "workspace_finalize"),- eq(workspaceOperations.status, "succeeded"),- )).limit(1);- if (exported.length) adapterFinalizeOutcome = "succeeded";- else await restoreNativeWorkspaceBestEffort({- db, runId: run.id, assertOwnership: ownership?.assertHeld,- restore: () => nativeWorkspaceSync!.restoreWorkspace(ownership?.assertHeld),- });- }- await ownership?.assertHeld();- await db- .update(heartbeatRuns)- .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })- .where(- and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.status, "running"),- ),- );- const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded";- await recordWorkspaceFinalize(workspaceFinalizeStatus);- if (adapterResult.nativeFinalization) {- adapterResult.nativeFinalization.workspaceFinalizeStatus =- workspaceFinalizeStatus;- try {- const finalized = await finalizeNativeRun({- db,- runId: run.id,- workspaceFinalizeStatus,- preserveProviderAttempt: Boolean(nativeWorkspaceSync),- });- await dispatchPendingNativeStatusWakeups({- companyId: run.companyId,- });- if (finalized.phase === "committed") {- await nativeWorkspaceSync?.cleanup();- }- } catch (finalizeErr) {- logger.warn(- { err: finalizeErr, runId: run.id },- "native result persisted but finalization did not apply; the reconciliation loop will retry",- );- }- }- } catch (error) {- if (ownership) {- await ownership.assertHeld();- await recordWorkspaceFinalize("failed");- }- throw error;- }- };- if (nativeWorkspaceSync) {- const owned = await withNativeWorkspaceFinalizationOwnership({- db, companyId: run.companyId, runId: run.id,- }, completeWorkspace);- if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError();- } else {- await completeWorkspace();- }- } catch (adapterErr) {- if (adapterErr instanceof NativeCancellationPendingRecoveryError) {- // Durable cancellation is settled by the outer recovery handler;- // it does not imply a failed workspace or a persisted run result.- throw adapterErr;- }- if (adapterErr instanceof NativeWorkspaceFinalizationBusyError- || adapterErr instanceof NativeWorkspaceFinalizationOwnershipLostError) {- nativeWorkspaceFinalizeScheduled = true;- throw adapterErr;- }- if (adapterErr instanceof NativeControllerDetachedForRestartError) {- // Preserve the provider and its run for the new controller. This- // also keeps generic teardown from terminalizing/releasing its lease.- nativeSessionResumeScheduled = true;- throw adapterErr;- }- if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {- nativeOwnershipHeld = true;- throw adapterErr;- }- await db- .update(heartbeatRuns)- .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })- .where(- and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.status, "running"),- ),- );- if (- issueRef &&- context.resumeSessionGoalHeartbeat === true &&- !runGoalControlRequestId- ) {- await blockRunnerGoalRecovery(- db,- {- companyId: run.companyId,- issueId: issueRef.id,- agentId: agent.id,- adapterType: agent.adapterType,- },- "provider_session_goal_recovery_failed",- ).catch(() => undefined);- }- if (issueRef && runGoalControlRequestId) {- await failRunnerGoalAction(- db,- {- companyId: run.companyId,- issueId: issueRef.id,- agentId: agent.id,- adapterType: agent.adapterType,- },- runGoalControlRequestId,- adapterErr instanceof Error- ? adapterErr.message- : "session_goal_control_failed",- ).catch(() => undefined);- }- const nativeResumeScheduled =- nativeRuntimeResolution.kind === "native"- ? await db- .select({- phase: nativeRunFinalizations.phase,- resultId: nativeRunFinalizations.resultId,- })- .from(nativeRunFinalizations)- .where(eq(nativeRunFinalizations.runId, run.id))- .limit(1)- .then(- (rows) =>- rows[0]?.phase === "retryable_failure" &&- rows[0]?.resultId === null,- )- : false;- if (nativeResumeScheduled) {- nativeSessionResumeScheduled = true;- throw new NativeSessionResumeScheduledError(adapterErr);- }- // Adapter (or its restore finally) threw — or the finalize record- // write itself threw. Either way the workspace may be in a partial- // state. Best-effort record finalize=failed so the dependent readiness- // check keeps the gate closed instead of waking on stale local state,- // and surface the original error to the caller.- try {- await recordWorkspaceFinalize("failed", {- errorMessage:- adapterErr instanceof Error- ? adapterErr.message- : String(adapterErr),- });- } catch (recordErr) {- logger.warn(- {- err: recordErr,- runId: run.id,- executionWorkspaceId: persistedExecutionWorkspace?.id ?? null,- },- "failed to record workspace_finalize=failed operation; dependents may remain gated",- );- }- if (nativeRuntimeResolution.kind === "native") {- const proposedResult = await db- .select({ resultId: nativeRunFinalizations.resultId })- .from(nativeRunFinalizations)- .where(eq(nativeRunFinalizations.runId, run.id))- .limit(1)- .then((rows) => rows[0]?.resultId ?? null);- if (proposedResult && nativeWorkspaceSync) {- const workspaceFailureMessage =- adapterErr instanceof Error ? adapterErr.message : "";- const unrecoverable =- workspaceFailureMessage ===- "workspace_sync_out_unrecoverable" ||- workspaceFailureMessage.includes("daytona_sandbox_not_found");- const failure = await recordNativeFinalizationFailure({- db,- runId: run.id,- error: new Error(- unrecoverable- ? "native_workspace_sync_out_unrecoverable"- : "native_workspace_sync_out_failed",- ),- projectRunStatus: true,- failureScope: "workspace",- permanent: unrecoverable,- });- nativeWorkspaceFinalizeScheduled = true;- throw new NativeWorkspaceFinalizeScheduledError(- adapterErr,- failure.phase === "terminal_failure",- unrecoverable- ? "workspace_sync_out_unrecoverable"- : "workspace_sync_out_failed",- );- }- try {- await finalizeNativeRun({- db,- runId: run.id,- workspaceFinalizeStatus: "failed",- });- await dispatchPendingNativeStatusWakeups({- companyId: run.companyId,- });- } catch (finalizeErr) {- logger.warn(- { err: finalizeErr, runId: run.id },- "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results",- );- }- }- throw adapterErr;- } finally {- try {- await revokeHeartbeatRunGatewayTokens({- db,- companyId: agent.companyId,- runId: run.id,- });- } catch (revokeErr) {- logger.warn(- { err: revokeErr, runId: run.id, companyId: agent.companyId },- "failed to revoke heartbeat-run MCP gateway tokens",- );- }- await nativeInstructionReservation?.release();- await withAdapterExecutionPhase(executionPhaseContext, "instruction_cleanup", releaseInstructionCopy);- }+ const execution = await executeHeartbeatRuntime(db, {+ run, agent, options,+ task: {+ issueRef,+ issueContext,+ context,+ issueId,+ taskKey,+ executionContinuation,+ },+ runtime: {+ nativeRuntimeResolution,+ nativeExecution,+ nativeRunnerInstanceId,+ adapter,+ runtimeForAdapter,+ getNativeFreshSessionHandoff,+ getFreshSessionHandoff,+ runOptions,+ },+ workspace: {+ persistedExecutionWorkspace,+ executionWorkspace,+ executionTarget,+ workspaceOperationRecorder,+ nativeWorkspaceSync,+ workspaceRestoreSource,+ remoteExecution,+ },+ config: {+ runtimeConfig,+ resolvedConfig,+ resolvedInstanceSettings,+ configuredTaskEnvironment,+ adapterEnv,+ agentIdentity,+ managedAiRuntime,+ runtimeEnv,+ useHostGitHub,+ githubSelection,+ },+ session: {+ runtimeSessionParamsForAdapter,+ configuredModel,+ sessionConfigMetadata,+ goalCheckpointSession,+ },+ instructions: {+ nativeInstructionWorkingCopy,+ collectStoppedInstructions,+ nativeInstructionReservation,+ releaseInstructionCopy,+ get instructionCopy() { return instructionCopy; },+ getInstructionSave: () => instructionSave,+ },+ trace: {+ providerTraceCapture,+ nativeRunnerPreparationSpans,+ attestedQuestionResponseAtMs,+ attemptStartedAtMs,+ environmentAcquireStartedAtMs,+ environmentRealizeEndedAtMs,+ },+ control: {+ executionControl,+ executionPhaseContext,+ dispatchResolvedInteractionContinuationWithAtomicGate,+ },+ output: {+ identityRedactor,+ appendIdentityRedactedLog,+ onLog,+ onAdapterEvent,+ onAdapterMeta,+ },+ services: {+ executionWorkspacesSvc,+ instanceSettings,+ envOrchestrator,+ upsertTaskSession,+ getRun,+ cancelRunInternal,+ enqueueWakeup,+ persistRunProcessMetadata,+ recordCurrentHeartbeatRunRuntimeProgress,+ dispatchPendingNativeStatusWakeups,+ },+ effects: {+ onUsageCaptureReady: (value) => { persistUsageCaptureFailure = value; },+ onNativeDispatchStarted: () => { nativeDispatchStarted = true; },+ onLegacyAdapterEntered: () => { legacyAdapterEntered = true; },+ onNativeWorkspaceFinalizeScheduled: () => { nativeWorkspaceFinalizeScheduled = true; },+ onNativeSessionResumeScheduled: () => { nativeSessionResumeScheduled = true; },+ onNativeOwnershipHeld: () => { nativeOwnershipHeld = true; },+ onProviderResourceDisposition: (value) => { providerResourceDispositionForRun = value; },+ onWorkspaceRestoreFailure: (evidence) => { requiredWorkspaceRestoreEvidence = evidence; },+ },+ });+ if (!execution.dispatched) return;+ const { adapterResult } = execution;// Reconcile the referenced-project set against the real remote staging outcome. A referenced// project can pass authorization and clone locally at run prep, then fail to stage into the// sandbox during execution. The run-prep observability above counts such a project as synced,diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.tsnew file mode 100644index 0000000000..6fdf8fa0e9--- /dev/null+++ b/server/src/services/heartbeat/runtime-execution.test.ts@@ -0,0 +1,252 @@+import { randomUUID } from "node:crypto";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";+import { eq, sql } from "drizzle-orm";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import * as legacyRecovery from "../legacy-execution-recovery.js";+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";+import { createAdapterExecutionControl } from "../adapter-execution-control.js";+import { executionWorkspaceService } from "../execution-workspaces.js";+import { workspaceOperationService } from "../workspace-operations.js";+import { instanceSettingsService } from "../instance-settings.js";+import { executePaperclipNativeSession, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError } from "../native-runtime/native-session-executor.js";+import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js";+import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError } from "../native-runtime/native-workspace-finalization-ownership.js";+import { resolveHeartbeatNativeRuntimeMode } from "../native-runtime/runtime-mode.js";+import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js";+import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";+import { createHeartbeatRunState } from "./run-state.js";+import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js";+import { revokeHeartbeatRunGatewayTokens } from "./run-preparation.js";+import { executeHeartbeatRuntime, type HeartbeatRuntimeExecutionInput } from "./runtime-execution.js";++// Keep real accounting, workspace barriers, and task-session persistence. Providers+// and gateway delivery are external boundaries; browser acceptance covers their wiring.+vi.mock("../native-runtime/native-session-executor.js", async original => ({+ ...await original<typeof import("../native-runtime/native-session-executor.js")>(),+ executePaperclipNativeSession: vi.fn(),+}));+vi.mock("./run-preparation.js", async original => ({+ ...await original<typeof import("./run-preparation.js")>(),+ buildPaperclipRuntimeMcpServers: vi.fn(async () => []),+ createAdapterRuntimeToolAccess: vi.fn(() => null),+ createManagedMcpRunConfig: vi.fn(async () => null),+ revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}),+}));++const support = await getEmbeddedPostgresTestSupport();+describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: Db;+ let home: string;+ beforeAll(async () => {+ home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-"));+ vi.stubEnv("PAPERCLIP_HOME", home);+ database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-execution");+ db = createDb(database.connectionString);+ await db.execute(sql`set client_min_messages = warning`);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); });+ beforeEach(async () => {+ await db.execute(sql`truncate table companies restart identity cascade`);+ vi.clearAllMocks();+ vi.mocked(executePaperclipNativeSession).mockResolvedValue({ exitCode: 0, signal: null, timedOut: false });+ });++ async function fixture(native = false) {+ const [company] = await db.insert(companies).values({ name: "Runtime execution", issuePrefix: "EXE" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Executing agent", status: "running", adapterType: native ? "paperclip_runner" : "process", adapterConfig: { provider: "codex" } }).returning();+ const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Complete this task", assigneeAgentId: agent.id }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: native ? "native" : "legacy" }).returning();+ const state = createHeartbeatRunState(db);+ const executionControl = createAdapterExecutionControl();+ const adapter = { type: "process", label: "Test process", testEnvironment: vi.fn(), execute: vi.fn<HeartbeatRuntimeExecutionInput["runtime"]["adapter"]["execute"]>(async () => ({ exitCode: 0, signal: null, timedOut: false })) };+ let instructionSave: Record<string, unknown> | null = null;+ const nativeExecution = native ? buildNativeExecutionInput({+ companyId: company.id, agentId: agent.id, runId: run.id, issue,+ taskPrompt: issue.title, workspace: { id: run.id, cwd: home, repoUrl: null, repoRef: null, branchName: null },+ normalizedSessionId: null, provider: "codex", runtimeContext: nativeRuntimeContextFixture(),+ completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: issue.title, criteria: [{ id: "output", requirement: issue.title }] } },+ }) : null;+ const input: HeartbeatRuntimeExecutionInput = {+ run, agent, options: {},+ task: { issueRef: issue, issueContext: null, context: {}, issueId: issue.id, taskKey: issue.id, executionContinuation: null },+ runtime: { nativeExecution, nativeRunnerInstanceId: native ? randomUUID() : null, getNativeFreshSessionHandoff: undefined,+ nativeRuntimeResolution: resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: native, runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: null, workspaceId: null }),+ adapter, runtimeForAdapter: { taskKey: issue.id, sessionId: null, sessionParams: null, sessionDisplayId: null }, getFreshSessionHandoff: undefined, runOptions: {},+ },+ workspace: { persistedExecutionWorkspace: null, executionWorkspace: { baseCwd: home, cwd: home, source: "agent_home", projectId: null, workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false },+ executionTarget: null, nativeWorkspaceSync: null, workspaceRestoreSource: undefined, remoteExecution: null,+ workspaceOperationRecorder: workspaceOperationService(db).createRecorder({ companyId: company.id, heartbeatRunId: run.id, issueId: issue.id }),+ },+ config: { runtimeConfig: {}, resolvedConfig: {}, managedAiRuntime: undefined,+ resolvedInstanceSettings: { experimental: { enableWorkspaceBranchReconcileForward: false, enableWorkspaceDirtyQuarantineRepair: false } },+ configuredTaskEnvironment: {}, adapterEnv: {}, agentIdentity: undefined, runtimeEnv: {}, useHostGitHub: false, githubSelection: { configured: false },+ },+ session: { runtimeSessionParamsForAdapter: null, configuredModel: null,+ sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }), goalCheckpointSession: { current: null },+ },+ instructions: { instructionCopy: null, getInstructionSave: () => instructionSave, nativeInstructionWorkingCopy: () => undefined,+ collectStoppedInstructions: async () => { instructionSave = { state: "saved", revisionId: "during-execution" }; },+ nativeInstructionReservation: null, releaseInstructionCopy: vi.fn(async () => {}),+ },+ trace: { providerTraceCapture: null, nativeRunnerPreparationSpans: [], attestedQuestionResponseAtMs: null, attemptStartedAtMs: Date.now(), environmentAcquireStartedAtMs: Date.now(), environmentRealizeEndedAtMs: Date.now() },+ control: { executionControl, executionPhaseContext: { onExecutionPhase: executionControl.phases.enter },+ dispatchResolvedInteractionContinuationWithAtomicGate: async dispatch => ({ dispatched: true, resultPromise: dispatch(vi.fn()) }),+ },+ output: { identityRedactor: createAgentIdentityRedactor(), appendIdentityRedactedLog: vi.fn(async () => {}), onLog: vi.fn(async () => {}), onAdapterEvent: vi.fn(async () => {}), onAdapterMeta: vi.fn(async () => {}) },+ services: { ...state, executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db),+ envOrchestrator: { releaseForRun: vi.fn() }, cancelRunInternal: vi.fn(), enqueueWakeup: vi.fn(),+ persistRunProcessMetadata: vi.fn(), recordCurrentHeartbeatRunRuntimeProgress: vi.fn(), dispatchPendingNativeStatusWakeups: vi.fn(),+ },+ effects: { onUsageCaptureReady: vi.fn(), onNativeDispatchStarted: vi.fn(), onLegacyAdapterEntered: vi.fn(), onWorkspaceRestoreFailure: vi.fn(), onProviderResourceDisposition: vi.fn(), onNativeWorkspaceFinalizeScheduled: vi.fn(), onNativeSessionResumeScheduled: vi.fn(), onNativeOwnershipHeld: vi.fn() },+ };+ return { input, adapter, run };+ }+ const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);+ const finalizations = () => db.select().from(workspaceOperations).where(eq(workspaceOperations.phase, "workspace_finalize"));+ function expectCleanup(input: HeartbeatRuntimeExecutionInput) {+ expect(revokeHeartbeatRunGatewayTokens).toHaveBeenCalledWith({ db, companyId: input.agent.companyId, runId: input.run.id });+ expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce();+ }++ it.each([false, true])("preserves the dispatch gate's early return and cleanup (native: %s)", async native => {+ const { input, adapter } = await fixture(native);+ input.control.dispatchResolvedInteractionContinuationWithAtomicGate = async () => ({ dispatched: false });+ expect(await executeHeartbeatRuntime(db, input)).toEqual({ dispatched: false });+ expect(adapter.execute).not.toHaveBeenCalled();+ expect(executePaperclipNativeSession).not.toHaveBeenCalled();+ expect(input.effects.onNativeDispatchStarted).not.toHaveBeenCalled();+ expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled();+ expect(await finalizations()).toHaveLength(0);+ expectCleanup(input);+ });++ it("captures usage and live instruction saves before finalizing a legacy workspace", async () => {+ const { input, adapter, run } = await fixture();+ adapter.execute.mockImplementationOnce(async ctx => {+ expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce();+ expect(input.effects.onUsageCaptureReady).toHaveBeenCalledOnce();+ await ctx.onProviderStopped?.();+ return { exitCode: 0, signal: null, timedOut: false, usage: { inputTokens: 10, outputTokens: 4 }, costUsd: 0.01, provider: "test" };+ });+ expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usageComplete: true, resultJson: { instructionSave: { state: "saved", revisionId: "during-execution" } } } });+ expect(await read(run.id)).toMatchObject({ status: "running", costAccountingPending: true, usageJson: { inputTokens: 10, outputTokens: 4, accountingReceiptReady: true } });+ expect(await db.select().from(runUsageReceipts)).toHaveLength(1);+ expect(await finalizations()).toMatchObject([{ status: "succeeded" }]);+ expectCleanup(input);+ });++ it("preserves Stop metadata and marks no-work accounting without manufacturing a receipt", async () => {+ const { input, adapter, run } = await fixture();+ await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { stoppedByUser: true } }).where(eq(heartbeatRuns.id, run.id));+ adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } });+ await executeHeartbeatRuntime(db, input);+ expect(await read(run.id)).toMatchObject({ status: "cancelled", resultJson: { stoppedByUser: true }, usageJson: { accountingProviderWorkStarted: false } });+ expect(await db.select().from(runUsageReceipts)).toHaveLength(0);+ expectCleanup(input);+ });++ it("records the failed workspace and keeps the original provider error", async () => {+ const { input, adapter } = await fixture();+ const error = new Error("provider failed");+ adapter.execute.mockRejectedValueOnce(error);+ await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error);+ expect(await finalizations()).toMatchObject([{ status: "failed", metadata: { errorMessage: "provider failed" } }]);+ expect(input.effects.onLegacyAdapterEntered).toHaveBeenCalledOnce();+ expectCleanup(input);+ });++ it.each([+ [new NativeCancellationPendingRecoveryError(), null],+ [new NativeControllerDetachedForRestartError(), "onNativeSessionResumeScheduled"],+ [new NativeRunnerOwnershipUnverifiedError(), "onNativeOwnershipHeld"],+ [new NativeWorkspaceFinalizationBusyError(), "onNativeWorkspaceFinalizeScheduled"],+ [new NativeWorkspaceFinalizationOwnershipLostError(), "onNativeWorkspaceFinalizeScheduled"],+ ] as const)("transfers recovery ownership for %s before cleanup", async (error, effect) => {+ const { input } = await fixture(true);+ vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error);+ input.instructions.releaseInstructionCopy = vi.fn(async () => {+ if (effect) expect(input.effects[effect]).toHaveBeenCalledOnce();+ });+ await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error);+ expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce();+ expect(input.effects.onLegacyAdapterEntered).not.toHaveBeenCalled();+ expect(await finalizations()).toHaveLength(0);+ expectCleanup(input);+ });++ it("dispatches native execution with the selected session and persists its usage", async () => {+ const { input, adapter, run } = await fixture(true);+ vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => {+ expect(native.execution).toBe(input.runtime.nativeExecution);+ expect(native.runnerInstanceId).toBe(input.runtime.nativeRunnerInstanceId);+ await native.onUsage?.({ usage: { inputTokens: 7, outputTokens: 3 }, complete: true });+ return { exitCode: 0, signal: null, timedOut: false, usageComplete: false };+ });+ expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true, adapterResult: { usage: { inputTokens: 7, outputTokens: 3 }, usageComplete: true } });+ expect(adapter.execute).not.toHaveBeenCalled();+ expect(input.effects.onNativeDispatchStarted).toHaveBeenCalledOnce();+ expect(await read(run.id)).toMatchObject({ status: "running", usageJson: { inputTokens: 7, outputTokens: 3, accountingReceiptReady: true } });+ expect(await finalizations()).toMatchObject([{ status: "succeeded" }]);+ expectCleanup(input);+ });++ it("retains the same-run retry signal without recording a failed workspace", async () => {+ const { input, run } = await fixture(true);+ await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id));+ await db.insert(nativeRunFinalizations).values({ companyId: run.companyId, issueId: input.task.issueId!, runId: run.id, phase: "retryable_failure" });+ const error = new Error("session connection lost");+ vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error);+ await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({+ name: "NativeSessionResumeScheduledError", original: error,+ });+ expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce();+ expect(await finalizations()).toHaveLength(0);+ expectCleanup(input);+ });++ it("records a failed barrier if the successful finalization write fails", async () => {+ const { input } = await fixture();+ const error = new Error("finalization write failed");+ vi.spyOn(input.workspace.workspaceOperationRecorder, "recordOperation").mockRejectedValueOnce(error);+ await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error);+ expect(await finalizations()).toMatchObject([{ status: "failed" }]);+ expectCleanup(input);+ });++ it("retains a failed sandbox restore even when persisting its receipt throws", async () => {+ const { input, adapter, run } = await fixture();+ input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home };+ const evidence = { workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: { message: "copy-back failed" } };+ adapter.execute.mockResolvedValueOnce({ exitCode: 0, signal: null, timedOut: false, resultJson: evidence });+ const order: string[] = [];+ input.effects.onWorkspaceRestoreFailure = vi.fn(() => { order.push("restore evidence"); });+ input.effects.onProviderResourceDisposition = vi.fn(() => { order.push("retain sandbox"); });+ input.instructions.releaseInstructionCopy = vi.fn(async () => { order.push("cleanup"); });+ const error = new Error("restore receipt write failed");+ const record = vi.spyOn(legacyRecovery, "recordLegacyWorkspaceRestoreFailure").mockImplementationOnce(async () => {+ order.push("receipt write");+ throw error;+ });+ try {+ await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error);+ expect(record).toHaveBeenCalledWith(db, run, evidence, input.workspace.workspaceRestoreSource);+ expect(input.effects.onWorkspaceRestoreFailure).toHaveBeenCalledWith(evidence);+ expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain");+ expect(order).toEqual(["restore evidence", "retain sandbox", "receipt write", "cleanup"]);+ expect(await finalizations()).toMatchObject([{ status: "failed" }]);+ expectCleanup(input);+ } finally { record.mockRestore(); }+ });++ it("runs instruction cleanup even when gateway revocation fails", async () => {+ const { input } = await fixture();+ vi.mocked(revokeHeartbeatRunGatewayTokens).mockRejectedValueOnce(new Error("gateway unavailable"));+ expect(await executeHeartbeatRuntime(db, input)).toMatchObject({ dispatched: true });+ expectCleanup(input);+ });+});diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.tsnew file mode 100644index 0000000000..814ffa2e70--- /dev/null+++ b/server/src/services/heartbeat/runtime-execution.ts@@ -0,0 +1,1150 @@+import { parseNativeSessionGoalControl } from "./scheduling.js";+import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";+import {+ PAPERCLIP_WAKE_PAYLOAD_KEY,+ configuredPaperclipApiBaseUrl,+ buildPaperclipRuntimeMcpServers,+ createAdapterRuntimeToolAccess,+ paperclipApiBaseUrl,+ createAdapterRuntimeMcpAccess,+ createManagedMcpRunConfig,+ revokeHeartbeatRunGatewayTokens,+} from "./run-preparation.js";+import {+ WorkspaceValidationFailure,+ isWorkspaceValidationFailure,+ fingerprintFinalizeWorkspaceBranchValidation,+ attachPaperclipSessionMetadataToSessionParams,+} from "./workspaces.js";+import { recordLegacyWorkspaceRestoreFailure } from "../legacy-execution-recovery.js";+import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js";+import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils";+import { externalObjectService } from "../external-objects.js";+import { isAgentDirectoryCopy } from "../agent-directory-working-copies.js";+import { restoreNativeWorkspaceBestEffort } from "../native-runtime/native-workspace-best-effort.js";+import {+ withNativeWorkspaceFinalizationOwnership,+ NativeWorkspaceFinalizationBusyError,+ NativeWorkspaceFinalizationOwnershipLostError,+ type NativeWorkspaceFinalizationOwnership,+} from "../native-runtime/native-workspace-finalization-ownership.js";+import { createRunUsageRecorder } from "../usage-receipts.js";+import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";+import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";+import { isConversation } from "../agent-conversations.js";+import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase";+import { remoteExecutionHasStopped } from "../remote-execution-termination.js";+import { registerAdapterExecutionControl } from "../adapter-execution-control.js";+import { resolveManagedOpenAiBilling } from "@paperclipai/adapter-utils";+import { and, eq, isNull, sql } from "drizzle-orm";+import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db";+import { conflict } from "../../errors.js";+import { getStartupTraceContext } from "../../instrumentation.js";+import { logger } from "../../middleware/logger.js";+import {+ buildNativeProviderEnvironment,+ executePaperclipNativeSession,+ finalizeNativeRun,+ isRunnerIngressAuthorized,+ materializeLegacyQuestionResponseWakeProjection,+ NativeCancellationPendingRecoveryError,+ NativeControllerDetachedForRestartError,+ recordNativeFinalizationFailure,+} from "../native-runtime/index.js";+import {+ buildNativeHeartbeatPreparationSpans,+ buildNativeWakeIngressSpan,+} from "../native-runtime/native-run-trace.js";+import { PROVIDER_TRACE_MAX_BYTES } from "../provider-trace-store.js";+import type { AdapterExecutionResult } from "../../adapters/index.js";+import { createLocalAgentJwt } from "../../agent-auth-jwt.js";+import { parseObject, asNumber } from "../../adapters/utils.js";+import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js";+import {+ ensureGitWorktreeBranchCoherent,+ formatManagedGitWorktreeBranchInspection,+ inspectManagedGitWorktreeBranch,+} from "../workspace-runtime.js";+import {+ blockRunnerGoalRecovery,+ failRunnerGoalAction,+ isRunnerGoalActionCompleted,+ settleLiveRunnerGoalBeforeInterrupt,+} from "../runner-goals.js";+import type { Db, agents, executionWorkspaces } from "@paperclipai/db";+import type { AdapterExecutionContext, ServerAdapterModule } from "../../adapters/index.js";+import type { HeartbeatRuntimeSelectionInput, HeartbeatRuntimeSelectionResult } from "./runtime-selection.js";+import type { createHeartbeatRunState } from "./run-state.js";+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";+import type { createHeartbeatRunControl } from "./run-control.js";+import type { createHeartbeatQueue } from "./queue.js";+import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";+import type { createAdapterExecutionControl } from "../adapter-execution-control.js";+import type { executionWorkspaceService } from "../execution-workspaces.js";+import type { workspaceOperationService } from "../workspace-operations.js";+import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";+import type { instanceSettingsService } from "../instance-settings.js";+import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js";+import type { ProviderResourceDisposition } from "../environment-runtime.js";+import type { NativeExecutionInput, NativeSessionBackend } from "../../vendor/paperclip-runner/index.js";+import type { PluginWorkerManager } from "../plugin-worker-manager.js";+import type { RealizedExecutionWorkspace } from "../workspace-runtime.js";+import type { reserveWarmNativeInstructionDirectory } from "../native-runtime/index.js";++type Selection = HeartbeatRuntimeSelectionInput;+type Selected = Extract<HeartbeatRuntimeSelectionResult, { selected: true }>;+type NativeSessionInput = Parameters<typeof executePaperclipNativeSession>[0];+type RunState = ReturnType<typeof createHeartbeatRunState>;+type Lifecycle = ReturnType<typeof createHeartbeatLifecycle>;++/** Dispatches a prepared run; admission and terminal run status remain owned by the executor. */+export interface HeartbeatRuntimeExecutionInput {+ run: typeof heartbeatRuns.$inferSelect;+ agent: typeof agents.$inferSelect;+ options: {+ nativeSessionBackendFactory?: (execution: NativeExecutionInput) => NativeSessionBackend;+ pluginWorkerManager?: PluginWorkerManager;+ };+ task: Pick<Selection["task"], "issueContext" | "context" | "executionContinuation"> & {+ issueRef: (NonNullable<Selection["task"]["issueRef"]> & { projectId: string | null }) | null;+ issueId: string | null;+ taskKey: string | null;+ };+ runtime: Pick<Selected, "nativeExecution" | "nativeRunnerInstanceId" | "getNativeFreshSessionHandoff"> & {+ nativeRuntimeResolution: Selection["nativeRuntimeResolution"];+ adapter: ServerAdapterModule;+ runtimeForAdapter: AdapterExecutionContext["runtime"];+ getFreshSessionHandoff: AdapterExecutionContext["getFreshSessionHandoff"];+ runOptions: Selection["recovery"];+ };+ workspace: {+ persistedExecutionWorkspace: typeof executionWorkspaces.$inferSelect | null;+ executionWorkspace: RealizedExecutionWorkspace;+ executionTarget: Selection["workspace"]["executionTarget"];+ workspaceOperationRecorder: ReturnType<ReturnType<typeof workspaceOperationService>["createRecorder"]>;+ nativeWorkspaceSync: Selected["nativeWorkspaceSync"];+ workspaceRestoreSource: Parameters<typeof recordLegacyWorkspaceRestoreFailure>[3];+ remoteExecution: unknown;+ };+ config: Pick<Selection["config"], "runtimeConfig" | "resolvedConfig" | "managedAiRuntime"> & {+ resolvedInstanceSettings: { experimental: Pick<Awaited<ReturnType<ReturnType<typeof instanceSettingsService>["getExperimental"]>>, "enableWorkspaceBranchReconcileForward" | "enableWorkspaceDirtyQuarantineRepair"> };+ configuredTaskEnvironment: Parameters<typeof configuredEnvironmentProjection>[0];+ adapterEnv: Parameters<typeof buildNativeProviderEnvironment>[0];+ agentIdentity: AdapterExecutionContext["agentIdentity"];+ runtimeEnv: Selection["runtimeEnv"];+ useHostGitHub: boolean;+ githubSelection: { configured: boolean };+ };+ session: {+ runtimeSessionParamsForAdapter: Record<string, unknown> | null;+ configuredModel: string | null;+ sessionConfigMetadata: EffectiveRunSessionConfigMetadata;+ goalCheckpointSession: { current: { params: Record<string, unknown>; displayId: string } | null };+ };+ instructions: {+ // Instruction callbacks can replace the copy and receipt while a provider is running.+ readonly instructionCopy: Selection["config"]["instructionCopy"];+ getInstructionSave: () => Record<string, unknown> | null;+ nativeInstructionWorkingCopy: () => NativeSessionInput["instructionWorkingCopy"];+ collectStoppedInstructions: AdapterExecutionContext["onProviderStopped"];+ nativeInstructionReservation: Awaited<ReturnType<typeof reserveWarmNativeInstructionDirectory>>;+ releaseInstructionCopy: () => Promise<void>;+ };+ trace: {+ providerTraceCapture: { path: string } | null;+ nativeRunnerPreparationSpans: NonNullable<NativeSessionInput["preparationSpans"]>;+ attestedQuestionResponseAtMs: Parameters<typeof buildNativeWakeIngressSpan>[0]["attestedQuestionResponseAtMs"];+ attemptStartedAtMs: number;+ environmentAcquireStartedAtMs: number;+ environmentRealizeEndedAtMs: number;+ };+ control: {+ executionControl: ReturnType<typeof createAdapterExecutionControl>;+ executionPhaseContext: Parameters<typeof withAdapterExecutionPhase>[0];+ dispatchResolvedInteractionContinuationWithAtomicGate: <T>(dispatch: (markDispatchStarted: () => void) => Promise<T>) => Promise<+ { dispatched: true; resultPromise: Promise<T> } | { dispatched: false }+ >;+ };+ output: Pick<AdapterExecutionContext, "onLog"> & {+ onAdapterEvent: NonNullable<AdapterExecutionContext["onEvent"]>;+ onAdapterMeta: AdapterExecutionContext["onMeta"];+ identityRedactor: ReturnType<typeof createAgentIdentityRedactor>;+ appendIdentityRedactedLog: AdapterExecutionContext["onLog"];+ };+ services: Pick<RunState, "upsertTaskSession" | "getRun"> &+ Pick<Lifecycle, "persistRunProcessMetadata" | "recordCurrentHeartbeatRunRuntimeProgress"> &+ Pick<ReturnType<typeof createHeartbeatRunControl>, "cancelRunInternal"> &+ Pick<ReturnType<typeof createHeartbeatQueue>, "enqueueWakeup" | "dispatchPendingNativeStatusWakeups"> & {+ executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "getById">;+ instanceSettings: Pick<ReturnType<typeof instanceSettingsService>, "getExperimental">;+ envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "releaseForRun">;+ };+ // Teardown must observe these updates immediately, including when execution later throws.+ effects: {+ onUsageCaptureReady: (persistFailure: () => Promise<void>) => void;+ onNativeDispatchStarted: () => void;+ onLegacyAdapterEntered: () => void;+ onWorkspaceRestoreFailure: (evidence: Record<string, unknown>) => void;+ onProviderResourceDisposition: (disposition: ProviderResourceDisposition) => void;+ onNativeWorkspaceFinalizeScheduled: () => void;+ onNativeSessionResumeScheduled: () => void;+ onNativeOwnershipHeld: () => void;+ };+}++export type HeartbeatRuntimeExecutionResult = { dispatched: false } | {+ dispatched: true;+ adapterResult: AdapterExecutionResult;+};++export class NativeSessionResumeScheduledError extends Error {+ constructor(readonly original: unknown) {+ super("Native session recovery has been scheduled for the same run.");+ this.name = "NativeSessionResumeScheduledError";+ }+}++export class NativeWorkspaceFinalizeScheduledError extends Error {+ constructor(+ readonly original: unknown,+ readonly terminalFailure: boolean,+ readonly reasonCode:+ "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",+ ) {+ super("Native workspace finalization recovery has been scheduled.");+ this.name = "NativeWorkspaceFinalizeScheduledError";+ }+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExecutionInput): Promise<HeartbeatRuntimeExecutionResult> {+ const { run, agent, options } = input;+ const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task;+ const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime;+ const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace;+ const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config;+ const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session;+ const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;+ const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace;+ const { executionControl, executionPhaseContext, dispatchResolvedInteractionContinuationWithAtomicGate } = input.control;+ const { identityRedactor, appendIdentityRedactedLog, onLog, onAdapterEvent, onAdapterMeta } = input.output;+ const { executionWorkspacesSvc, instanceSettings, envOrchestrator, upsertTaskSession, getRun, cancelRunInternal, enqueueWakeup, persistRunProcessMetadata, recordCurrentHeartbeatRunRuntimeProgress, dispatchPendingNativeStatusWakeups } = input.services;+ const localAgentJwtScope =+ issueRef?.workMode === "skill_test"+ ? { kind: "skill_test" as const, issueId: issueRef.id }+ : { kind: "standard" as const };+ const authToken =+ nativeRuntimeResolution.kind === "legacy" &&+ adapter.supportsLocalAgentJwt+ ? createLocalAgentJwt(+ agent.id,+ agent.companyId,+ agent.adapterType,+ run.id,+ run.responsibleUserId,+ localAgentJwtScope,+ )+ : null;+ if (+ nativeRuntimeResolution.kind === "legacy" &&+ adapter.supportsLocalAgentJwt &&+ !authToken+ ) {+ logger.warn(+ {+ companyId: agent.companyId,+ agentId: agent.id,+ runId: run.id,+ adapterType: agent.adapterType,+ },+ "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY",+ );+ }+ let adapterFinalizeOutcome: "succeeded" | "failed" | null = null;+ const inspectFinalizeWorkspaceBranch = async () => {+ const workspaceRecord = persistedExecutionWorkspace?.id+ ? await executionWorkspacesSvc.getById(+ persistedExecutionWorkspace.id,+ )+ : persistedExecutionWorkspace;+ if (workspaceRecord?.strategyType !== "git_worktree") return null;++ const worktreePath =+ readNonEmptyString(workspaceRecord.providerRef) ??+ readNonEmptyString(workspaceRecord.cwd) ??+ readNonEmptyString(executionWorkspace.worktreePath) ??+ readNonEmptyString(executionWorkspace.cwd);+ const expectedBranchName =+ readNonEmptyString(workspaceRecord.branchName) ??+ readNonEmptyString(executionWorkspace.branchName);+ if (!worktreePath || !expectedBranchName) return null;++ const inspection = await inspectManagedGitWorktreeBranch({+ worktreePath,+ expectedBranchName,+ });+ return { workspaceRecord, inspection };+ };+ const recordWorkspaceFinalize = async (+ status: "succeeded" | "failed",+ metadata?: Record<string, unknown>,+ ) => {+ if (adapterFinalizeOutcome) return;+ let finalizeBranchMetadata: Record<string, unknown> | null = null;+ let finalizeBranchRepairMetadata: Record<string, unknown> | null =+ null;+ if (status === "succeeded") {+ const branchInspection = await inspectFinalizeWorkspaceBranch();+ if (branchInspection) {+ let inspection = branchInspection.inspection;+ const initialManagedGitWorktreeBranch =+ formatManagedGitWorktreeBranchInspection(inspection);+ if (+ !inspection.valid &&+ inspection.reasonCode === "branch_mismatch" &&+ inspection.repoRoot+ ) {+ let repairedExpectedBranchName = inspection.expectedBranchName;+ try {+ const coherence = await ensureGitWorktreeBranchCoherent({+ db,+ repoRoot: inspection.repoRoot,+ worktreePath: inspection.worktreePath,+ expectedBranchName: inspection.expectedBranchName,+ actualBranchName: inspection.actualBranchName,+ sourceIssue: issueRef+ ? {+ id: issueRef.id,+ identifier: issueRef.identifier,+ title: issueRef.title,+ workMode: issueRef.workMode,+ }+ : null,+ executionWorkspaceId: branchInspection.workspaceRecord.id,+ heartbeatRunId: run.id,+ enableWorkspaceBranchReconcileForward:+ resolvedInstanceSettings.experimental+ .enableWorkspaceBranchReconcileForward,+ enableWorkspaceDirtyQuarantineRepair:+ resolvedInstanceSettings.experimental+ .enableWorkspaceDirtyQuarantineRepair,+ persistForwardReconcile: false,+ reconcileOperationPhase: "workspace_finalize",+ recorder: workspaceOperationRecorder,+ });+ if (+ coherence.branchName &&+ coherence.branchName !==+ branchInspection.workspaceRecord.branchName+ ) {+ repairedExpectedBranchName = coherence.branchName;+ executionWorkspace.branchName = coherence.branchName;+ executionWorkspace.warnings.push(...coherence.warnings);+ }+ } catch (repairErr) {+ const workspaceValidationFailure =+ isWorkspaceValidationFailure(repairErr) ? repairErr : null;+ finalizeBranchMetadata = {+ executionWorkspaceId: branchInspection.workspaceRecord.id,+ ...initialManagedGitWorktreeBranch,+ };+ finalizeBranchRepairMetadata = {+ attempted: true,+ succeeded: false,+ initial: initialManagedGitWorktreeBranch,+ reason:+ repairErr instanceof Error+ ? repairErr.message+ : String(repairErr),+ };+ await workspaceOperationRecorder.recordOperation({+ phase: "workspace_finalize",+ cwd: executionWorkspace.cwd,+ metadata: {+ adapterType: agent.adapterType,+ executionTargetKind: executionTarget?.kind ?? "local",+ ...metadata,+ managedGitWorktreeBranch: finalizeBranchMetadata,+ managedGitWorktreeBranchRepair:+ finalizeBranchRepairMetadata,+ ...(workspaceValidationFailure?.resultJson+ ? {+ workspaceValidation:+ workspaceValidationFailure.resultJson+ .workspaceValidation ??+ workspaceValidationFailure.resultJson,+ }+ : {}),+ },+ run: async () => ({+ status: "failed",+ stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`,+ }),+ });+ adapterFinalizeOutcome = "failed";+ throw repairErr;+ }++ const repairedInspection =+ await inspectManagedGitWorktreeBranch({+ worktreePath: inspection.worktreePath,+ expectedBranchName: repairedExpectedBranchName,+ repoRoot: inspection.repoRoot,+ });+ finalizeBranchRepairMetadata = {+ attempted: true,+ succeeded: repairedInspection.valid,+ initial: initialManagedGitWorktreeBranch,+ repaired:+ formatManagedGitWorktreeBranchInspection(+ repairedInspection,+ ),+ };+ inspection = repairedInspection;+ }++ const managedGitWorktreeBranch =+ formatManagedGitWorktreeBranchInspection(inspection);+ finalizeBranchMetadata = {+ executionWorkspaceId: branchInspection.workspaceRecord.id,+ ...managedGitWorktreeBranch,+ };+ if (!inspection.valid) {+ const workspaceValidationFingerprint =+ fingerprintFinalizeWorkspaceBranchValidation({+ issueId: issueRef?.id ?? null,+ executionWorkspaceId: branchInspection.workspaceRecord.id,+ inspection: managedGitWorktreeBranch,+ });+ await workspaceOperationRecorder.recordOperation({+ phase: "workspace_finalize",+ cwd: executionWorkspace.cwd,+ metadata: {+ adapterType: agent.adapterType,+ executionTargetKind: executionTarget?.kind ?? "local",+ ...metadata,+ managedGitWorktreeBranch: finalizeBranchMetadata,+ ...(finalizeBranchRepairMetadata+ ? {+ managedGitWorktreeBranchRepair:+ finalizeBranchRepairMetadata,+ }+ : {}),+ },+ run: async () => ({+ status: "failed",+ stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`,+ }),+ });+ adapterFinalizeOutcome = "failed";+ throw new WorkspaceValidationFailure(+ `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`,+ {+ workspaceValidation: {+ reason: "git_worktree_branch_incoherence",+ fingerprint: workspaceValidationFingerprint,+ adapterType: agent.adapterType,+ issueId: issueRef?.id ?? null,+ issueIdentifier: issueRef?.identifier ?? null,+ persistedExecutionWorkspaceId:+ branchInspection.workspaceRecord.id,+ executionWorkspaceCwd: executionWorkspace.cwd,+ managedGitWorktreeBranch: finalizeBranchMetadata,+ },+ },+ );+ }+ }+ }+ await workspaceOperationRecorder.recordOperation({+ phase: "workspace_finalize",+ cwd: executionWorkspace.cwd,+ metadata: {+ adapterType: agent.adapterType,+ executionTargetKind: executionTarget?.kind ?? "local",+ ...metadata,+ ...(finalizeBranchMetadata+ ? { managedGitWorktreeBranch: finalizeBranchMetadata }+ : {}),+ ...(finalizeBranchRepairMetadata+ ? {+ managedGitWorktreeBranchRepair:+ finalizeBranchRepairMetadata,+ }+ : {}),+ },+ run: async () => ({ status }),+ });+ // Only mark the outcome after the row landed, so a transient write+ // failure on the succeeded path can still be recovered by recording+ // finalize=failed from the catch path below.+ adapterFinalizeOutcome = status;+ };++ const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType });+ input.effects.onUsageCaptureReady(usageRecorder.persistFailure);+ let adapterResult: AdapterExecutionResult;+ const runGoalControlRequestId = readNonEmptyString(+ context.goalControlRequestId,+ );+ try {+ if (nativeRuntimeResolution.kind === "native") {+ if (!nativeExecution || !nativeRunnerInstanceId)+ throw new Error("native_runtime_selection_not_persisted");+ const expectedNativeMcpDigest =+ "runtimeContext" in nativeExecution &&+ nativeExecution.runtimeContext.mcp.bindingId+ ? nativeExecution.runtimeContext.mcp.digest+ : null;+ const nativeMcpServers = await buildPaperclipRuntimeMcpServers({+ db,+ agent,+ runId: run.id,+ expectedAssignmentDigest: expectedNativeMcpDigest,+ });+ if ("runtimeContext" in nativeExecution) {+ if (nativeMcpServers.length > 1)+ throw new Error(+ "native MCP realization must produce one aggregate gateway",+ );+ const server = nativeMcpServers[0] ?? null;+ const digest = server?.connectionId.startsWith("assignment:")+ ? server.connectionId.slice("assignment:".length)+ : null;+ if (digest && digest !== expectedNativeMcpDigest) {+ throw new Error("native MCP assignment digest mismatch");+ }+ }+ const nativeMcpServer = nativeMcpServers[0] ?? null;+ let sessionGoalControl = parseNativeSessionGoalControl(+ context.runnerGoalControl,+ );+ if (runGoalControlRequestId && !sessionGoalControl) {+ throw new Error("session_goal_control_payload_invalid");+ }+ // A hard restart replays the heartbeat context, not a new user+ // action. Do not repeat a completed create/replace/edit (which+ // could reactivate or clear a goal that finished while detached).+ const completedGoalControl =+ sessionGoalControl !== null &&+ taskKey !== null &&+ (await isRunnerGoalActionCompleted(+ db,+ {+ companyId: agent.companyId,+ agentId: agent.id,+ issueId: taskKey,+ },+ sessionGoalControl.requestId,+ ));+ if (completedGoalControl) sessionGoalControl = null;+ const nativeDispatchAtMs = Date.now();+ const runCreatedAtMs = run.createdAt.getTime();+ const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();+ const wakeComments = Array.isArray(+ parseObject(context.paperclipWake).comments,+ )+ ? (parseObject(context.paperclipWake).comments as unknown[])+ : [];+ const wakeIngressSpan = buildNativeWakeIngressSpan({+ runCreatedAtMs,+ wakeComments,+ attestedQuestionResponseAtMs,+ });+ if (wakeIngressSpan)+ nativeRunnerPreparationSpans.unshift(wakeIngressSpan);+ nativeRunnerPreparationSpans.push(+ ...buildNativeHeartbeatPreparationSpans({+ runCreatedAtMs,+ runStartedAtMs,+ attemptStartedAtMs,+ environmentAcquireStartedAtMs,+ environmentRealizeEndedAtMs,+ nativeDispatchAtMs,+ }),+ );+ const guardedDispatch =+ await dispatchResolvedInteractionContinuationWithAtomicGate(+ (markDispatchStarted) => {+ return executePaperclipNativeSession({+ db,+ execution: nativeExecution,+ getFreshSessionHandoff: getNativeFreshSessionHandoff,+ refreshTools: context.refreshTools === true,+ conversationMode: isConversation(issueContext),+ turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000,+ runnerInstanceId: nativeRunnerInstanceId,+ leaseOwner: runOptions.nativeLeaseOwner,+ restartRecovery: runOptions.nativeRestartRecovery,+ backend:+ options.nativeSessionBackendFactory?.(nativeExecution),+ useRunnerd: agent.adapterType === "paperclip_runner",+ adapterType: agent.adapterType,+ sessionGoalControl,+ resumeSessionGoalHeartbeat:+ context.resumeSessionGoalHeartbeat === true ||+ completedGoalControl,+ onGoalCheckpoint: async (snapshot) => {+ if (!taskKey) return;+ const params =+ attachPaperclipSessionMetadataToSessionParams(+ {+ ...runtimeSessionParamsForAdapter,+ sessionId: snapshot.identity.sessionId,+ cwd: executionWorkspace.cwd,+ },+ configuredModel,+ sessionConfigMetadata,+ )!;+ const displayId =+ snapshot.providerSessionId ?? snapshot.sessionId;+ await upsertTaskSession({+ companyId: agent.companyId,+ agentId: agent.id,+ adapterType: agent.adapterType,+ taskKey,+ sessionParamsJson: params,+ sessionDisplayId: displayId,+ lastRunId: run.id,+ lastError: null,+ });+ goalCheckpointSession.current = { params, displayId };+ },+ onLog,+ onEvent: onAdapterEvent,+ instructionWorkingCopy: nativeInstructionWorkingCopy(),++ onUsage: async receipt => { await usageRecorder.capture(receipt); },+ preparationSpans: nativeRunnerPreparationSpans,+ // Bootstrap with executable/home discovery while keeping+ // configured provider values and the server-selected+ // workspace boundary authoritative.+ managedGitHub: !useHostGitHub && githubSelection.configured,+ billingIdentity: managedAiRuntime ? { provider: managedAiRuntime.attribution.provider, biller: managedAiRuntime.attribution.provider === "openai" ? resolveManagedOpenAiBilling(managedAiRuntime.config.managedAiRouting)?.biller ?? managedAiRuntime.attribution.provider : managedAiRuntime.attribution.provider, billingType: managedAiRuntime.attribution.method === "subscription" ? "subscription_included" : "metered_api" } : undefined,+ managedAiCredentialIdentity: managedAiRuntime?.identity,+ managedAiCredentialHome: managedAiRuntime ? String((managedAiRuntime.config.env as Record<string, unknown>).CODEX_HOME) : undefined,+ dotWorkspaceRoot: nativeExecution.provider.kind === "openai_dot" && resolvedConfig.dotWorkspaceAccess === true ? executionWorkspace.cwd : undefined,+ runnerEnvironment: {+ ...configuredEnvironmentProjection(configuredTaskEnvironment),+ ...buildNativeProviderEnvironment(+ adapterEnv,+ process.env,+ executionWorkspace.cwd,+ ),+ ...buildAgentIdentityEnv(agentIdentity),+ ...(input.instructions.instructionCopy && isAgentDirectoryCopy(input.instructions.instructionCopy) ? { AGENT_HOME: input.instructions.instructionCopy.executionRoot } : {}),+ ...(nativeMcpServer+ ? {+ PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name,+ PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url,+ PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token,+ }+ : {}),+ ...(providerTraceCapture+ ? {+ PAPERCLIP_PROVIDER_TRACE_PATH:+ providerTraceCapture.path,+ PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(+ PROVIDER_TRACE_MAX_BYTES,+ ),+ }+ : {}),+ },+ runnerExecutionTarget: executionTarget,+ runnerIngressAuthorized: isRunnerIngressAuthorized(+ nativeRuntimeResolution,+ ),+ runnerPublicUrl:+ runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null,+ runnerCaBundlePath:+ runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() ||+ null,+ runnerRemoteBinaryPath:+ runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() ||+ null,+ runnerRemoteCodexPath:+ runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() ||+ null,+ runnerRemoteCodexNpmSpec:+ runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() ||+ null,+ runnerRemoteProviderPackPath:+ runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() ||+ null,+ stopTaskForReassignment: async (target) => {+ await settleLiveRunnerGoalBeforeInterrupt(db, target);+ if (!target.runId) return;+ const prior = await getRun(target.runId);+ if (!prior || prior.companyId !== target.companyId || prior.agentId !== target.agentId) {+ throw conflict("Reassignment run binding changed");+ }+ const stopped = await cancelRunInternal(target.runId, "Cancelled for task reassignment", {+ errorCode: "issue_reassigned", suppressImmediateRecovery: true,+ resultJson: { reassignmentStopConfirmed: true },+ });+ if (stopped && ["running", "queued", "scheduled_retry"].includes(stopped.status)) {+ throw conflict("The previous run did not stop; reassignment was not applied");+ }+ },+ enqueueWakeup,+ syncIssueExternalObjects: externalObjectService(db, {+ pluginWorkerManager: options.pluginWorkerManager,+ enabled: async () => (await instanceSettings.getExperimental()).enableExternalObjects === true,+ }).syncIssueSafely,+ onSpawn: async (meta) => {+ markDispatchStarted();+ await persistRunProcessMetadata(run.id, { ...meta, targetKind: executionTarget?.kind ?? "local" });+ },+ });+ },+ );+ if (!guardedDispatch.dispatched) return { dispatched: false };+ input.effects.onNativeDispatchStarted();+ adapterResult = await guardedDispatch.resultPromise;+ } else {+ const interactionId = readNonEmptyString(context.interactionId);+ const legacyQuestionResponse =+ issueRef &&+ interactionId &&+ readNonEmptyString(context.interactionKind) ===+ "ask_user_questions" &&+ readNonEmptyString(context.interactionStatus) === "answered"+ ? await materializeLegacyQuestionResponseWakeProjection({+ db,+ companyId: agent.companyId,+ issueId: issueRef.id,+ runId: run.id,+ agentId: agent.id,+ interactionId,+ })+ : null;+ // Do not write the answer projection back to `context`: legacy+ // adapters need it in their prompt, but the authoritative answers+ // remain on the interaction instead of being duplicated in the+ // heartbeat run snapshot.+ const adapterContext: Record<string, unknown> = {+ ...context,+ ...(legacyQuestionResponse+ ? {+ [PAPERCLIP_WAKE_PAYLOAD_KEY]: {+ ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]),+ questionResponse: legacyQuestionResponse,+ },+ }+ : {}),+ };+ const runtimeTools = createAdapterRuntimeToolAccess({+ agentId: agent.id,+ companyId: agent.companyId,+ runId: run.id,+ responsibleUserId: run.responsibleUserId,+ });+ if (!runtimeTools) {+ logger.warn(+ {+ companyId: agent.companyId,+ agentId: agent.id,+ runId: run.id,+ },+ "runtime connection tools could not be delivered",+ );+ }+ const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({+ db,+ agent,+ runId: run.id,+ });+ const runtimeToolDelivery =+ adapter.runtimeToolDelivery ?? "invocation_context";+ if (runtimeTools && runtimeToolDelivery === "native_mcp") {+ runtimeMcpServers.unshift({+ name: "Paperclip connections",+ url: runtimeTools.mcpEndpoint,+ token: runtimeTools.bearerToken,+ connectionId: "paperclip-runtime-tools",+ });+ }+ if (authToken && configuredPaperclipApiBaseUrl() && issueRef) {+ runtimeMcpServers.unshift({ name: "Paperclip projects", url: `${paperclipApiBaseUrl()}/api/mcp/project-tools`,+ token: authToken, connectionId: "paperclip-project-tools" });+ }+ const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers);+ if (runtimeTools && runtimeToolDelivery === "invocation_context") {+ adapterContext.paperclipRuntimeTools = runtimeTools;+ }+ const managedMcpConfig = await createManagedMcpRunConfig({+ db,+ agent,+ runId: run.id,+ config: runtimeConfig,+ projectId: issueRef?.projectId ?? null,+ issueId: issueRef?.id ?? null,+ });+ if (managedMcpConfig) {+ adapterContext.paperclipManagedMcp = managedMcpConfig;+ }+ const guardedDispatch =+ await dispatchResolvedInteractionContinuationWithAtomicGate(+ (markDispatchStarted) => {+ input.effects.onLegacyAdapterEntered();+ return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({+ getFreshSessionHandoff,+ agentIdentity,+ runId: run.id,+ agent,+ runtime: runtimeForAdapter,+ config: runtimeConfig,+ context: adapterContext,+ executionContinuation: executionContinuation ?? null,+ runtimeCommandSpec:+ adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null,+ executionTarget,+ executionTransport: remoteExecution+ ? {+ remoteExecution: remoteExecution as unknown as Record<+ string,+ unknown+ >,+ }+ : undefined,+ runtimeMcp,+ runtimeTools,+ onLog,+ onMeta: onAdapterMeta,+ onEvent: onAdapterEvent,+ onUsage: async receipt => { await usageRecorder.capture(receipt); },+ onExecutionPhase: executionControl.phases.enter,+ startupTraceContext: getStartupTraceContext(),+ onRuntimeProgress: async (progress) => {+ await recordCurrentHeartbeatRunRuntimeProgress(+ run,+ progress,+ issueId,+ );+ },+ onProviderStopped: collectStoppedInstructions,+ onDispatch: markDispatchStarted,+ signal: executionControl.controller.signal,+ ...(executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" ? {+ stopRemoteStartup: async () => {+ // Scope comes from the running host invocation, never agent+ // config. Keep adapter ownership until setup has unwound.+ if (!executionControl.controller.signal.aborted) {+ throw new Error("Remote startup stop requires a cancelled run");+ }+ const release = await envOrchestrator.releaseForRun({+ heartbeatRunId: run.id,+ companyId: agent.companyId,+ agentId: agent.id,+ status: "released",+ providerResourceDisposition: "stop_and_retain",+ cancelActiveWork: true,+ });+ if (release.errors.length || !await remoteExecutionHasStopped(db, agent.companyId, run.id)) {+ throw new Error("Could not verify remote startup stopped");+ }+ },+ } : {}),+ onCancellationReady: async () => {+ await registerAdapterExecutionControl(run.id, executionControl);+ const current = await getRun(run.id);+ if (!current || isHeartbeatRunTerminalStatus(current.status)) {+ executionControl.controller.abort(new Error("Run stopped before provider startup"));+ }+ },+ onSpawn: async (meta) => {+ markDispatchStarted();+ await persistRunProcessMetadata(run.id, {+ pid: meta.pid,+ processGroupId:+ "processGroupId" in meta &&+ typeof meta.processGroupId === "number"+ ? meta.processGroupId+ : null,+ startedAt: meta.startedAt,+ });+ },+ authToken: authToken ?? undefined,+ }));+ },+ );+ if (!guardedDispatch.dispatched) return { dispatched: false };+ adapterResult = await guardedDispatch.resultPromise;+ }+ adapterResult = identityRedactor.redact(adapterResult);+ if (run.runtimeMode === "legacy" && hasWorkspaceRestoreFailure(adapterResult.resultJson)+ && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") {+ const requiredWorkspaceRestoreEvidence = {+ workspaceRestoreFailure: adapterResult.resultJson!.workspaceRestoreFailure,+ ...(adapterResult.resultJson?.workspaceRestoreDiagnostic ? { workspaceRestoreDiagnostic: adapterResult.resultJson.workspaceRestoreDiagnostic } : {}),+ };+ input.effects.onWorkspaceRestoreFailure(requiredWorkspaceRestoreEvidence);+ // Retention is the fallback even if recording this receipt fails.+ input.effects.onProviderResourceDisposition("stop_and_retain");+ await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);+ }+ for (const stream of ["stdout", "stderr"] as const) {+ const tail = identityRedactor.finish(stream);+ if (tail) await appendIdentityRedactedLog(stream, tail);+ }+ const instructionSave = input.instructions.getInstructionSave();+ if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };++ if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {+ const captured = await usageRecorder.complete(adapterResult);+ adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };+ } else {+ // Stop may already own the terminal result. Preserve its metadata+ // while durably recording the proof needed to release admission.+ await db.update(heartbeatRuns).set({ costAccountingPending: true,+ usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingProviderWorkStarted":false}'::jsonb`,+ }).where(and(eq(heartbeatRuns.id, run.id), isNull(heartbeatRuns.costAccountedAt)));+ }+ adapterResult = applyWorkspaceRestoreFailure(adapterResult);+ // A returned result can include a failed restore. Keep the workspace+ // barrier closed until required files have been restored.+ // If recording the barrier itself fails, propagate as a run failure+ // rather than silently leaving dependents stranded behind a missing+ // finalize row.+ const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => {+ try {+ if (nativeWorkspaceSync) {+ const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(+ eq(workspaceOperations.companyId, run.companyId),+ eq(workspaceOperations.heartbeatRunId, run.id),+ eq(workspaceOperations.phase, "workspace_finalize"),+ eq(workspaceOperations.status, "succeeded"),+ )).limit(1);+ if (exported.length) adapterFinalizeOutcome = "succeeded";+ else await restoreNativeWorkspaceBestEffort({+ db, runId: run.id, assertOwnership: ownership?.assertHeld,+ restore: () => nativeWorkspaceSync!.restoreWorkspace(ownership?.assertHeld),+ });+ }+ await ownership?.assertHeld();+ await db+ .update(heartbeatRuns)+ .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.status, "running"),+ ),+ );+ const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded";+ await recordWorkspaceFinalize(workspaceFinalizeStatus);+ if (adapterResult.nativeFinalization) {+ adapterResult.nativeFinalization.workspaceFinalizeStatus =+ workspaceFinalizeStatus;+ try {+ const finalized = await finalizeNativeRun({+ db,+ runId: run.id,+ workspaceFinalizeStatus,+ preserveProviderAttempt: Boolean(nativeWorkspaceSync),+ });+ await dispatchPendingNativeStatusWakeups({+ companyId: run.companyId,+ });+ if (finalized.phase === "committed") {+ await nativeWorkspaceSync?.cleanup();+ }+ } catch (finalizeErr) {+ logger.warn(+ { err: finalizeErr, runId: run.id },+ "native result persisted but finalization did not apply; the reconciliation loop will retry",+ );+ }+ }+ } catch (error) {+ if (ownership) {+ await ownership.assertHeld();+ await recordWorkspaceFinalize("failed");+ }+ throw error;+ }+ };+ if (nativeWorkspaceSync) {+ const owned = await withNativeWorkspaceFinalizationOwnership({+ db, companyId: run.companyId, runId: run.id,+ }, completeWorkspace);+ if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError();+ } else {+ await completeWorkspace();+ }+ } catch (adapterErr) {+ if (adapterErr instanceof NativeCancellationPendingRecoveryError) {+ // Durable cancellation is settled by the outer recovery handler;+ // it does not imply a failed workspace or a persisted run result.+ throw adapterErr;+ }+ if (adapterErr instanceof NativeWorkspaceFinalizationBusyError+ || adapterErr instanceof NativeWorkspaceFinalizationOwnershipLostError) {+ input.effects.onNativeWorkspaceFinalizeScheduled();+ throw adapterErr;+ }+ if (adapterErr instanceof NativeControllerDetachedForRestartError) {+ // Preserve the provider and its run for the new controller. This+ // also keeps generic teardown from terminalizing/releasing its lease.+ input.effects.onNativeSessionResumeScheduled();+ throw adapterErr;+ }+ if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {+ input.effects.onNativeOwnershipHeld();+ throw adapterErr;+ }+ await db+ .update(heartbeatRuns)+ .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.status, "running"),+ ),+ );+ if (+ issueRef &&+ context.resumeSessionGoalHeartbeat === true &&+ !runGoalControlRequestId+ ) {+ await blockRunnerGoalRecovery(+ db,+ {+ companyId: run.companyId,+ issueId: issueRef.id,+ agentId: agent.id,+ adapterType: agent.adapterType,+ },+ "provider_session_goal_recovery_failed",+ ).catch(() => undefined);+ }+ if (issueRef && runGoalControlRequestId) {+ await failRunnerGoalAction(+ db,+ {+ companyId: run.companyId,+ issueId: issueRef.id,+ agentId: agent.id,+ adapterType: agent.adapterType,+ },+ runGoalControlRequestId,+ adapterErr instanceof Error+ ? adapterErr.message+ : "session_goal_control_failed",+ ).catch(() => undefined);+ }+ const nativeResumeScheduled =+ nativeRuntimeResolution.kind === "native"+ ? await db+ .select({+ phase: nativeRunFinalizations.phase,+ resultId: nativeRunFinalizations.resultId,+ })+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, run.id))+ .limit(1)+ .then(+ (rows) =>+ rows[0]?.phase === "retryable_failure" &&+ rows[0]?.resultId === null,+ )+ : false;+ if (nativeResumeScheduled) {+ input.effects.onNativeSessionResumeScheduled();+ throw new NativeSessionResumeScheduledError(adapterErr);+ }+ // Adapter (or its restore finally) threw — or the finalize record+ // write itself threw. Either way the workspace may be in a partial+ // state. Best-effort record finalize=failed so the dependent readiness+ // check keeps the gate closed instead of waking on stale local state,+ // and surface the original error to the caller.+ try {+ await recordWorkspaceFinalize("failed", {+ errorMessage:+ adapterErr instanceof Error+ ? adapterErr.message+ : String(adapterErr),+ });+ } catch (recordErr) {+ logger.warn(+ {+ err: recordErr,+ runId: run.id,+ executionWorkspaceId: persistedExecutionWorkspace?.id ?? null,+ },+ "failed to record workspace_finalize=failed operation; dependents may remain gated",+ );+ }+ if (nativeRuntimeResolution.kind === "native") {+ const proposedResult = await db+ .select({ resultId: nativeRunFinalizations.resultId })+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, run.id))+ .limit(1)+ .then((rows) => rows[0]?.resultId ?? null);+ if (proposedResult && nativeWorkspaceSync) {+ const workspaceFailureMessage =+ adapterErr instanceof Error ? adapterErr.message : "";+ const unrecoverable =+ workspaceFailureMessage ===+ "workspace_sync_out_unrecoverable" ||+ workspaceFailureMessage.includes("daytona_sandbox_not_found");+ const failure = await recordNativeFinalizationFailure({+ db,+ runId: run.id,+ error: new Error(+ unrecoverable+ ? "native_workspace_sync_out_unrecoverable"+ : "native_workspace_sync_out_failed",+ ),+ projectRunStatus: true,+ failureScope: "workspace",+ permanent: unrecoverable,+ });+ input.effects.onNativeWorkspaceFinalizeScheduled();+ throw new NativeWorkspaceFinalizeScheduledError(+ adapterErr,+ failure.phase === "terminal_failure",+ unrecoverable+ ? "workspace_sync_out_unrecoverable"+ : "workspace_sync_out_failed",+ );+ }+ try {+ await finalizeNativeRun({+ db,+ runId: run.id,+ workspaceFinalizeStatus: "failed",+ });+ await dispatchPendingNativeStatusWakeups({+ companyId: run.companyId,+ });+ } catch (finalizeErr) {+ logger.warn(+ { err: finalizeErr, runId: run.id },+ "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results",+ );+ }+ }+ throw adapterErr;+ } finally {+ try {+ await revokeHeartbeatRunGatewayTokens({+ db,+ companyId: agent.companyId,+ runId: run.id,+ });+ } catch (revokeErr) {+ logger.warn(+ { err: revokeErr, runId: run.id, companyId: agent.companyId },+ "failed to revoke heartbeat-run MCP gateway tokens",+ );+ }+ await nativeInstructionReservation?.release();+ await withAdapterExecutionPhase(executionPhaseContext, "instruction_cleanup", releaseInstructionCopy);+ }+ return { dispatched: true, adapterResult };+}diff --git a/tests/e2e/fixtures/composer-stop-codex-loader.mjs b/tests/e2e/fixtures/composer-stop-codex-loader.mjsnew file mode 100644index 0000000000..0815a0c62b--- /dev/null+++ b/tests/e2e/fixtures/composer-stop-codex-loader.mjs@@ -0,0 +1,23 @@+// The installed Codex dependency takes precedence over PATH. Pin only this+// acceptance server's command resolver to its deterministic protocol fixture.+import { accessSync, constants } from "node:fs";+import { registerHooks } from "node:module";+import { isAbsolute } from "node:path";++const command = process.env.PAPERCLIP_STOP_CODEX_COMMAND;+if (process.env.NODE_ENV !== "test" || !command || !isAbsolute(command)) {+ throw new Error("Composer Stop requires an absolute fixture command in NODE_ENV=test");+}+accessSync(command, constants.X_OK);+registerHooks({+ load(url, context, nextLoad) {+ if (/\/packages\/paperclip-runner\/(src|dist)\/drivers\/codex\/codex-command\.(js|ts)$/.test(url)) {+ return {+ format: "module",+ shortCircuit: true,+ source: `export const resolveCodexCommand = () => ${JSON.stringify(command)}; export const resolvePinnedCodexCommand = resolveCodexCommand;`,+ };+ }+ return nextLoad(url, context);+ },+});diff --git a/tests/e2e/fixtures/composer-stop-codex-loader.test.mjs b/tests/e2e/fixtures/composer-stop-codex-loader.test.mjsnew file mode 100644index 0000000000..35a0de116e--- /dev/null+++ b/tests/e2e/fixtures/composer-stop-codex-loader.test.mjs@@ -0,0 +1,41 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { test } from "node:test";++const loader = fileURLToPath(new URL("./composer-stop-codex-loader.mjs", import.meta.url));+const resolver = new URL("../../../packages/paperclip-runner/src/drivers/codex/codex-command.ts", import.meta.url).href;+function launch(env, source = "") {+ return spawnSync(process.execPath, ["--import", loader, "--input-type=module", "-e", source], {+ encoding: "utf8", env: { ...process.env, NODE_ENV: "test", PAPERCLIP_STOP_CODEX_COMMAND: "", ...env },+ });+}++test("pins the fixture even when a bundled Codex dependency is installed", () => {+ const dir = mkdtempSync(join(tmpdir(), "composer-stop-loader-"));+ const command = join(dir, "codex");+ try {+ writeFileSync(command, "#!/bin/sh\nprintf 'composer-stop-fixture'\n", { mode: 0o755 });+ const result = launch({ PAPERCLIP_STOP_CODEX_COMMAND: command }, `+ import { resolveCodexCommand, resolvePinnedCodexCommand } from ${JSON.stringify(resolver)};+ import { execFileSync } from "node:child_process";+ import assert from "node:assert/strict";+ assert.equal(resolveCodexCommand(), ${JSON.stringify(command)});+ assert.equal(resolvePinnedCodexCommand(), ${JSON.stringify(command)});+ process.stdout.write(execFileSync(resolveCodexCommand(), ["--version"]));+ `);+ assert.equal(result.status, 0, result.stderr);+ assert.equal(result.stdout, "composer-stop-fixture");+ } finally { rmSync(dir, { recursive: true, force: true }); }+});++for (const env of [{}, { PAPERCLIP_STOP_CODEX_COMMAND: "codex" }, { NODE_ENV: "production", PAPERCLIP_STOP_CODEX_COMMAND: process.execPath }]) {+ test(`rejects a missing or unsafe fixture configuration: ${JSON.stringify(env)}`, () => {+ const result = launch(env);+ assert.notEqual(result.status, 0);+ assert.match(result.stderr, /Composer Stop requires an absolute fixture command/);+ });+}diff --git a/tests/e2e/playwright-composer-stop.config.ts b/tests/e2e/playwright-composer-stop.config.tsindex 41d0319468..a58f850703 100644--- a/tests/e2e/playwright-composer-stop.config.ts+++ b/tests/e2e/playwright-composer-stop.config.ts@@ -19,6 +19,7 @@ if (fixture) {throw new Error("PAPERCLIP_STOP_FAKE_CODEX must name the built fake-codex-app-server binary",);+ fs.mkdirSync(path.join(fixtureDir, "codex-home"));const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;fs.writeFileSync(path.join(fixtureDir, "codex"),@@ -37,6 +38,14 @@ export default defineConfig({...server?.env,HEARTBEAT_SCHEDULER_INTERVAL_MS: "10000",PATH: `${fixtureDir}${path.delimiter}${process.env.PATH ?? ""}`,+ ...(fixture ? {+ // Provider selection prefers the installed dependency over PATH.+ // Scope this override and its empty credential home to the test server.+ PAPERCLIP_STOP_CODEX_COMMAND: path.join(fixtureDir, "codex"),+ CODEX_HOME: path.join(fixtureDir, "codex-home"),+ OPENAI_API_KEY: "composer-stop-fixture-key",+ NODE_OPTIONS: `${server?.env?.NODE_OPTIONS ?? ""} --import=${path.resolve(import.meta.dirname, "fixtures/composer-stop-codex-loader.mjs")}`,+ } : {}),},},});