The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(server): extract heartbeat run completion (#15732)
Dottac27ee928b0b2
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 624c833467..9c3d79a07c 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1331,6 +1331,16 @@ claims remain database-scoped so separate scheduler instances cannot dispatchthe same claim. The public heartbeat methods keep their existing signatures.Keep scheduling policy changes separate from this extraction and execution.+Run completion is in `server/src/services/heartbeat/run-completion.ts`.+`createHeartbeatRunCompletion` binds terminal result persistence, accounting,+response presentation, task-session updates, and execution/setup failure handling.+The executor calls these handlers from the original try/catch boundaries. It+retains native recovery exceptions, dispatch ownership, and the outer cleanup.+Completion reads the shared Stop barriers, uses live output getters, and reports+trace finalization immediately so a later error cannot lose cleanup state.+Existing public failure helpers remain re-exported by `heartbeat.ts`.+Keep completion policy changes separate from this extraction and dispatch.+## Wake Context DeliveryBuilt-in adapters deliver wake context through the run prompt, including structureddiff --git a/server/src/services/chat-publication-reconciliation.test.ts b/server/src/services/chat-publication-reconciliation.test.tsindex 3af863db2d..aa3fad2a36 100644--- a/server/src/services/chat-publication-reconciliation.test.ts+++ b/server/src/services/chat-publication-reconciliation.test.ts@@ -239,14 +239,14 @@ describe("chat publication commit signals", () => {expect(persistedEvent).toBeGreaterThan(appendRunEventStart);expect(emittedEvent).toBeGreaterThan(persistedEvent);- const heartbeatSource = readFileSync(- new URL("./heartbeat.ts", import.meta.url),+ const completionSource = readFileSync(+ new URL("./heartbeat/run-completion.ts", import.meta.url),"utf8",);- const presentationMarker = heartbeatSource.indexOf(+ const presentationMarker = completionSource.indexOf('eventType: "run.presentation.resolved"',);- const committedComment = heartbeatSource.lastIndexOf(+ const committedComment = completionSource.lastIndexOf("await issuesSvc.addComment",presentationMarker,);diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex b39ece2ebb..b533b9e2d7 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,3 +1,10 @@+import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";+export {+ MAX_TURN_CONTINUATION_WAKE_REASON,+ parseSandboxProviderPluginNotReadyFailureMessage,+ isConfigurationIncompleteFailedRun,+ isWorkspaceSyncConflictFailure,+} from "./heartbeat/run-completion.js";import {createHeartbeatScheduling,formatIssueIdentifierLink,@@ -17,7 +24,6 @@ export {cancelHeartbeatNativeRun,} from "./heartbeat/run-control.js";import {- NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,isHeartbeatRunTerminalStatus,isHeartbeatRunRuntimeStatusActive,publishHeartbeatRunRuntimeProgress,@@ -31,7 +37,6 @@ export {resolveSkillTestRunCompletionForHeartbeatOutcome,} from "./heartbeat/run-lifecycle.js";import {- ChatControlRecoveryUnresolvedError,shouldDeferFollowupWakeForSameIssue,shouldQueueFollowupForRunningIssueWake,DEFERRED_WAKE_CONTEXT_KEY,@@ -48,10 +53,7 @@ import {import {BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,WorkspaceBusyDeferral,- readTransientRecoveryContractFromRun,- readHeartbeatRunErrorFamily,isWorkspaceBusyDeferral,- isTransientWorkspaceGitScanCode,createHeartbeatRetries,} from "./heartbeat/retries.js";export {@@ -67,7 +69,6 @@ export {computeBoundedTransientHeartbeatRetrySchedule,} from "./heartbeat/retries.js";import {- type RunSessionOutcome,EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,deriveTaskKeyWithHeartbeatFallback,getAdapterSessionCodec,@@ -80,11 +81,8 @@ import {isCanonicalSessionIdForAdapter,requiresCanonicalSessionIds,resolveNextSessionState,- normalizeUsageTotals,resolveCacheAdjustedCostUsd,- resolveLedgerBiller,resolveLedgerCostStatus,- normalizeLedgerBillingType,createHeartbeatRunState,} from "./heartbeat/run-state.js";export {@@ -103,9 +101,8 @@ export {normalizeSessionParams,resolveNextSessionState,} from "./heartbeat/run-state.js";-import { readGitConnectionFailure } from "./git-connection-failure.js";+import {- CONFIGURATION_INCOMPLETE_FAILURE_CODE,ConfigurationIncompleteFailure,deriveTaskKey,type WakeupOptions,@@ -144,7 +141,6 @@ export {buildPaperclipWakePayload,} from "./heartbeat/run-preparation.js";import {- isWorkspaceValidationFailedRun,resolveNativeRecoveryExecutionWorkspaceBinding,resolveExecutionWorkspaceReuseRequestForIssue,buildExecutionWorkspaceConfigSnapshot,@@ -247,7 +243,7 @@ export {} from "./heartbeat/run-log.js";import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";-import { preserveWorkspaceRestoreRecoveryMetadataSql } from "./legacy-workspace-restore-recovery.js";+import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";import {@@ -277,10 +273,7 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./ageimport type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js";-import {- cancellationResultJson,- readRunCancellation,-} from "./run-cancellation.js";+import {withNativeWorkspaceFinalizationOwnership,NativeWorkspaceFinalizationBusyError,@@ -342,7 +335,6 @@ import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./import { buildNativeReviewRequest } from "./native-runtime/native-review-prompt.js";import {- legacyExecutionNeedsReconciliationWithEvidence,settleInterruptedNativeBootstrap,} from "./legacy-execution-recovery.js";import {@@ -356,7 +348,7 @@ export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-import { buildExecutionContinuation, StaleExecutionContinuationError } from "./execution-continuation.js";import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils";-import { isWorkspaceGitScanError } from "./workspace-git-operation-scheduler.js";+import {initializeRunIdentity,@@ -492,7 +484,7 @@ import {type NativeSessionBackend,} from "../vendor/paperclip-runner/index.js";import { createNativeSessionHandoffLoader } from "./native-runtime/native-session-handoff.js";-import { normalizeResponsibleUserDenialCode } from "./responsible-user-denial-run-outcomes.js";+import { getRunLogStore, type RunLogHandle } from "./run-log-store.js";import {providerTraceStore,@@ -549,18 +541,10 @@ import {} from "../home-paths.js";import {buildHeartbeatRunIssueComment,- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,- isExternalChatPresentationContext,- mergeHeartbeatRunResultJson,- resolveHeartbeatRunResponse,- type RunPresentationDecision,} from "./heartbeat-run-summary.js";-import {- normalizeMaxTurnStopReason,-} from "./heartbeat-stop-metadata.js";+import {CHAT_CONTROL_RECOVERY_ADMISSION_KEY,- CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,} from "./chat-control-recovery-stop.js";import {@@ -580,12 +564,9 @@ import {persistAdapterManagedRuntimeServices,realizeExecutionWorkspace,releaseRuntimeServicesForRun,- isUnresolvedWorkspaceBaseRefError,- readUnresolvedWorkspaceBaseRefDiagnostic,type ExecutionWorkspaceInput,type RealizedExecutionWorkspace,type RuntimeServiceRef,- type UnresolvedWorkspaceBaseRefError,} from "./workspace-runtime.js";import {@@ -597,10 +578,9 @@ import {blockRunnerGoalRecovery,failRunnerGoalAction,isRunnerGoalActionCompleted,- runnerGoalService,settleLiveRunnerGoalBeforeInterrupt,} from "./runner-goals.js";-import { resolveChatRunPresentationAuthorizationReason } from "./chat-run-publications.js";+import {parseIssueExecutionState,@@ -661,7 +641,6 @@ import {buildExecutionReviewParticipantRecoveryNoticeSeed,buildImmediateExecutionPathRecoveryNoticeSeed,buildWorkspaceValidationRecoveryNoticeSeed,- SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,type StrandedRecoveryNoticeSeed,} from "./recovery/stranded-notice.js";@@ -773,146 +752,12 @@ const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON =const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE ="execution_review_participant_recovery";-function nonRetryablePreflightFailureCode(error: unknown): string | null {- if (error instanceof ChatControlRecoveryUnresolvedError)- return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;- if (- error instanceof HttpError &&- error.status === 409 &&- parseObject(error.details).code === "chat_failed_run_retry_not_authorized"- ) {- return "chat_failed_run_retry_not_authorized";- }- if (!(error instanceof HttpError) || error.status !== 422) return null;- const code = readNonEmptyString(parseObject(error.details).code);- return code && NON_RETRYABLE_PREFLIGHT_FAILURE_CODES.has(code) ? code : null;-}export { MAX_TURN_CONTINUATION_RETRY_REASON };-export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";export { WORKSPACE_BUSY_RETRY_REASON };-// Build the configuration-incomplete result payload for a workspace base ref-// that never resolved to a commit. The setup catch maps this to errorCode-// `configuration_incomplete`, so the recovery path routes it to a human owner-// instead of a dispatched-then-failed run. The `fingerprint` uses the canonical-// remote ref, not the operator spelling. Two equivalent spellings of one remote-// branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated-// failure reuses one active recovery action and does not reset the attempt-// count or post a duplicate notice. A different branch makes a new action.-// Build the configuration-incomplete result payload for a sandbox provider-// plugin that is installed but not `ready`. The `fingerprint` is the plugin-// key plus its status, so every run that hits the same stuck plugin reuses-// one active recovery action instead of posting a fresh notice per attempt,-// while a status change (say `error` -> `disabled`) makes a new one.-function buildSandboxProviderPluginNotReadyResultJson(- run: typeof heartbeatRuns.$inferSelect,- failure: { provider: string; pluginKey: string; pluginStatus: string },-): Record<string, unknown> {- const context = parseObject(run.contextSnapshot);- return {- configurationIncomplete: {- reason: SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,- companyId: run.companyId,- agentId: run.agentId,- issueId: readNonEmptyString(context.issueId) ?? null,- projectId: readNonEmptyString(context.projectId) ?? null,- sandboxProvider: failure.provider,- pluginKey: failure.pluginKey,- pluginStatus: failure.pluginStatus,- fingerprint: `sandbox_provider_plugin:${failure.pluginKey}:${failure.pluginStatus}`,- missingBindings: [],- },- };-}--function buildUnresolvedWorkspaceBaseRefResultJson(- run: typeof heartbeatRuns.$inferSelect,- error: UnresolvedWorkspaceBaseRefError,-): Record<string, unknown> {- const context = parseObject(run.contextSnapshot);- const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);- return {- configurationIncomplete: {- reason: "workspace_base_ref_unresolved",- ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),- companyId: run.companyId,- agentId: run.agentId,- issueId: readNonEmptyString(context.issueId) ?? null,- projectId: readNonEmptyString(context.projectId) ?? null,- requestedRef: error.requestedRef,- defaultBranch: error.defaultBranch,- attemptedRefs: error.attemptedRefs,- fetchError: error.fetchError,- fingerprint: `workspace_base_ref:${error.recoveryIdentityRef}`,- missingBindings: [],- },- };-}-export { isNonAssigneeWorkspaceBusyRetry };-function isMaxTurnExhaustionRun(- run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,-) {- const resultJson = parseObject(run.resultJson);- return Boolean(- normalizeMaxTurnStopReason(resultJson.stopReason) ??- normalizeMaxTurnStopReason(run.errorCode),- );-}--// environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message,-// e.g. 'Sandbox provider "kubernetes" is installed via plugin-// "acme.kubernetes-sandbox-provider", but that plugin is currently error.'-// The plugin row exists but its status is `error` (a failed activation),-// `disabled` (an operator switched it off) or `upgrade_pending`. Unlike the-// worker restart window above, nothing on the run path ever changes that-// status: only an operator enabling the plugin, or a server boot that-// re-activates a bundled plugin, does. Re-running the agent produces the-// identical failure every time, so the setup catch classifies it as-// `configuration_incomplete` (routed to a human owner) instead of a retryable-// `setup_failed` that the scheduler would keep re-dispatching.-const SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE =- /sandbox provider "([^"]*)" is installed via plugin "([^"]*)", but that plugin is currently (error|disabled|upgrade_pending)\b/i;--export function parseSandboxProviderPluginNotReadyFailureMessage(- value: unknown,-): { provider: string; pluginKey: string; pluginStatus: string } | null {- if (typeof value !== "string") return null;- const match = SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE.exec(value);- if (!match) return null;- return {- provider: match[1] ?? "",- pluginKey: match[2] ?? "",- pluginStatus: (match[3] ?? "").toLowerCase(),- };-}--function mergeAdapterRecoveryMetadata(input: {- resultJson: Record<string, unknown> | null | undefined;- errorFamily?: string | null;- retryNotBefore?: string | null;-}) {- const errorFamily = readNonEmptyString(input.errorFamily);- const retryNotBefore = readNonEmptyString(input.retryNotBefore);- if (!input.resultJson && !errorFamily && !retryNotBefore)- return input.resultJson ?? null;-- return {- ...(input.resultJson ?? {}),- ...(errorFamily ? { errorFamily } : {}),- ...(retryNotBefore- ? {- retryNotBefore,- transientRetryNotBefore: retryNotBefore,- ...(errorFamily === "provider_quota"- ? { providerQuotaRetryNotBefore: retryNotBefore }- : {}),- }- : {}),- };-}// Routes and the scheduler construct separate heartbeatService instances, but// they must agree on in-process adapter executions when reaping stale runs.const activeRunExecutions = new Set<string>();@@ -1047,21 +892,6 @@ export function resolveNativeSandboxLifecycle(input: {return resolveReusableSandboxLifecycle(input);}-function isConfigurationIncompleteFailure(- error: unknown,-): error is ConfigurationIncompleteFailure {- return error instanceof ConfigurationIncompleteFailure;-}--export function isConfigurationIncompleteFailedRun(- run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode"> | null | undefined,-) {- return (- run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE ||- run?.errorCode === "model_not_found"- );-}-interface ParsedIssueAssigneeAdapterOverrides {adapterConfig: Record<string, unknown> | null;useProjectWorkspace: boolean | null;@@ -1124,30 +954,6 @@ export function filterZombieCoalesceTarget<return target && isZombieRun(target, tracked) ? null : target;}-/**- * Failure signatures from sandbox→host git workspace reconciliation. These- * describe the state of the SHARED workspace (divergent histories written by- * different runs), not a defect in the agent that happened to run last —- * putting the agent into a sticky `error` state over them removes a healthy- * agent from rotation while leaving the actual problem (the workspace)- * untouched. The run still fails and carries the full message.- */-const WORKSPACE_SYNC_CONFLICT_SIGNATURES = [- "Failed to merge concurrent remote git histories",- "Failed to integrate concurrent remote git history",- "did not send all necessary objects",- "lacks these prerequisite commits",-];--export function isWorkspaceSyncConflictFailure(- message: string | null | undefined,-): boolean {- if (!message) return false;- return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) =>- message.includes(signature),- );-}-export function shouldAutoCheckoutIssueForWake(input: {contextSnapshot: Record<string, unknown> | null | undefined;issueStatus: string | null;@@ -2328,6 +2134,43 @@ export function heartbeatService(nativeSessionResumeDispatchTimers.set(runId, timer);}+ const runCompletion = createHeartbeatRunCompletion(db, {+ getRun,+ getAgent,+ resolveNormalizedUsageForSession,+ clearTaskSessions,+ upsertTaskSession,+ mergeRunStopMetadataForAgent,+ setRunStatusIfRunning,+ setWakeupStatus,+ appendRunEvent,+ classifyAndPersistRunLiveness,+ completeSkillTestRunForHeartbeatOutcome,+ refreshContinuationSummaryForRun,+ findRunIssueComment,+ findLatestCompletedFinalAgentMessage,+ parseMaxTurnContinuationPolicy,+ finalizeIssueCommentPolicy,+ handleIssueReviewPathDisposition,+ handleRunLivenessContinuation,+ handleSuccessfulRunHandoff,+ updateRuntimeState,+ scheduleBoundedRetryForRun,+ scheduleInteractionContinuationInfrastructureRetryIfEligible,+ timerClaimWasFirstHeartbeat,+ releaseIssueExecutionAndPromote,+ finalizeAgentStatus,+ enqueueWakeup,+ issuesSvc,+ recovery,+ getCurrentUserRedactionOptions,+ budgetHooks,+ runLogStore,+ traceStore,+ processRunCancellationSettlements,+ failedProcessRunCancellations,+ });+async function executeRun(runId: string,runOptions: {@@ -7647,650 +7490,20 @@ export function heartbeatService(}}}- const processCancellation =- processRunCancellationSettlements.get(run.id) ??- failedProcessRunCancellations.get(run.id);- await processCancellation?.settled;- let outcome: RunSessionOutcome;- const latestRun = await getRun(run.id);- if (isHeartbeatRunTerminalStatus(latestRun?.status)) {- outcome = latestRun.status;- } else if (executionControl.controller.signal.aborted) {- outcome = "cancelled";- } else if (adapterResult.nativeFinalization) {- const nativeTerminal =- adapterResult.nativeFinalization.terminal.runTerminalState;- outcome =- nativeTerminal === "succeeded"- ? "succeeded"- : nativeTerminal === "cancelled"- ? "cancelled"- : "failed";- } else if (adapterResult.timedOut) {- outcome = "timed_out";- } else if (adapterResult.resultJson?.status === "cancelled") {- outcome = "cancelled";- } else if (- (adapterResult.exitCode ?? 0) === 0 &&- !adapterResult.errorMessage &&- !adapterResult.signal &&- !processCancellation?.failed- ) {- outcome = "succeeded";- } else {- outcome = "failed";- }-- const nextSessionState = resolveNextSessionState({- adapterType: agent.adapterType,- codec: sessionCodec,- adapterResult,- outcome,- previousParams: previousSessionParams,- previousDisplayId: runtimeForAdapter.sessionDisplayId,- previousLegacySessionId: runtimeForAdapter.sessionId,- });- const rawUsage = normalizeUsageTotals(adapterResult.usage);- const sessionUsageResolution = await resolveNormalizedUsageForSession({- agentId: agent.id,- runId: run.id,- sessionId:- nextSessionState.displayId ?? nextSessionState.legacySessionId,- rawUsage,- usageBasis: adapterResult.usageBasis ?? null,- });- const normalizedUsage = sessionUsageResolution.normalizedUsage;- const runErrorMessage =- outcome === "cancelled"- ? redactCurrentUserText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled", currentUserRedactionOptions)- : outcome === "succeeded"- ? null- : redactCurrentUserText(- adapterResult.errorMessage ??- (outcome === "timed_out" ? "Timed out" : "Adapter failed"),- currentUserRedactionOptions,- );- const recordedResponsibleUserDenialCode =- normalizeResponsibleUserDenialCode(latestRun?.errorCode);- const runErrorCode =- outcome === "timed_out"- ? "timeout"- : outcome === "cancelled"- ? (latestRun?.errorCode ?? "cancelled")- : outcome === "failed"- ? (adapterResult.errorCode ??- recordedResponsibleUserDenialCode ??- "adapter_failed")- : null;-- let logSummary: {- bytes: number | null;- sha256?: string | null;- compressed: boolean;- } | null = null;- if (handle) {- logSummary = await runLogStore.finalize(handle);- }- const finalLogBytes = logSummary?.bytes;- if (outputProgressState.pending && typeof finalLogBytes === "number") {- outputProgressState.pending.bytes = finalLogBytes;- }- await flushOutputProgress({ force: true });-- if (providerTraceCapture) {- try {- await traceStore.finalize(run.id, run.companyId);- providerTraceFinalized = true;- } catch (error) {- logger.warn(- { error, runId: run.id },- "provider trace finalization failed without affecting run outcome",- );- }- }-- const status =- outcome === "succeeded"- ? "succeeded"- : outcome === "cancelled"- ? "cancelled"- : outcome === "timed_out"- ? "timed_out"- : "failed";-- const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null- ? null : resolveCacheAdjustedCostUsd(adapterResult);- const usageJson: Record<string, unknown> = {- accountingReceiptReady: adapterResult.usageComplete !== false,- costUsdExact: adapterResult.costUsdExact ?? null,- providerRequestId: adapterResult.providerRequestId ?? null,- ...(normalizedUsage ?? {}),- ...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}),- ...(rawUsage- ? {- rawInputTokens: rawUsage.inputTokens,- rawInputIncludesCached: false,- rawCachedInputTokens: rawUsage.cachedInputTokens,- rawOutputTokens: rawUsage.outputTokens,- }- : {}),- ...(sessionUsageResolution.derivedFromSessionTotals- ? { usageSource: "session_delta" }- : adapterResult.usageBasis === "per_run"- ? { usageSource: "per_run" }- : {}),- ...((nextSessionState.displayId ??- nextSessionState.legacySessionId)- ? {- persistedSessionId:- nextSessionState.displayId ??- nextSessionState.legacySessionId,- }- : {}),- sessionReused:- runtimeForAdapter.sessionId != null ||- runtimeForAdapter.sessionDisplayId != null,- taskSessionReused: taskSessionForRun != null,- freshSession:- runtimeForAdapter.sessionId == null &&- runtimeForAdapter.sessionDisplayId == null,- sessionRotated: sessionCompaction.rotate,- sessionRotationReason: sessionCompaction.reason,- configFreshness: configFreshnessResultMetadata,- provider:- readNonEmptyString(adapterResult.provider) ?? "unknown",- biller: resolveLedgerBiller(adapterResult),- model: readNonEmptyString(adapterResult.model) ?? "unknown",- ...(adapterResult.costUsd != null- ? { costUsd: adapterResult.costUsd }- : {}),- ...(cacheAdjustedCostUsd != null- ? { cacheAdjustedCostUsd }- : {}),- pricingProvenance: adapterResult.pricingProvenance,- costStatus: adapterResult.costStatus ?? resolveLedgerCostStatus({- costUsd: cacheAdjustedCostUsd ?? (adapterResult.costUsdExact != null ? Number(adapterResult.costUsdExact) : null),- billingType: normalizeLedgerBillingType(adapterResult.billingType),- inputTokens: normalizedUsage?.inputTokens ?? 0,- cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,- outputTokens: normalizedUsage?.outputTokens ?? 0,- }),- billingType: normalizeLedgerBillingType(- adapterResult.billingType,- ),- };-- const persistedResultJson = cancellationResultJson(latestRun ?? run, outcome, mergeHeartbeatRunResultJson(- mergeRunStopMetadataForAgent(agent, outcome, {- resultJson: mergeAdapterRecoveryMetadata({- resultJson: {- ...(adapterResult.nativeFinalization || outcome === "cancelled"- ? parseObject(latestRun?.resultJson)- : {}),- ...parseObject(adapterResult.resultJson),- ...(adapterResult.executionRecovery- ? { executionRecovery: adapterResult.executionRecovery }- : {}),- configFreshness: configFreshnessResultMetadata,- },- errorFamily: adapterResult.errorFamily ?? null,- retryNotBefore: adapterResult.retryNotBefore ?? null,- }),- errorCode: runErrorCode,- errorMessage: runErrorMessage,- }),- adapterResult.summary ?? null,- ), runErrorCode, runErrorMessage);-- const ledgerScope = runLedgerScope;- const finalRunPatch: Partial<typeof heartbeatRuns.$inferInsert> = {- // Accounting must acknowledge even a proven pre-provider failure:- // that transaction releases the reservation without creating a charge.- costAccountingPending: true,- finishedAt: new Date(),- error: runErrorMessage,- errorCode: runErrorCode,- exitCode: adapterResult.exitCode,- signal: adapterResult.signal,- usageJson: { ...usageJson, ledgerScope },- resultJson: persistedResultJson,- sessionIdAfter:- nextSessionState.displayId ?? nextSessionState.legacySessionId,- stdoutExcerpt,- stderrExcerpt,- logBytes: logSummary?.bytes,- logSha256: logSummary?.sha256,- logCompressed: logSummary?.compressed ?? false,- };- const persistedRunWrite = await setRunStatusIfRunning(- run.id,- status,- finalRunPatch,- { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },- );- let persistedRun: typeof heartbeatRuns.$inferSelect | null =- persistedRunWrite.run;- if (!persistedRunWrite.updated) {- persistedRun = null;- // Native reconciliation can commit and project the terminal status in- // the narrow window between adapter completion and this live write.- // The status is authoritative, but it must not make us discard the- // adapter's semantic result, usage, logs, or presentation decision.- // Only complete the late metadata write when the reconciler chose the- // same terminal status; a conflicting terminal outcome remains owned- // by the path that won the compare-and-set. Owned legacy cancellation- // likewise keeps the provider session, logs, and usage after Stop wins.- if (- (adapterResult.nativeFinalization ||- (processCancellation && !processCancellation.failed && status === "cancelled")) &&- persistedRunWrite.run?.status === status- ) {- persistedRun = await db- .update(heartbeatRuns)- .set({- ...finalRunPatch,- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(- cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,- ),-- usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },- finishedAt:- persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,- updatedAt: new Date(),- })- .where(- and(- eq(heartbeatRuns.id, run.id),- eq(heartbeatRuns.status, status),- ),- )- .returning()- .then((rows) => rows[0] ?? null);- }- if (!persistedRun) {- await accountRunCost(db, run.id, budgetHooks).catch((err) => {- logger.error({ err, runId: run.id }, "Late run accounting queued for recovery");- });- logger.info(- {- runId: run.id,- attemptedStatus: status,- currentStatus: persistedRunWrite.run?.status ?? null,- },- "skipping late run finalization because the run already left running state",- );- return;- }- }- if (persistedRun) {- // Accounting recovery is independent of workspace or issue finalization.- await accountRunCost(db, persistedRun.id, budgetHooks).catch((err) => {- logger.error({ err, runId: run.id }, "Run accounting queued for recovery");- });- persistedRun =- (await classifyAndPersistRunLiveness(- persistedRun,- persistedResultJson,- )) ?? persistedRun;- }-- await setWakeupStatus(- run.wakeupRequestId,- outcome === "succeeded" ? "completed" : status,- {- finishedAt: new Date(),- error: runErrorMessage,+ await runCompletion.completeRun({+ run, agent, issueId, issueRef, signal: executionControl.controller.signal,+ output: {+ handle, outputProgressState, flushOutputProgress,+ get stdoutExcerpt() { return stdoutExcerpt; },+ get stderrExcerpt() { return stderrExcerpt; },},- );-- const finalizedRun = persistedRun ?? (await getRun(run.id));- if (finalizedRun) {- await appendRunEvent(finalizedRun, {- eventType: "lifecycle",- stream: "system",- level: outcome === "succeeded" ? "info" : "error",- message: `run ${outcome}`,- payload: {- status,- exitCode: adapterResult.exitCode,- ...(readRunCancellation(finalizedRun.resultJson) ? { cancellation: readRunCancellation(finalizedRun.resultJson) } : {}),- },- });- try {- await completeSkillTestRunForHeartbeatOutcome({- run: finalizedRun,- issueId,- issueWorkMode: issueRef?.workMode ?? null,- outcome,- error: runErrorMessage,- });- } catch (err) {- logger.warn(- { err, runId: finalizedRun.id, issueId },- "failed to complete skill test run after heartbeat finalization",- );- await onLog(- "stderr",- `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`,- );- }- const livenessRun = finalizedRun;- await refreshContinuationSummaryForRun(livenessRun, agent);- const skipRunIssueComment =- parseObject(livenessRun.contextSnapshot).skipIssueComment === true;- let resolvedPresentationDecision: RunPresentationDecision | null =- null;- try {- const existingRunComment = issueId- ? await findRunIssueComment(- livenessRun.id,- livenessRun.companyId,- issueId,- persistedResultJson,- )- : null;- const finalAgentMessage =- await findLatestCompletedFinalAgentMessage(- livenessRun.id,- livenessRun.companyId,- );- const externalChatPresentationCandidate =- isExternalChatPresentationContext(livenessRun.contextSnapshot) ||- parseObject(livenessRun.contextSnapshot).source === "tool_action_review" ||- String(parseObject(livenessRun.contextSnapshot).source ?? "").startsWith("issue.comment") ||- parseObject(livenessRun.contextSnapshot).source === "issue.update";- const externalChatPresentationAuthorization =- issueId && externalChatPresentationCandidate- ? await resolveChatRunPresentationAuthorizationReason(db, {- companyId: livenessRun.companyId,- issueId,- runId: livenessRun.id,- })- : null;- const externalChatPresentationContext = isExternalChatPresentationContext(- livenessRun.contextSnapshot,- externalChatPresentationAuthorization === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,- );- const resolved = resolveHeartbeatRunResponse({- resultJson: persistedResultJson,- conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded"- && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished",- existingComment: existingRunComment,- finalAgentMessage,- preferFinalResponseOverExistingComment:- externalChatPresentationContext,- externalChatReviewResponseSummaryAuthorized:- persistedResultJson?.finalizationReasonCode ===- "governed_response_waiting" &&- externalChatPresentationAuthorization ===- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,- externalChatResponseWakeSummaryAuthorized:- Boolean(adapterResult.nativeFinalization) &&- externalChatPresentationAuthorization ===- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,- });- let presentationDecision: RunPresentationDecision =- resolved.decision;-- if (- issueId &&- !skipRunIssueComment &&- presentationDecision.commentAction === "create" &&- resolved.text- ) {- // The presentation resolver exposes only the final assistant- // surface selected from completed final messages or accepted- // semantic results. For an exactly bound external-chat run,- // authorize that narrow presentation as the provider reply;- // ordinary internal runs retain the private default.- const presentationAuthorizationReason =- await resolveChatRunPresentationAuthorizationReason(db, {- companyId: livenessRun.companyId,- issueId,- runId: livenessRun.id,- });- const comment = await issuesSvc.addComment(- issueId,- resolved.text,- { agentId: agent.id, runId: livenessRun.id },- { authorizationReason: presentationAuthorizationReason, completionReply: true },- );- presentationDecision = {- ...presentationDecision,- commentId: comment.id,- reasonCodes: [- ...presentationDecision.reasonCodes,- "resolved_response_materialized",- ],- };- await logActivity(db, {- companyId: livenessRun.companyId,- actorType: "agent",- actorId: agent.id,- agentId: agent.id,- runId: livenessRun.id,- issueId,- action: "issue.comment_added",- entityType: "issue",- entityId: issueId,- details: {- commentId: comment.id,- bodySnippet: comment.body.slice(0, 120),- identifier: issueRef?.identifier ?? null,- issueTitle: issueRef?.title ?? null,- authorizationReason: presentationAuthorizationReason,- source: "run_presentation_resolver",- presentationSource: presentationDecision.chosenSource,- },- });- } else if (presentationDecision.commentAction === "create") {- presentationDecision = {- ...presentationDecision,- commentAction: "none",- reasonCodes: [- ...presentationDecision.reasonCodes,- skipRunIssueComment- ? "issue_comment_suppressed"- : "run_has_no_issue",- ],- };- }-- await db- .update(heartbeatRuns)- .set({- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ presentationDecision })}::jsonb`,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, livenessRun.id));- await appendRunEvent(livenessRun, {- eventType: "run.presentation.resolved",- stream: "system",- level: "info",- message: "run presentation resolved",- payload: { presentationDecision },- });- resolvedPresentationDecision = presentationDecision;- } catch (err) {- await onLog(- "stderr",- `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`,- );- }- if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {- const policy = parseMaxTurnContinuationPolicy(agent);- if (policy.enabled && policy.maxAttempts > 0) {- await scheduleBoundedRetryForRun(livenessRun, agent, {- retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,- wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,- maxAttempts: policy.maxAttempts,- delayMs: policy.delayMs,- });- } else {- await appendRunEvent(livenessRun, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message:- "Max-turn continuation suppressed because the policy is disabled",- payload: {- retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,- policy,- },- });- }- } else if (- outcome === "failed" &&- readTransientRecoveryContractFromRun(livenessRun)- ) {- await scheduleBoundedRetryForRun(livenessRun, agent);- } else if (- outcome === "failed" &&- !(await legacyExecutionNeedsReconciliationWithEvidence(db, livenessRun))- ) {- await scheduleInteractionContinuationInfrastructureRetryIfEligible(- livenessRun,- agent,- );- }- const issueCommentPolicyResult = await finalizeIssueCommentPolicy(- livenessRun,- agent,- resolvedPresentationDecision,- );- const conversationSettled = await settleConversationTurn(db, livenessRun);- await releaseIssueExecutionAndPromote(livenessRun, {- suppressImmediateRecovery: conversationSettled ||- readNonEmptyString(- parseObject(livenessRun.contextSnapshot).goalControlRequestId,- ) !== null ||- parseObject(livenessRun.contextSnapshot)- .resumeSessionGoalHeartbeat === true,- });- if (!conversationSettled) {- await handleIssueReviewPathDisposition(livenessRun);- if (livenessRun.runtimeMode !== "native") {- await recovery.reconcileLegacyContinuation(livenessRun.id);- } else {- await handleRunLivenessContinuation(livenessRun);- await handleSuccessfulRunHandoff(- issueCommentPolicyResult.outcome === "retry_queued" ||- issueCommentPolicyResult.outcome === "retry_exhausted"- ? { ...livenessRun, issueCommentStatus: issueCommentPolicyResult.outcome }- : livenessRun,- agent,- );- }- }- if (- outcome === "succeeded" &&- issueId &&- parseObject(adapterResult.resultJson).goalRolloverRequired === true- ) {- const rolloverProjection = await runnerGoalService(db).projection(- livenessRun.companyId,- issueId,- agent.id,- );- if (rolloverProjection?.goal?.status === "active") {- await enqueueWakeup(agent.id, {- source: "automation",- triggerDetail: "system",- reason: "goal_control",- payload: {- issueId,- intent: "goal_rollover",- predecessorRunId: livenessRun.id,- },- idempotencyKey: `goal_rollover:${livenessRun.id}`,- requestedByActorType: "system",- contextSnapshot: {- issueId,- taskKey: issueId,- resumeSessionGoalHeartbeat: true,- skipIssueComment: true,- goalRolloverFromRunId: livenessRun.id,- },- });- }- }-- // Dependency wake re-check: if this run's issue was marked done mid-run,- // the route-time `issue_blockers_resolved` wake may have been gated by- // workspace finalization or merged into this run. Reuse the level-triggered- // dependency backstop so finalize and periodic recovery share idempotency,- // readiness, active-path, and observability rules.- if (issueId && finalizedRun) {- try {- const blockerIssueStatus = await db- .select({ status: issues.status })- .from(issues)- .where(eq(issues.id, issueId))- .then((rows) => rows[0]?.status ?? null);- if (blockerIssueStatus === "done") {- await recovery.reconcileResolvedDependencyWakeBackstop({- runId: finalizedRun.id,- companyId: finalizedRun.companyId,- blockerIssueId: issueId,- source: "workspace.finalize",- });- }- } catch (finalizeWakeErr) {- logger.warn(- { err: finalizeWakeErr, runId: run.id, issueId },- "failed to evaluate dependent wakes after workspace_finalize",- );- }- }- }-- if (finalizedRun) {- await updateRuntimeState(- agent,- finalizedRun,- adapterResult,- {- legacySessionId: nextSessionState.legacySessionId,- },- normalizedUsage,- );- if (taskKey) {- if (- adapterResult.clearSession ||- (!nextSessionState.params && !nextSessionState.displayId)- ) {- await clearTaskSessions(agent.companyId, agent.id, {- taskKey,- adapterType: agent.adapterType,- expectedRunId: finalizedRun.id,- });- } else {- await upsertTaskSession({- companyId: agent.companyId,- agentId: agent.id,- adapterType: agent.adapterType,- taskKey,- sessionParamsJson:- attachPaperclipSessionMetadataToSessionParams(- nextSessionState.params,- configuredModel,- sessionConfigMetadata,- ),- sessionDisplayId: nextSessionState.displayId,- lastRunId: finalizedRun.id,- lastError: runErrorMessage,- });- }- }- }- await finalizeAgentStatus(agent.id, outcome, runErrorMessage, {- keepIdleOnFailure:- outcome === "failed" &&- ((finalizedRun- ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota"- : runErrorCode === "provider_quota") ||- isWorkspaceSyncConflictFailure(adapterResult.errorMessage)),- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ session: { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata },+ readFailureReportSecrets,+ adapterResult, sessionCodec, taskSessionForRun, sessionCompaction,+ configFreshnessResultMetadata, runLedgerScope, currentUserRedactionOptions,+ providerTraceCapture: providerTraceCapture !== null,+ onProviderTraceFinalized: () => { providerTraceFinalized = true; },+ issueContext, onLog,});} catch (err) {await persistUsageCaptureFailure?.();@@ -8412,226 +7625,17 @@ export function heartbeatService(}return;}- // A process adapter may throw while its owned Stop is joining the- // child. Let the cancellation write settle before attempting failure.- await processRunCancellationSettlements.get(run.id)?.settled;- const message = redactCurrentUserText(- identityRedactor.redact(err instanceof Error ? err.message : "Unknown adapter failure"),- await getCurrentUserRedactionOptions(),- );- const workspaceValidationFailure = isWorkspaceValidationFailure(err)- ? err- : null;- const configurationIncompleteFailure = isConfigurationIncompleteFailure(- err,- )- ? err- : null;- const recordedResponsibleUserDenialCode =- normalizeResponsibleUserDenialCode(- (await getRun(run.id).catch(() => null))?.errorCode,- );- // The runtime resolution is scoped to the adapter try block. The- // durable coordinator is also the stronger authority here: legacy- // runs simply have no row, while native result-less exhaustion keeps- // its named failure instead of being flattened to `adapter_failed`.- const nativeTerminalFailureCode = await db- .select({- phase: nativeRunFinalizations.phase,- resultId: nativeRunFinalizations.resultId,- failureCode: nativeRunFinalizations.failureCode,- })- .from(nativeRunFinalizations)- .where(eq(nativeRunFinalizations.runId, run.id))- .limit(1)- .then((rows) => {- const coordinator = rows[0];- return coordinator?.phase === "terminal_failure" &&- coordinator.resultId === null- ? coordinator.failureCode- : null;- })- .catch(() => null);- const failureErrorCode =- workspaceValidationFailure?.code ??- configurationIncompleteFailure?.code ??- nonRetryablePreflightFailureCode(err) ??- recordedResponsibleUserDenialCode ??- nativeTerminalFailureCode ??- "adapter_failed";- logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId }, "heartbeat execution failed");-- let logSummary: {- bytes: number | null;- sha256?: string | null;- compressed: boolean;- } | null = null;- if (handle) {- try {- logSummary = await runLogStore.finalize(handle);- } catch (finalizeErr) {- logger.warn(- { err: finalizeErr, runId },- "failed to finalize run log after error",- );- }- }- const finalLogBytes = logSummary?.bytes;- if (outputProgressState.pending && typeof finalLogBytes === "number") {- outputProgressState.pending.bytes = finalLogBytes;- }- await flushOutputProgress({ force: true }).catch((flushErr) => {- logger.warn(- { err: flushErr, runId },- "failed to flush run output progress after error",- );- });-- const stoppedDuringFailure = executionControl.controller.signal.aborted;- const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null;- const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed";- const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, {- error: message,- errorCode: stopSnapshot?.errorCode ?? failureErrorCode,- finishedAt: new Date(),- resultJson: mergeRunStopMetadataForAgent(agent, failureOutcome, {- errorCode: failureErrorCode,- errorMessage: message,- resultJson: {- ...parseObject(stopSnapshot?.resultJson),- ...requiredWorkspaceRestoreEvidence,- ...(workspaceValidationFailure?.resultJson ??- configurationIncompleteFailure?.resultJson ??- {}),- ...(!legacyAdapterEntered && run.runtimeMode !== "native"- ? {- executionRecovery: {- kind: "bootstrap",- providerWorkStarted: false,- },- }- : {}),- },- }),- stdoutExcerpt,- stderrExcerpt,- logBytes: logSummary?.bytes,- logSha256: logSummary?.sha256,- logCompressed: logSummary?.compressed ?? false,- }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });- if (- !failedRunWrite.updated &&- !(- nativeTerminalFailureCode && failedRunWrite.run?.status === "failed"- )- ) {- logger.info(- {- runId: run.id,- attemptedStatus: "failed",- currentStatus: failedRunWrite.run?.status ?? null,- },- "skipping late adapter failure finalization because the run already left running state",- );- return;- }-- const failedRun = failedRunWrite.run;- await setWakeupStatus(run.wakeupRequestId, "failed", {- finishedAt: new Date(),- error: message,- });-- if (failedRun) {- await appendRunEvent(failedRun, {- eventType: "error",- stream: "system",- level: "error",- message,- });- const livenessRun =- (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun;- try {- await completeSkillTestRunForHeartbeatOutcome({- run: livenessRun,- issueId,- issueWorkMode: issueRef?.workMode ?? null,- outcome: "failed",- error: message,- });- } catch (err) {- logger.warn(- { err, runId: livenessRun.id, issueId },- "failed to complete skill test run after heartbeat adapter failure",- );- }- await refreshContinuationSummaryForRun(livenessRun, agent);- if (- !isWorkspaceValidationFailedRun(livenessRun) &&- !isConfigurationIncompleteFailedRun(livenessRun)- ) {- await finalizeIssueCommentPolicy(livenessRun, agent);- }- await scheduleInteractionContinuationInfrastructureRetryIfEligible(- livenessRun,- agent,- );- await releaseIssueExecutionAndPromote(livenessRun, {- // Native recovery owns the original heartbeat run through- // exhaustion. Once its durable coordinator has classified a- // terminal failure, generic issue recovery must not create a- // replacement retryOfRunId chain for the same provider work.- suppressImmediateRecovery: nativeTerminalFailureCode !== null,- });- await handleIssueReviewPathDisposition(livenessRun);-- await updateRuntimeState(- agent,- livenessRun,- {- exitCode: null,- signal: null,- timedOut: false,- errorMessage: message,- },- {- legacySessionId: runtimeForAdapter.sessionId,- },- );-- if (- taskKey &&- (goalCheckpointSession.current ||- previousSessionParams ||- previousSessionDisplayId ||- taskSession)- ) {- await upsertTaskSession({- companyId: agent.companyId,- agentId: agent.id,- adapterType: agent.adapterType,- taskKey,- sessionParamsJson:- goalCheckpointSession.current?.params ??- attachPaperclipSessionMetadataToSessionParams(- previousSessionParams,- configuredModel,- sessionConfigMetadata,- ),- sessionDisplayId:- goalCheckpointSession.current?.displayId ??- previousSessionDisplayId,- lastRunId: failedRun.id,- lastError: message,- });- }- }-- await finalizeAgentStatus(agent.id, "failed", message, {- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),- keepIdleOnFailure:- Boolean(nonRetryablePreflightFailureCode(err)) ||- isWorkspaceSyncConflictFailure(message),+ await runCompletion.failRun({+ run, agent, issueId, issueRef, signal: executionControl.controller.signal,+ output: {+ handle, outputProgressState, flushOutputProgress,+ get stdoutExcerpt() { return stdoutExcerpt; },+ get stderrExcerpt() { return stderrExcerpt; },+ },+ session: { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata },+ readFailureReportSecrets,+ runId, err, identityRedactor, requiredWorkspaceRestoreEvidence, legacyAdapterEntered,+ goalCheckpointSession, previousSessionDisplayId, taskSession,});}} catch (outerErr) {@@ -8671,223 +7675,9 @@ export function heartbeatService(},);} else {- // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).- // The inner catch did not fire, so we must record the failure here.- const message = redactCurrentUserText(- identityRedactor.redact(outerErr instanceof Error- ? outerErr.message- : "Unknown setup failure"),- await getCurrentUserRedactionOptions(),- );- // A missing secret/env binding is a known pre-dispatch configuration gap,- // not an opaque setup crash. Surface it with its own errorCode so the- // recovery path routes it to a human owner instead of looping retries.- const workspaceValidationSetupFailure = isWorkspaceValidationFailure(- outerErr,- )- ? outerErr- : null;- const configurationIncompleteSetupFailure =- isConfigurationIncompleteFailure(outerErr) ? outerErr : null;- const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(- outerErr,- )- ? outerErr- : null;- // A sandbox provider plugin stuck in error/disabled/upgrade_pending- // fails every lease the same way until an operator acts, so it is a- // configuration gap, not a transient setup failure.- const sandboxProviderPluginNotReadySetupFailure =- parseSandboxProviderPluginNotReadyFailureMessage(- outerErr instanceof Error ? outerErr.message : null,- );- const recordedResponsibleUserDenialCode =- normalizeResponsibleUserDenialCode(- (await getRun(runId).catch(() => null))?.errorCode,- );- const nonRetryablePreflightCode =- nonRetryablePreflightFailureCode(outerErr);- const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null;- const setupFailureErrorCode =- workspaceGitScanFailure?.code ??- workspaceValidationSetupFailure?.code ??- configurationIncompleteSetupFailure?.code ??- (unresolvedBaseRefSetupFailure ||- sandboxProviderPluginNotReadySetupFailure- ? CONFIGURATION_INCOMPLETE_FAILURE_CODE- : null) ??- recordedResponsibleUserDenialCode ??- nonRetryablePreflightCode ??- "setup_failed";- logger.error(- { err: identityRedactor.redact({ message: outerErr instanceof Error ? outerErr.message : String(outerErr), stack: outerErr instanceof Error ? outerErr.stack : undefined }), runId },- "heartbeat execution setup failed",- );- const setupFailureAgent = await getAgent(run.agentId).catch(() => null);- // The structured failure payload drives the recovery notice and next- // action, so it is persisted even when the agent lookup failed and the- // agent-scoped stop metadata cannot be merged in.- const setupFailureDetails =- (workspaceGitScanFailure ? {- workspaceGitScan: {- code: workspaceGitScanFailure.code,- phase: "workspace_setup",- retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code),- },- } : null) ??- workspaceValidationSetupFailure?.resultJson ??- configurationIncompleteSetupFailure?.resultJson ??- (unresolvedBaseRefSetupFailure- ? buildUnresolvedWorkspaceBaseRefResultJson(- run,- unresolvedBaseRefSetupFailure,- )- : null) ??- (sandboxProviderPluginNotReadySetupFailure- ? buildSandboxProviderPluginNotReadyResultJson(- run,- sandboxProviderPluginNotReadySetupFailure,- )- : null);- const connectionFailure = readGitConnectionFailure(outerErr);- const setupFailureResultJson = {- ...setupFailureDetails,- ...(connectionFailure ? { connectionFailure } : {}),- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },- };- const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", {- error: message,- errorCode: setupFailureErrorCode,- finishedAt: new Date(),- ...(setupFailureAgent- ? {- resultJson: mergeRunStopMetadataForAgent(- setupFailureAgent,- "failed",- {- errorCode: setupFailureErrorCode,- errorMessage: message,- resultJson: setupFailureResultJson,- },- ),- }- : setupFailureResultJson- ? { resultJson: setupFailureResultJson }- : {}),- }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));- if (!setupFailureWrite.updated) {- logger.info(- {- runId,- attemptedStatus: "failed",- currentStatus: setupFailureWrite.run?.status ?? null,- },- "skipping late setup failure finalization because the run already left running state",- );- } else {- await setWakeupStatus(run.wakeupRequestId, "failed", {- finishedAt: new Date(),- error: message,- }).catch(() => undefined);- }- const failedRun = await getRun(runId).catch(() => null);- if (setupFailureWrite.updated && failedRun) {- // Emit a run-log event so the failure is visible in the run timeline,- // consistent with what the inner catch block does for adapter failures.- await appendRunEvent(failedRun, {- eventType: "error",- stream: "system",- level: "error",- message,- }).catch(() => undefined);- const livenessRun = await classifyAndPersistRunLiveness(- failedRun,- ).catch(() => failedRun);- const setupFailureIssueId = readNonEmptyString(- parseObject(livenessRun.contextSnapshot).issueId,- );- if (setupFailureIssueId) {- await completeSkillTestRunForHeartbeatOutcome({- run: livenessRun,- issueId: setupFailureIssueId,- outcome: "failed",- error: message,- }).catch((completionErr) => {- logger.warn(- {- err: completionErr,- runId: livenessRun.id,- issueId: setupFailureIssueId,- },- "failed to complete skill test run after heartbeat setup failure",- );- });- }- const failedAgent =- setupFailureAgent ??- (await getAgent(run.agentId).catch(() => null));- if (failedAgent) {- await refreshContinuationSummaryForRun(- livenessRun,- failedAgent,- ).catch(() => undefined);- if (- !isWorkspaceValidationFailedRun(livenessRun) &&- !isConfigurationIncompleteFailedRun(livenessRun)- ) {- await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(- () => undefined,- );- }- // No provider work began. Retry temporary host scan failures with- // the existing durable failure budget, before releasing execution.- // Generic recovery must not grant a second budget on exhaustion.- await (isTransientWorkspaceGitScanCode(livenessRun.errorCode)- ? scheduleBoundedRetryForRun(livenessRun, failedAgent)- : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent)- ).catch((retryError) => {- logger.warn(- { err: retryError, runId: livenessRun.id },- "failed to schedule interaction continuation retry after setup failure",- );- });- }- await releaseIssueExecutionAndPromote(livenessRun, {- suppressImmediateRecovery:- readNonEmptyString(- parseObject(livenessRun.contextSnapshot).goalControlRequestId,- ) !== null ||- parseObject(livenessRun.contextSnapshot)- .resumeSessionGoalHeartbeat === true,- }).catch((releaseError) => {- logger.error(- { err: releaseError, runId },- "failed to release issue execution after heartbeat setup failure",- );- });- await handleIssueReviewPathDisposition(livenessRun).catch(- (reviewPathError) => {- logger.error(- { err: reviewPathError, runId },- "failed to evaluate review-path disposition after heartbeat setup failure",- );- },- );- }- // Ensure the agent is not left stuck in "running" if the setup-failure- // path owned the terminal transition. If another path already finalized- // the run, keep that terminal outcome authoritative.- if (setupFailureWrite.updated) {- await finalizeAgentStatus(run.agentId, "failed", message, {- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),- // Low-trust admission failures are task/principal preconditions,- // not evidence that the immutable endpoint agent is unhealthy.- // Keep the failed run and its safe provider refusal authoritative,- // but return the agent to idle so clients do not also announce a- // misleading agent-wide error for the same rejected chat turn.- keepIdleOnFailure: Boolean(nonRetryablePreflightCode),- }).catch(() => undefined);- }+ await runCompletion.failRunSetup({+ run, runId, outerErr, identityRedactor, readFailureReportSecrets,+ });}} finally {await nativeInstructionReservation?.release().catch(error => logger.warn({ runId: run.id, err: error }, "Managed warm session preparation cleanup failed"));diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.tsnew file mode 100644index 0000000000..4388cd4e4e--- /dev/null+++ b/server/src/services/heartbeat/run-completion.test.ts@@ -0,0 +1,288 @@+import type { Db } from "@paperclipai/db";+import { agents, agentWakeupRequests, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";+import { eq, sql } from "drizzle-orm";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import * as usageReceipts from "../usage-receipts.js";+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";+import { UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";+import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js";+import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";+import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js";+import { createHeartbeatRunCompletion, type CompleteHeartbeatRunInput, type FailHeartbeatRunInput, type HeartbeatRunCompletionDependencies } from "./run-completion.js";++vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));++type Dependencies = HeartbeatRunCompletionDependencies;+function callbacks(db: Db) {+ const state = createHeartbeatRunState(db);+ const lifecycleDeps = {+ ...state, ...createHeartbeatRunPreparation(db),+ issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() },+ recovery: { escalateStrandedAssignedIssue: vi.fn() },+ companySkills: { completeTestRunForIssue: vi.fn() },+ budgets: { getInvocationBlock: vi.fn(async () => null) },+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },+ enqueueWakeup: vi.fn(),+ getCurrentUserRedactionOptions: vi.fn(async () => ({ enabled: false })),+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),+ emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {},+ } satisfies HeartbeatLifecycleDependencies;+ const lifecycle = createHeartbeatLifecycle(db, lifecycleDeps);+ return {+ ...state, ...lifecycle,+ setRunStatusIfRunning: vi.fn(lifecycle.setRunStatusIfRunning),+ setWakeupStatus: vi.fn(lifecycle.setWakeupStatus),+ upsertTaskSession: vi.fn(state.upsertTaskSession),+ clearTaskSessions: vi.fn(state.clearTaskSessions),+ classifyAndPersistRunLiveness: vi.fn<Dependencies["classifyAndPersistRunLiveness"]>(async run => run),+ completeSkillTestRunForHeartbeatOutcome: vi.fn<Dependencies["completeSkillTestRunForHeartbeatOutcome"]>(async () => null),+ refreshContinuationSummaryForRun: vi.fn<Dependencies["refreshContinuationSummaryForRun"]>(async () => null),+ findRunIssueComment: vi.fn<Dependencies["findRunIssueComment"]>(async () => null),+ findLatestCompletedFinalAgentMessage: vi.fn<Dependencies["findLatestCompletedFinalAgentMessage"]>(async () => null),+ finalizeIssueCommentPolicy: vi.fn<Dependencies["finalizeIssueCommentPolicy"]>(async () => ({ outcome: "not_applicable", queuedRun: null })),+ handleIssueReviewPathDisposition: vi.fn<Dependencies["handleIssueReviewPathDisposition"]>(async () => undefined),+ handleRunLivenessContinuation: vi.fn<Dependencies["handleRunLivenessContinuation"]>(),+ handleSuccessfulRunHandoff: vi.fn<Dependencies["handleSuccessfulRunHandoff"]>(),+ updateRuntimeState: vi.fn<Dependencies["updateRuntimeState"]>(),+ scheduleBoundedRetryForRun: vi.fn<Dependencies["scheduleBoundedRetryForRun"]>(),+ scheduleInteractionContinuationInfrastructureRetryIfEligible: vi.fn<Dependencies["scheduleInteractionContinuationInfrastructureRetryIfEligible"]>(async () => null),+ timerClaimWasFirstHeartbeat: vi.fn<Dependencies["timerClaimWasFirstHeartbeat"]>(() => undefined),+ releaseIssueExecutionAndPromote: vi.fn<Dependencies["releaseIssueExecutionAndPromote"]>(async () => undefined),+ finalizeAgentStatus: vi.fn<Dependencies["finalizeAgentStatus"]>(async () => undefined),+ enqueueWakeup: vi.fn<Dependencies["enqueueWakeup"]>(),+ issuesSvc: { addComment: vi.fn<Dependencies["issuesSvc"]["addComment"]>() },+ recovery: { reconcileLegacyContinuation: vi.fn<Dependencies["recovery"]["reconcileLegacyContinuation"]>(), reconcileResolvedDependencyWakeBackstop: vi.fn<Dependencies["recovery"]["reconcileResolvedDependencyWakeBackstop"]>() },+ getCurrentUserRedactionOptions: lifecycleDeps.getCurrentUserRedactionOptions,+ budgetHooks: {},+ runLogStore: { finalize: vi.fn<Dependencies["runLogStore"]["finalize"]>(async () => ({ bytes: 123, sha256: "log-hash", compressed: true })) },+ traceStore: { finalize: vi.fn<Dependencies["traceStore"]["finalize"]>() },+ processRunCancellationSettlements: new Map<string, { settled: Promise<void>; failed: boolean }>(),+ failedProcessRunCancellations: new Map<string, { settled: Promise<void>; failed: boolean }>(),+ } satisfies Dependencies;+}++it("constructs completion handlers without querying or starting work", () => {+ const access = vi.fn(() => { throw new Error("Unexpected database access"); });+ const db = new Proxy({}, { get: access }) as Db;+ const deps = callbacks(db);+ createHeartbeatRunCompletion(db, deps);+ expect(access).not.toHaveBeenCalled();+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();+});++const support = await getEmbeddedPostgresTestSupport();+describe.skipIf(!support.supported)("heartbeat run completion boundary", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: Db;+ beforeAll(async () => {+ // Keep the accounting transaction real without reading provider receipt files.+ vi.spyOn(usageReceipts, "indexPendingUsageReceipts").mockResolvedValue(new Map());+ vi.spyOn(usageReceipts, "recoverPendingRunUsageReceipts").mockResolvedValue([]);+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-completion");+ db = createDb(database.connectionString);+ await db.execute(sql`set client_min_messages = warning`);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.restoreAllMocks(); });+ beforeEach(async () => { await db.execute(sql`truncate table companies restart identity cascade`); });++ async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {+ const [company] = await db.insert(companies).values({ name: "Completion", issuePrefix: "CMP" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Completing agent", adapterType: "process" }).returning();+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "on_demand", status: "claimed" }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "legacy", ...patch }).returning();+ const deps = callbacks(db);+ const controller = new AbortController();+ const input: CompleteHeartbeatRunInput = {+ run, agent, issueId: null, issueRef: null, signal: controller.signal,+ output: { handle: { store: "local_file", logRef: "test-log" }, outputProgressState: { pending: { bytes: 7 } }, flushOutputProgress: vi.fn(async () => {}), stdoutExcerpt: "answer", stderrExcerpt: "diagnostic" },+ session: { runtimeForAdapter: { sessionId: null, sessionDisplayId: null }, previousSessionParams: null, taskKey: "completion-task", configuredModel: "test-model",+ sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }),+ },+ readFailureReportSecrets: () => [],+ adapterResult: { exitCode: 0, signal: null, timedOut: false, sessionId: "provider-session", sessionParams: { sessionId: "provider-session" }, usage: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8 }, usageBasis: "per_run", costUsd: 0.01, provider: "test", model: "test-model", resultJson: { finalResponse: "Completed the work." } },+ sessionCodec: getAdapterSessionCodec(agent.adapterType), taskSessionForRun: null, sessionCompaction: { rotate: false, reason: null }, configFreshnessResultMetadata: { version: 1 }, runLedgerScope: {}, currentUserRedactionOptions: { enabled: false }, providerTraceCapture: true, onProviderTraceFinalized: vi.fn(), issueContext: null, onLog: vi.fn(async () => {}),+ };+ const completion = createHeartbeatRunCompletion(db, deps);+ const failure = (err: unknown): FailHeartbeatRunInput => ({ ...input, runId: run.id, err, identityRedactor: createAgentIdentityRedactor("private-identity-material"), requiredWorkspaceRestoreEvidence: { workspaceRestoreFailure: { reason: "missing_base" } }, legacyAdapterEntered: true, goalCheckpointSession: { current: null }, previousSessionDisplayId: "previous-session", taskSession: null });+ const setup = (outerErr: unknown) => ({ run, runId: run.id, outerErr, identityRedactor: createAgentIdentityRedactor(), readFailureReportSecrets: () => [] });+ return { run, agent, wake, deps, controller, input, completion, failure, setup };+ }+ const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);++ it("persists usage, accounting, final logs, presentation and the next task session", async () => {+ const f = await fixture();+ await f.completion.completeRun(f.input);+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", sessionIdAfter: "provider-session", logBytes: 123, logSha256: "log-hash", logCompressed: true, stdoutExcerpt: "answer", costAccountingPending: false, costAccountedAt: expect.any(Date), usageJson: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8, usageSource: "per_run", configFreshness: { version: 1 } }, resultJson: { presentationDecision: { commentAction: "none" } } });+ expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(1);+ expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("provider-session");+ expect(f.input.output.outputProgressState.pending?.bytes).toBe(123);+ expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();+ expect(f.deps.setWakeupStatus).toHaveBeenCalledWith(f.wake.id, "completed", expect.any(Object));+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));+ });++ it.each([+ { timedOut: true, expected: "timed_out", errorCode: "timeout" },+ { signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },+ { errorMessage: "provider quota", errorCode: "provider_quota", expected: "failed" },+ ])("keeps adapter outcome classification ($expected, $errorCode)", async ({ expected, ...result }) => {+ const f = await fixture();+ Object.assign(f.input.adapterResult, result);+ await f.completion.completeRun(f.input);+ expect(await read(f.run.id)).toMatchObject({ status: expected, errorCode: result.errorCode });+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, expected, expect.any(String), expect.objectContaining({ keepIdleOnFailure: result.errorCode === "provider_quota" }));+ });++ it.each(["completeRun", "failRun"] as const)("waits for an owned Stop before %s can write a competing outcome", async operation => {+ const f = await fixture();+ let settle!: () => void;+ const settled = new Promise<void>(resolve => { settle = resolve; });+ f.deps.processRunCancellationSettlements.set(f.run.id, { settled, failed: false });+ const completion = operation === "completeRun" ? f.completion.completeRun(f.input) : f.completion.failRun(f.failure(new Error("late adapter error")));+ await Promise.resolve();+ expect(f.deps.setRunStatusIfRunning).not.toHaveBeenCalled();+ await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));+ f.controller.abort();+ settle();+ await completion;+ expect((await read(f.run.id)).status).toBe("cancelled");+ if (operation === "completeRun") {+ expect((await read(f.run.id)).sessionIdAfter).toBe("provider-session");+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "cancelled", expect.any(String), expect.any(Object));+ } else expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();+ });++ it("does not publish or overwrite a conflicting terminal transition that wins the compare-and-set", async () => {+ const f = await fixture();+ const write = f.deps.setRunStatusIfRunning.getMockImplementation()!;+ f.deps.setRunStatusIfRunning.mockImplementationOnce(async (...args) => {+ await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: { winner: "Stop" } }).where(eq(heartbeatRuns.id, f.run.id));+ return write(...args);+ });+ await f.completion.completeRun(f.input);+ expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: { winner: "Stop" }, sessionIdAfter: null });+ expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();+ expect(f.deps.updateRuntimeState).not.toHaveBeenCalled();+ expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();+ expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();+ });++ it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {+ const finishedAt = new Date("2026-10-09T12:00:00Z");+ const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });+ f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };+ await f.completion.completeRun(f.input);+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });+ expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();+ });++ it("reports trace completion before a later write throws and reads output after log finalization", async () => {+ const f = await fixture();+ let excerpt = "before finalize";+ Object.defineProperty(f.input.output, "stdoutExcerpt", { get: () => excerpt });+ f.deps.runLogStore.finalize.mockImplementationOnce(async () => { excerpt = "last provider output"; return { bytes: 99, compressed: false }; });+ const writeError = new Error("write failed");+ f.deps.setRunStatusIfRunning.mockRejectedValueOnce(writeError);+ await expect(f.completion.completeRun(f.input)).rejects.toBe(writeError);+ expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();+ expect(f.deps.setRunStatusIfRunning).toHaveBeenCalledWith(f.run.id, "succeeded", expect.objectContaining({ stdoutExcerpt: "last provider output", logBytes: 99 }), expect.any(Object));+ });++ it("keeps a failed trace finalization eligible for the executor's cleanup retry", async () => {+ const f = await fixture();+ f.deps.traceStore.finalize.mockRejectedValueOnce(new Error("trace store unavailable"));+ await f.completion.completeRun(f.input);+ expect((await read(f.run.id)).status).toBe("succeeded");+ expect(f.input.onProviderTraceFinalized).not.toHaveBeenCalled();+ });++ it("keeps completion and session persistence moving after a response comment fails", async () => {+ const f = await fixture();+ f.input.adapterResult.resultJson = { finalResponseDisposition: "final", finalResponse: "Completed the work." };+ const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Report the result" }).returning();+ f.input.issueId = issue.id;+ f.input.issueRef = issue;+ f.input.issueContext = await createHeartbeatRunPreparation(db).getIssueExecutionContext(f.run.companyId, issue.id);+ f.deps.issuesSvc.addComment.mockRejectedValueOnce(new Error("comment unavailable"));+ await f.completion.completeRun(f.input);+ expect(f.deps.issuesSvc.addComment).toHaveBeenCalledOnce();+ expect(f.input.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("comment unavailable"));+ expect((await read(f.run.id)).status).toBe("succeeded");+ expect(f.deps.upsertTaskSession).toHaveBeenCalledOnce();+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();+ });++ it("clears only the completed task session when the adapter requests a reset", async () => {+ const f = await fixture();+ f.input.adapterResult.clearSession = true;+ await f.completion.completeRun(f.input);+ expect(f.deps.clearTaskSessions).toHaveBeenCalledWith(f.run.companyId, f.agent.id, { taskKey: "completion-task", adapterType: "process", expectedRunId: f.run.id });+ expect(f.deps.upsertTaskSession).not.toHaveBeenCalled();+ });++ it("redacts execution failures, retains restore evidence and resumes the latest checkpoint", async () => {+ const f = await fixture();+ const failure = f.failure(new Error("private-identity-material crashed"));+ failure.goalCheckpointSession.current = { params: { sessionId: "checkpoint" }, displayId: "checkpoint" };+ await f.completion.failRun(failure);+ const run = await read(f.run.id);+ expect(run).toMatchObject({ status: "failed", errorCode: "adapter_failed", logBytes: 123, resultJson: { workspaceRestoreFailure: { reason: "missing_base" } } });+ expect(run.error).not.toContain("private-identity-material");+ expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("checkpoint");+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();+ });++ it("still records failure and releases issue ownership when log finalization and progress flushing fail", async () => {+ const f = await fixture();+ f.deps.runLogStore.finalize.mockRejectedValueOnce(new Error("log unavailable"));+ f.input.output.flushOutputProgress = vi.fn(async () => { throw new Error("progress unavailable"); });+ await f.completion.failRun(f.failure(new Error("adapter failed")));+ expect((await read(f.run.id)).status).toBe("failed");+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();+ });++ it("keeps durable result-less native failure authoritative and suppresses generic replacement recovery", async () => {+ const f = await fixture({ runtimeMode: "native", status: "failed", errorCode: "native_result_missing" });+ const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Native recovery" }).returning();+ await db.update(heartbeatRuns).set({ nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));+ await db.insert(nativeRunFinalizations).values({ runId: f.run.id, companyId: f.run.companyId, issueId: issue.id, phase: "terminal_failure", failureCode: "native_result_missing" });+ await f.completion.failRun(f.failure(new Error("transport ended")));+ expect((await read(f.run.id)).errorCode).toBe("native_result_missing");+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), { suppressImmediateRecovery: true });+ });++ it("records a setup configuration gap without inventing provider work or applying comment policy", async () => {+ const f = await fixture();+ await f.completion.failRunSetup(f.setup(new ConfigurationIncompleteFailure("missing credential", { configurationIncomplete: { reason: "missing_binding" } })));+ expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { reason: "missing_binding" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });+ expect(f.deps.finalizeIssueCommentPolicy).not.toHaveBeenCalled();+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();+ });++ it("retains the default branch needed to repair an unresolved workspace base ref", async () => {+ const f = await fixture();+ const error = new UnresolvedWorkspaceBaseRefError({+ requestedRef: "main", recoveryIdentityRef: "origin/main", attemptedRefs: ["origin/main"], defaultBranch: "master",+ });+ await f.completion.failRunSetup(f.setup(error));+ expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "main", defaultBranch: "master", attemptedRefs: ["origin/main"], fingerprint: "workspace_base_ref:origin/main" },+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },+ } });+ });++ it("leaves a winning terminal outcome and agent state untouched after a late setup error", async () => {+ const f = await fixture({ status: "succeeded", resultJson: { winner: "completion" } });+ await f.completion.failRunSetup(f.setup(new Error("late setup error")));+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", resultJson: { winner: "completion" } });+ expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();+ expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();+ expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();+ });+});diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.tsnew file mode 100644index 0000000000..abda3c7f21--- /dev/null+++ b/server/src/services/heartbeat/run-completion.ts@@ -0,0 +1,1460 @@+import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";+import { ChatControlRecoveryUnresolvedError } from "./queue.js";+import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTransientWorkspaceGitScanCode } from "./retries.js";+import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js";+import { readGitConnectionFailure } from "../git-connection-failure.js";+import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, ConfigurationIncompleteFailure } from "./run-preparation.js";+import { isWorkspaceValidationFailedRun, isWorkspaceValidationFailure, attachPaperclipSessionMetadataToSessionParams } from "./workspaces.js";+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";+import { cancellationResultJson, readRunCancellation } from "../run-cancellation.js";+import { accountRunCost } from "../run-cost-accounting.js";+import { isConversation, settleConversationTurn } from "../agent-conversations.js";+import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";+import { isWorkspaceGitScanError } from "../workspace-git-operation-scheduler.js";+import { and, eq, sql } from "drizzle-orm";+import { heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";+import { HttpError } from "../../errors.js";+import { logger } from "../../middleware/logger.js";+import { normalizeResponsibleUserDenialCode } from "../responsible-user-denial-run-outcomes.js";+import { parseObject } from "../../adapters/utils.js";+import { CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, isExternalChatPresentationContext, mergeHeartbeatRunResultJson, resolveHeartbeatRunResponse, type RunPresentationDecision } from "../heartbeat-run-summary.js";+import { normalizeMaxTurnStopReason } from "../heartbeat-stop-metadata.js";+import { CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";+import { logActivity } from "../activity-log.js";+import { isUnresolvedWorkspaceBaseRefError, readUnresolvedWorkspaceBaseRefDiagnostic, type UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";+import { runnerGoalService } from "../runner-goals.js";+import { resolveChatRunPresentationAuthorizationReason } from "../chat-run-publications.js";+import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON } from "../recovery/stranded-notice.js";+import { MAX_TURN_CONTINUATION_RETRY_REASON } from "../../modules/run-dispatch/index.js";+import { redactCurrentUserText } from "../../log-redaction.js";+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";+import type { createHeartbeatRunPreparation } from "./run-preparation.js";+import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js";+import type { createHeartbeatQueue } from "./queue.js";+import type { createHeartbeatRunControl, HeartbeatRunControlDependencies } from "./run-control.js";+import type { issueService } from "../issues.js";+import type { recoveryService } from "../recovery/index.js";+import type { BudgetServiceHooks } from "../budgets.js";+import type { providerTraceStore } from "../provider-trace-store.js";+import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";+import type { CurrentUserRedactionOptions } from "../../log-redaction.js";+import type { AdapterExecutionResult } from "@paperclipai/adapter-utils";+import type { getRunLogStore, RunLogHandle } from "../run-log-store.js";+import type { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";+import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js";+import type { Db, agents } from "@paperclipai/db";++type Run = typeof heartbeatRuns.$inferSelect;+type Agent = typeof agents.$inferSelect;+type RunState = ReturnType<typeof createHeartbeatRunState>;+type IssueContext = Awaited<ReturnType<ReturnType<typeof createHeartbeatRunPreparation>["getIssueExecutionContext"]>> | null;+type TaskSession = Awaited<ReturnType<RunState["getTaskSession"]>> | null;++/** Completion persists terminal outcomes; the executor retains dispatch and cleanup ownership. */+export interface HeartbeatRunCompletionDependencies extends Pick<RunState,+ "getRun" | "getAgent" | "resolveNormalizedUsageForSession" | "clearTaskSessions" | "upsertTaskSession">,+ Pick<ReturnType<typeof createHeartbeatLifecycle>,+ "mergeRunStopMetadataForAgent" | "setRunStatusIfRunning" | "setWakeupStatus" | "appendRunEvent" |+ "classifyAndPersistRunLiveness" | "completeSkillTestRunForHeartbeatOutcome" | "refreshContinuationSummaryForRun" |+ "findRunIssueComment" | "findLatestCompletedFinalAgentMessage" | "parseMaxTurnContinuationPolicy" |+ "finalizeIssueCommentPolicy" | "handleIssueReviewPathDisposition" | "handleRunLivenessContinuation" |+ "handleSuccessfulRunHandoff" | "updateRuntimeState">,+ Pick<ReturnType<typeof createHeartbeatRetries>,+ "scheduleBoundedRetryForRun" | "scheduleInteractionContinuationInfrastructureRetryIfEligible" | "timerClaimWasFirstHeartbeat">,+ Pick<ReturnType<typeof createHeartbeatRunControl>, "releaseIssueExecutionAndPromote">,+ Pick<HeartbeatRetryDependencies, "finalizeAgentStatus">,+ Pick<ReturnType<typeof createHeartbeatQueue>, "enqueueWakeup"> {+ issuesSvc: Pick<ReturnType<typeof issueService>, "addComment">;+ recovery: Pick<ReturnType<typeof recoveryService>, "reconcileLegacyContinuation" | "reconcileResolvedDependencyWakeBackstop">;+ getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;+ budgetHooks: BudgetServiceHooks;+ runLogStore: Pick<ReturnType<typeof getRunLogStore>, "finalize">;+ traceStore: Pick<ReturnType<typeof providerTraceStore>, "finalize">;+ processRunCancellationSettlements: Pick<HeartbeatRunControlDependencies["processRunCancellationSettlements"], "get">;+ failedProcessRunCancellations: HeartbeatRunCompletionDependencies["processRunCancellationSettlements"];+}++interface RunCompletionOutput {+ handle: RunLogHandle | null;+ outputProgressState: { pending: { bytes: number } | null };+ flushOutputProgress: (options?: { force?: boolean }) => Promise<void>;+ // Getters in the executor retain output that arrives while finalization awaits I/O.+ readonly stdoutExcerpt: string;+ readonly stderrExcerpt: string;+}++interface RunCompletionSession {+ runtimeForAdapter: { sessionId: string | null; sessionDisplayId: string | null };+ previousSessionParams: Record<string, unknown> | null;+ taskKey: string | null;+ configuredModel: string | null;+ sessionConfigMetadata: EffectiveRunSessionConfigMetadata;+}++interface RunCompletionInput {+ run: Run;+ agent: Agent;+ issueId: string | null;+ issueRef: Pick<NonNullable<IssueContext>, "workMode" | "identifier" | "title"> | null;+ signal: AbortSignal;+ output: RunCompletionOutput;+ session: RunCompletionSession;+ readFailureReportSecrets: () => string[];+}++export interface CompleteHeartbeatRunInput extends RunCompletionInput {+ adapterResult: AdapterExecutionResult;+ sessionCodec: ReturnType<typeof getAdapterSessionCodec>;+ taskSessionForRun: TaskSession;+ sessionCompaction: Pick<Awaited<ReturnType<RunState["evaluateSessionCompaction"]>>, "rotate" | "reason">;+ configFreshnessResultMetadata: Record<string, unknown>;+ runLedgerScope: Record<string, unknown>;+ currentUserRedactionOptions: CurrentUserRedactionOptions;+ providerTraceCapture: boolean;+ // Update the executor immediately: a later completion step can still throw.+ onProviderTraceFinalized: () => void;+ issueContext: IssueContext;+ onLog: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;+}++export interface FailHeartbeatRunInput extends RunCompletionInput {+ runId: string;+ err: unknown;+ identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "redact">;+ requiredWorkspaceRestoreEvidence: Record<string, unknown> | null;+ legacyAdapterEntered: boolean;+ goalCheckpointSession: { current: { params: Record<string, unknown>; displayId: string } | null };+ previousSessionDisplayId: string | null;+ taskSession: TaskSession;+}++export interface FailHeartbeatRunSetupInput {+ run: Run;+ runId: string;+ outerErr: unknown;+ identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "redact">;+ readFailureReportSecrets: () => string[];+}++function nonRetryablePreflightFailureCode(error: unknown): string | null {+ if (error instanceof ChatControlRecoveryUnresolvedError)+ return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;+ if (+ error instanceof HttpError &&+ error.status === 409 &&+ parseObject(error.details).code === "chat_failed_run_retry_not_authorized"+ ) {+ return "chat_failed_run_retry_not_authorized";+ }+ if (!(error instanceof HttpError) || error.status !== 422) return null;+ const code = readNonEmptyString(parseObject(error.details).code);+ return code && NON_RETRYABLE_PREFLIGHT_FAILURE_CODES.has(code) ? code : null;+}++export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";++// Build the configuration-incomplete result payload for a workspace base ref+// that never resolved to a commit. The setup catch maps this to errorCode+// `configuration_incomplete`, so the recovery path routes it to a human owner+// instead of a dispatched-then-failed run. The `fingerprint` uses the canonical+// remote ref, not the operator spelling. Two equivalent spellings of one remote+// branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated+// failure reuses one active recovery action and does not reset the attempt+// count or post a duplicate notice. A different branch makes a new action.+// Build the configuration-incomplete result payload for a sandbox provider+// plugin that is installed but not `ready`. The `fingerprint` is the plugin+// key plus its status, so every run that hits the same stuck plugin reuses+// one active recovery action instead of posting a fresh notice per attempt,+// while a status change (say `error` -> `disabled`) makes a new one.+function buildSandboxProviderPluginNotReadyResultJson(+ run: typeof heartbeatRuns.$inferSelect,+ failure: { provider: string; pluginKey: string; pluginStatus: string },+): Record<string, unknown> {+ const context = parseObject(run.contextSnapshot);+ return {+ configurationIncomplete: {+ reason: SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,+ companyId: run.companyId,+ agentId: run.agentId,+ issueId: readNonEmptyString(context.issueId) ?? null,+ projectId: readNonEmptyString(context.projectId) ?? null,+ sandboxProvider: failure.provider,+ pluginKey: failure.pluginKey,+ pluginStatus: failure.pluginStatus,+ fingerprint: `sandbox_provider_plugin:${failure.pluginKey}:${failure.pluginStatus}`,+ missingBindings: [],+ },+ };+}++function buildUnresolvedWorkspaceBaseRefResultJson(+ run: typeof heartbeatRuns.$inferSelect,+ error: UnresolvedWorkspaceBaseRefError,+): Record<string, unknown> {+ const context = parseObject(run.contextSnapshot);+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);+ return {+ configurationIncomplete: {+ reason: "workspace_base_ref_unresolved",+ ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),+ companyId: run.companyId,+ agentId: run.agentId,+ issueId: readNonEmptyString(context.issueId) ?? null,+ projectId: readNonEmptyString(context.projectId) ?? null,+ requestedRef: error.requestedRef,+ defaultBranch: error.defaultBranch,+ attemptedRefs: error.attemptedRefs,+ fetchError: error.fetchError,+ fingerprint: `workspace_base_ref:${error.recoveryIdentityRef}`,+ missingBindings: [],+ },+ };+}++function isMaxTurnExhaustionRun(+ run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,+) {+ const resultJson = parseObject(run.resultJson);+ return Boolean(+ normalizeMaxTurnStopReason(resultJson.stopReason) ??+ normalizeMaxTurnStopReason(run.errorCode),+ );+}++// environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message,+// e.g. 'Sandbox provider "kubernetes" is installed via plugin+// "acme.kubernetes-sandbox-provider", but that plugin is currently error.'+// The plugin row exists but its status is `error` (a failed activation),+// `disabled` (an operator switched it off) or `upgrade_pending`. Unlike the+// worker restart window above, nothing on the run path ever changes that+// status: only an operator enabling the plugin, or a server boot that+// re-activates a bundled plugin, does. Re-running the agent produces the+// identical failure every time, so the setup catch classifies it as+// `configuration_incomplete` (routed to a human owner) instead of a retryable+// `setup_failed` that the scheduler would keep re-dispatching.+const SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE =+ /sandbox provider "([^"]*)" is installed via plugin "([^"]*)", but that plugin is currently (error|disabled|upgrade_pending)\b/i;++export function parseSandboxProviderPluginNotReadyFailureMessage(+ value: unknown,+): { provider: string; pluginKey: string; pluginStatus: string } | null {+ if (typeof value !== "string") return null;+ const match = SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE.exec(value);+ if (!match) return null;+ return {+ provider: match[1] ?? "",+ pluginKey: match[2] ?? "",+ pluginStatus: (match[3] ?? "").toLowerCase(),+ };+}++function mergeAdapterRecoveryMetadata(input: {+ resultJson: Record<string, unknown> | null | undefined;+ errorFamily?: string | null;+ retryNotBefore?: string | null;+}) {+ const errorFamily = readNonEmptyString(input.errorFamily);+ const retryNotBefore = readNonEmptyString(input.retryNotBefore);+ if (!input.resultJson && !errorFamily && !retryNotBefore)+ return input.resultJson ?? null;++ return {+ ...(input.resultJson ?? {}),+ ...(errorFamily ? { errorFamily } : {}),+ ...(retryNotBefore+ ? {+ retryNotBefore,+ transientRetryNotBefore: retryNotBefore,+ ...(errorFamily === "provider_quota"+ ? { providerQuotaRetryNotBefore: retryNotBefore }+ : {}),+ }+ : {}),+ };+}++function isConfigurationIncompleteFailure(+ error: unknown,+): error is ConfigurationIncompleteFailure {+ return error instanceof ConfigurationIncompleteFailure;+}++export function isConfigurationIncompleteFailedRun(+ run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode"> | null | undefined,+) {+ return (+ run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE ||+ run?.errorCode === "model_not_found"+ );+}++/**+ * Failure signatures from sandbox→host git workspace reconciliation. These+ * describe the state of the SHARED workspace (divergent histories written by+ * different runs), not a defect in the agent that happened to run last —+ * putting the agent into a sticky `error` state over them removes a healthy+ * agent from rotation while leaving the actual problem (the workspace)+ * untouched. The run still fails and carries the full message.+ */+const WORKSPACE_SYNC_CONFLICT_SIGNATURES = [+ "Failed to merge concurrent remote git histories",+ "Failed to integrate concurrent remote git history",+ "did not send all necessary objects",+ "lacks these prerequisite commits",+];++export function isWorkspaceSyncConflictFailure(+ message: string | null | undefined,+): boolean {+ if (!message) return false;+ return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) =>+ message.includes(signature),+ );+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunCompletionDependencies) {+ const {+ getRun,+ getAgent,+ resolveNormalizedUsageForSession,+ clearTaskSessions,+ upsertTaskSession,+ mergeRunStopMetadataForAgent,+ setRunStatusIfRunning,+ setWakeupStatus,+ appendRunEvent,+ classifyAndPersistRunLiveness,+ completeSkillTestRunForHeartbeatOutcome,+ refreshContinuationSummaryForRun,+ findRunIssueComment,+ findLatestCompletedFinalAgentMessage,+ parseMaxTurnContinuationPolicy,+ finalizeIssueCommentPolicy,+ handleIssueReviewPathDisposition,+ handleRunLivenessContinuation,+ handleSuccessfulRunHandoff,+ updateRuntimeState,+ scheduleBoundedRetryForRun,+ scheduleInteractionContinuationInfrastructureRetryIfEligible,+ timerClaimWasFirstHeartbeat,+ releaseIssueExecutionAndPromote,+ finalizeAgentStatus,+ enqueueWakeup,+ issuesSvc,+ recovery,+ getCurrentUserRedactionOptions,+ budgetHooks,+ runLogStore,+ traceStore,+ processRunCancellationSettlements,+ failedProcessRunCancellations,+ } = dependencies;++ async function completeRun(input: CompleteHeartbeatRunInput) {+ const { run, agent, issueId, issueRef, signal, output, session, readFailureReportSecrets, adapterResult, sessionCodec, taskSessionForRun, sessionCompaction,+ configFreshnessResultMetadata, runLedgerScope, currentUserRedactionOptions,+ providerTraceCapture, onProviderTraceFinalized, issueContext, onLog } = input;+ const { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata } = session;+ const { handle, outputProgressState, flushOutputProgress } = output;+ const processCancellation =+ processRunCancellationSettlements.get(run.id) ??+ failedProcessRunCancellations.get(run.id);+ await processCancellation?.settled;+ let outcome: RunSessionOutcome;+ const latestRun = await getRun(run.id);+ if (isHeartbeatRunTerminalStatus(latestRun?.status)) {+ outcome = latestRun.status;+ } else if (signal.aborted) {+ outcome = "cancelled";+ } else if (adapterResult.nativeFinalization) {+ const nativeTerminal =+ adapterResult.nativeFinalization.terminal.runTerminalState;+ outcome =+ nativeTerminal === "succeeded"+ ? "succeeded"+ : nativeTerminal === "cancelled"+ ? "cancelled"+ : "failed";+ } else if (adapterResult.timedOut) {+ outcome = "timed_out";+ } else if (adapterResult.resultJson?.status === "cancelled") {+ outcome = "cancelled";+ } else if (+ (adapterResult.exitCode ?? 0) === 0 &&+ !adapterResult.errorMessage &&+ !adapterResult.signal &&+ !processCancellation?.failed+ ) {+ outcome = "succeeded";+ } else {+ outcome = "failed";+ }++ const nextSessionState = resolveNextSessionState({+ adapterType: agent.adapterType,+ codec: sessionCodec,+ adapterResult,+ outcome,+ previousParams: previousSessionParams,+ previousDisplayId: runtimeForAdapter.sessionDisplayId,+ previousLegacySessionId: runtimeForAdapter.sessionId,+ });+ const rawUsage = normalizeUsageTotals(adapterResult.usage);+ const sessionUsageResolution = await resolveNormalizedUsageForSession({+ agentId: agent.id,+ runId: run.id,+ sessionId:+ nextSessionState.displayId ?? nextSessionState.legacySessionId,+ rawUsage,+ usageBasis: adapterResult.usageBasis ?? null,+ });+ const normalizedUsage = sessionUsageResolution.normalizedUsage;+ const runErrorMessage =+ outcome === "cancelled"+ ? redactCurrentUserText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled", currentUserRedactionOptions)+ : outcome === "succeeded"+ ? null+ : redactCurrentUserText(+ adapterResult.errorMessage ??+ (outcome === "timed_out" ? "Timed out" : "Adapter failed"),+ currentUserRedactionOptions,+ );+ const recordedResponsibleUserDenialCode =+ normalizeResponsibleUserDenialCode(latestRun?.errorCode);+ const runErrorCode =+ outcome === "timed_out"+ ? "timeout"+ : outcome === "cancelled"+ ? (latestRun?.errorCode ?? "cancelled")+ : outcome === "failed"+ ? (adapterResult.errorCode ??+ recordedResponsibleUserDenialCode ??+ "adapter_failed")+ : null;++ let logSummary: {+ bytes: number | null;+ sha256?: string | null;+ compressed: boolean;+ } | null = null;+ if (handle) {+ logSummary = await runLogStore.finalize(handle);+ }+ const finalLogBytes = logSummary?.bytes;+ if (outputProgressState.pending && typeof finalLogBytes === "number") {+ outputProgressState.pending.bytes = finalLogBytes;+ }+ await flushOutputProgress({ force: true });++ if (providerTraceCapture) {+ try {+ await traceStore.finalize(run.id, run.companyId);+ onProviderTraceFinalized();+ } catch (error) {+ logger.warn(+ { error, runId: run.id },+ "provider trace finalization failed without affecting run outcome",+ );+ }+ }++ const status =+ outcome === "succeeded"+ ? "succeeded"+ : outcome === "cancelled"+ ? "cancelled"+ : outcome === "timed_out"+ ? "timed_out"+ : "failed";++ const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null+ ? null : resolveCacheAdjustedCostUsd(adapterResult);+ const usageJson: Record<string, unknown> = {+ accountingReceiptReady: adapterResult.usageComplete !== false,+ costUsdExact: adapterResult.costUsdExact ?? null,+ providerRequestId: adapterResult.providerRequestId ?? null,+ ...(normalizedUsage ?? {}),+ ...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}),+ ...(rawUsage+ ? {+ rawInputTokens: rawUsage.inputTokens,+ rawInputIncludesCached: false,+ rawCachedInputTokens: rawUsage.cachedInputTokens,+ rawOutputTokens: rawUsage.outputTokens,+ }+ : {}),+ ...(sessionUsageResolution.derivedFromSessionTotals+ ? { usageSource: "session_delta" }+ : adapterResult.usageBasis === "per_run"+ ? { usageSource: "per_run" }+ : {}),+ ...((nextSessionState.displayId ??+ nextSessionState.legacySessionId)+ ? {+ persistedSessionId:+ nextSessionState.displayId ??+ nextSessionState.legacySessionId,+ }+ : {}),+ sessionReused:+ runtimeForAdapter.sessionId != null ||+ runtimeForAdapter.sessionDisplayId != null,+ taskSessionReused: taskSessionForRun != null,+ freshSession:+ runtimeForAdapter.sessionId == null &&+ runtimeForAdapter.sessionDisplayId == null,+ sessionRotated: sessionCompaction.rotate,+ sessionRotationReason: sessionCompaction.reason,+ configFreshness: configFreshnessResultMetadata,+ provider:+ readNonEmptyString(adapterResult.provider) ?? "unknown",+ biller: resolveLedgerBiller(adapterResult),+ model: readNonEmptyString(adapterResult.model) ?? "unknown",+ ...(adapterResult.costUsd != null+ ? { costUsd: adapterResult.costUsd }+ : {}),+ ...(cacheAdjustedCostUsd != null+ ? { cacheAdjustedCostUsd }+ : {}),+ pricingProvenance: adapterResult.pricingProvenance,+ costStatus: adapterResult.costStatus ?? resolveLedgerCostStatus({+ costUsd: cacheAdjustedCostUsd ?? (adapterResult.costUsdExact != null ? Number(adapterResult.costUsdExact) : null),+ billingType: normalizeLedgerBillingType(adapterResult.billingType),+ inputTokens: normalizedUsage?.inputTokens ?? 0,+ cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,+ outputTokens: normalizedUsage?.outputTokens ?? 0,+ }),+ billingType: normalizeLedgerBillingType(+ adapterResult.billingType,+ ),+ };++ const persistedResultJson = cancellationResultJson(latestRun ?? run, outcome, mergeHeartbeatRunResultJson(+ mergeRunStopMetadataForAgent(agent, outcome, {+ resultJson: mergeAdapterRecoveryMetadata({+ resultJson: {+ ...(adapterResult.nativeFinalization || outcome === "cancelled"+ ? parseObject(latestRun?.resultJson)+ : {}),+ ...parseObject(adapterResult.resultJson),+ ...(adapterResult.executionRecovery+ ? { executionRecovery: adapterResult.executionRecovery }+ : {}),+ configFreshness: configFreshnessResultMetadata,+ },+ errorFamily: adapterResult.errorFamily ?? null,+ retryNotBefore: adapterResult.retryNotBefore ?? null,+ }),+ errorCode: runErrorCode,+ errorMessage: runErrorMessage,+ }),+ adapterResult.summary ?? null,+ ), runErrorCode, runErrorMessage);++ const ledgerScope = runLedgerScope;+ const finalRunPatch: Partial<typeof heartbeatRuns.$inferInsert> = {+ // Accounting must acknowledge even a proven pre-provider failure:+ // that transaction releases the reservation without creating a charge.+ costAccountingPending: true,+ finishedAt: new Date(),+ error: runErrorMessage,+ errorCode: runErrorCode,+ exitCode: adapterResult.exitCode,+ signal: adapterResult.signal,+ usageJson: { ...usageJson, ledgerScope },+ resultJson: persistedResultJson,+ sessionIdAfter:+ nextSessionState.displayId ?? nextSessionState.legacySessionId,+ stdoutExcerpt: output.stdoutExcerpt,+ stderrExcerpt: output.stderrExcerpt,+ logBytes: logSummary?.bytes,+ logSha256: logSummary?.sha256,+ logCompressed: logSummary?.compressed ?? false,+ };+ const persistedRunWrite = await setRunStatusIfRunning(+ run.id,+ status,+ finalRunPatch,+ { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },+ );+ let persistedRun: typeof heartbeatRuns.$inferSelect | null =+ persistedRunWrite.run;+ if (!persistedRunWrite.updated) {+ persistedRun = null;+ // Native reconciliation can commit and project the terminal status in+ // the narrow window between adapter completion and this live write.+ // The status is authoritative, but it must not make us discard the+ // adapter's semantic result, usage, logs, or presentation decision.+ // Only complete the late metadata write when the reconciler chose the+ // same terminal status; a conflicting terminal outcome remains owned+ // by the path that won the compare-and-set. Owned legacy cancellation+ // likewise keeps the provider session, logs, and usage after Stop wins.+ if (+ (adapterResult.nativeFinalization ||+ (processCancellation && !processCancellation.failed && status === "cancelled")) &&+ persistedRunWrite.run?.status === status+ ) {+ persistedRun = await db+ .update(heartbeatRuns)+ .set({+ ...finalRunPatch,+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(+ cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,+ ),++ usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },+ finishedAt:+ persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(heartbeatRuns.id, run.id),+ eq(heartbeatRuns.status, status),+ ),+ )+ .returning()+ .then((rows) => rows[0] ?? null);+ }+ if (!persistedRun) {+ await accountRunCost(db, run.id, budgetHooks).catch((err) => {+ logger.error({ err, runId: run.id }, "Late run accounting queued for recovery");+ });+ logger.info(+ {+ runId: run.id,+ attemptedStatus: status,+ currentStatus: persistedRunWrite.run?.status ?? null,+ },+ "skipping late run finalization because the run already left running state",+ );+ return;+ }+ }+ if (persistedRun) {+ // Accounting recovery is independent of workspace or issue finalization.+ await accountRunCost(db, persistedRun.id, budgetHooks).catch((err) => {+ logger.error({ err, runId: run.id }, "Run accounting queued for recovery");+ });+ persistedRun =+ (await classifyAndPersistRunLiveness(+ persistedRun,+ persistedResultJson,+ )) ?? persistedRun;+ }++ await setWakeupStatus(+ run.wakeupRequestId,+ outcome === "succeeded" ? "completed" : status,+ {+ finishedAt: new Date(),+ error: runErrorMessage,+ },+ );++ const finalizedRun = persistedRun ?? (await getRun(run.id));+ if (finalizedRun) {+ await appendRunEvent(finalizedRun, {+ eventType: "lifecycle",+ stream: "system",+ level: outcome === "succeeded" ? "info" : "error",+ message: `run ${outcome}`,+ payload: {+ status,+ exitCode: adapterResult.exitCode,+ ...(readRunCancellation(finalizedRun.resultJson) ? { cancellation: readRunCancellation(finalizedRun.resultJson) } : {}),+ },+ });+ try {+ await completeSkillTestRunForHeartbeatOutcome({+ run: finalizedRun,+ issueId,+ issueWorkMode: issueRef?.workMode ?? null,+ outcome,+ error: runErrorMessage,+ });+ } catch (err) {+ logger.warn(+ { err, runId: finalizedRun.id, issueId },+ "failed to complete skill test run after heartbeat finalization",+ );+ await onLog(+ "stderr",+ `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`,+ );+ }+ const livenessRun = finalizedRun;+ await refreshContinuationSummaryForRun(livenessRun, agent);+ const skipRunIssueComment =+ parseObject(livenessRun.contextSnapshot).skipIssueComment === true;+ let resolvedPresentationDecision: RunPresentationDecision | null =+ null;+ try {+ const existingRunComment = issueId+ ? await findRunIssueComment(+ livenessRun.id,+ livenessRun.companyId,+ issueId,+ persistedResultJson,+ )+ : null;+ const finalAgentMessage =+ await findLatestCompletedFinalAgentMessage(+ livenessRun.id,+ livenessRun.companyId,+ );+ const externalChatPresentationCandidate =+ isExternalChatPresentationContext(livenessRun.contextSnapshot) ||+ parseObject(livenessRun.contextSnapshot).source === "tool_action_review" ||+ String(parseObject(livenessRun.contextSnapshot).source ?? "").startsWith("issue.comment") ||+ parseObject(livenessRun.contextSnapshot).source === "issue.update";+ const externalChatPresentationAuthorization =+ issueId && externalChatPresentationCandidate+ ? await resolveChatRunPresentationAuthorizationReason(db, {+ companyId: livenessRun.companyId,+ issueId,+ runId: livenessRun.id,+ })+ : null;+ const externalChatPresentationContext = isExternalChatPresentationContext(+ livenessRun.contextSnapshot,+ externalChatPresentationAuthorization === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,+ );+ const resolved = resolveHeartbeatRunResponse({+ resultJson: persistedResultJson,+ conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded"+ && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished",+ existingComment: existingRunComment,+ finalAgentMessage,+ preferFinalResponseOverExistingComment:+ externalChatPresentationContext,+ externalChatReviewResponseSummaryAuthorized:+ persistedResultJson?.finalizationReasonCode ===+ "governed_response_waiting" &&+ externalChatPresentationAuthorization ===+ CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,+ externalChatResponseWakeSummaryAuthorized:+ Boolean(adapterResult.nativeFinalization) &&+ externalChatPresentationAuthorization ===+ CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,+ });+ let presentationDecision: RunPresentationDecision =+ resolved.decision;++ if (+ issueId &&+ !skipRunIssueComment &&+ presentationDecision.commentAction === "create" &&+ resolved.text+ ) {+ // The presentation resolver exposes only the final assistant+ // surface selected from completed final messages or accepted+ // semantic results. For an exactly bound external-chat run,+ // authorize that narrow presentation as the provider reply;+ // ordinary internal runs retain the private default.+ const presentationAuthorizationReason =+ await resolveChatRunPresentationAuthorizationReason(db, {+ companyId: livenessRun.companyId,+ issueId,+ runId: livenessRun.id,+ });+ const comment = await issuesSvc.addComment(+ issueId,+ resolved.text,+ { agentId: agent.id, runId: livenessRun.id },+ { authorizationReason: presentationAuthorizationReason, completionReply: true },+ );+ presentationDecision = {+ ...presentationDecision,+ commentId: comment.id,+ reasonCodes: [+ ...presentationDecision.reasonCodes,+ "resolved_response_materialized",+ ],+ };+ await logActivity(db, {+ companyId: livenessRun.companyId,+ actorType: "agent",+ actorId: agent.id,+ agentId: agent.id,+ runId: livenessRun.id,+ issueId,+ action: "issue.comment_added",+ entityType: "issue",+ entityId: issueId,+ details: {+ commentId: comment.id,+ bodySnippet: comment.body.slice(0, 120),+ identifier: issueRef?.identifier ?? null,+ issueTitle: issueRef?.title ?? null,+ authorizationReason: presentationAuthorizationReason,+ source: "run_presentation_resolver",+ presentationSource: presentationDecision.chosenSource,+ },+ });+ } else if (presentationDecision.commentAction === "create") {+ presentationDecision = {+ ...presentationDecision,+ commentAction: "none",+ reasonCodes: [+ ...presentationDecision.reasonCodes,+ skipRunIssueComment+ ? "issue_comment_suppressed"+ : "run_has_no_issue",+ ],+ };+ }++ await db+ .update(heartbeatRuns)+ .set({+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ presentationDecision })}::jsonb`,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, livenessRun.id));+ await appendRunEvent(livenessRun, {+ eventType: "run.presentation.resolved",+ stream: "system",+ level: "info",+ message: "run presentation resolved",+ payload: { presentationDecision },+ });+ resolvedPresentationDecision = presentationDecision;+ } catch (err) {+ await onLog(+ "stderr",+ `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`,+ );+ }+ if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {+ const policy = parseMaxTurnContinuationPolicy(agent);+ if (policy.enabled && policy.maxAttempts > 0) {+ await scheduleBoundedRetryForRun(livenessRun, agent, {+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,+ wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,+ maxAttempts: policy.maxAttempts,+ delayMs: policy.delayMs,+ });+ } else {+ await appendRunEvent(livenessRun, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message:+ "Max-turn continuation suppressed because the policy is disabled",+ payload: {+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,+ policy,+ },+ });+ }+ } else if (+ outcome === "failed" &&+ readTransientRecoveryContractFromRun(livenessRun)+ ) {+ await scheduleBoundedRetryForRun(livenessRun, agent);+ } else if (+ outcome === "failed" &&+ !(await legacyExecutionNeedsReconciliationWithEvidence(db, livenessRun))+ ) {+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(+ livenessRun,+ agent,+ );+ }+ const issueCommentPolicyResult = await finalizeIssueCommentPolicy(+ livenessRun,+ agent,+ resolvedPresentationDecision,+ );+ const conversationSettled = await settleConversationTurn(db, livenessRun);+ await releaseIssueExecutionAndPromote(livenessRun, {+ suppressImmediateRecovery: conversationSettled ||+ readNonEmptyString(+ parseObject(livenessRun.contextSnapshot).goalControlRequestId,+ ) !== null ||+ parseObject(livenessRun.contextSnapshot)+ .resumeSessionGoalHeartbeat === true,+ });+ if (!conversationSettled) {+ await handleIssueReviewPathDisposition(livenessRun);+ if (livenessRun.runtimeMode !== "native") {+ await recovery.reconcileLegacyContinuation(livenessRun.id);+ } else {+ await handleRunLivenessContinuation(livenessRun);+ await handleSuccessfulRunHandoff(+ issueCommentPolicyResult.outcome === "retry_queued" ||+ issueCommentPolicyResult.outcome === "retry_exhausted"+ ? { ...livenessRun, issueCommentStatus: issueCommentPolicyResult.outcome }+ : livenessRun,+ agent,+ );+ }+ }+ if (+ outcome === "succeeded" &&+ issueId &&+ parseObject(adapterResult.resultJson).goalRolloverRequired === true+ ) {+ const rolloverProjection = await runnerGoalService(db).projection(+ livenessRun.companyId,+ issueId,+ agent.id,+ );+ if (rolloverProjection?.goal?.status === "active") {+ await enqueueWakeup(agent.id, {+ source: "automation",+ triggerDetail: "system",+ reason: "goal_control",+ payload: {+ issueId,+ intent: "goal_rollover",+ predecessorRunId: livenessRun.id,+ },+ idempotencyKey: `goal_rollover:${livenessRun.id}`,+ requestedByActorType: "system",+ contextSnapshot: {+ issueId,+ taskKey: issueId,+ resumeSessionGoalHeartbeat: true,+ skipIssueComment: true,+ goalRolloverFromRunId: livenessRun.id,+ },+ });+ }+ }++ // Dependency wake re-check: if this run's issue was marked done mid-run,+ // the route-time `issue_blockers_resolved` wake may have been gated by+ // workspace finalization or merged into this run. Reuse the level-triggered+ // dependency backstop so finalize and periodic recovery share idempotency,+ // readiness, active-path, and observability rules.+ if (issueId && finalizedRun) {+ try {+ const blockerIssueStatus = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0]?.status ?? null);+ if (blockerIssueStatus === "done") {+ await recovery.reconcileResolvedDependencyWakeBackstop({+ runId: finalizedRun.id,+ companyId: finalizedRun.companyId,+ blockerIssueId: issueId,+ source: "workspace.finalize",+ });+ }+ } catch (finalizeWakeErr) {+ logger.warn(+ { err: finalizeWakeErr, runId: run.id, issueId },+ "failed to evaluate dependent wakes after workspace_finalize",+ );+ }+ }+ }++ if (finalizedRun) {+ await updateRuntimeState(+ agent,+ finalizedRun,+ adapterResult,+ {+ legacySessionId: nextSessionState.legacySessionId,+ },+ normalizedUsage,+ );+ if (taskKey) {+ if (+ adapterResult.clearSession ||+ (!nextSessionState.params && !nextSessionState.displayId)+ ) {+ await clearTaskSessions(agent.companyId, agent.id, {+ taskKey,+ adapterType: agent.adapterType,+ expectedRunId: finalizedRun.id,+ });+ } else {+ await upsertTaskSession({+ companyId: agent.companyId,+ agentId: agent.id,+ adapterType: agent.adapterType,+ taskKey,+ sessionParamsJson:+ attachPaperclipSessionMetadataToSessionParams(+ nextSessionState.params,+ configuredModel,+ sessionConfigMetadata,+ ),+ sessionDisplayId: nextSessionState.displayId,+ lastRunId: finalizedRun.id,+ lastError: runErrorMessage,+ });+ }+ }+ }+ await finalizeAgentStatus(agent.id, outcome, runErrorMessage, {+ keepIdleOnFailure:+ outcome === "failed" &&+ ((finalizedRun+ ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota"+ : runErrorCode === "provider_quota") ||+ isWorkspaceSyncConflictFailure(adapterResult.errorMessage)),+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ });+ }++ async function failRun(input: FailHeartbeatRunInput) {+ const { run, agent, issueId, issueRef, signal, output, session, readFailureReportSecrets, runId, err, identityRedactor, requiredWorkspaceRestoreEvidence,+ legacyAdapterEntered, goalCheckpointSession, previousSessionDisplayId, taskSession } = input;+ const { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata } = session;+ const { handle, outputProgressState, flushOutputProgress } = output;+ // A process adapter may throw while its owned Stop is joining the+ // child. Let the cancellation write settle before attempting failure.+ await processRunCancellationSettlements.get(run.id)?.settled;+ const message = redactCurrentUserText(+ identityRedactor.redact(err instanceof Error ? err.message : "Unknown adapter failure"),+ await getCurrentUserRedactionOptions(),+ );+ const workspaceValidationFailure = isWorkspaceValidationFailure(err)+ ? err+ : null;+ const configurationIncompleteFailure = isConfigurationIncompleteFailure(+ err,+ )+ ? err+ : null;+ const recordedResponsibleUserDenialCode =+ normalizeResponsibleUserDenialCode(+ (await getRun(run.id).catch(() => null))?.errorCode,+ );+ // The runtime resolution is scoped to the adapter try block. The+ // durable coordinator is also the stronger authority here: legacy+ // runs simply have no row, while native result-less exhaustion keeps+ // its named failure instead of being flattened to `adapter_failed`.+ const nativeTerminalFailureCode = await db+ .select({+ phase: nativeRunFinalizations.phase,+ resultId: nativeRunFinalizations.resultId,+ failureCode: nativeRunFinalizations.failureCode,+ })+ .from(nativeRunFinalizations)+ .where(eq(nativeRunFinalizations.runId, run.id))+ .limit(1)+ .then((rows) => {+ const coordinator = rows[0];+ return coordinator?.phase === "terminal_failure" &&+ coordinator.resultId === null+ ? coordinator.failureCode+ : null;+ })+ .catch(() => null);+ const failureErrorCode =+ workspaceValidationFailure?.code ??+ configurationIncompleteFailure?.code ??+ nonRetryablePreflightFailureCode(err) ??+ recordedResponsibleUserDenialCode ??+ nativeTerminalFailureCode ??+ "adapter_failed";+ logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId }, "heartbeat execution failed");++ let logSummary: {+ bytes: number | null;+ sha256?: string | null;+ compressed: boolean;+ } | null = null;+ if (handle) {+ try {+ logSummary = await runLogStore.finalize(handle);+ } catch (finalizeErr) {+ logger.warn(+ { err: finalizeErr, runId },+ "failed to finalize run log after error",+ );+ }+ }+ const finalLogBytes = logSummary?.bytes;+ if (outputProgressState.pending && typeof finalLogBytes === "number") {+ outputProgressState.pending.bytes = finalLogBytes;+ }+ await flushOutputProgress({ force: true }).catch((flushErr) => {+ logger.warn(+ { err: flushErr, runId },+ "failed to flush run output progress after error",+ );+ });++ const stoppedDuringFailure = signal.aborted;+ const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null;+ const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed";+ const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, {+ error: message,+ errorCode: stopSnapshot?.errorCode ?? failureErrorCode,+ finishedAt: new Date(),+ resultJson: mergeRunStopMetadataForAgent(agent, failureOutcome, {+ errorCode: failureErrorCode,+ errorMessage: message,+ resultJson: {+ ...parseObject(stopSnapshot?.resultJson),+ ...requiredWorkspaceRestoreEvidence,+ ...(workspaceValidationFailure?.resultJson ??+ configurationIncompleteFailure?.resultJson ??+ {}),+ ...(!legacyAdapterEntered && run.runtimeMode !== "native"+ ? {+ executionRecovery: {+ kind: "bootstrap",+ providerWorkStarted: false,+ },+ }+ : {}),+ },+ }),+ stdoutExcerpt: output.stdoutExcerpt,+ stderrExcerpt: output.stderrExcerpt,+ logBytes: logSummary?.bytes,+ logSha256: logSummary?.sha256,+ logCompressed: logSummary?.compressed ?? false,+ }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });+ if (+ !failedRunWrite.updated &&+ !(+ nativeTerminalFailureCode && failedRunWrite.run?.status === "failed"+ )+ ) {+ logger.info(+ {+ runId: run.id,+ attemptedStatus: "failed",+ currentStatus: failedRunWrite.run?.status ?? null,+ },+ "skipping late adapter failure finalization because the run already left running state",+ );+ return;+ }++ const failedRun = failedRunWrite.run;+ await setWakeupStatus(run.wakeupRequestId, "failed", {+ finishedAt: new Date(),+ error: message,+ });++ if (failedRun) {+ await appendRunEvent(failedRun, {+ eventType: "error",+ stream: "system",+ level: "error",+ message,+ });+ const livenessRun =+ (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun;+ try {+ await completeSkillTestRunForHeartbeatOutcome({+ run: livenessRun,+ issueId,+ issueWorkMode: issueRef?.workMode ?? null,+ outcome: "failed",+ error: message,+ });+ } catch (err) {+ logger.warn(+ { err, runId: livenessRun.id, issueId },+ "failed to complete skill test run after heartbeat adapter failure",+ );+ }+ await refreshContinuationSummaryForRun(livenessRun, agent);+ if (+ !isWorkspaceValidationFailedRun(livenessRun) &&+ !isConfigurationIncompleteFailedRun(livenessRun)+ ) {+ await finalizeIssueCommentPolicy(livenessRun, agent);+ }+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(+ livenessRun,+ agent,+ );+ await releaseIssueExecutionAndPromote(livenessRun, {+ // Native recovery owns the original heartbeat run through+ // exhaustion. Once its durable coordinator has classified a+ // terminal failure, generic issue recovery must not create a+ // replacement retryOfRunId chain for the same provider work.+ suppressImmediateRecovery: nativeTerminalFailureCode !== null,+ });+ await handleIssueReviewPathDisposition(livenessRun);++ await updateRuntimeState(+ agent,+ livenessRun,+ {+ exitCode: null,+ signal: null,+ timedOut: false,+ errorMessage: message,+ },+ {+ legacySessionId: runtimeForAdapter.sessionId,+ },+ );++ if (+ taskKey &&+ (goalCheckpointSession.current ||+ previousSessionParams ||+ previousSessionDisplayId ||+ taskSession)+ ) {+ await upsertTaskSession({+ companyId: agent.companyId,+ agentId: agent.id,+ adapterType: agent.adapterType,+ taskKey,+ sessionParamsJson:+ goalCheckpointSession.current?.params ??+ attachPaperclipSessionMetadataToSessionParams(+ previousSessionParams,+ configuredModel,+ sessionConfigMetadata,+ ),+ sessionDisplayId:+ goalCheckpointSession.current?.displayId ??+ previousSessionDisplayId,+ lastRunId: failedRun.id,+ lastError: message,+ });+ }+ }++ await finalizeAgentStatus(agent.id, "failed", message, {+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ keepIdleOnFailure:+ Boolean(nonRetryablePreflightFailureCode(err)) ||+ isWorkspaceSyncConflictFailure(message),+ });+ }++ async function failRunSetup(input: FailHeartbeatRunSetupInput) {+ const { run, runId, outerErr, identityRedactor, readFailureReportSecrets } = input;+ // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).+ // The inner catch did not fire, so we must record the failure here.+ const message = redactCurrentUserText(+ identityRedactor.redact(outerErr instanceof Error+ ? outerErr.message+ : "Unknown setup failure"),+ await getCurrentUserRedactionOptions(),+ );+ // A missing secret/env binding is a known pre-dispatch configuration gap,+ // not an opaque setup crash. Surface it with its own errorCode so the+ // recovery path routes it to a human owner instead of looping retries.+ const workspaceValidationSetupFailure = isWorkspaceValidationFailure(+ outerErr,+ )+ ? outerErr+ : null;+ const configurationIncompleteSetupFailure =+ isConfigurationIncompleteFailure(outerErr) ? outerErr : null;+ const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(+ outerErr,+ )+ ? outerErr+ : null;+ // A sandbox provider plugin stuck in error/disabled/upgrade_pending+ // fails every lease the same way until an operator acts, so it is a+ // configuration gap, not a transient setup failure.+ const sandboxProviderPluginNotReadySetupFailure =+ parseSandboxProviderPluginNotReadyFailureMessage(+ outerErr instanceof Error ? outerErr.message : null,+ );+ const recordedResponsibleUserDenialCode =+ normalizeResponsibleUserDenialCode(+ (await getRun(runId).catch(() => null))?.errorCode,+ );+ const nonRetryablePreflightCode =+ nonRetryablePreflightFailureCode(outerErr);+ const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null;+ const setupFailureErrorCode =+ workspaceGitScanFailure?.code ??+ workspaceValidationSetupFailure?.code ??+ configurationIncompleteSetupFailure?.code ??+ (unresolvedBaseRefSetupFailure ||+ sandboxProviderPluginNotReadySetupFailure+ ? CONFIGURATION_INCOMPLETE_FAILURE_CODE+ : null) ??+ recordedResponsibleUserDenialCode ??+ nonRetryablePreflightCode ??+ "setup_failed";+ logger.error(+ { err: identityRedactor.redact({ message: outerErr instanceof Error ? outerErr.message : String(outerErr), stack: outerErr instanceof Error ? outerErr.stack : undefined }), runId },+ "heartbeat execution setup failed",+ );+ const setupFailureAgent = await getAgent(run.agentId).catch(() => null);+ // The structured failure payload drives the recovery notice and next+ // action, so it is persisted even when the agent lookup failed and the+ // agent-scoped stop metadata cannot be merged in.+ const setupFailureDetails =+ (workspaceGitScanFailure ? {+ workspaceGitScan: {+ code: workspaceGitScanFailure.code,+ phase: "workspace_setup",+ retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code),+ },+ } : null) ??+ workspaceValidationSetupFailure?.resultJson ??+ configurationIncompleteSetupFailure?.resultJson ??+ (unresolvedBaseRefSetupFailure+ ? buildUnresolvedWorkspaceBaseRefResultJson(+ run,+ unresolvedBaseRefSetupFailure,+ )+ : null) ??+ (sandboxProviderPluginNotReadySetupFailure+ ? buildSandboxProviderPluginNotReadyResultJson(+ run,+ sandboxProviderPluginNotReadySetupFailure,+ )+ : null);+ const connectionFailure = readGitConnectionFailure(outerErr);+ const setupFailureResultJson = {+ ...setupFailureDetails,+ ...(connectionFailure ? { connectionFailure } : {}),+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },+ };+ const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", {+ error: message,+ errorCode: setupFailureErrorCode,+ finishedAt: new Date(),+ ...(setupFailureAgent+ ? {+ resultJson: mergeRunStopMetadataForAgent(+ setupFailureAgent,+ "failed",+ {+ errorCode: setupFailureErrorCode,+ errorMessage: message,+ resultJson: setupFailureResultJson,+ },+ ),+ }+ : setupFailureResultJson+ ? { resultJson: setupFailureResultJson }+ : {}),+ }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));+ if (!setupFailureWrite.updated) {+ logger.info(+ {+ runId,+ attemptedStatus: "failed",+ currentStatus: setupFailureWrite.run?.status ?? null,+ },+ "skipping late setup failure finalization because the run already left running state",+ );+ } else {+ await setWakeupStatus(run.wakeupRequestId, "failed", {+ finishedAt: new Date(),+ error: message,+ }).catch(() => undefined);+ }+ const failedRun = await getRun(runId).catch(() => null);+ if (setupFailureWrite.updated && failedRun) {+ // Emit a run-log event so the failure is visible in the run timeline,+ // consistent with what the inner catch block does for adapter failures.+ await appendRunEvent(failedRun, {+ eventType: "error",+ stream: "system",+ level: "error",+ message,+ }).catch(() => undefined);+ const livenessRun = await classifyAndPersistRunLiveness(+ failedRun,+ ).catch(() => failedRun);+ const setupFailureIssueId = readNonEmptyString(+ parseObject(livenessRun.contextSnapshot).issueId,+ );+ if (setupFailureIssueId) {+ await completeSkillTestRunForHeartbeatOutcome({+ run: livenessRun,+ issueId: setupFailureIssueId,+ outcome: "failed",+ error: message,+ }).catch((completionErr) => {+ logger.warn(+ {+ err: completionErr,+ runId: livenessRun.id,+ issueId: setupFailureIssueId,+ },+ "failed to complete skill test run after heartbeat setup failure",+ );+ });+ }+ const failedAgent =+ setupFailureAgent ??+ (await getAgent(run.agentId).catch(() => null));+ if (failedAgent) {+ await refreshContinuationSummaryForRun(+ livenessRun,+ failedAgent,+ ).catch(() => undefined);+ if (+ !isWorkspaceValidationFailedRun(livenessRun) &&+ !isConfigurationIncompleteFailedRun(livenessRun)+ ) {+ await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(+ () => undefined,+ );+ }+ // No provider work began. Retry temporary host scan failures with+ // the existing durable failure budget, before releasing execution.+ // Generic recovery must not grant a second budget on exhaustion.+ await (isTransientWorkspaceGitScanCode(livenessRun.errorCode)+ ? scheduleBoundedRetryForRun(livenessRun, failedAgent)+ : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent)+ ).catch((retryError) => {+ logger.warn(+ { err: retryError, runId: livenessRun.id },+ "failed to schedule interaction continuation retry after setup failure",+ );+ });+ }+ await releaseIssueExecutionAndPromote(livenessRun, {+ suppressImmediateRecovery:+ readNonEmptyString(+ parseObject(livenessRun.contextSnapshot).goalControlRequestId,+ ) !== null ||+ parseObject(livenessRun.contextSnapshot)+ .resumeSessionGoalHeartbeat === true,+ }).catch((releaseError) => {+ logger.error(+ { err: releaseError, runId },+ "failed to release issue execution after heartbeat setup failure",+ );+ });+ await handleIssueReviewPathDisposition(livenessRun).catch(+ (reviewPathError) => {+ logger.error(+ { err: reviewPathError, runId },+ "failed to evaluate review-path disposition after heartbeat setup failure",+ );+ },+ );+ }+ // Ensure the agent is not left stuck in "running" if the setup-failure+ // path owned the terminal transition. If another path already finalized+ // the run, keep that terminal outcome authoritative.+ if (setupFailureWrite.updated) {+ await finalizeAgentStatus(run.agentId, "failed", message, {+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),+ // Low-trust admission failures are task/principal preconditions,+ // not evidence that the immutable endpoint agent is unhealthy.+ // Keep the failed run and its safe provider refusal authoritative,+ // but return the agent to idle so clients do not also announce a+ // misleading agent-wide error for the same rejected chat turn.+ keepIdleOnFailure: Boolean(nonRetryablePreflightCode),+ }).catch(() => undefined);+ }+ }++ return { completeRun, failRun, failRunSetup };+}