Skip to content

The open-source app everyone uses to manage agents at work

refactor(server): extract heartbeat run completion (#15732)

Dottac27ee928b0b2
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index 624c833467..9c3d79a07c 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -1331,6 +1331,16 @@ claims remain database-scoped so separate scheduler instances cannot dispatch
the same claim. The public heartbeat methods keep their existing signatures.
Keep scheduling policy changes separate from this extraction and execution.
+Run completion is in `server/src/services/heartbeat/run-completion.ts`.
+`createHeartbeatRunCompletion` binds terminal result persistence, accounting,
+response presentation, task-session updates, and execution/setup failure handling.
+The executor calls these handlers from the original try/catch boundaries. It
+retains native recovery exceptions, dispatch ownership, and the outer cleanup.
+Completion reads the shared Stop barriers, uses live output getters, and reports
+trace finalization immediately so a later error cannot lose cleanup state.
+Existing public failure helpers remain re-exported by `heartbeat.ts`.
+Keep completion policy changes separate from this extraction and dispatch.
+
## Wake Context Delivery
Built-in adapters deliver wake context through the run prompt, including structured
diff --git a/server/src/services/chat-publication-reconciliation.test.ts b/server/src/services/chat-publication-reconciliation.test.ts
index 3af863db2d..aa3fad2a36 100644
--- a/server/src/services/chat-publication-reconciliation.test.ts
+++ b/server/src/services/chat-publication-reconciliation.test.ts
@@ -239,14 +239,14 @@ describe("chat publication commit signals", () => {
expect(persistedEvent).toBeGreaterThan(appendRunEventStart);
expect(emittedEvent).toBeGreaterThan(persistedEvent);
- const heartbeatSource = readFileSync(
- new URL("./heartbeat.ts", import.meta.url),
+ const completionSource = readFileSync(
+ new URL("./heartbeat/run-completion.ts", import.meta.url),
"utf8",
);
- const presentationMarker = heartbeatSource.indexOf(
+ const presentationMarker = completionSource.indexOf(
'eventType: "run.presentation.resolved"',
);
- const committedComment = heartbeatSource.lastIndexOf(
+ const committedComment = completionSource.lastIndexOf(
"await issuesSvc.addComment",
presentationMarker,
);
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index b39ece2ebb..b533b9e2d7 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -1,3 +1,10 @@
+import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";
+export {
+ MAX_TURN_CONTINUATION_WAKE_REASON,
+ parseSandboxProviderPluginNotReadyFailureMessage,
+ isConfigurationIncompleteFailedRun,
+ isWorkspaceSyncConflictFailure,
+} from "./heartbeat/run-completion.js";
import {
createHeartbeatScheduling,
formatIssueIdentifierLink,
@@ -17,7 +24,6 @@ export {
cancelHeartbeatNativeRun,
} from "./heartbeat/run-control.js";
import {
- NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
isHeartbeatRunTerminalStatus,
isHeartbeatRunRuntimeStatusActive,
publishHeartbeatRunRuntimeProgress,
@@ -31,7 +37,6 @@ export {
resolveSkillTestRunCompletionForHeartbeatOutcome,
} from "./heartbeat/run-lifecycle.js";
import {
- ChatControlRecoveryUnresolvedError,
shouldDeferFollowupWakeForSameIssue,
shouldQueueFollowupForRunningIssueWake,
DEFERRED_WAKE_CONTEXT_KEY,
@@ -48,10 +53,7 @@ import {
import {
BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
WorkspaceBusyDeferral,
- readTransientRecoveryContractFromRun,
- readHeartbeatRunErrorFamily,
isWorkspaceBusyDeferral,
- isTransientWorkspaceGitScanCode,
createHeartbeatRetries,
} from "./heartbeat/retries.js";
export {
@@ -67,7 +69,6 @@ export {
computeBoundedTransientHeartbeatRetrySchedule,
} from "./heartbeat/retries.js";
import {
- type RunSessionOutcome,
EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
deriveTaskKeyWithHeartbeatFallback,
getAdapterSessionCodec,
@@ -80,11 +81,8 @@ import {
isCanonicalSessionIdForAdapter,
requiresCanonicalSessionIds,
resolveNextSessionState,
- normalizeUsageTotals,
resolveCacheAdjustedCostUsd,
- resolveLedgerBiller,
resolveLedgerCostStatus,
- normalizeLedgerBillingType,
createHeartbeatRunState,
} from "./heartbeat/run-state.js";
export {
@@ -103,9 +101,8 @@ export {
normalizeSessionParams,
resolveNextSessionState,
} from "./heartbeat/run-state.js";
-import { readGitConnectionFailure } from "./git-connection-failure.js";
+
import {
- CONFIGURATION_INCOMPLETE_FAILURE_CODE,
ConfigurationIncompleteFailure,
deriveTaskKey,
type WakeupOptions,
@@ -144,7 +141,6 @@ export {
buildPaperclipWakePayload,
} from "./heartbeat/run-preparation.js";
import {
- isWorkspaceValidationFailedRun,
resolveNativeRecoveryExecutionWorkspaceBinding,
resolveExecutionWorkspaceReuseRequestForIssue,
buildExecutionWorkspaceConfigSnapshot,
@@ -247,7 +243,7 @@ export {
} from "./heartbeat/run-log.js";
import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";
export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";
-import { preserveWorkspaceRestoreRecoveryMetadataSql } from "./legacy-workspace-restore-recovery.js";
+
import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
import {
@@ -277,10 +273,7 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./age
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js";
-import {
- cancellationResultJson,
- readRunCancellation,
-} from "./run-cancellation.js";
+
import {
withNativeWorkspaceFinalizationOwnership,
NativeWorkspaceFinalizationBusyError,
@@ -342,7 +335,6 @@ import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./
import { buildNativeReviewRequest } from "./native-runtime/native-review-prompt.js";
import {
- legacyExecutionNeedsReconciliationWithEvidence,
settleInterruptedNativeBootstrap,
} from "./legacy-execution-recovery.js";
import {
@@ -356,7 +348,7 @@ export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-
import { buildExecutionContinuation, StaleExecutionContinuationError } from "./execution-continuation.js";
import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils";
-import { isWorkspaceGitScanError } from "./workspace-git-operation-scheduler.js";
+
import {
initializeRunIdentity,
@@ -492,7 +484,7 @@ import {
type NativeSessionBackend,
} from "../vendor/paperclip-runner/index.js";
import { createNativeSessionHandoffLoader } from "./native-runtime/native-session-handoff.js";
-import { normalizeResponsibleUserDenialCode } from "./responsible-user-denial-run-outcomes.js";
+
import { getRunLogStore, type RunLogHandle } from "./run-log-store.js";
import {
providerTraceStore,
@@ -549,18 +541,10 @@ import {
} from "../home-paths.js";
import {
buildHeartbeatRunIssueComment,
- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
- isExternalChatPresentationContext,
- mergeHeartbeatRunResultJson,
- resolveHeartbeatRunResponse,
- type RunPresentationDecision,
} from "./heartbeat-run-summary.js";
-import {
- normalizeMaxTurnStopReason,
-} from "./heartbeat-stop-metadata.js";
+
import {
CHAT_CONTROL_RECOVERY_ADMISSION_KEY,
- CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
} from "./chat-control-recovery-stop.js";
import {
@@ -580,12 +564,9 @@ import {
persistAdapterManagedRuntimeServices,
realizeExecutionWorkspace,
releaseRuntimeServicesForRun,
- isUnresolvedWorkspaceBaseRefError,
- readUnresolvedWorkspaceBaseRefDiagnostic,
type ExecutionWorkspaceInput,
type RealizedExecutionWorkspace,
type RuntimeServiceRef,
- type UnresolvedWorkspaceBaseRefError,
} from "./workspace-runtime.js";
import {
@@ -597,10 +578,9 @@ import {
blockRunnerGoalRecovery,
failRunnerGoalAction,
isRunnerGoalActionCompleted,
- runnerGoalService,
settleLiveRunnerGoalBeforeInterrupt,
} from "./runner-goals.js";
-import { resolveChatRunPresentationAuthorizationReason } from "./chat-run-publications.js";
+
import {
parseIssueExecutionState,
@@ -661,7 +641,6 @@ import {
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildImmediateExecutionPathRecoveryNoticeSeed,
buildWorkspaceValidationRecoveryNoticeSeed,
- SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,
type StrandedRecoveryNoticeSeed,
} from "./recovery/stranded-notice.js";
@@ -773,146 +752,12 @@ const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON =
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE =
"execution_review_participant_recovery";
-function nonRetryablePreflightFailureCode(error: unknown): string | null {
- if (error instanceof ChatControlRecoveryUnresolvedError)
- return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;
- if (
- error instanceof HttpError &&
- error.status === 409 &&
- parseObject(error.details).code === "chat_failed_run_retry_not_authorized"
- ) {
- return "chat_failed_run_retry_not_authorized";
- }
- if (!(error instanceof HttpError) || error.status !== 422) return null;
- const code = readNonEmptyString(parseObject(error.details).code);
- return code && NON_RETRYABLE_PREFLIGHT_FAILURE_CODES.has(code) ? code : null;
-}
export { MAX_TURN_CONTINUATION_RETRY_REASON };
-export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
export { WORKSPACE_BUSY_RETRY_REASON };
-// Build the configuration-incomplete result payload for a workspace base ref
-// that never resolved to a commit. The setup catch maps this to errorCode
-// `configuration_incomplete`, so the recovery path routes it to a human owner
-// instead of a dispatched-then-failed run. The `fingerprint` uses the canonical
-// remote ref, not the operator spelling. Two equivalent spellings of one remote
-// branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated
-// failure reuses one active recovery action and does not reset the attempt
-// count or post a duplicate notice. A different branch makes a new action.
-// Build the configuration-incomplete result payload for a sandbox provider
-// plugin that is installed but not `ready`. The `fingerprint` is the plugin
-// key plus its status, so every run that hits the same stuck plugin reuses
-// one active recovery action instead of posting a fresh notice per attempt,
-// while a status change (say `error` -> `disabled`) makes a new one.
-function buildSandboxProviderPluginNotReadyResultJson(
- run: typeof heartbeatRuns.$inferSelect,
- failure: { provider: string; pluginKey: string; pluginStatus: string },
-): Record<string, unknown> {
- const context = parseObject(run.contextSnapshot);
- return {
- configurationIncomplete: {
- reason: SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,
- companyId: run.companyId,
- agentId: run.agentId,
- issueId: readNonEmptyString(context.issueId) ?? null,
- projectId: readNonEmptyString(context.projectId) ?? null,
- sandboxProvider: failure.provider,
- pluginKey: failure.pluginKey,
- pluginStatus: failure.pluginStatus,
- fingerprint: `sandbox_provider_plugin:${failure.pluginKey}:${failure.pluginStatus}`,
- missingBindings: [],
- },
- };
-}
-
-function buildUnresolvedWorkspaceBaseRefResultJson(
- run: typeof heartbeatRuns.$inferSelect,
- error: UnresolvedWorkspaceBaseRefError,
-): Record<string, unknown> {
- const context = parseObject(run.contextSnapshot);
- const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);
- return {
- configurationIncomplete: {
- reason: "workspace_base_ref_unresolved",
- ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),
- companyId: run.companyId,
- agentId: run.agentId,
- issueId: readNonEmptyString(context.issueId) ?? null,
- projectId: readNonEmptyString(context.projectId) ?? null,
- requestedRef: error.requestedRef,
- defaultBranch: error.defaultBranch,
- attemptedRefs: error.attemptedRefs,
- fetchError: error.fetchError,
- fingerprint: `workspace_base_ref:${error.recoveryIdentityRef}`,
- missingBindings: [],
- },
- };
-}
-
export { isNonAssigneeWorkspaceBusyRetry };
-function isMaxTurnExhaustionRun(
- run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
-) {
- const resultJson = parseObject(run.resultJson);
- return Boolean(
- normalizeMaxTurnStopReason(resultJson.stopReason) ??
- normalizeMaxTurnStopReason(run.errorCode),
- );
-}
-
-// environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message,
-// e.g. 'Sandbox provider "kubernetes" is installed via plugin
-// "acme.kubernetes-sandbox-provider", but that plugin is currently error.'
-// The plugin row exists but its status is `error` (a failed activation),
-// `disabled` (an operator switched it off) or `upgrade_pending`. Unlike the
-// worker restart window above, nothing on the run path ever changes that
-// status: only an operator enabling the plugin, or a server boot that
-// re-activates a bundled plugin, does. Re-running the agent produces the
-// identical failure every time, so the setup catch classifies it as
-// `configuration_incomplete` (routed to a human owner) instead of a retryable
-// `setup_failed` that the scheduler would keep re-dispatching.
-const SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE =
- /sandbox provider "([^"]*)" is installed via plugin "([^"]*)", but that plugin is currently (error|disabled|upgrade_pending)\b/i;
-
-export function parseSandboxProviderPluginNotReadyFailureMessage(
- value: unknown,
-): { provider: string; pluginKey: string; pluginStatus: string } | null {
- if (typeof value !== "string") return null;
- const match = SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE.exec(value);
- if (!match) return null;
- return {
- provider: match[1] ?? "",
- pluginKey: match[2] ?? "",
- pluginStatus: (match[3] ?? "").toLowerCase(),
- };
-}
-
-function mergeAdapterRecoveryMetadata(input: {
- resultJson: Record<string, unknown> | null | undefined;
- errorFamily?: string | null;
- retryNotBefore?: string | null;
-}) {
- const errorFamily = readNonEmptyString(input.errorFamily);
- const retryNotBefore = readNonEmptyString(input.retryNotBefore);
- if (!input.resultJson && !errorFamily && !retryNotBefore)
- return input.resultJson ?? null;
-
- return {
- ...(input.resultJson ?? {}),
- ...(errorFamily ? { errorFamily } : {}),
- ...(retryNotBefore
- ? {
- retryNotBefore,
- transientRetryNotBefore: retryNotBefore,
- ...(errorFamily === "provider_quota"
- ? { providerQuotaRetryNotBefore: retryNotBefore }
- : {}),
- }
- : {}),
- };
-}
// Routes and the scheduler construct separate heartbeatService instances, but
// they must agree on in-process adapter executions when reaping stale runs.
const activeRunExecutions = new Set<string>();
@@ -1047,21 +892,6 @@ export function resolveNativeSandboxLifecycle(input: {
return resolveReusableSandboxLifecycle(input);
}
-function isConfigurationIncompleteFailure(
- error: unknown,
-): error is ConfigurationIncompleteFailure {
- return error instanceof ConfigurationIncompleteFailure;
-}
-
-export function isConfigurationIncompleteFailedRun(
- run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode"> | null | undefined,
-) {
- return (
- run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE ||
- run?.errorCode === "model_not_found"
- );
-}
-
interface ParsedIssueAssigneeAdapterOverrides {
adapterConfig: Record<string, unknown> | null;
useProjectWorkspace: boolean | null;
@@ -1124,30 +954,6 @@ export function filterZombieCoalesceTarget<
return target && isZombieRun(target, tracked) ? null : target;
}
-/**
- * Failure signatures from sandbox→host git workspace reconciliation. These
- * describe the state of the SHARED workspace (divergent histories written by
- * different runs), not a defect in the agent that happened to run last —
- * putting the agent into a sticky `error` state over them removes a healthy
- * agent from rotation while leaving the actual problem (the workspace)
- * untouched. The run still fails and carries the full message.
- */
-const WORKSPACE_SYNC_CONFLICT_SIGNATURES = [
- "Failed to merge concurrent remote git histories",
- "Failed to integrate concurrent remote git history",
- "did not send all necessary objects",
- "lacks these prerequisite commits",
-];
-
-export function isWorkspaceSyncConflictFailure(
- message: string | null | undefined,
-): boolean {
- if (!message) return false;
- return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) =>
- message.includes(signature),
- );
-}
-
export function shouldAutoCheckoutIssueForWake(input: {
contextSnapshot: Record<string, unknown> | null | undefined;
issueStatus: string | null;
@@ -2328,6 +2134,43 @@ export function heartbeatService(
nativeSessionResumeDispatchTimers.set(runId, timer);
}
+ const runCompletion = createHeartbeatRunCompletion(db, {
+ getRun,
+ getAgent,
+ resolveNormalizedUsageForSession,
+ clearTaskSessions,
+ upsertTaskSession,
+ mergeRunStopMetadataForAgent,
+ setRunStatusIfRunning,
+ setWakeupStatus,
+ appendRunEvent,
+ classifyAndPersistRunLiveness,
+ completeSkillTestRunForHeartbeatOutcome,
+ refreshContinuationSummaryForRun,
+ findRunIssueComment,
+ findLatestCompletedFinalAgentMessage,
+ parseMaxTurnContinuationPolicy,
+ finalizeIssueCommentPolicy,
+ handleIssueReviewPathDisposition,
+ handleRunLivenessContinuation,
+ handleSuccessfulRunHandoff,
+ updateRuntimeState,
+ scheduleBoundedRetryForRun,
+ scheduleInteractionContinuationInfrastructureRetryIfEligible,
+ timerClaimWasFirstHeartbeat,
+ releaseIssueExecutionAndPromote,
+ finalizeAgentStatus,
+ enqueueWakeup,
+ issuesSvc,
+ recovery,
+ getCurrentUserRedactionOptions,
+ budgetHooks,
+ runLogStore,
+ traceStore,
+ processRunCancellationSettlements,
+ failedProcessRunCancellations,
+ });
+
async function executeRun(
runId: string,
runOptions: {
@@ -7647,650 +7490,20 @@ export function heartbeatService(
}
}
}
- const processCancellation =
- processRunCancellationSettlements.get(run.id) ??
- failedProcessRunCancellations.get(run.id);
- await processCancellation?.settled;
- let outcome: RunSessionOutcome;
- const latestRun = await getRun(run.id);
- if (isHeartbeatRunTerminalStatus(latestRun?.status)) {
- outcome = latestRun.status;
- } else if (executionControl.controller.signal.aborted) {
- outcome = "cancelled";
- } else if (adapterResult.nativeFinalization) {
- const nativeTerminal =
- adapterResult.nativeFinalization.terminal.runTerminalState;
- outcome =
- nativeTerminal === "succeeded"
- ? "succeeded"
- : nativeTerminal === "cancelled"
- ? "cancelled"
- : "failed";
- } else if (adapterResult.timedOut) {
- outcome = "timed_out";
- } else if (adapterResult.resultJson?.status === "cancelled") {
- outcome = "cancelled";
- } else if (
- (adapterResult.exitCode ?? 0) === 0 &&
- !adapterResult.errorMessage &&
- !adapterResult.signal &&
- !processCancellation?.failed
- ) {
- outcome = "succeeded";
- } else {
- outcome = "failed";
- }
-
- const nextSessionState = resolveNextSessionState({
- adapterType: agent.adapterType,
- codec: sessionCodec,
- adapterResult,
- outcome,
- previousParams: previousSessionParams,
- previousDisplayId: runtimeForAdapter.sessionDisplayId,
- previousLegacySessionId: runtimeForAdapter.sessionId,
- });
- const rawUsage = normalizeUsageTotals(adapterResult.usage);
- const sessionUsageResolution = await resolveNormalizedUsageForSession({
- agentId: agent.id,
- runId: run.id,
- sessionId:
- nextSessionState.displayId ?? nextSessionState.legacySessionId,
- rawUsage,
- usageBasis: adapterResult.usageBasis ?? null,
- });
- const normalizedUsage = sessionUsageResolution.normalizedUsage;
- const runErrorMessage =
- outcome === "cancelled"
- ? redactCurrentUserText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled", currentUserRedactionOptions)
- : outcome === "succeeded"
- ? null
- : redactCurrentUserText(
- adapterResult.errorMessage ??
- (outcome === "timed_out" ? "Timed out" : "Adapter failed"),
- currentUserRedactionOptions,
- );
- const recordedResponsibleUserDenialCode =
- normalizeResponsibleUserDenialCode(latestRun?.errorCode);
- const runErrorCode =
- outcome === "timed_out"
- ? "timeout"
- : outcome === "cancelled"
- ? (latestRun?.errorCode ?? "cancelled")
- : outcome === "failed"
- ? (adapterResult.errorCode ??
- recordedResponsibleUserDenialCode ??
- "adapter_failed")
- : null;
-
- let logSummary: {
- bytes: number | null;
- sha256?: string | null;
- compressed: boolean;
- } | null = null;
- if (handle) {
- logSummary = await runLogStore.finalize(handle);
- }
- const finalLogBytes = logSummary?.bytes;
- if (outputProgressState.pending && typeof finalLogBytes === "number") {
- outputProgressState.pending.bytes = finalLogBytes;
- }
- await flushOutputProgress({ force: true });
-
- if (providerTraceCapture) {
- try {
- await traceStore.finalize(run.id, run.companyId);
- providerTraceFinalized = true;
- } catch (error) {
- logger.warn(
- { error, runId: run.id },
- "provider trace finalization failed without affecting run outcome",
- );
- }
- }
-
- const status =
- outcome === "succeeded"
- ? "succeeded"
- : outcome === "cancelled"
- ? "cancelled"
- : outcome === "timed_out"
- ? "timed_out"
- : "failed";
-
- const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null
- ? null : resolveCacheAdjustedCostUsd(adapterResult);
- const usageJson: Record<string, unknown> = {
- accountingReceiptReady: adapterResult.usageComplete !== false,
- costUsdExact: adapterResult.costUsdExact ?? null,
- providerRequestId: adapterResult.providerRequestId ?? null,
- ...(normalizedUsage ?? {}),
- ...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}),
- ...(rawUsage
- ? {
- rawInputTokens: rawUsage.inputTokens,
- rawInputIncludesCached: false,
- rawCachedInputTokens: rawUsage.cachedInputTokens,
- rawOutputTokens: rawUsage.outputTokens,
- }
- : {}),
- ...(sessionUsageResolution.derivedFromSessionTotals
- ? { usageSource: "session_delta" }
- : adapterResult.usageBasis === "per_run"
- ? { usageSource: "per_run" }
- : {}),
- ...((nextSessionState.displayId ??
- nextSessionState.legacySessionId)
- ? {
- persistedSessionId:
- nextSessionState.displayId ??
- nextSessionState.legacySessionId,
- }
- : {}),
- sessionReused:
- runtimeForAdapter.sessionId != null ||
- runtimeForAdapter.sessionDisplayId != null,
- taskSessionReused: taskSessionForRun != null,
- freshSession:
- runtimeForAdapter.sessionId == null &&
- runtimeForAdapter.sessionDisplayId == null,
- sessionRotated: sessionCompaction.rotate,
- sessionRotationReason: sessionCompaction.reason,
- configFreshness: configFreshnessResultMetadata,
- provider:
- readNonEmptyString(adapterResult.provider) ?? "unknown",
- biller: resolveLedgerBiller(adapterResult),
- model: readNonEmptyString(adapterResult.model) ?? "unknown",
- ...(adapterResult.costUsd != null
- ? { costUsd: adapterResult.costUsd }
- : {}),
- ...(cacheAdjustedCostUsd != null
- ? { cacheAdjustedCostUsd }
- : {}),
- pricingProvenance: adapterResult.pricingProvenance,
- costStatus: adapterResult.costStatus ?? resolveLedgerCostStatus({
- costUsd: cacheAdjustedCostUsd ?? (adapterResult.costUsdExact != null ? Number(adapterResult.costUsdExact) : null),
- billingType: normalizeLedgerBillingType(adapterResult.billingType),
- inputTokens: normalizedUsage?.inputTokens ?? 0,
- cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,
- outputTokens: normalizedUsage?.outputTokens ?? 0,
- }),
- billingType: normalizeLedgerBillingType(
- adapterResult.billingType,
- ),
- };
-
- const persistedResultJson = cancellationResultJson(latestRun ?? run, outcome, mergeHeartbeatRunResultJson(
- mergeRunStopMetadataForAgent(agent, outcome, {
- resultJson: mergeAdapterRecoveryMetadata({
- resultJson: {
- ...(adapterResult.nativeFinalization || outcome === "cancelled"
- ? parseObject(latestRun?.resultJson)
- : {}),
- ...parseObject(adapterResult.resultJson),
- ...(adapterResult.executionRecovery
- ? { executionRecovery: adapterResult.executionRecovery }
- : {}),
- configFreshness: configFreshnessResultMetadata,
- },
- errorFamily: adapterResult.errorFamily ?? null,
- retryNotBefore: adapterResult.retryNotBefore ?? null,
- }),
- errorCode: runErrorCode,
- errorMessage: runErrorMessage,
- }),
- adapterResult.summary ?? null,
- ), runErrorCode, runErrorMessage);
-
- const ledgerScope = runLedgerScope;
- const finalRunPatch: Partial<typeof heartbeatRuns.$inferInsert> = {
- // Accounting must acknowledge even a proven pre-provider failure:
- // that transaction releases the reservation without creating a charge.
- costAccountingPending: true,
- finishedAt: new Date(),
- error: runErrorMessage,
- errorCode: runErrorCode,
- exitCode: adapterResult.exitCode,
- signal: adapterResult.signal,
- usageJson: { ...usageJson, ledgerScope },
- resultJson: persistedResultJson,
- sessionIdAfter:
- nextSessionState.displayId ?? nextSessionState.legacySessionId,
- stdoutExcerpt,
- stderrExcerpt,
- logBytes: logSummary?.bytes,
- logSha256: logSummary?.sha256,
- logCompressed: logSummary?.compressed ?? false,
- };
- const persistedRunWrite = await setRunStatusIfRunning(
- run.id,
- status,
- finalRunPatch,
- { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },
- );
- let persistedRun: typeof heartbeatRuns.$inferSelect | null =
- persistedRunWrite.run;
- if (!persistedRunWrite.updated) {
- persistedRun = null;
- // Native reconciliation can commit and project the terminal status in
- // the narrow window between adapter completion and this live write.
- // The status is authoritative, but it must not make us discard the
- // adapter's semantic result, usage, logs, or presentation decision.
- // Only complete the late metadata write when the reconciler chose the
- // same terminal status; a conflicting terminal outcome remains owned
- // by the path that won the compare-and-set. Owned legacy cancellation
- // likewise keeps the provider session, logs, and usage after Stop wins.
- if (
- (adapterResult.nativeFinalization ||
- (processCancellation && !processCancellation.failed && status === "cancelled")) &&
- persistedRunWrite.run?.status === status
- ) {
- persistedRun = await db
- .update(heartbeatRuns)
- .set({
- ...finalRunPatch,
- resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(
- cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,
- ),
-
- usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
- finishedAt:
- persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
- updatedAt: new Date(),
- })
- .where(
- and(
- eq(heartbeatRuns.id, run.id),
- eq(heartbeatRuns.status, status),
- ),
- )
- .returning()
- .then((rows) => rows[0] ?? null);
- }
- if (!persistedRun) {
- await accountRunCost(db, run.id, budgetHooks).catch((err) => {
- logger.error({ err, runId: run.id }, "Late run accounting queued for recovery");
- });
- logger.info(
- {
- runId: run.id,
- attemptedStatus: status,
- currentStatus: persistedRunWrite.run?.status ?? null,
- },
- "skipping late run finalization because the run already left running state",
- );
- return;
- }
- }
- if (persistedRun) {
- // Accounting recovery is independent of workspace or issue finalization.
- await accountRunCost(db, persistedRun.id, budgetHooks).catch((err) => {
- logger.error({ err, runId: run.id }, "Run accounting queued for recovery");
- });
- persistedRun =
- (await classifyAndPersistRunLiveness(
- persistedRun,
- persistedResultJson,
- )) ?? persistedRun;
- }
-
- await setWakeupStatus(
- run.wakeupRequestId,
- outcome === "succeeded" ? "completed" : status,
- {
- finishedAt: new Date(),
- error: runErrorMessage,
+ await runCompletion.completeRun({
+ run, agent, issueId, issueRef, signal: executionControl.controller.signal,
+ output: {
+ handle, outputProgressState, flushOutputProgress,
+ get stdoutExcerpt() { return stdoutExcerpt; },
+ get stderrExcerpt() { return stderrExcerpt; },
},
- );
-
- const finalizedRun = persistedRun ?? (await getRun(run.id));
- if (finalizedRun) {
- await appendRunEvent(finalizedRun, {
- eventType: "lifecycle",
- stream: "system",
- level: outcome === "succeeded" ? "info" : "error",
- message: `run ${outcome}`,
- payload: {
- status,
- exitCode: adapterResult.exitCode,
- ...(readRunCancellation(finalizedRun.resultJson) ? { cancellation: readRunCancellation(finalizedRun.resultJson) } : {}),
- },
- });
- try {
- await completeSkillTestRunForHeartbeatOutcome({
- run: finalizedRun,
- issueId,
- issueWorkMode: issueRef?.workMode ?? null,
- outcome,
- error: runErrorMessage,
- });
- } catch (err) {
- logger.warn(
- { err, runId: finalizedRun.id, issueId },
- "failed to complete skill test run after heartbeat finalization",
- );
- await onLog(
- "stderr",
- `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`,
- );
- }
- const livenessRun = finalizedRun;
- await refreshContinuationSummaryForRun(livenessRun, agent);
- const skipRunIssueComment =
- parseObject(livenessRun.contextSnapshot).skipIssueComment === true;
- let resolvedPresentationDecision: RunPresentationDecision | null =
- null;
- try {
- const existingRunComment = issueId
- ? await findRunIssueComment(
- livenessRun.id,
- livenessRun.companyId,
- issueId,
- persistedResultJson,
- )
- : null;
- const finalAgentMessage =
- await findLatestCompletedFinalAgentMessage(
- livenessRun.id,
- livenessRun.companyId,
- );
- const externalChatPresentationCandidate =
- isExternalChatPresentationContext(livenessRun.contextSnapshot) ||
- parseObject(livenessRun.contextSnapshot).source === "tool_action_review" ||
- String(parseObject(livenessRun.contextSnapshot).source ?? "").startsWith("issue.comment") ||
- parseObject(livenessRun.contextSnapshot).source === "issue.update";
- const externalChatPresentationAuthorization =
- issueId && externalChatPresentationCandidate
- ? await resolveChatRunPresentationAuthorizationReason(db, {
- companyId: livenessRun.companyId,
- issueId,
- runId: livenessRun.id,
- })
- : null;
- const externalChatPresentationContext = isExternalChatPresentationContext(
- livenessRun.contextSnapshot,
- externalChatPresentationAuthorization === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
- );
- const resolved = resolveHeartbeatRunResponse({
- resultJson: persistedResultJson,
- conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded"
- && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished",
- existingComment: existingRunComment,
- finalAgentMessage,
- preferFinalResponseOverExistingComment:
- externalChatPresentationContext,
- externalChatReviewResponseSummaryAuthorized:
- persistedResultJson?.finalizationReasonCode ===
- "governed_response_waiting" &&
- externalChatPresentationAuthorization ===
- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
- externalChatResponseWakeSummaryAuthorized:
- Boolean(adapterResult.nativeFinalization) &&
- externalChatPresentationAuthorization ===
- CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
- });
- let presentationDecision: RunPresentationDecision =
- resolved.decision;
-
- if (
- issueId &&
- !skipRunIssueComment &&
- presentationDecision.commentAction === "create" &&
- resolved.text
- ) {
- // The presentation resolver exposes only the final assistant
- // surface selected from completed final messages or accepted
- // semantic results. For an exactly bound external-chat run,
- // authorize that narrow presentation as the provider reply;
- // ordinary internal runs retain the private default.
- const presentationAuthorizationReason =
- await resolveChatRunPresentationAuthorizationReason(db, {
- companyId: livenessRun.companyId,
- issueId,
- runId: livenessRun.id,
- });
- const comment = await issuesSvc.addComment(
- issueId,
- resolved.text,
- { agentId: agent.id, runId: livenessRun.id },
- { authorizationReason: presentationAuthorizationReason, completionReply: true },
- );
- presentationDecision = {
- ...presentationDecision,
- commentId: comment.id,
- reasonCodes: [
- ...presentationDecision.reasonCodes,
- "resolved_response_materialized",
- ],
- };
- await logActivity(db, {
- companyId: livenessRun.companyId,
- actorType: "agent",
- actorId: agent.id,
- agentId: agent.id,
- runId: livenessRun.id,
- issueId,
- action: "issue.comment_added",
- entityType: "issue",
- entityId: issueId,
- details: {
- commentId: comment.id,
- bodySnippet: comment.body.slice(0, 120),
- identifier: issueRef?.identifier ?? null,
- issueTitle: issueRef?.title ?? null,
- authorizationReason: presentationAuthorizationReason,
- source: "run_presentation_resolver",
- presentationSource: presentationDecision.chosenSource,
- },
- });
- } else if (presentationDecision.commentAction === "create") {
- presentationDecision = {
- ...presentationDecision,
- commentAction: "none",
- reasonCodes: [
- ...presentationDecision.reasonCodes,
- skipRunIssueComment
- ? "issue_comment_suppressed"
- : "run_has_no_issue",
- ],
- };
- }
-
- await db
- .update(heartbeatRuns)
- .set({
- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ presentationDecision })}::jsonb`,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, livenessRun.id));
- await appendRunEvent(livenessRun, {
- eventType: "run.presentation.resolved",
- stream: "system",
- level: "info",
- message: "run presentation resolved",
- payload: { presentationDecision },
- });
- resolvedPresentationDecision = presentationDecision;
- } catch (err) {
- await onLog(
- "stderr",
- `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`,
- );
- }
- if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
- const policy = parseMaxTurnContinuationPolicy(agent);
- if (policy.enabled && policy.maxAttempts > 0) {
- await scheduleBoundedRetryForRun(livenessRun, agent, {
- retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
- wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,
- maxAttempts: policy.maxAttempts,
- delayMs: policy.delayMs,
- });
- } else {
- await appendRunEvent(livenessRun, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message:
- "Max-turn continuation suppressed because the policy is disabled",
- payload: {
- retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
- policy,
- },
- });
- }
- } else if (
- outcome === "failed" &&
- readTransientRecoveryContractFromRun(livenessRun)
- ) {
- await scheduleBoundedRetryForRun(livenessRun, agent);
- } else if (
- outcome === "failed" &&
- !(await legacyExecutionNeedsReconciliationWithEvidence(db, livenessRun))
- ) {
- await scheduleInteractionContinuationInfrastructureRetryIfEligible(
- livenessRun,
- agent,
- );
- }
- const issueCommentPolicyResult = await finalizeIssueCommentPolicy(
- livenessRun,
- agent,
- resolvedPresentationDecision,
- );
- const conversationSettled = await settleConversationTurn(db, livenessRun);
- await releaseIssueExecutionAndPromote(livenessRun, {
- suppressImmediateRecovery: conversationSettled ||
- readNonEmptyString(
- parseObject(livenessRun.contextSnapshot).goalControlRequestId,
- ) !== null ||
- parseObject(livenessRun.contextSnapshot)
- .resumeSessionGoalHeartbeat === true,
- });
- if (!conversationSettled) {
- await handleIssueReviewPathDisposition(livenessRun);
- if (livenessRun.runtimeMode !== "native") {
- await recovery.reconcileLegacyContinuation(livenessRun.id);
- } else {
- await handleRunLivenessContinuation(livenessRun);
- await handleSuccessfulRunHandoff(
- issueCommentPolicyResult.outcome === "retry_queued" ||
- issueCommentPolicyResult.outcome === "retry_exhausted"
- ? { ...livenessRun, issueCommentStatus: issueCommentPolicyResult.outcome }
- : livenessRun,
- agent,
- );
- }
- }
- if (
- outcome === "succeeded" &&
- issueId &&
- parseObject(adapterResult.resultJson).goalRolloverRequired === true
- ) {
- const rolloverProjection = await runnerGoalService(db).projection(
- livenessRun.companyId,
- issueId,
- agent.id,
- );
- if (rolloverProjection?.goal?.status === "active") {
- await enqueueWakeup(agent.id, {
- source: "automation",
- triggerDetail: "system",
- reason: "goal_control",
- payload: {
- issueId,
- intent: "goal_rollover",
- predecessorRunId: livenessRun.id,
- },
- idempotencyKey: `goal_rollover:${livenessRun.id}`,
- requestedByActorType: "system",
- contextSnapshot: {
- issueId,
- taskKey: issueId,
- resumeSessionGoalHeartbeat: true,
- skipIssueComment: true,
- goalRolloverFromRunId: livenessRun.id,
- },
- });
- }
- }
-
- // Dependency wake re-check: if this run's issue was marked done mid-run,
- // the route-time `issue_blockers_resolved` wake may have been gated by
- // workspace finalization or merged into this run. Reuse the level-triggered
- // dependency backstop so finalize and periodic recovery share idempotency,
- // readiness, active-path, and observability rules.
- if (issueId && finalizedRun) {
- try {
- const blockerIssueStatus = await db
- .select({ status: issues.status })
- .from(issues)
- .where(eq(issues.id, issueId))
- .then((rows) => rows[0]?.status ?? null);
- if (blockerIssueStatus === "done") {
- await recovery.reconcileResolvedDependencyWakeBackstop({
- runId: finalizedRun.id,
- companyId: finalizedRun.companyId,
- blockerIssueId: issueId,
- source: "workspace.finalize",
- });
- }
- } catch (finalizeWakeErr) {
- logger.warn(
- { err: finalizeWakeErr, runId: run.id, issueId },
- "failed to evaluate dependent wakes after workspace_finalize",
- );
- }
- }
- }
-
- if (finalizedRun) {
- await updateRuntimeState(
- agent,
- finalizedRun,
- adapterResult,
- {
- legacySessionId: nextSessionState.legacySessionId,
- },
- normalizedUsage,
- );
- if (taskKey) {
- if (
- adapterResult.clearSession ||
- (!nextSessionState.params && !nextSessionState.displayId)
- ) {
- await clearTaskSessions(agent.companyId, agent.id, {
- taskKey,
- adapterType: agent.adapterType,
- expectedRunId: finalizedRun.id,
- });
- } else {
- await upsertTaskSession({
- companyId: agent.companyId,
- agentId: agent.id,
- adapterType: agent.adapterType,
- taskKey,
- sessionParamsJson:
- attachPaperclipSessionMetadataToSessionParams(
- nextSessionState.params,
- configuredModel,
- sessionConfigMetadata,
- ),
- sessionDisplayId: nextSessionState.displayId,
- lastRunId: finalizedRun.id,
- lastError: runErrorMessage,
- });
- }
- }
- }
- await finalizeAgentStatus(agent.id, outcome, runErrorMessage, {
- keepIdleOnFailure:
- outcome === "failed" &&
- ((finalizedRun
- ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota"
- : runErrorCode === "provider_quota") ||
- isWorkspaceSyncConflictFailure(adapterResult.errorMessage)),
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ session: { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata },
+ readFailureReportSecrets,
+ adapterResult, sessionCodec, taskSessionForRun, sessionCompaction,
+ configFreshnessResultMetadata, runLedgerScope, currentUserRedactionOptions,
+ providerTraceCapture: providerTraceCapture !== null,
+ onProviderTraceFinalized: () => { providerTraceFinalized = true; },
+ issueContext, onLog,
});
} catch (err) {
await persistUsageCaptureFailure?.();
@@ -8412,226 +7625,17 @@ export function heartbeatService(
}
return;
}
- // A process adapter may throw while its owned Stop is joining the
- // child. Let the cancellation write settle before attempting failure.
- await processRunCancellationSettlements.get(run.id)?.settled;
- const message = redactCurrentUserText(
- identityRedactor.redact(err instanceof Error ? err.message : "Unknown adapter failure"),
- await getCurrentUserRedactionOptions(),
- );
- const workspaceValidationFailure = isWorkspaceValidationFailure(err)
- ? err
- : null;
- const configurationIncompleteFailure = isConfigurationIncompleteFailure(
- err,
- )
- ? err
- : null;
- const recordedResponsibleUserDenialCode =
- normalizeResponsibleUserDenialCode(
- (await getRun(run.id).catch(() => null))?.errorCode,
- );
- // The runtime resolution is scoped to the adapter try block. The
- // durable coordinator is also the stronger authority here: legacy
- // runs simply have no row, while native result-less exhaustion keeps
- // its named failure instead of being flattened to `adapter_failed`.
- const nativeTerminalFailureCode = await db
- .select({
- phase: nativeRunFinalizations.phase,
- resultId: nativeRunFinalizations.resultId,
- failureCode: nativeRunFinalizations.failureCode,
- })
- .from(nativeRunFinalizations)
- .where(eq(nativeRunFinalizations.runId, run.id))
- .limit(1)
- .then((rows) => {
- const coordinator = rows[0];
- return coordinator?.phase === "terminal_failure" &&
- coordinator.resultId === null
- ? coordinator.failureCode
- : null;
- })
- .catch(() => null);
- const failureErrorCode =
- workspaceValidationFailure?.code ??
- configurationIncompleteFailure?.code ??
- nonRetryablePreflightFailureCode(err) ??
- recordedResponsibleUserDenialCode ??
- nativeTerminalFailureCode ??
- "adapter_failed";
- logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId }, "heartbeat execution failed");
-
- let logSummary: {
- bytes: number | null;
- sha256?: string | null;
- compressed: boolean;
- } | null = null;
- if (handle) {
- try {
- logSummary = await runLogStore.finalize(handle);
- } catch (finalizeErr) {
- logger.warn(
- { err: finalizeErr, runId },
- "failed to finalize run log after error",
- );
- }
- }
- const finalLogBytes = logSummary?.bytes;
- if (outputProgressState.pending && typeof finalLogBytes === "number") {
- outputProgressState.pending.bytes = finalLogBytes;
- }
- await flushOutputProgress({ force: true }).catch((flushErr) => {
- logger.warn(
- { err: flushErr, runId },
- "failed to flush run output progress after error",
- );
- });
-
- const stoppedDuringFailure = executionControl.controller.signal.aborted;
- const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null;
- const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed";
- const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, {
- error: message,
- errorCode: stopSnapshot?.errorCode ?? failureErrorCode,
- finishedAt: new Date(),
- resultJson: mergeRunStopMetadataForAgent(agent, failureOutcome, {
- errorCode: failureErrorCode,
- errorMessage: message,
- resultJson: {
- ...parseObject(stopSnapshot?.resultJson),
- ...requiredWorkspaceRestoreEvidence,
- ...(workspaceValidationFailure?.resultJson ??
- configurationIncompleteFailure?.resultJson ??
- {}),
- ...(!legacyAdapterEntered && run.runtimeMode !== "native"
- ? {
- executionRecovery: {
- kind: "bootstrap",
- providerWorkStarted: false,
- },
- }
- : {}),
- },
- }),
- stdoutExcerpt,
- stderrExcerpt,
- logBytes: logSummary?.bytes,
- logSha256: logSummary?.sha256,
- logCompressed: logSummary?.compressed ?? false,
- }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });
- if (
- !failedRunWrite.updated &&
- !(
- nativeTerminalFailureCode && failedRunWrite.run?.status === "failed"
- )
- ) {
- logger.info(
- {
- runId: run.id,
- attemptedStatus: "failed",
- currentStatus: failedRunWrite.run?.status ?? null,
- },
- "skipping late adapter failure finalization because the run already left running state",
- );
- return;
- }
-
- const failedRun = failedRunWrite.run;
- await setWakeupStatus(run.wakeupRequestId, "failed", {
- finishedAt: new Date(),
- error: message,
- });
-
- if (failedRun) {
- await appendRunEvent(failedRun, {
- eventType: "error",
- stream: "system",
- level: "error",
- message,
- });
- const livenessRun =
- (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun;
- try {
- await completeSkillTestRunForHeartbeatOutcome({
- run: livenessRun,
- issueId,
- issueWorkMode: issueRef?.workMode ?? null,
- outcome: "failed",
- error: message,
- });
- } catch (err) {
- logger.warn(
- { err, runId: livenessRun.id, issueId },
- "failed to complete skill test run after heartbeat adapter failure",
- );
- }
- await refreshContinuationSummaryForRun(livenessRun, agent);
- if (
- !isWorkspaceValidationFailedRun(livenessRun) &&
- !isConfigurationIncompleteFailedRun(livenessRun)
- ) {
- await finalizeIssueCommentPolicy(livenessRun, agent);
- }
- await scheduleInteractionContinuationInfrastructureRetryIfEligible(
- livenessRun,
- agent,
- );
- await releaseIssueExecutionAndPromote(livenessRun, {
- // Native recovery owns the original heartbeat run through
- // exhaustion. Once its durable coordinator has classified a
- // terminal failure, generic issue recovery must not create a
- // replacement retryOfRunId chain for the same provider work.
- suppressImmediateRecovery: nativeTerminalFailureCode !== null,
- });
- await handleIssueReviewPathDisposition(livenessRun);
-
- await updateRuntimeState(
- agent,
- livenessRun,
- {
- exitCode: null,
- signal: null,
- timedOut: false,
- errorMessage: message,
- },
- {
- legacySessionId: runtimeForAdapter.sessionId,
- },
- );
-
- if (
- taskKey &&
- (goalCheckpointSession.current ||
- previousSessionParams ||
- previousSessionDisplayId ||
- taskSession)
- ) {
- await upsertTaskSession({
- companyId: agent.companyId,
- agentId: agent.id,
- adapterType: agent.adapterType,
- taskKey,
- sessionParamsJson:
- goalCheckpointSession.current?.params ??
- attachPaperclipSessionMetadataToSessionParams(
- previousSessionParams,
- configuredModel,
- sessionConfigMetadata,
- ),
- sessionDisplayId:
- goalCheckpointSession.current?.displayId ??
- previousSessionDisplayId,
- lastRunId: failedRun.id,
- lastError: message,
- });
- }
- }
-
- await finalizeAgentStatus(agent.id, "failed", message, {
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
- keepIdleOnFailure:
- Boolean(nonRetryablePreflightFailureCode(err)) ||
- isWorkspaceSyncConflictFailure(message),
+ await runCompletion.failRun({
+ run, agent, issueId, issueRef, signal: executionControl.controller.signal,
+ output: {
+ handle, outputProgressState, flushOutputProgress,
+ get stdoutExcerpt() { return stdoutExcerpt; },
+ get stderrExcerpt() { return stderrExcerpt; },
+ },
+ session: { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata },
+ readFailureReportSecrets,
+ runId, err, identityRedactor, requiredWorkspaceRestoreEvidence, legacyAdapterEntered,
+ goalCheckpointSession, previousSessionDisplayId, taskSession,
});
}
} catch (outerErr) {
@@ -8671,223 +7675,9 @@ export function heartbeatService(
},
);
} else {
- // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
- // The inner catch did not fire, so we must record the failure here.
- const message = redactCurrentUserText(
- identityRedactor.redact(outerErr instanceof Error
- ? outerErr.message
- : "Unknown setup failure"),
- await getCurrentUserRedactionOptions(),
- );
- // A missing secret/env binding is a known pre-dispatch configuration gap,
- // not an opaque setup crash. Surface it with its own errorCode so the
- // recovery path routes it to a human owner instead of looping retries.
- const workspaceValidationSetupFailure = isWorkspaceValidationFailure(
- outerErr,
- )
- ? outerErr
- : null;
- const configurationIncompleteSetupFailure =
- isConfigurationIncompleteFailure(outerErr) ? outerErr : null;
- const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(
- outerErr,
- )
- ? outerErr
- : null;
- // A sandbox provider plugin stuck in error/disabled/upgrade_pending
- // fails every lease the same way until an operator acts, so it is a
- // configuration gap, not a transient setup failure.
- const sandboxProviderPluginNotReadySetupFailure =
- parseSandboxProviderPluginNotReadyFailureMessage(
- outerErr instanceof Error ? outerErr.message : null,
- );
- const recordedResponsibleUserDenialCode =
- normalizeResponsibleUserDenialCode(
- (await getRun(runId).catch(() => null))?.errorCode,
- );
- const nonRetryablePreflightCode =
- nonRetryablePreflightFailureCode(outerErr);
- const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null;
- const setupFailureErrorCode =
- workspaceGitScanFailure?.code ??
- workspaceValidationSetupFailure?.code ??
- configurationIncompleteSetupFailure?.code ??
- (unresolvedBaseRefSetupFailure ||
- sandboxProviderPluginNotReadySetupFailure
- ? CONFIGURATION_INCOMPLETE_FAILURE_CODE
- : null) ??
- recordedResponsibleUserDenialCode ??
- nonRetryablePreflightCode ??
- "setup_failed";
- logger.error(
- { err: identityRedactor.redact({ message: outerErr instanceof Error ? outerErr.message : String(outerErr), stack: outerErr instanceof Error ? outerErr.stack : undefined }), runId },
- "heartbeat execution setup failed",
- );
- const setupFailureAgent = await getAgent(run.agentId).catch(() => null);
- // The structured failure payload drives the recovery notice and next
- // action, so it is persisted even when the agent lookup failed and the
- // agent-scoped stop metadata cannot be merged in.
- const setupFailureDetails =
- (workspaceGitScanFailure ? {
- workspaceGitScan: {
- code: workspaceGitScanFailure.code,
- phase: "workspace_setup",
- retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code),
- },
- } : null) ??
- workspaceValidationSetupFailure?.resultJson ??
- configurationIncompleteSetupFailure?.resultJson ??
- (unresolvedBaseRefSetupFailure
- ? buildUnresolvedWorkspaceBaseRefResultJson(
- run,
- unresolvedBaseRefSetupFailure,
- )
- : null) ??
- (sandboxProviderPluginNotReadySetupFailure
- ? buildSandboxProviderPluginNotReadyResultJson(
- run,
- sandboxProviderPluginNotReadySetupFailure,
- )
- : null);
- const connectionFailure = readGitConnectionFailure(outerErr);
- const setupFailureResultJson = {
- ...setupFailureDetails,
- ...(connectionFailure ? { connectionFailure } : {}),
- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
- };
- const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", {
- error: message,
- errorCode: setupFailureErrorCode,
- finishedAt: new Date(),
- ...(setupFailureAgent
- ? {
- resultJson: mergeRunStopMetadataForAgent(
- setupFailureAgent,
- "failed",
- {
- errorCode: setupFailureErrorCode,
- errorMessage: message,
- resultJson: setupFailureResultJson,
- },
- ),
- }
- : setupFailureResultJson
- ? { resultJson: setupFailureResultJson }
- : {}),
- }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));
- if (!setupFailureWrite.updated) {
- logger.info(
- {
- runId,
- attemptedStatus: "failed",
- currentStatus: setupFailureWrite.run?.status ?? null,
- },
- "skipping late setup failure finalization because the run already left running state",
- );
- } else {
- await setWakeupStatus(run.wakeupRequestId, "failed", {
- finishedAt: new Date(),
- error: message,
- }).catch(() => undefined);
- }
- const failedRun = await getRun(runId).catch(() => null);
- if (setupFailureWrite.updated && failedRun) {
- // Emit a run-log event so the failure is visible in the run timeline,
- // consistent with what the inner catch block does for adapter failures.
- await appendRunEvent(failedRun, {
- eventType: "error",
- stream: "system",
- level: "error",
- message,
- }).catch(() => undefined);
- const livenessRun = await classifyAndPersistRunLiveness(
- failedRun,
- ).catch(() => failedRun);
- const setupFailureIssueId = readNonEmptyString(
- parseObject(livenessRun.contextSnapshot).issueId,
- );
- if (setupFailureIssueId) {
- await completeSkillTestRunForHeartbeatOutcome({
- run: livenessRun,
- issueId: setupFailureIssueId,
- outcome: "failed",
- error: message,
- }).catch((completionErr) => {
- logger.warn(
- {
- err: completionErr,
- runId: livenessRun.id,
- issueId: setupFailureIssueId,
- },
- "failed to complete skill test run after heartbeat setup failure",
- );
- });
- }
- const failedAgent =
- setupFailureAgent ??
- (await getAgent(run.agentId).catch(() => null));
- if (failedAgent) {
- await refreshContinuationSummaryForRun(
- livenessRun,
- failedAgent,
- ).catch(() => undefined);
- if (
- !isWorkspaceValidationFailedRun(livenessRun) &&
- !isConfigurationIncompleteFailedRun(livenessRun)
- ) {
- await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(
- () => undefined,
- );
- }
- // No provider work began. Retry temporary host scan failures with
- // the existing durable failure budget, before releasing execution.
- // Generic recovery must not grant a second budget on exhaustion.
- await (isTransientWorkspaceGitScanCode(livenessRun.errorCode)
- ? scheduleBoundedRetryForRun(livenessRun, failedAgent)
- : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent)
- ).catch((retryError) => {
- logger.warn(
- { err: retryError, runId: livenessRun.id },
- "failed to schedule interaction continuation retry after setup failure",
- );
- });
- }
- await releaseIssueExecutionAndPromote(livenessRun, {
- suppressImmediateRecovery:
- readNonEmptyString(
- parseObject(livenessRun.contextSnapshot).goalControlRequestId,
- ) !== null ||
- parseObject(livenessRun.contextSnapshot)
- .resumeSessionGoalHeartbeat === true,
- }).catch((releaseError) => {
- logger.error(
- { err: releaseError, runId },
- "failed to release issue execution after heartbeat setup failure",
- );
- });
- await handleIssueReviewPathDisposition(livenessRun).catch(
- (reviewPathError) => {
- logger.error(
- { err: reviewPathError, runId },
- "failed to evaluate review-path disposition after heartbeat setup failure",
- );
- },
- );
- }
- // Ensure the agent is not left stuck in "running" if the setup-failure
- // path owned the terminal transition. If another path already finalized
- // the run, keep that terminal outcome authoritative.
- if (setupFailureWrite.updated) {
- await finalizeAgentStatus(run.agentId, "failed", message, {
- wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
- // Low-trust admission failures are task/principal preconditions,
- // not evidence that the immutable endpoint agent is unhealthy.
- // Keep the failed run and its safe provider refusal authoritative,
- // but return the agent to idle so clients do not also announce a
- // misleading agent-wide error for the same rejected chat turn.
- keepIdleOnFailure: Boolean(nonRetryablePreflightCode),
- }).catch(() => undefined);
- }
+ await runCompletion.failRunSetup({
+ run, runId, outerErr, identityRedactor, readFailureReportSecrets,
+ });
}
} finally {
await nativeInstructionReservation?.release().catch(error => logger.warn({ runId: run.id, err: error }, "Managed warm session preparation cleanup failed"));
diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts
new file mode 100644
index 0000000000..4388cd4e4e
--- /dev/null
+++ b/server/src/services/heartbeat/run-completion.test.ts
@@ -0,0 +1,288 @@
+import type { Db } from "@paperclipai/db";
+import { agents, agentWakeupRequests, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import * as usageReceipts from "../usage-receipts.js";
+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
+import { UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";
+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";
+import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js";
+import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";
+import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js";
+import { createHeartbeatRunCompletion, type CompleteHeartbeatRunInput, type FailHeartbeatRunInput, type HeartbeatRunCompletionDependencies } from "./run-completion.js";
+
+vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
+
+type Dependencies = HeartbeatRunCompletionDependencies;
+function callbacks(db: Db) {
+ const state = createHeartbeatRunState(db);
+ const lifecycleDeps = {
+ ...state, ...createHeartbeatRunPreparation(db),
+ issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() },
+ recovery: { escalateStrandedAssignedIssue: vi.fn() },
+ companySkills: { completeTestRunForIssue: vi.fn() },
+ budgets: { getInvocationBlock: vi.fn(async () => null) },
+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
+ enqueueWakeup: vi.fn(),
+ getCurrentUserRedactionOptions: vi.fn(async () => ({ enabled: false })),
+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),
+ emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {},
+ } satisfies HeartbeatLifecycleDependencies;
+ const lifecycle = createHeartbeatLifecycle(db, lifecycleDeps);
+ return {
+ ...state, ...lifecycle,
+ setRunStatusIfRunning: vi.fn(lifecycle.setRunStatusIfRunning),
+ setWakeupStatus: vi.fn(lifecycle.setWakeupStatus),
+ upsertTaskSession: vi.fn(state.upsertTaskSession),
+ clearTaskSessions: vi.fn(state.clearTaskSessions),
+ classifyAndPersistRunLiveness: vi.fn<Dependencies["classifyAndPersistRunLiveness"]>(async run => run),
+ completeSkillTestRunForHeartbeatOutcome: vi.fn<Dependencies["completeSkillTestRunForHeartbeatOutcome"]>(async () => null),
+ refreshContinuationSummaryForRun: vi.fn<Dependencies["refreshContinuationSummaryForRun"]>(async () => null),
+ findRunIssueComment: vi.fn<Dependencies["findRunIssueComment"]>(async () => null),
+ findLatestCompletedFinalAgentMessage: vi.fn<Dependencies["findLatestCompletedFinalAgentMessage"]>(async () => null),
+ finalizeIssueCommentPolicy: vi.fn<Dependencies["finalizeIssueCommentPolicy"]>(async () => ({ outcome: "not_applicable", queuedRun: null })),
+ handleIssueReviewPathDisposition: vi.fn<Dependencies["handleIssueReviewPathDisposition"]>(async () => undefined),
+ handleRunLivenessContinuation: vi.fn<Dependencies["handleRunLivenessContinuation"]>(),
+ handleSuccessfulRunHandoff: vi.fn<Dependencies["handleSuccessfulRunHandoff"]>(),
+ updateRuntimeState: vi.fn<Dependencies["updateRuntimeState"]>(),
+ scheduleBoundedRetryForRun: vi.fn<Dependencies["scheduleBoundedRetryForRun"]>(),
+ scheduleInteractionContinuationInfrastructureRetryIfEligible: vi.fn<Dependencies["scheduleInteractionContinuationInfrastructureRetryIfEligible"]>(async () => null),
+ timerClaimWasFirstHeartbeat: vi.fn<Dependencies["timerClaimWasFirstHeartbeat"]>(() => undefined),
+ releaseIssueExecutionAndPromote: vi.fn<Dependencies["releaseIssueExecutionAndPromote"]>(async () => undefined),
+ finalizeAgentStatus: vi.fn<Dependencies["finalizeAgentStatus"]>(async () => undefined),
+ enqueueWakeup: vi.fn<Dependencies["enqueueWakeup"]>(),
+ issuesSvc: { addComment: vi.fn<Dependencies["issuesSvc"]["addComment"]>() },
+ recovery: { reconcileLegacyContinuation: vi.fn<Dependencies["recovery"]["reconcileLegacyContinuation"]>(), reconcileResolvedDependencyWakeBackstop: vi.fn<Dependencies["recovery"]["reconcileResolvedDependencyWakeBackstop"]>() },
+ getCurrentUserRedactionOptions: lifecycleDeps.getCurrentUserRedactionOptions,
+ budgetHooks: {},
+ runLogStore: { finalize: vi.fn<Dependencies["runLogStore"]["finalize"]>(async () => ({ bytes: 123, sha256: "log-hash", compressed: true })) },
+ traceStore: { finalize: vi.fn<Dependencies["traceStore"]["finalize"]>() },
+ processRunCancellationSettlements: new Map<string, { settled: Promise<void>; failed: boolean }>(),
+ failedProcessRunCancellations: new Map<string, { settled: Promise<void>; failed: boolean }>(),
+ } satisfies Dependencies;
+}
+
+it("constructs completion handlers without querying or starting work", () => {
+ const access = vi.fn(() => { throw new Error("Unexpected database access"); });
+ const db = new Proxy({}, { get: access }) as Db;
+ const deps = callbacks(db);
+ createHeartbeatRunCompletion(db, deps);
+ expect(access).not.toHaveBeenCalled();
+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+describe.skipIf(!support.supported)("heartbeat run completion boundary", () => {
+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+ let db: Db;
+ beforeAll(async () => {
+ // Keep the accounting transaction real without reading provider receipt files.
+ vi.spyOn(usageReceipts, "indexPendingUsageReceipts").mockResolvedValue(new Map());
+ vi.spyOn(usageReceipts, "recoverPendingRunUsageReceipts").mockResolvedValue([]);
+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-completion");
+ db = createDb(database.connectionString);
+ await db.execute(sql`set client_min_messages = warning`);
+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+ afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.restoreAllMocks(); });
+ beforeEach(async () => { await db.execute(sql`truncate table companies restart identity cascade`); });
+
+ async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
+ const [company] = await db.insert(companies).values({ name: "Completion", issuePrefix: "CMP" }).returning();
+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Completing agent", adapterType: "process" }).returning();
+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "on_demand", status: "claimed" }).returning();
+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "legacy", ...patch }).returning();
+ const deps = callbacks(db);
+ const controller = new AbortController();
+ const input: CompleteHeartbeatRunInput = {
+ run, agent, issueId: null, issueRef: null, signal: controller.signal,
+ output: { handle: { store: "local_file", logRef: "test-log" }, outputProgressState: { pending: { bytes: 7 } }, flushOutputProgress: vi.fn(async () => {}), stdoutExcerpt: "answer", stderrExcerpt: "diagnostic" },
+ session: { runtimeForAdapter: { sessionId: null, sessionDisplayId: null }, previousSessionParams: null, taskKey: "completion-task", configuredModel: "test-model",
+ sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }),
+ },
+ readFailureReportSecrets: () => [],
+ adapterResult: { exitCode: 0, signal: null, timedOut: false, sessionId: "provider-session", sessionParams: { sessionId: "provider-session" }, usage: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8 }, usageBasis: "per_run", costUsd: 0.01, provider: "test", model: "test-model", resultJson: { finalResponse: "Completed the work." } },
+ sessionCodec: getAdapterSessionCodec(agent.adapterType), taskSessionForRun: null, sessionCompaction: { rotate: false, reason: null }, configFreshnessResultMetadata: { version: 1 }, runLedgerScope: {}, currentUserRedactionOptions: { enabled: false }, providerTraceCapture: true, onProviderTraceFinalized: vi.fn(), issueContext: null, onLog: vi.fn(async () => {}),
+ };
+ const completion = createHeartbeatRunCompletion(db, deps);
+ const failure = (err: unknown): FailHeartbeatRunInput => ({ ...input, runId: run.id, err, identityRedactor: createAgentIdentityRedactor("private-identity-material"), requiredWorkspaceRestoreEvidence: { workspaceRestoreFailure: { reason: "missing_base" } }, legacyAdapterEntered: true, goalCheckpointSession: { current: null }, previousSessionDisplayId: "previous-session", taskSession: null });
+ const setup = (outerErr: unknown) => ({ run, runId: run.id, outerErr, identityRedactor: createAgentIdentityRedactor(), readFailureReportSecrets: () => [] });
+ return { run, agent, wake, deps, controller, input, completion, failure, setup };
+ }
+ const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);
+
+ it("persists usage, accounting, final logs, presentation and the next task session", async () => {
+ const f = await fixture();
+ await f.completion.completeRun(f.input);
+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", sessionIdAfter: "provider-session", logBytes: 123, logSha256: "log-hash", logCompressed: true, stdoutExcerpt: "answer", costAccountingPending: false, costAccountedAt: expect.any(Date), usageJson: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8, usageSource: "per_run", configFreshness: { version: 1 } }, resultJson: { presentationDecision: { commentAction: "none" } } });
+ expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(1);
+ expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("provider-session");
+ expect(f.input.output.outputProgressState.pending?.bytes).toBe(123);
+ expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();
+ expect(f.deps.setWakeupStatus).toHaveBeenCalledWith(f.wake.id, "completed", expect.any(Object));
+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));
+ });
+
+ it.each([
+ { timedOut: true, expected: "timed_out", errorCode: "timeout" },
+ { signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },
+ { errorMessage: "provider quota", errorCode: "provider_quota", expected: "failed" },
+ ])("keeps adapter outcome classification ($expected, $errorCode)", async ({ expected, ...result }) => {
+ const f = await fixture();
+ Object.assign(f.input.adapterResult, result);
+ await f.completion.completeRun(f.input);
+ expect(await read(f.run.id)).toMatchObject({ status: expected, errorCode: result.errorCode });
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, expected, expect.any(String), expect.objectContaining({ keepIdleOnFailure: result.errorCode === "provider_quota" }));
+ });
+
+ it.each(["completeRun", "failRun"] as const)("waits for an owned Stop before %s can write a competing outcome", async operation => {
+ const f = await fixture();
+ let settle!: () => void;
+ const settled = new Promise<void>(resolve => { settle = resolve; });
+ f.deps.processRunCancellationSettlements.set(f.run.id, { settled, failed: false });
+ const completion = operation === "completeRun" ? f.completion.completeRun(f.input) : f.completion.failRun(f.failure(new Error("late adapter error")));
+ await Promise.resolve();
+ expect(f.deps.setRunStatusIfRunning).not.toHaveBeenCalled();
+ await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
+ f.controller.abort();
+ settle();
+ await completion;
+ expect((await read(f.run.id)).status).toBe("cancelled");
+ if (operation === "completeRun") {
+ expect((await read(f.run.id)).sessionIdAfter).toBe("provider-session");
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "cancelled", expect.any(String), expect.any(Object));
+ } else expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
+ });
+
+ it("does not publish or overwrite a conflicting terminal transition that wins the compare-and-set", async () => {
+ const f = await fixture();
+ const write = f.deps.setRunStatusIfRunning.getMockImplementation()!;
+ f.deps.setRunStatusIfRunning.mockImplementationOnce(async (...args) => {
+ await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: { winner: "Stop" } }).where(eq(heartbeatRuns.id, f.run.id));
+ return write(...args);
+ });
+ await f.completion.completeRun(f.input);
+ expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: { winner: "Stop" }, sessionIdAfter: null });
+ expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();
+ expect(f.deps.updateRuntimeState).not.toHaveBeenCalled();
+ expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();
+ expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
+ });
+
+ it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {
+ const finishedAt = new Date("2026-10-09T12:00:00Z");
+ const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });
+ f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
+ await f.completion.completeRun(f.input);
+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
+ expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();
+ });
+
+ it("reports trace completion before a later write throws and reads output after log finalization", async () => {
+ const f = await fixture();
+ let excerpt = "before finalize";
+ Object.defineProperty(f.input.output, "stdoutExcerpt", { get: () => excerpt });
+ f.deps.runLogStore.finalize.mockImplementationOnce(async () => { excerpt = "last provider output"; return { bytes: 99, compressed: false }; });
+ const writeError = new Error("write failed");
+ f.deps.setRunStatusIfRunning.mockRejectedValueOnce(writeError);
+ await expect(f.completion.completeRun(f.input)).rejects.toBe(writeError);
+ expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();
+ expect(f.deps.setRunStatusIfRunning).toHaveBeenCalledWith(f.run.id, "succeeded", expect.objectContaining({ stdoutExcerpt: "last provider output", logBytes: 99 }), expect.any(Object));
+ });
+
+ it("keeps a failed trace finalization eligible for the executor's cleanup retry", async () => {
+ const f = await fixture();
+ f.deps.traceStore.finalize.mockRejectedValueOnce(new Error("trace store unavailable"));
+ await f.completion.completeRun(f.input);
+ expect((await read(f.run.id)).status).toBe("succeeded");
+ expect(f.input.onProviderTraceFinalized).not.toHaveBeenCalled();
+ });
+
+ it("keeps completion and session persistence moving after a response comment fails", async () => {
+ const f = await fixture();
+ f.input.adapterResult.resultJson = { finalResponseDisposition: "final", finalResponse: "Completed the work." };
+ const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Report the result" }).returning();
+ f.input.issueId = issue.id;
+ f.input.issueRef = issue;
+ f.input.issueContext = await createHeartbeatRunPreparation(db).getIssueExecutionContext(f.run.companyId, issue.id);
+ f.deps.issuesSvc.addComment.mockRejectedValueOnce(new Error("comment unavailable"));
+ await f.completion.completeRun(f.input);
+ expect(f.deps.issuesSvc.addComment).toHaveBeenCalledOnce();
+ expect(f.input.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("comment unavailable"));
+ expect((await read(f.run.id)).status).toBe("succeeded");
+ expect(f.deps.upsertTaskSession).toHaveBeenCalledOnce();
+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
+ });
+
+ it("clears only the completed task session when the adapter requests a reset", async () => {
+ const f = await fixture();
+ f.input.adapterResult.clearSession = true;
+ await f.completion.completeRun(f.input);
+ expect(f.deps.clearTaskSessions).toHaveBeenCalledWith(f.run.companyId, f.agent.id, { taskKey: "completion-task", adapterType: "process", expectedRunId: f.run.id });
+ expect(f.deps.upsertTaskSession).not.toHaveBeenCalled();
+ });
+
+ it("redacts execution failures, retains restore evidence and resumes the latest checkpoint", async () => {
+ const f = await fixture();
+ const failure = f.failure(new Error("private-identity-material crashed"));
+ failure.goalCheckpointSession.current = { params: { sessionId: "checkpoint" }, displayId: "checkpoint" };
+ await f.completion.failRun(failure);
+ const run = await read(f.run.id);
+ expect(run).toMatchObject({ status: "failed", errorCode: "adapter_failed", logBytes: 123, resultJson: { workspaceRestoreFailure: { reason: "missing_base" } } });
+ expect(run.error).not.toContain("private-identity-material");
+ expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("checkpoint");
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
+ });
+
+ it("still records failure and releases issue ownership when log finalization and progress flushing fail", async () => {
+ const f = await fixture();
+ f.deps.runLogStore.finalize.mockRejectedValueOnce(new Error("log unavailable"));
+ f.input.output.flushOutputProgress = vi.fn(async () => { throw new Error("progress unavailable"); });
+ await f.completion.failRun(f.failure(new Error("adapter failed")));
+ expect((await read(f.run.id)).status).toBe("failed");
+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
+ });
+
+ it("keeps durable result-less native failure authoritative and suppresses generic replacement recovery", async () => {
+ const f = await fixture({ runtimeMode: "native", status: "failed", errorCode: "native_result_missing" });
+ const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Native recovery" }).returning();
+ await db.update(heartbeatRuns).set({ nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
+ await db.insert(nativeRunFinalizations).values({ runId: f.run.id, companyId: f.run.companyId, issueId: issue.id, phase: "terminal_failure", failureCode: "native_result_missing" });
+ await f.completion.failRun(f.failure(new Error("transport ended")));
+ expect((await read(f.run.id)).errorCode).toBe("native_result_missing");
+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), { suppressImmediateRecovery: true });
+ });
+
+ it("records a setup configuration gap without inventing provider work or applying comment policy", async () => {
+ const f = await fixture();
+ await f.completion.failRunSetup(f.setup(new ConfigurationIncompleteFailure("missing credential", { configurationIncomplete: { reason: "missing_binding" } })));
+ expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { reason: "missing_binding" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
+ expect(f.deps.finalizeIssueCommentPolicy).not.toHaveBeenCalled();
+ expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
+ expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
+ });
+
+ it("retains the default branch needed to repair an unresolved workspace base ref", async () => {
+ const f = await fixture();
+ const error = new UnresolvedWorkspaceBaseRefError({
+ requestedRef: "main", recoveryIdentityRef: "origin/main", attemptedRefs: ["origin/main"], defaultBranch: "master",
+ });
+ await f.completion.failRunSetup(f.setup(error));
+ expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {
+ configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "main", defaultBranch: "master", attemptedRefs: ["origin/main"], fingerprint: "workspace_base_ref:origin/main" },
+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
+ } });
+ });
+
+ it("leaves a winning terminal outcome and agent state untouched after a late setup error", async () => {
+ const f = await fixture({ status: "succeeded", resultJson: { winner: "completion" } });
+ await f.completion.failRunSetup(f.setup(new Error("late setup error")));
+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", resultJson: { winner: "completion" } });
+ expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();
+ expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();
+ expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts
new file mode 100644
index 0000000000..abda3c7f21
--- /dev/null
+++ b/server/src/services/heartbeat/run-completion.ts
@@ -0,0 +1,1460 @@
+import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";
+import { ChatControlRecoveryUnresolvedError } from "./queue.js";
+import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTransientWorkspaceGitScanCode } from "./retries.js";
+import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js";
+import { readGitConnectionFailure } from "../git-connection-failure.js";
+import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, ConfigurationIncompleteFailure } from "./run-preparation.js";
+import { isWorkspaceValidationFailedRun, isWorkspaceValidationFailure, attachPaperclipSessionMetadataToSessionParams } from "./workspaces.js";
+import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js";
+import { cancellationResultJson, readRunCancellation } from "../run-cancellation.js";
+import { accountRunCost } from "../run-cost-accounting.js";
+import { isConversation, settleConversationTurn } from "../agent-conversations.js";
+import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js";
+import { isWorkspaceGitScanError } from "../workspace-git-operation-scheduler.js";
+import { and, eq, sql } from "drizzle-orm";
+import { heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
+import { HttpError } from "../../errors.js";
+import { logger } from "../../middleware/logger.js";
+import { normalizeResponsibleUserDenialCode } from "../responsible-user-denial-run-outcomes.js";
+import { parseObject } from "../../adapters/utils.js";
+import { CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, isExternalChatPresentationContext, mergeHeartbeatRunResultJson, resolveHeartbeatRunResponse, type RunPresentationDecision } from "../heartbeat-run-summary.js";
+import { normalizeMaxTurnStopReason } from "../heartbeat-stop-metadata.js";
+import { CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";
+import { logActivity } from "../activity-log.js";
+import { isUnresolvedWorkspaceBaseRefError, readUnresolvedWorkspaceBaseRefDiagnostic, type UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";
+import { runnerGoalService } from "../runner-goals.js";
+import { resolveChatRunPresentationAuthorizationReason } from "../chat-run-publications.js";
+import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON } from "../recovery/stranded-notice.js";
+import { MAX_TURN_CONTINUATION_RETRY_REASON } from "../../modules/run-dispatch/index.js";
+import { redactCurrentUserText } from "../../log-redaction.js";
+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
+import type { createHeartbeatRunPreparation } from "./run-preparation.js";
+import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js";
+import type { createHeartbeatQueue } from "./queue.js";
+import type { createHeartbeatRunControl, HeartbeatRunControlDependencies } from "./run-control.js";
+import type { issueService } from "../issues.js";
+import type { recoveryService } from "../recovery/index.js";
+import type { BudgetServiceHooks } from "../budgets.js";
+import type { providerTraceStore } from "../provider-trace-store.js";
+import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
+import type { CurrentUserRedactionOptions } from "../../log-redaction.js";
+import type { AdapterExecutionResult } from "@paperclipai/adapter-utils";
+import type { getRunLogStore, RunLogHandle } from "../run-log-store.js";
+import type { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";
+import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js";
+import type { Db, agents } from "@paperclipai/db";
+
+type Run = typeof heartbeatRuns.$inferSelect;
+type Agent = typeof agents.$inferSelect;
+type RunState = ReturnType<typeof createHeartbeatRunState>;
+type IssueContext = Awaited<ReturnType<ReturnType<typeof createHeartbeatRunPreparation>["getIssueExecutionContext"]>> | null;
+type TaskSession = Awaited<ReturnType<RunState["getTaskSession"]>> | null;
+
+/** Completion persists terminal outcomes; the executor retains dispatch and cleanup ownership. */
+export interface HeartbeatRunCompletionDependencies extends Pick<RunState,
+ "getRun" | "getAgent" | "resolveNormalizedUsageForSession" | "clearTaskSessions" | "upsertTaskSession">,
+ Pick<ReturnType<typeof createHeartbeatLifecycle>,
+ "mergeRunStopMetadataForAgent" | "setRunStatusIfRunning" | "setWakeupStatus" | "appendRunEvent" |
+ "classifyAndPersistRunLiveness" | "completeSkillTestRunForHeartbeatOutcome" | "refreshContinuationSummaryForRun" |
+ "findRunIssueComment" | "findLatestCompletedFinalAgentMessage" | "parseMaxTurnContinuationPolicy" |
+ "finalizeIssueCommentPolicy" | "handleIssueReviewPathDisposition" | "handleRunLivenessContinuation" |
+ "handleSuccessfulRunHandoff" | "updateRuntimeState">,
+ Pick<ReturnType<typeof createHeartbeatRetries>,
+ "scheduleBoundedRetryForRun" | "scheduleInteractionContinuationInfrastructureRetryIfEligible" | "timerClaimWasFirstHeartbeat">,
+ Pick<ReturnType<typeof createHeartbeatRunControl>, "releaseIssueExecutionAndPromote">,
+ Pick<HeartbeatRetryDependencies, "finalizeAgentStatus">,
+ Pick<ReturnType<typeof createHeartbeatQueue>, "enqueueWakeup"> {
+ issuesSvc: Pick<ReturnType<typeof issueService>, "addComment">;
+ recovery: Pick<ReturnType<typeof recoveryService>, "reconcileLegacyContinuation" | "reconcileResolvedDependencyWakeBackstop">;
+ getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;
+ budgetHooks: BudgetServiceHooks;
+ runLogStore: Pick<ReturnType<typeof getRunLogStore>, "finalize">;
+ traceStore: Pick<ReturnType<typeof providerTraceStore>, "finalize">;
+ processRunCancellationSettlements: Pick<HeartbeatRunControlDependencies["processRunCancellationSettlements"], "get">;
+ failedProcessRunCancellations: HeartbeatRunCompletionDependencies["processRunCancellationSettlements"];
+}
+
+interface RunCompletionOutput {
+ handle: RunLogHandle | null;
+ outputProgressState: { pending: { bytes: number } | null };
+ flushOutputProgress: (options?: { force?: boolean }) => Promise<void>;
+ // Getters in the executor retain output that arrives while finalization awaits I/O.
+ readonly stdoutExcerpt: string;
+ readonly stderrExcerpt: string;
+}
+
+interface RunCompletionSession {
+ runtimeForAdapter: { sessionId: string | null; sessionDisplayId: string | null };
+ previousSessionParams: Record<string, unknown> | null;
+ taskKey: string | null;
+ configuredModel: string | null;
+ sessionConfigMetadata: EffectiveRunSessionConfigMetadata;
+}
+
+interface RunCompletionInput {
+ run: Run;
+ agent: Agent;
+ issueId: string | null;
+ issueRef: Pick<NonNullable<IssueContext>, "workMode" | "identifier" | "title"> | null;
+ signal: AbortSignal;
+ output: RunCompletionOutput;
+ session: RunCompletionSession;
+ readFailureReportSecrets: () => string[];
+}
+
+export interface CompleteHeartbeatRunInput extends RunCompletionInput {
+ adapterResult: AdapterExecutionResult;
+ sessionCodec: ReturnType<typeof getAdapterSessionCodec>;
+ taskSessionForRun: TaskSession;
+ sessionCompaction: Pick<Awaited<ReturnType<RunState["evaluateSessionCompaction"]>>, "rotate" | "reason">;
+ configFreshnessResultMetadata: Record<string, unknown>;
+ runLedgerScope: Record<string, unknown>;
+ currentUserRedactionOptions: CurrentUserRedactionOptions;
+ providerTraceCapture: boolean;
+ // Update the executor immediately: a later completion step can still throw.
+ onProviderTraceFinalized: () => void;
+ issueContext: IssueContext;
+ onLog: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
+}
+
+export interface FailHeartbeatRunInput extends RunCompletionInput {
+ runId: string;
+ err: unknown;
+ identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "redact">;
+ requiredWorkspaceRestoreEvidence: Record<string, unknown> | null;
+ legacyAdapterEntered: boolean;
+ goalCheckpointSession: { current: { params: Record<string, unknown>; displayId: string } | null };
+ previousSessionDisplayId: string | null;
+ taskSession: TaskSession;
+}
+
+export interface FailHeartbeatRunSetupInput {
+ run: Run;
+ runId: string;
+ outerErr: unknown;
+ identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "redact">;
+ readFailureReportSecrets: () => string[];
+}
+
+function nonRetryablePreflightFailureCode(error: unknown): string | null {
+ if (error instanceof ChatControlRecoveryUnresolvedError)
+ return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;
+ if (
+ error instanceof HttpError &&
+ error.status === 409 &&
+ parseObject(error.details).code === "chat_failed_run_retry_not_authorized"
+ ) {
+ return "chat_failed_run_retry_not_authorized";
+ }
+ if (!(error instanceof HttpError) || error.status !== 422) return null;
+ const code = readNonEmptyString(parseObject(error.details).code);
+ return code && NON_RETRYABLE_PREFLIGHT_FAILURE_CODES.has(code) ? code : null;
+}
+
+export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
+
+// Build the configuration-incomplete result payload for a workspace base ref
+// that never resolved to a commit. The setup catch maps this to errorCode
+// `configuration_incomplete`, so the recovery path routes it to a human owner
+// instead of a dispatched-then-failed run. The `fingerprint` uses the canonical
+// remote ref, not the operator spelling. Two equivalent spellings of one remote
+// branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated
+// failure reuses one active recovery action and does not reset the attempt
+// count or post a duplicate notice. A different branch makes a new action.
+// Build the configuration-incomplete result payload for a sandbox provider
+// plugin that is installed but not `ready`. The `fingerprint` is the plugin
+// key plus its status, so every run that hits the same stuck plugin reuses
+// one active recovery action instead of posting a fresh notice per attempt,
+// while a status change (say `error` -> `disabled`) makes a new one.
+function buildSandboxProviderPluginNotReadyResultJson(
+ run: typeof heartbeatRuns.$inferSelect,
+ failure: { provider: string; pluginKey: string; pluginStatus: string },
+): Record<string, unknown> {
+ const context = parseObject(run.contextSnapshot);
+ return {
+ configurationIncomplete: {
+ reason: SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON,
+ companyId: run.companyId,
+ agentId: run.agentId,
+ issueId: readNonEmptyString(context.issueId) ?? null,
+ projectId: readNonEmptyString(context.projectId) ?? null,
+ sandboxProvider: failure.provider,
+ pluginKey: failure.pluginKey,
+ pluginStatus: failure.pluginStatus,
+ fingerprint: `sandbox_provider_plugin:${failure.pluginKey}:${failure.pluginStatus}`,
+ missingBindings: [],
+ },
+ };
+}
+
+function buildUnresolvedWorkspaceBaseRefResultJson(
+ run: typeof heartbeatRuns.$inferSelect,
+ error: UnresolvedWorkspaceBaseRefError,
+): Record<string, unknown> {
+ const context = parseObject(run.contextSnapshot);
+ const diagnostic = readUnresolvedWorkspaceBaseRefDiagnostic(error);
+ return {
+ configurationIncomplete: {
+ reason: "workspace_base_ref_unresolved",
+ ...(diagnostic ? { baseRefDiagnostic: diagnostic } : {}),
+ companyId: run.companyId,
+ agentId: run.agentId,
+ issueId: readNonEmptyString(context.issueId) ?? null,
+ projectId: readNonEmptyString(context.projectId) ?? null,
+ requestedRef: error.requestedRef,
+ defaultBranch: error.defaultBranch,
+ attemptedRefs: error.attemptedRefs,
+ fetchError: error.fetchError,
+ fingerprint: `workspace_base_ref:${error.recoveryIdentityRef}`,
+ missingBindings: [],
+ },
+ };
+}
+
+function isMaxTurnExhaustionRun(
+ run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,
+) {
+ const resultJson = parseObject(run.resultJson);
+ return Boolean(
+ normalizeMaxTurnStopReason(resultJson.stopReason) ??
+ normalizeMaxTurnStopReason(run.errorCode),
+ );
+}
+
+// environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message,
+// e.g. 'Sandbox provider "kubernetes" is installed via plugin
+// "acme.kubernetes-sandbox-provider", but that plugin is currently error.'
+// The plugin row exists but its status is `error` (a failed activation),
+// `disabled` (an operator switched it off) or `upgrade_pending`. Unlike the
+// worker restart window above, nothing on the run path ever changes that
+// status: only an operator enabling the plugin, or a server boot that
+// re-activates a bundled plugin, does. Re-running the agent produces the
+// identical failure every time, so the setup catch classifies it as
+// `configuration_incomplete` (routed to a human owner) instead of a retryable
+// `setup_failed` that the scheduler would keep re-dispatching.
+const SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE =
+ /sandbox provider "([^"]*)" is installed via plugin "([^"]*)", but that plugin is currently (error|disabled|upgrade_pending)\b/i;
+
+export function parseSandboxProviderPluginNotReadyFailureMessage(
+ value: unknown,
+): { provider: string; pluginKey: string; pluginStatus: string } | null {
+ if (typeof value !== "string") return null;
+ const match = SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE.exec(value);
+ if (!match) return null;
+ return {
+ provider: match[1] ?? "",
+ pluginKey: match[2] ?? "",
+ pluginStatus: (match[3] ?? "").toLowerCase(),
+ };
+}
+
+function mergeAdapterRecoveryMetadata(input: {
+ resultJson: Record<string, unknown> | null | undefined;
+ errorFamily?: string | null;
+ retryNotBefore?: string | null;
+}) {
+ const errorFamily = readNonEmptyString(input.errorFamily);
+ const retryNotBefore = readNonEmptyString(input.retryNotBefore);
+ if (!input.resultJson && !errorFamily && !retryNotBefore)
+ return input.resultJson ?? null;
+
+ return {
+ ...(input.resultJson ?? {}),
+ ...(errorFamily ? { errorFamily } : {}),
+ ...(retryNotBefore
+ ? {
+ retryNotBefore,
+ transientRetryNotBefore: retryNotBefore,
+ ...(errorFamily === "provider_quota"
+ ? { providerQuotaRetryNotBefore: retryNotBefore }
+ : {}),
+ }
+ : {}),
+ };
+}
+
+function isConfigurationIncompleteFailure(
+ error: unknown,
+): error is ConfigurationIncompleteFailure {
+ return error instanceof ConfigurationIncompleteFailure;
+}
+
+export function isConfigurationIncompleteFailedRun(
+ run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode"> | null | undefined,
+) {
+ return (
+ run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE ||
+ run?.errorCode === "model_not_found"
+ );
+}
+
+/**
+ * Failure signatures from sandbox→host git workspace reconciliation. These
+ * describe the state of the SHARED workspace (divergent histories written by
+ * different runs), not a defect in the agent that happened to run last —
+ * putting the agent into a sticky `error` state over them removes a healthy
+ * agent from rotation while leaving the actual problem (the workspace)
+ * untouched. The run still fails and carries the full message.
+ */
+const WORKSPACE_SYNC_CONFLICT_SIGNATURES = [
+ "Failed to merge concurrent remote git histories",
+ "Failed to integrate concurrent remote git history",
+ "did not send all necessary objects",
+ "lacks these prerequisite commits",
+];
+
+export function isWorkspaceSyncConflictFailure(
+ message: string | null | undefined,
+): boolean {
+ if (!message) return false;
+ return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) =>
+ message.includes(signature),
+ );
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunCompletionDependencies) {
+ const {
+ getRun,
+ getAgent,
+ resolveNormalizedUsageForSession,
+ clearTaskSessions,
+ upsertTaskSession,
+ mergeRunStopMetadataForAgent,
+ setRunStatusIfRunning,
+ setWakeupStatus,
+ appendRunEvent,
+ classifyAndPersistRunLiveness,
+ completeSkillTestRunForHeartbeatOutcome,
+ refreshContinuationSummaryForRun,
+ findRunIssueComment,
+ findLatestCompletedFinalAgentMessage,
+ parseMaxTurnContinuationPolicy,
+ finalizeIssueCommentPolicy,
+ handleIssueReviewPathDisposition,
+ handleRunLivenessContinuation,
+ handleSuccessfulRunHandoff,
+ updateRuntimeState,
+ scheduleBoundedRetryForRun,
+ scheduleInteractionContinuationInfrastructureRetryIfEligible,
+ timerClaimWasFirstHeartbeat,
+ releaseIssueExecutionAndPromote,
+ finalizeAgentStatus,
+ enqueueWakeup,
+ issuesSvc,
+ recovery,
+ getCurrentUserRedactionOptions,
+ budgetHooks,
+ runLogStore,
+ traceStore,
+ processRunCancellationSettlements,
+ failedProcessRunCancellations,
+ } = dependencies;
+
+ async function completeRun(input: CompleteHeartbeatRunInput) {
+ const { run, agent, issueId, issueRef, signal, output, session, readFailureReportSecrets, adapterResult, sessionCodec, taskSessionForRun, sessionCompaction,
+ configFreshnessResultMetadata, runLedgerScope, currentUserRedactionOptions,
+ providerTraceCapture, onProviderTraceFinalized, issueContext, onLog } = input;
+ const { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata } = session;
+ const { handle, outputProgressState, flushOutputProgress } = output;
+ const processCancellation =
+ processRunCancellationSettlements.get(run.id) ??
+ failedProcessRunCancellations.get(run.id);
+ await processCancellation?.settled;
+ let outcome: RunSessionOutcome;
+ const latestRun = await getRun(run.id);
+ if (isHeartbeatRunTerminalStatus(latestRun?.status)) {
+ outcome = latestRun.status;
+ } else if (signal.aborted) {
+ outcome = "cancelled";
+ } else if (adapterResult.nativeFinalization) {
+ const nativeTerminal =
+ adapterResult.nativeFinalization.terminal.runTerminalState;
+ outcome =
+ nativeTerminal === "succeeded"
+ ? "succeeded"
+ : nativeTerminal === "cancelled"
+ ? "cancelled"
+ : "failed";
+ } else if (adapterResult.timedOut) {
+ outcome = "timed_out";
+ } else if (adapterResult.resultJson?.status === "cancelled") {
+ outcome = "cancelled";
+ } else if (
+ (adapterResult.exitCode ?? 0) === 0 &&
+ !adapterResult.errorMessage &&
+ !adapterResult.signal &&
+ !processCancellation?.failed
+ ) {
+ outcome = "succeeded";
+ } else {
+ outcome = "failed";
+ }
+
+ const nextSessionState = resolveNextSessionState({
+ adapterType: agent.adapterType,
+ codec: sessionCodec,
+ adapterResult,
+ outcome,
+ previousParams: previousSessionParams,
+ previousDisplayId: runtimeForAdapter.sessionDisplayId,
+ previousLegacySessionId: runtimeForAdapter.sessionId,
+ });
+ const rawUsage = normalizeUsageTotals(adapterResult.usage);
+ const sessionUsageResolution = await resolveNormalizedUsageForSession({
+ agentId: agent.id,
+ runId: run.id,
+ sessionId:
+ nextSessionState.displayId ?? nextSessionState.legacySessionId,
+ rawUsage,
+ usageBasis: adapterResult.usageBasis ?? null,
+ });
+ const normalizedUsage = sessionUsageResolution.normalizedUsage;
+ const runErrorMessage =
+ outcome === "cancelled"
+ ? redactCurrentUserText(latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled", currentUserRedactionOptions)
+ : outcome === "succeeded"
+ ? null
+ : redactCurrentUserText(
+ adapterResult.errorMessage ??
+ (outcome === "timed_out" ? "Timed out" : "Adapter failed"),
+ currentUserRedactionOptions,
+ );
+ const recordedResponsibleUserDenialCode =
+ normalizeResponsibleUserDenialCode(latestRun?.errorCode);
+ const runErrorCode =
+ outcome === "timed_out"
+ ? "timeout"
+ : outcome === "cancelled"
+ ? (latestRun?.errorCode ?? "cancelled")
+ : outcome === "failed"
+ ? (adapterResult.errorCode ??
+ recordedResponsibleUserDenialCode ??
+ "adapter_failed")
+ : null;
+
+ let logSummary: {
+ bytes: number | null;
+ sha256?: string | null;
+ compressed: boolean;
+ } | null = null;
+ if (handle) {
+ logSummary = await runLogStore.finalize(handle);
+ }
+ const finalLogBytes = logSummary?.bytes;
+ if (outputProgressState.pending && typeof finalLogBytes === "number") {
+ outputProgressState.pending.bytes = finalLogBytes;
+ }
+ await flushOutputProgress({ force: true });
+
+ if (providerTraceCapture) {
+ try {
+ await traceStore.finalize(run.id, run.companyId);
+ onProviderTraceFinalized();
+ } catch (error) {
+ logger.warn(
+ { error, runId: run.id },
+ "provider trace finalization failed without affecting run outcome",
+ );
+ }
+ }
+
+ const status =
+ outcome === "succeeded"
+ ? "succeeded"
+ : outcome === "cancelled"
+ ? "cancelled"
+ : outcome === "timed_out"
+ ? "timed_out"
+ : "failed";
+
+ const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null
+ ? null : resolveCacheAdjustedCostUsd(adapterResult);
+ const usageJson: Record<string, unknown> = {
+ accountingReceiptReady: adapterResult.usageComplete !== false,
+ costUsdExact: adapterResult.costUsdExact ?? null,
+ providerRequestId: adapterResult.providerRequestId ?? null,
+ ...(normalizedUsage ?? {}),
+ ...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}),
+ ...(rawUsage
+ ? {
+ rawInputTokens: rawUsage.inputTokens,
+ rawInputIncludesCached: false,
+ rawCachedInputTokens: rawUsage.cachedInputTokens,
+ rawOutputTokens: rawUsage.outputTokens,
+ }
+ : {}),
+ ...(sessionUsageResolution.derivedFromSessionTotals
+ ? { usageSource: "session_delta" }
+ : adapterResult.usageBasis === "per_run"
+ ? { usageSource: "per_run" }
+ : {}),
+ ...((nextSessionState.displayId ??
+ nextSessionState.legacySessionId)
+ ? {
+ persistedSessionId:
+ nextSessionState.displayId ??
+ nextSessionState.legacySessionId,
+ }
+ : {}),
+ sessionReused:
+ runtimeForAdapter.sessionId != null ||
+ runtimeForAdapter.sessionDisplayId != null,
+ taskSessionReused: taskSessionForRun != null,
+ freshSession:
+ runtimeForAdapter.sessionId == null &&
+ runtimeForAdapter.sessionDisplayId == null,
+ sessionRotated: sessionCompaction.rotate,
+ sessionRotationReason: sessionCompaction.reason,
+ configFreshness: configFreshnessResultMetadata,
+ provider:
+ readNonEmptyString(adapterResult.provider) ?? "unknown",
+ biller: resolveLedgerBiller(adapterResult),
+ model: readNonEmptyString(adapterResult.model) ?? "unknown",
+ ...(adapterResult.costUsd != null
+ ? { costUsd: adapterResult.costUsd }
+ : {}),
+ ...(cacheAdjustedCostUsd != null
+ ? { cacheAdjustedCostUsd }
+ : {}),
+ pricingProvenance: adapterResult.pricingProvenance,
+ costStatus: adapterResult.costStatus ?? resolveLedgerCostStatus({
+ costUsd: cacheAdjustedCostUsd ?? (adapterResult.costUsdExact != null ? Number(adapterResult.costUsdExact) : null),
+ billingType: normalizeLedgerBillingType(adapterResult.billingType),
+ inputTokens: normalizedUsage?.inputTokens ?? 0,
+ cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0,
+ outputTokens: normalizedUsage?.outputTokens ?? 0,
+ }),
+ billingType: normalizeLedgerBillingType(
+ adapterResult.billingType,
+ ),
+ };
+
+ const persistedResultJson = cancellationResultJson(latestRun ?? run, outcome, mergeHeartbeatRunResultJson(
+ mergeRunStopMetadataForAgent(agent, outcome, {
+ resultJson: mergeAdapterRecoveryMetadata({
+ resultJson: {
+ ...(adapterResult.nativeFinalization || outcome === "cancelled"
+ ? parseObject(latestRun?.resultJson)
+ : {}),
+ ...parseObject(adapterResult.resultJson),
+ ...(adapterResult.executionRecovery
+ ? { executionRecovery: adapterResult.executionRecovery }
+ : {}),
+ configFreshness: configFreshnessResultMetadata,
+ },
+ errorFamily: adapterResult.errorFamily ?? null,
+ retryNotBefore: adapterResult.retryNotBefore ?? null,
+ }),
+ errorCode: runErrorCode,
+ errorMessage: runErrorMessage,
+ }),
+ adapterResult.summary ?? null,
+ ), runErrorCode, runErrorMessage);
+
+ const ledgerScope = runLedgerScope;
+ const finalRunPatch: Partial<typeof heartbeatRuns.$inferInsert> = {
+ // Accounting must acknowledge even a proven pre-provider failure:
+ // that transaction releases the reservation without creating a charge.
+ costAccountingPending: true,
+ finishedAt: new Date(),
+ error: runErrorMessage,
+ errorCode: runErrorCode,
+ exitCode: adapterResult.exitCode,
+ signal: adapterResult.signal,
+ usageJson: { ...usageJson, ledgerScope },
+ resultJson: persistedResultJson,
+ sessionIdAfter:
+ nextSessionState.displayId ?? nextSessionState.legacySessionId,
+ stdoutExcerpt: output.stdoutExcerpt,
+ stderrExcerpt: output.stderrExcerpt,
+ logBytes: logSummary?.bytes,
+ logSha256: logSummary?.sha256,
+ logCompressed: logSummary?.compressed ?? false,
+ };
+ const persistedRunWrite = await setRunStatusIfRunning(
+ run.id,
+ status,
+ finalRunPatch,
+ { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },
+ );
+ let persistedRun: typeof heartbeatRuns.$inferSelect | null =
+ persistedRunWrite.run;
+ if (!persistedRunWrite.updated) {
+ persistedRun = null;
+ // Native reconciliation can commit and project the terminal status in
+ // the narrow window between adapter completion and this live write.
+ // The status is authoritative, but it must not make us discard the
+ // adapter's semantic result, usage, logs, or presentation decision.
+ // Only complete the late metadata write when the reconciler chose the
+ // same terminal status; a conflicting terminal outcome remains owned
+ // by the path that won the compare-and-set. Owned legacy cancellation
+ // likewise keeps the provider session, logs, and usage after Stop wins.
+ if (
+ (adapterResult.nativeFinalization ||
+ (processCancellation && !processCancellation.failed && status === "cancelled")) &&
+ persistedRunWrite.run?.status === status
+ ) {
+ persistedRun = await db
+ .update(heartbeatRuns)
+ .set({
+ ...finalRunPatch,
+ resultJson: preserveWorkspaceRestoreRecoveryMetadataSql(
+ cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,
+ ),
+
+ usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
+ finishedAt:
+ persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, run.id),
+ eq(heartbeatRuns.status, status),
+ ),
+ )
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ }
+ if (!persistedRun) {
+ await accountRunCost(db, run.id, budgetHooks).catch((err) => {
+ logger.error({ err, runId: run.id }, "Late run accounting queued for recovery");
+ });
+ logger.info(
+ {
+ runId: run.id,
+ attemptedStatus: status,
+ currentStatus: persistedRunWrite.run?.status ?? null,
+ },
+ "skipping late run finalization because the run already left running state",
+ );
+ return;
+ }
+ }
+ if (persistedRun) {
+ // Accounting recovery is independent of workspace or issue finalization.
+ await accountRunCost(db, persistedRun.id, budgetHooks).catch((err) => {
+ logger.error({ err, runId: run.id }, "Run accounting queued for recovery");
+ });
+ persistedRun =
+ (await classifyAndPersistRunLiveness(
+ persistedRun,
+ persistedResultJson,
+ )) ?? persistedRun;
+ }
+
+ await setWakeupStatus(
+ run.wakeupRequestId,
+ outcome === "succeeded" ? "completed" : status,
+ {
+ finishedAt: new Date(),
+ error: runErrorMessage,
+ },
+ );
+
+ const finalizedRun = persistedRun ?? (await getRun(run.id));
+ if (finalizedRun) {
+ await appendRunEvent(finalizedRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: outcome === "succeeded" ? "info" : "error",
+ message: `run ${outcome}`,
+ payload: {
+ status,
+ exitCode: adapterResult.exitCode,
+ ...(readRunCancellation(finalizedRun.resultJson) ? { cancellation: readRunCancellation(finalizedRun.resultJson) } : {}),
+ },
+ });
+ try {
+ await completeSkillTestRunForHeartbeatOutcome({
+ run: finalizedRun,
+ issueId,
+ issueWorkMode: issueRef?.workMode ?? null,
+ outcome,
+ error: runErrorMessage,
+ });
+ } catch (err) {
+ logger.warn(
+ { err, runId: finalizedRun.id, issueId },
+ "failed to complete skill test run after heartbeat finalization",
+ );
+ await onLog(
+ "stderr",
+ `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`,
+ );
+ }
+ const livenessRun = finalizedRun;
+ await refreshContinuationSummaryForRun(livenessRun, agent);
+ const skipRunIssueComment =
+ parseObject(livenessRun.contextSnapshot).skipIssueComment === true;
+ let resolvedPresentationDecision: RunPresentationDecision | null =
+ null;
+ try {
+ const existingRunComment = issueId
+ ? await findRunIssueComment(
+ livenessRun.id,
+ livenessRun.companyId,
+ issueId,
+ persistedResultJson,
+ )
+ : null;
+ const finalAgentMessage =
+ await findLatestCompletedFinalAgentMessage(
+ livenessRun.id,
+ livenessRun.companyId,
+ );
+ const externalChatPresentationCandidate =
+ isExternalChatPresentationContext(livenessRun.contextSnapshot) ||
+ parseObject(livenessRun.contextSnapshot).source === "tool_action_review" ||
+ String(parseObject(livenessRun.contextSnapshot).source ?? "").startsWith("issue.comment") ||
+ parseObject(livenessRun.contextSnapshot).source === "issue.update";
+ const externalChatPresentationAuthorization =
+ issueId && externalChatPresentationCandidate
+ ? await resolveChatRunPresentationAuthorizationReason(db, {
+ companyId: livenessRun.companyId,
+ issueId,
+ runId: livenessRun.id,
+ })
+ : null;
+ const externalChatPresentationContext = isExternalChatPresentationContext(
+ livenessRun.contextSnapshot,
+ externalChatPresentationAuthorization === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
+ );
+ const resolved = resolveHeartbeatRunResponse({
+ resultJson: persistedResultJson,
+ conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded"
+ && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished",
+ existingComment: existingRunComment,
+ finalAgentMessage,
+ preferFinalResponseOverExistingComment:
+ externalChatPresentationContext,
+ externalChatReviewResponseSummaryAuthorized:
+ persistedResultJson?.finalizationReasonCode ===
+ "governed_response_waiting" &&
+ externalChatPresentationAuthorization ===
+ CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
+ externalChatResponseWakeSummaryAuthorized:
+ Boolean(adapterResult.nativeFinalization) &&
+ externalChatPresentationAuthorization ===
+ CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
+ });
+ let presentationDecision: RunPresentationDecision =
+ resolved.decision;
+
+ if (
+ issueId &&
+ !skipRunIssueComment &&
+ presentationDecision.commentAction === "create" &&
+ resolved.text
+ ) {
+ // The presentation resolver exposes only the final assistant
+ // surface selected from completed final messages or accepted
+ // semantic results. For an exactly bound external-chat run,
+ // authorize that narrow presentation as the provider reply;
+ // ordinary internal runs retain the private default.
+ const presentationAuthorizationReason =
+ await resolveChatRunPresentationAuthorizationReason(db, {
+ companyId: livenessRun.companyId,
+ issueId,
+ runId: livenessRun.id,
+ });
+ const comment = await issuesSvc.addComment(
+ issueId,
+ resolved.text,
+ { agentId: agent.id, runId: livenessRun.id },
+ { authorizationReason: presentationAuthorizationReason, completionReply: true },
+ );
+ presentationDecision = {
+ ...presentationDecision,
+ commentId: comment.id,
+ reasonCodes: [
+ ...presentationDecision.reasonCodes,
+ "resolved_response_materialized",
+ ],
+ };
+ await logActivity(db, {
+ companyId: livenessRun.companyId,
+ actorType: "agent",
+ actorId: agent.id,
+ agentId: agent.id,
+ runId: livenessRun.id,
+ issueId,
+ action: "issue.comment_added",
+ entityType: "issue",
+ entityId: issueId,
+ details: {
+ commentId: comment.id,
+ bodySnippet: comment.body.slice(0, 120),
+ identifier: issueRef?.identifier ?? null,
+ issueTitle: issueRef?.title ?? null,
+ authorizationReason: presentationAuthorizationReason,
+ source: "run_presentation_resolver",
+ presentationSource: presentationDecision.chosenSource,
+ },
+ });
+ } else if (presentationDecision.commentAction === "create") {
+ presentationDecision = {
+ ...presentationDecision,
+ commentAction: "none",
+ reasonCodes: [
+ ...presentationDecision.reasonCodes,
+ skipRunIssueComment
+ ? "issue_comment_suppressed"
+ : "run_has_no_issue",
+ ],
+ };
+ }
+
+ await db
+ .update(heartbeatRuns)
+ .set({
+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ presentationDecision })}::jsonb`,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, livenessRun.id));
+ await appendRunEvent(livenessRun, {
+ eventType: "run.presentation.resolved",
+ stream: "system",
+ level: "info",
+ message: "run presentation resolved",
+ payload: { presentationDecision },
+ });
+ resolvedPresentationDecision = presentationDecision;
+ } catch (err) {
+ await onLog(
+ "stderr",
+ `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`,
+ );
+ }
+ if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) {
+ const policy = parseMaxTurnContinuationPolicy(agent);
+ if (policy.enabled && policy.maxAttempts > 0) {
+ await scheduleBoundedRetryForRun(livenessRun, agent, {
+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
+ wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON,
+ maxAttempts: policy.maxAttempts,
+ delayMs: policy.delayMs,
+ });
+ } else {
+ await appendRunEvent(livenessRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message:
+ "Max-turn continuation suppressed because the policy is disabled",
+ payload: {
+ retryReason: MAX_TURN_CONTINUATION_RETRY_REASON,
+ policy,
+ },
+ });
+ }
+ } else if (
+ outcome === "failed" &&
+ readTransientRecoveryContractFromRun(livenessRun)
+ ) {
+ await scheduleBoundedRetryForRun(livenessRun, agent);
+ } else if (
+ outcome === "failed" &&
+ !(await legacyExecutionNeedsReconciliationWithEvidence(db, livenessRun))
+ ) {
+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(
+ livenessRun,
+ agent,
+ );
+ }
+ const issueCommentPolicyResult = await finalizeIssueCommentPolicy(
+ livenessRun,
+ agent,
+ resolvedPresentationDecision,
+ );
+ const conversationSettled = await settleConversationTurn(db, livenessRun);
+ await releaseIssueExecutionAndPromote(livenessRun, {
+ suppressImmediateRecovery: conversationSettled ||
+ readNonEmptyString(
+ parseObject(livenessRun.contextSnapshot).goalControlRequestId,
+ ) !== null ||
+ parseObject(livenessRun.contextSnapshot)
+ .resumeSessionGoalHeartbeat === true,
+ });
+ if (!conversationSettled) {
+ await handleIssueReviewPathDisposition(livenessRun);
+ if (livenessRun.runtimeMode !== "native") {
+ await recovery.reconcileLegacyContinuation(livenessRun.id);
+ } else {
+ await handleRunLivenessContinuation(livenessRun);
+ await handleSuccessfulRunHandoff(
+ issueCommentPolicyResult.outcome === "retry_queued" ||
+ issueCommentPolicyResult.outcome === "retry_exhausted"
+ ? { ...livenessRun, issueCommentStatus: issueCommentPolicyResult.outcome }
+ : livenessRun,
+ agent,
+ );
+ }
+ }
+ if (
+ outcome === "succeeded" &&
+ issueId &&
+ parseObject(adapterResult.resultJson).goalRolloverRequired === true
+ ) {
+ const rolloverProjection = await runnerGoalService(db).projection(
+ livenessRun.companyId,
+ issueId,
+ agent.id,
+ );
+ if (rolloverProjection?.goal?.status === "active") {
+ await enqueueWakeup(agent.id, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: "goal_control",
+ payload: {
+ issueId,
+ intent: "goal_rollover",
+ predecessorRunId: livenessRun.id,
+ },
+ idempotencyKey: `goal_rollover:${livenessRun.id}`,
+ requestedByActorType: "system",
+ contextSnapshot: {
+ issueId,
+ taskKey: issueId,
+ resumeSessionGoalHeartbeat: true,
+ skipIssueComment: true,
+ goalRolloverFromRunId: livenessRun.id,
+ },
+ });
+ }
+ }
+
+ // Dependency wake re-check: if this run's issue was marked done mid-run,
+ // the route-time `issue_blockers_resolved` wake may have been gated by
+ // workspace finalization or merged into this run. Reuse the level-triggered
+ // dependency backstop so finalize and periodic recovery share idempotency,
+ // readiness, active-path, and observability rules.
+ if (issueId && finalizedRun) {
+ try {
+ const blockerIssueStatus = await db
+ .select({ status: issues.status })
+ .from(issues)
+ .where(eq(issues.id, issueId))
+ .then((rows) => rows[0]?.status ?? null);
+ if (blockerIssueStatus === "done") {
+ await recovery.reconcileResolvedDependencyWakeBackstop({
+ runId: finalizedRun.id,
+ companyId: finalizedRun.companyId,
+ blockerIssueId: issueId,
+ source: "workspace.finalize",
+ });
+ }
+ } catch (finalizeWakeErr) {
+ logger.warn(
+ { err: finalizeWakeErr, runId: run.id, issueId },
+ "failed to evaluate dependent wakes after workspace_finalize",
+ );
+ }
+ }
+ }
+
+ if (finalizedRun) {
+ await updateRuntimeState(
+ agent,
+ finalizedRun,
+ adapterResult,
+ {
+ legacySessionId: nextSessionState.legacySessionId,
+ },
+ normalizedUsage,
+ );
+ if (taskKey) {
+ if (
+ adapterResult.clearSession ||
+ (!nextSessionState.params && !nextSessionState.displayId)
+ ) {
+ await clearTaskSessions(agent.companyId, agent.id, {
+ taskKey,
+ adapterType: agent.adapterType,
+ expectedRunId: finalizedRun.id,
+ });
+ } else {
+ await upsertTaskSession({
+ companyId: agent.companyId,
+ agentId: agent.id,
+ adapterType: agent.adapterType,
+ taskKey,
+ sessionParamsJson:
+ attachPaperclipSessionMetadataToSessionParams(
+ nextSessionState.params,
+ configuredModel,
+ sessionConfigMetadata,
+ ),
+ sessionDisplayId: nextSessionState.displayId,
+ lastRunId: finalizedRun.id,
+ lastError: runErrorMessage,
+ });
+ }
+ }
+ }
+ await finalizeAgentStatus(agent.id, outcome, runErrorMessage, {
+ keepIdleOnFailure:
+ outcome === "failed" &&
+ ((finalizedRun
+ ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota"
+ : runErrorCode === "provider_quota") ||
+ isWorkspaceSyncConflictFailure(adapterResult.errorMessage)),
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ });
+ }
+
+ async function failRun(input: FailHeartbeatRunInput) {
+ const { run, agent, issueId, issueRef, signal, output, session, readFailureReportSecrets, runId, err, identityRedactor, requiredWorkspaceRestoreEvidence,
+ legacyAdapterEntered, goalCheckpointSession, previousSessionDisplayId, taskSession } = input;
+ const { runtimeForAdapter, previousSessionParams, taskKey, configuredModel, sessionConfigMetadata } = session;
+ const { handle, outputProgressState, flushOutputProgress } = output;
+ // A process adapter may throw while its owned Stop is joining the
+ // child. Let the cancellation write settle before attempting failure.
+ await processRunCancellationSettlements.get(run.id)?.settled;
+ const message = redactCurrentUserText(
+ identityRedactor.redact(err instanceof Error ? err.message : "Unknown adapter failure"),
+ await getCurrentUserRedactionOptions(),
+ );
+ const workspaceValidationFailure = isWorkspaceValidationFailure(err)
+ ? err
+ : null;
+ const configurationIncompleteFailure = isConfigurationIncompleteFailure(
+ err,
+ )
+ ? err
+ : null;
+ const recordedResponsibleUserDenialCode =
+ normalizeResponsibleUserDenialCode(
+ (await getRun(run.id).catch(() => null))?.errorCode,
+ );
+ // The runtime resolution is scoped to the adapter try block. The
+ // durable coordinator is also the stronger authority here: legacy
+ // runs simply have no row, while native result-less exhaustion keeps
+ // its named failure instead of being flattened to `adapter_failed`.
+ const nativeTerminalFailureCode = await db
+ .select({
+ phase: nativeRunFinalizations.phase,
+ resultId: nativeRunFinalizations.resultId,
+ failureCode: nativeRunFinalizations.failureCode,
+ })
+ .from(nativeRunFinalizations)
+ .where(eq(nativeRunFinalizations.runId, run.id))
+ .limit(1)
+ .then((rows) => {
+ const coordinator = rows[0];
+ return coordinator?.phase === "terminal_failure" &&
+ coordinator.resultId === null
+ ? coordinator.failureCode
+ : null;
+ })
+ .catch(() => null);
+ const failureErrorCode =
+ workspaceValidationFailure?.code ??
+ configurationIncompleteFailure?.code ??
+ nonRetryablePreflightFailureCode(err) ??
+ recordedResponsibleUserDenialCode ??
+ nativeTerminalFailureCode ??
+ "adapter_failed";
+ logger.error({ err: identityRedactor.redact({ message: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }), runId }, "heartbeat execution failed");
+
+ let logSummary: {
+ bytes: number | null;
+ sha256?: string | null;
+ compressed: boolean;
+ } | null = null;
+ if (handle) {
+ try {
+ logSummary = await runLogStore.finalize(handle);
+ } catch (finalizeErr) {
+ logger.warn(
+ { err: finalizeErr, runId },
+ "failed to finalize run log after error",
+ );
+ }
+ }
+ const finalLogBytes = logSummary?.bytes;
+ if (outputProgressState.pending && typeof finalLogBytes === "number") {
+ outputProgressState.pending.bytes = finalLogBytes;
+ }
+ await flushOutputProgress({ force: true }).catch((flushErr) => {
+ logger.warn(
+ { err: flushErr, runId },
+ "failed to flush run output progress after error",
+ );
+ });
+
+ const stoppedDuringFailure = signal.aborted;
+ const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null;
+ const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed";
+ const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, {
+ error: message,
+ errorCode: stopSnapshot?.errorCode ?? failureErrorCode,
+ finishedAt: new Date(),
+ resultJson: mergeRunStopMetadataForAgent(agent, failureOutcome, {
+ errorCode: failureErrorCode,
+ errorMessage: message,
+ resultJson: {
+ ...parseObject(stopSnapshot?.resultJson),
+ ...requiredWorkspaceRestoreEvidence,
+ ...(workspaceValidationFailure?.resultJson ??
+ configurationIncompleteFailure?.resultJson ??
+ {}),
+ ...(!legacyAdapterEntered && run.runtimeMode !== "native"
+ ? {
+ executionRecovery: {
+ kind: "bootstrap",
+ providerWorkStarted: false,
+ },
+ }
+ : {}),
+ },
+ }),
+ stdoutExcerpt: output.stdoutExcerpt,
+ stderrExcerpt: output.stderrExcerpt,
+ logBytes: logSummary?.bytes,
+ logSha256: logSummary?.sha256,
+ logCompressed: logSummary?.compressed ?? false,
+ }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });
+ if (
+ !failedRunWrite.updated &&
+ !(
+ nativeTerminalFailureCode && failedRunWrite.run?.status === "failed"
+ )
+ ) {
+ logger.info(
+ {
+ runId: run.id,
+ attemptedStatus: "failed",
+ currentStatus: failedRunWrite.run?.status ?? null,
+ },
+ "skipping late adapter failure finalization because the run already left running state",
+ );
+ return;
+ }
+
+ const failedRun = failedRunWrite.run;
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
+ finishedAt: new Date(),
+ error: message,
+ });
+
+ if (failedRun) {
+ await appendRunEvent(failedRun, {
+ eventType: "error",
+ stream: "system",
+ level: "error",
+ message,
+ });
+ const livenessRun =
+ (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun;
+ try {
+ await completeSkillTestRunForHeartbeatOutcome({
+ run: livenessRun,
+ issueId,
+ issueWorkMode: issueRef?.workMode ?? null,
+ outcome: "failed",
+ error: message,
+ });
+ } catch (err) {
+ logger.warn(
+ { err, runId: livenessRun.id, issueId },
+ "failed to complete skill test run after heartbeat adapter failure",
+ );
+ }
+ await refreshContinuationSummaryForRun(livenessRun, agent);
+ if (
+ !isWorkspaceValidationFailedRun(livenessRun) &&
+ !isConfigurationIncompleteFailedRun(livenessRun)
+ ) {
+ await finalizeIssueCommentPolicy(livenessRun, agent);
+ }
+ await scheduleInteractionContinuationInfrastructureRetryIfEligible(
+ livenessRun,
+ agent,
+ );
+ await releaseIssueExecutionAndPromote(livenessRun, {
+ // Native recovery owns the original heartbeat run through
+ // exhaustion. Once its durable coordinator has classified a
+ // terminal failure, generic issue recovery must not create a
+ // replacement retryOfRunId chain for the same provider work.
+ suppressImmediateRecovery: nativeTerminalFailureCode !== null,
+ });
+ await handleIssueReviewPathDisposition(livenessRun);
+
+ await updateRuntimeState(
+ agent,
+ livenessRun,
+ {
+ exitCode: null,
+ signal: null,
+ timedOut: false,
+ errorMessage: message,
+ },
+ {
+ legacySessionId: runtimeForAdapter.sessionId,
+ },
+ );
+
+ if (
+ taskKey &&
+ (goalCheckpointSession.current ||
+ previousSessionParams ||
+ previousSessionDisplayId ||
+ taskSession)
+ ) {
+ await upsertTaskSession({
+ companyId: agent.companyId,
+ agentId: agent.id,
+ adapterType: agent.adapterType,
+ taskKey,
+ sessionParamsJson:
+ goalCheckpointSession.current?.params ??
+ attachPaperclipSessionMetadataToSessionParams(
+ previousSessionParams,
+ configuredModel,
+ sessionConfigMetadata,
+ ),
+ sessionDisplayId:
+ goalCheckpointSession.current?.displayId ??
+ previousSessionDisplayId,
+ lastRunId: failedRun.id,
+ lastError: message,
+ });
+ }
+ }
+
+ await finalizeAgentStatus(agent.id, "failed", message, {
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ keepIdleOnFailure:
+ Boolean(nonRetryablePreflightFailureCode(err)) ||
+ isWorkspaceSyncConflictFailure(message),
+ });
+ }
+
+ async function failRunSetup(input: FailHeartbeatRunSetupInput) {
+ const { run, runId, outerErr, identityRedactor, readFailureReportSecrets } = input;
+ // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun).
+ // The inner catch did not fire, so we must record the failure here.
+ const message = redactCurrentUserText(
+ identityRedactor.redact(outerErr instanceof Error
+ ? outerErr.message
+ : "Unknown setup failure"),
+ await getCurrentUserRedactionOptions(),
+ );
+ // A missing secret/env binding is a known pre-dispatch configuration gap,
+ // not an opaque setup crash. Surface it with its own errorCode so the
+ // recovery path routes it to a human owner instead of looping retries.
+ const workspaceValidationSetupFailure = isWorkspaceValidationFailure(
+ outerErr,
+ )
+ ? outerErr
+ : null;
+ const configurationIncompleteSetupFailure =
+ isConfigurationIncompleteFailure(outerErr) ? outerErr : null;
+ const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(
+ outerErr,
+ )
+ ? outerErr
+ : null;
+ // A sandbox provider plugin stuck in error/disabled/upgrade_pending
+ // fails every lease the same way until an operator acts, so it is a
+ // configuration gap, not a transient setup failure.
+ const sandboxProviderPluginNotReadySetupFailure =
+ parseSandboxProviderPluginNotReadyFailureMessage(
+ outerErr instanceof Error ? outerErr.message : null,
+ );
+ const recordedResponsibleUserDenialCode =
+ normalizeResponsibleUserDenialCode(
+ (await getRun(runId).catch(() => null))?.errorCode,
+ );
+ const nonRetryablePreflightCode =
+ nonRetryablePreflightFailureCode(outerErr);
+ const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null;
+ const setupFailureErrorCode =
+ workspaceGitScanFailure?.code ??
+ workspaceValidationSetupFailure?.code ??
+ configurationIncompleteSetupFailure?.code ??
+ (unresolvedBaseRefSetupFailure ||
+ sandboxProviderPluginNotReadySetupFailure
+ ? CONFIGURATION_INCOMPLETE_FAILURE_CODE
+ : null) ??
+ recordedResponsibleUserDenialCode ??
+ nonRetryablePreflightCode ??
+ "setup_failed";
+ logger.error(
+ { err: identityRedactor.redact({ message: outerErr instanceof Error ? outerErr.message : String(outerErr), stack: outerErr instanceof Error ? outerErr.stack : undefined }), runId },
+ "heartbeat execution setup failed",
+ );
+ const setupFailureAgent = await getAgent(run.agentId).catch(() => null);
+ // The structured failure payload drives the recovery notice and next
+ // action, so it is persisted even when the agent lookup failed and the
+ // agent-scoped stop metadata cannot be merged in.
+ const setupFailureDetails =
+ (workspaceGitScanFailure ? {
+ workspaceGitScan: {
+ code: workspaceGitScanFailure.code,
+ phase: "workspace_setup",
+ retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code),
+ },
+ } : null) ??
+ workspaceValidationSetupFailure?.resultJson ??
+ configurationIncompleteSetupFailure?.resultJson ??
+ (unresolvedBaseRefSetupFailure
+ ? buildUnresolvedWorkspaceBaseRefResultJson(
+ run,
+ unresolvedBaseRefSetupFailure,
+ )
+ : null) ??
+ (sandboxProviderPluginNotReadySetupFailure
+ ? buildSandboxProviderPluginNotReadyResultJson(
+ run,
+ sandboxProviderPluginNotReadySetupFailure,
+ )
+ : null);
+ const connectionFailure = readGitConnectionFailure(outerErr);
+ const setupFailureResultJson = {
+ ...setupFailureDetails,
+ ...(connectionFailure ? { connectionFailure } : {}),
+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
+ };
+ const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", {
+ error: message,
+ errorCode: setupFailureErrorCode,
+ finishedAt: new Date(),
+ ...(setupFailureAgent
+ ? {
+ resultJson: mergeRunStopMetadataForAgent(
+ setupFailureAgent,
+ "failed",
+ {
+ errorCode: setupFailureErrorCode,
+ errorMessage: message,
+ resultJson: setupFailureResultJson,
+ },
+ ),
+ }
+ : setupFailureResultJson
+ ? { resultJson: setupFailureResultJson }
+ : {}),
+ }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));
+ if (!setupFailureWrite.updated) {
+ logger.info(
+ {
+ runId,
+ attemptedStatus: "failed",
+ currentStatus: setupFailureWrite.run?.status ?? null,
+ },
+ "skipping late setup failure finalization because the run already left running state",
+ );
+ } else {
+ await setWakeupStatus(run.wakeupRequestId, "failed", {
+ finishedAt: new Date(),
+ error: message,
+ }).catch(() => undefined);
+ }
+ const failedRun = await getRun(runId).catch(() => null);
+ if (setupFailureWrite.updated && failedRun) {
+ // Emit a run-log event so the failure is visible in the run timeline,
+ // consistent with what the inner catch block does for adapter failures.
+ await appendRunEvent(failedRun, {
+ eventType: "error",
+ stream: "system",
+ level: "error",
+ message,
+ }).catch(() => undefined);
+ const livenessRun = await classifyAndPersistRunLiveness(
+ failedRun,
+ ).catch(() => failedRun);
+ const setupFailureIssueId = readNonEmptyString(
+ parseObject(livenessRun.contextSnapshot).issueId,
+ );
+ if (setupFailureIssueId) {
+ await completeSkillTestRunForHeartbeatOutcome({
+ run: livenessRun,
+ issueId: setupFailureIssueId,
+ outcome: "failed",
+ error: message,
+ }).catch((completionErr) => {
+ logger.warn(
+ {
+ err: completionErr,
+ runId: livenessRun.id,
+ issueId: setupFailureIssueId,
+ },
+ "failed to complete skill test run after heartbeat setup failure",
+ );
+ });
+ }
+ const failedAgent =
+ setupFailureAgent ??
+ (await getAgent(run.agentId).catch(() => null));
+ if (failedAgent) {
+ await refreshContinuationSummaryForRun(
+ livenessRun,
+ failedAgent,
+ ).catch(() => undefined);
+ if (
+ !isWorkspaceValidationFailedRun(livenessRun) &&
+ !isConfigurationIncompleteFailedRun(livenessRun)
+ ) {
+ await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(
+ () => undefined,
+ );
+ }
+ // No provider work began. Retry temporary host scan failures with
+ // the existing durable failure budget, before releasing execution.
+ // Generic recovery must not grant a second budget on exhaustion.
+ await (isTransientWorkspaceGitScanCode(livenessRun.errorCode)
+ ? scheduleBoundedRetryForRun(livenessRun, failedAgent)
+ : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent)
+ ).catch((retryError) => {
+ logger.warn(
+ { err: retryError, runId: livenessRun.id },
+ "failed to schedule interaction continuation retry after setup failure",
+ );
+ });
+ }
+ await releaseIssueExecutionAndPromote(livenessRun, {
+ suppressImmediateRecovery:
+ readNonEmptyString(
+ parseObject(livenessRun.contextSnapshot).goalControlRequestId,
+ ) !== null ||
+ parseObject(livenessRun.contextSnapshot)
+ .resumeSessionGoalHeartbeat === true,
+ }).catch((releaseError) => {
+ logger.error(
+ { err: releaseError, runId },
+ "failed to release issue execution after heartbeat setup failure",
+ );
+ });
+ await handleIssueReviewPathDisposition(livenessRun).catch(
+ (reviewPathError) => {
+ logger.error(
+ { err: reviewPathError, runId },
+ "failed to evaluate review-path disposition after heartbeat setup failure",
+ );
+ },
+ );
+ }
+ // Ensure the agent is not left stuck in "running" if the setup-failure
+ // path owned the terminal transition. If another path already finalized
+ // the run, keep that terminal outcome authoritative.
+ if (setupFailureWrite.updated) {
+ await finalizeAgentStatus(run.agentId, "failed", message, {
+ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run),
+ // Low-trust admission failures are task/principal preconditions,
+ // not evidence that the immutable endpoint agent is unhealthy.
+ // Keep the failed run and its safe provider refusal authoritative,
+ // but return the agent to idle so clients do not also announce a
+ // misleading agent-wide error for the same rejected chat turn.
+ keepIdleOnFailure: Boolean(nonRetryablePreflightCode),
+ }).catch(() => undefined);
+ }
+ }
+
+ return { completeRun, failRun, failRunSetup };
+}