The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(heartbeat): extract run lifecycle handling (#15672)
Dotta0b80ea17ef32
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 318fe32102..3fda91d3f6 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1250,9 +1250,9 @@ It owns bounded run projections, database encoding checks, task session reads anwrites, explicit resumes, session compaction, and usage/billing helpers.`createHeartbeatRunState(db)` binds these operations without doing database workduring construction. The encoding-check cache belongs to each factory instance.-`heartbeat.ts` keeps run execution, session-goal recovery, cost accounting writes,-and status transitions, and re-exports the existing public helpers. Keep session-policy changes separate from run orchestration changes.+`heartbeat.ts` keeps run execution and session-goal recovery, and re-exports the+existing public helpers. Keep session policy changes separate from run+orchestration changes.Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns boundedretry schedules, connection and workspace contention deferrals, shared-workspace@@ -1281,6 +1281,15 @@ its process-wide execution and wakeup promise sets. Forwarding callbacks preservconstruction order for retry and recovery services. Keep queue policy changesseparate from this extraction and from adapter execution changes.+Run lifecycle handling is in `server/src/services/heartbeat/run-lifecycle.ts`.+It owns status transitions, run events and progress, liveness classification,+completion handoffs, issue-comment finalization, and runtime/cost settlement.+`createHeartbeatLifecycle` binds the database and explicit reporting, recovery,+and wakeup callbacks without starting work. The service supplies terminal+reporting and forwarding callbacks preserve construction order for the other+heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`.+Keep lifecycle policy changes separate from this extraction and adapter execution.+## Wake Context DeliveryBuilt-in adapters deliver wake context through the run prompt, including structureddiff --git a/server/src/services/chat-publication-reconciliation.test.ts b/server/src/services/chat-publication-reconciliation.test.tsindex 46c27ab8ce..3af863db2d 100644--- a/server/src/services/chat-publication-reconciliation.test.ts+++ b/server/src/services/chat-publication-reconciliation.test.ts@@ -220,18 +220,18 @@ describe("createCoalescedAsyncTrigger", () => {describe("chat publication commit signals", () => {it("emits the accepted signals only after their durable source writes", () => {- const heartbeatSource = readFileSync(- new URL("./heartbeat.ts", import.meta.url),+ const lifecycleSource = readFileSync(+ new URL("./heartbeat/run-lifecycle.ts", import.meta.url),"utf8",);- const appendRunEventStart = heartbeatSource.indexOf(+ const appendRunEventStart = lifecycleSource.indexOf("async function appendRunEvent(",);- const persistedEvent = heartbeatSource.indexOf(+ const persistedEvent = lifecycleSource.indexOf("await appendHeartbeatRunEvent",appendRunEventStart,);- const emittedEvent = heartbeatSource.indexOf(+ const emittedEvent = lifecycleSource.indexOf("publishLiveEvent({",persistedEvent,);@@ -239,6 +239,10 @@ describe("chat publication commit signals", () => {expect(persistedEvent).toBeGreaterThan(appendRunEventStart);expect(emittedEvent).toBeGreaterThan(persistedEvent);+ const heartbeatSource = readFileSync(+ new URL("./heartbeat.ts", import.meta.url),+ "utf8",+ );const presentationMarker = heartbeatSource.indexOf('eventType: "run.presentation.resolved"',);diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 8d84f116e6..be77bf2f40 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,10 +1,22 @@+import {+ NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,+ isHeartbeatRunTerminalStatus,+ isHeartbeatRunRuntimeStatusActive,+ publishHeartbeatRunRuntimeProgress,+ decorateHeartbeatRunRuntimeStatus,+ createHeartbeatLifecycle,+} from "./heartbeat/run-lifecycle.js";+export {+ redactDetectedSuccessfulRunProgressSummaryForBoard,+ redactSuccessfulRunHandoffEvidence,+ persistHeartbeatRunProcessMetadata,+ resolveSkillTestRunCompletionForHeartbeatOutcome,+} from "./heartbeat/run-lifecycle.js";import {ChatControlRecoveryUnresolvedError,shouldDeferFollowupWakeForSameIssue,shouldQueueFollowupForRunningIssueWake,DEFERRED_WAKE_CONTEXT_KEY,- EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,- shouldRequireIssueCommentForWake,createHeartbeatQueue,} from "./heartbeat/queue.js";export {@@ -12,15 +24,12 @@ export {shouldQueueFollowupForRunningIssueWake,} from "./heartbeat/queue.js";import {- HEARTBEAT_RUN_TERMINAL_STATUSES,- DETACHED_PROCESS_ERROR_CODE,isProcessAlive,CANCELLABLE_HEARTBEAT_RUN_STATUSES,createHeartbeatRecovery,} from "./heartbeat/recovery.js";import {BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,- normalizeAgentNameKey,WorkspaceBusyDeferral,readTransientRecoveryContractFromRun,readHeartbeatRunErrorFamily,@@ -44,7 +53,6 @@ import {type RunSessionOutcome,EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,deriveTaskKeyWithHeartbeatFallback,- type UsageTotals,getAdapterSessionCodec,shouldResetTaskSessionForWake,normalizeSessionParams,@@ -119,7 +127,6 @@ export {buildPaperclipWakePayload,} from "./heartbeat/run-preparation.js";import {- WORKSPACE_VALIDATION_FAILURE_CODE,isWorkspaceValidationFailedRun,resolveNativeRecoveryExecutionWorkspaceBinding,resolveExecutionWorkspaceReuseRequestForIssue,@@ -224,13 +231,13 @@ export {import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";import { preserveWorkspaceRestoreRecoveryMetadataSql } from "./legacy-workspace-restore-recovery.js";-import { preserveWorkspaceRestoreRecoveryMetadata } from "./workspace-restore-recovery-state.js";+import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";import {configuredEnvironmentProjection,} from "../vendor/paperclip-runner/index.js";import { decisionModelService } from "./decision-models.js";-import { activeIssueInteractionCondition, TASK_QUESTION_GUIDANCE } from "./issue-question-context.js";+import { TASK_QUESTION_GUIDANCE } from "./issue-question-context.js";import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js";import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils";@@ -241,7 +248,11 @@ import { externalObjectService } from "./external-objects.js";import { resolvePaperclipInstanceRoot } from "../home-paths.js";import { dotRunnerBroker } from "./dot-runner-broker.js";import { isAiAuthenticationBlocked } from "./ai-auth-failure.js";-import { nativeRetryCancellationCommitCondition, rethrowNativeCancellationLockConflict, claimCancellationRequest, startupCancellationFence } from "./native-runtime/native-cancellation-request.js";+import {+ nativeRetryCancellationCommitCondition,+ claimCancellationRequest,+ startupCancellationFence,+} from "./native-runtime/native-cancellation-request.js";import {prepareChatCompletionTurn,chatCompletionInstruction,@@ -274,14 +285,18 @@ import { toolActionDeliveryService } from "./tool-action-delivery.js";import { readQueuedInteractionResponse } from "./queued-interaction-response.js";import {isConversation,- isWaitingConversation,prepareConversationTurn,settleConversationTurn,} from "./agent-conversations.js";import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase";import { getConversationConfirmationContext } from "./conversation-confirmation-context.js";-import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";-import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";+import {+ recordNativeLocalProcessStop,+} from "./native-local-process-stop.js";+import {+ isAcknowledgedNativeStop,+ acknowledgedNativeStopExecutionHasStopped,+} from "./acknowledged-native-stop.js";import {legacyControllerBootId,renewLegacyControllerLease,@@ -313,7 +328,12 @@ import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConimport { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js";import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js";import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js";-import { CONVERSATION_CONTINUATION_POLICY, claimedAdapterType, runUsedConversationAdapter, hasConversationContinuationPolicy, isConversationAdapter } from "./conversation-continuation.js";+import {+ CONVERSATION_CONTINUATION_POLICY,+ claimedAdapterType,+ runUsedConversationAdapter,+ hasConversationContinuationPolicy,+} from "./conversation-continuation.js";import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";@@ -381,48 +401,33 @@ import {import type { Db } from "@paperclipai/db";import {CHAT_PROVIDERS,- ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,ISSUE_DISPOSITION_REPAIR_RETRY_REASON,PROVIDER_QUOTA_MONITOR_SERVICE_NAME,isEnvironmentDriverSupportedForAdapter,type EnvironmentLeaseStatus,type ExecutionWorkspace,- type HeartbeatRunStatusPhase,type IssueExecutionMonitorClearReason,type IssueExecutionMonitorPolicy,type IssueExecutionMonitorRecoveryPolicy,- type RequestConfirmationResult,- type RunLivenessState,} from "@paperclipai/shared";import {agents,agentConfigRevisions,- agentRuntimeState,agentSessionGoalActions,agentTaskSessions,agentWakeupRequests,- activityLog,- approvals,chatActions,- companySkillTestRuns,companies,completionContracts,- documentRevisions,environmentLeases,- issueDocuments,heartbeatRunEvents,heartbeatRuns,- issueApprovals,issueComments,issueRecoveryActions,- issueRelations,- issueThreadInteractions,issues,- issueWorkProducts,nativeRunFinalizations,projects,projectWorkspaces,- routines,workspaceOperations,} from "@paperclipai/db";import { conflict, HttpError, notFound } from "../errors.js";@@ -442,10 +447,7 @@ import {// git-credentials module became its canonical home; existing importers keep working.export { scrubGitCredentialText };import { publishLiveEvent } from "./live-events.js";-import {- appendHeartbeatRunEvent,- type AppendHeartbeatRunEventInput,-} from "./heartbeat-run-events.js";+import {queuedCommentIdsFromWakePayload,queuedCommentIdsFromRunContext,@@ -528,7 +530,6 @@ import { createLocalAgentJwt } from "../agent-auth-jwt.js";import { createRuntimeToolsToken } from "../runtime-tools-token.js";import {parseObject,- asBoolean,asNumber,} from "../adapters/utils.js";@@ -539,7 +540,6 @@ import { materializeExternalChatQuestionResponseInput } from "./native-runtime/eimport {NativeRunnerOwnershipUnverifiedError,isNativeRunnerOwnershipHeld,- nativeRunnerOwnershipNotHeldCondition,NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,} from "./native-runtime/native-runner-ownership.js";import {@@ -570,32 +570,21 @@ import {import {buildHeartbeatRunIssueComment,CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,- findHeartbeatRunCompletionComment,- hasAcceptedSemanticResult,isExternalChatPresentationContext,mergeHeartbeatRunResultJson,- readCompletedAssistantMessageCandidate,resolveHeartbeatRunResponse,- selectHeartbeatRunFinalAgentMessage,type RunPresentationDecision,} from "./heartbeat-run-summary.js";import {- buildHeartbeatRunStopMetadata,- mergeHeartbeatRunStopMetadata,normalizeMaxTurnStopReason,} from "./heartbeat-stop-metadata.js";import {CHAT_CONTROL_RECOVERY_ADMISSION_KEY,CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,} from "./chat-control-recovery-stop.js";-import {- classifyRunLiveness,- type RunLivenessClassificationInput,-} from "./run-liveness.js";import {logActivity,- publishPluginDomainEvent,type LogActivityInput,} from "./activity-log.js";import {@@ -646,7 +635,6 @@ import {import {continuationSummaryParksExecutor,getIssueContinuationSummaryDocument,- refreshIssueContinuationSummary,} from "./issue-continuation-summary.js";import {buildPlanReviewContext,@@ -694,19 +682,6 @@ import {} from "./execution-allowlist.js";import {RECOVERY_ORIGIN_KINDS,- FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,- SUCCESSFUL_RUN_MISSING_STATE_REASON,- RUN_LIVENESS_CONTINUATION_REASON,- buildRunLivenessContinuationIdempotencyKey,- buildFinishSuccessfulRunHandoffIdempotencyKey,- buildSuccessfulRunHandoffRequiredNotice,- decideRunLivenessContinuation,- decideSuccessfulRunHandoff,- findExistingFinishSuccessfulRunHandoffWake,- findExistingRunLivenessContinuationWake,- isSuccessfulRunHandoffValidPathSkip,- SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,- readContinuationAttempt,} from "./recovery/index.js";import {buildConfigurationIncompleteRecoveryNoticeSeed,@@ -741,14 +716,9 @@ import {type RunSnapshot as WakeQueueRunSnapshot,} from "../modules/wake-queue/index.js";import {- buildIssueReviewPathLostIdempotencyKey,- decideIssueReviewPathRecovery,- ISSUE_REVIEW_PATH_LOST_WAKE_REASON,- isReviewPathRecoveryIdempotencyConflict,REVIEW_PATH_RECOVERY_INSTRUCTION,- reviewPathConsumedRefFromRun,} from "./recovery/review-path-recovery.js";-import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "./successful-run-handoff-state.js";+import { taskWatchdogService } from "./task-watchdogs.js";import {@@ -763,20 +733,15 @@ import {} from "./source-trust.js";import {redactCurrentUserText,- redactCurrentUserValue,- type CurrentUserRedactionOptions,} from "../log-redaction.js";-import { redactEventPayload, redactSensitiveText } from "../redaction.js";+import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js";import {resolvePaperclipRunnerIdleTimeoutMs,- type RuntimeStatusUpdate,} from "@paperclipai/adapter-utils";import {readPaperclipSkillSyncPreference,selectPaperclipTaskMarkdown,- UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,- UNMANAGED_BACKGROUND_TASK_STOP_REASON,} from "@paperclipai/adapter-utils/server-utils";import {isUuidLike,@@ -794,16 +759,10 @@ import { environmentRunOrchestrator } from "./environment-run-orchestrator.js";import {clearHeartbeatRunRuntimeStatus,- getHeartbeatRunRuntimeStatus,- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,- setHeartbeatRunRuntimeStatus,sweepExpiredHeartbeatRunRuntimeStatuses,touchHeartbeatRunRuntimeStatus,} from "./heartbeat-run-runtime-status.js";-import {- readProcessStartedAt,-} from "./hot-restart.js";+import {assertLowTrustRuntimeServicesAllowed,} from "./low-trust-runtime-containment.js";@@ -816,32 +775,6 @@ import { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain, readTaconst MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;-export function redactDetectedSuccessfulRunProgressSummaryForBoard(- summary: string,- currentUserRedactionOptions?: CurrentUserRedactionOptions,-) {- const normalized = summary.replace(/\s+/g, " ").trim();- const redacted = redactSensitiveText(- redactCurrentUserText(normalized, currentUserRedactionOptions),- );- return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`;-}--export function redactSuccessfulRunHandoffEvidence(- value: string | null,- currentUserRedactionOptions?: CurrentUserRedactionOptions,-) {- if (!value) return null;- return redactSensitiveText(- redactCurrentUserText(value, currentUserRedactionOptions),- );-}--const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [- "environment.lease_acquired",- "environment.lease_released",- "cost.reported",-];const ACCEPTED_PLAN_CONVERSION_SKILL_KEY ="paperclipai/paperclip/paperclip-converting-plans-to-tasks";@@ -873,25 +806,6 @@ const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON =const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE ="execution_review_participant_recovery";-const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set<string>([- "low_trust_isolation_unavailable",- "low_trust_requires_isolated_workspace",- "low_trust_boundary_mismatch",- "low_trust_requires_sandbox_environment",- "low_trust_runtime_services_denied",- "chat_failed_run_retry_not_authorized",- CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,-]);-// Error codes that mark a pre-dispatch setup failure. The adapter process never-// started, so no agent could post an issue comment. The setup catch writes one-// of these codes when a failure happens before `adapter.execute`.-const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([- "setup_failed",- CONFIGURATION_INCOMPLETE_FAILURE_CODE,- WORKSPACE_VALIDATION_FAILURE_CODE,- ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,-]);-function nonRetryablePreflightFailureCode(error: unknown): string | null {if (error instanceof ChatControlRecoveryUnresolvedError)return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;@@ -908,19 +822,9 @@ function nonRetryablePreflightFailureCode(error: unknown): string | null {}export { MAX_TURN_CONTINUATION_RETRY_REASON };export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";-const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;-const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;-const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;-const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;export { WORKSPACE_BUSY_RETRY_REASON };-interface MaxTurnContinuationPolicy {- enabled: boolean;- maxAttempts: number;- delayMs: number;-}-// Build the configuration-incomplete result payload for a workspace base ref// that never resolved to a commit. The setup catch maps this to errorCode// `configuration_incomplete`, so the recovery path routes it to a human owner@@ -1379,315 +1283,6 @@ function isSameTaskScope(left: string | null, right: string | null) {return (left ?? null) === (right ?? null);}-function isHeartbeatRunTerminalStatus(- status: string | null | undefined,-): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {- return HEARTBEAT_RUN_TERMINAL_STATUSES.includes(- status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number],- );-}--function isHeartbeatRunRuntimeStatusActive(- status: string | null | undefined,-): boolean {- return status === "queued" || status === "running";-}--type HeartbeatRunRuntimeStatusRunLike = {- id: string;- status?: string | null;- companyId?: string | null;- agentId?: string | null;- issueId?: string | null;- contextSnapshot?: Record<string, unknown> | null;-};--function readRuntimeStatusIssueIdCandidate(- run: HeartbeatRunRuntimeStatusRunLike,-): string | null | undefined {- if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null;- if ("contextSnapshot" in run) {- return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null;- }- return undefined;-}--function decorateHeartbeatRunRuntimeStatus<- T extends HeartbeatRunRuntimeStatusRunLike,->(- run: T,- expected: {- companyId?: string | null;- issueId?: string | null;- agentId?: string | null;- } = {},-): T & {- currentStatusMessage: string | null;- currentStatusUpdatedAt: Date | null;- currentToolName: string | null;- lastAssistantSnippet: string | null;- lastEventAt: Date | null;-} {- if (isHeartbeatRunTerminalStatus(run.status)) {- clearHeartbeatRunRuntimeStatus(run.id);- }-- const companyId = expected.companyId ?? run.companyId ?? null;- const agentId = expected.agentId ?? run.agentId ?? null;- const issueId =- expected.issueId !== undefined- ? expected.issueId- : readRuntimeStatusIssueIdCandidate(run);- const currentStatus =- isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId- ? getHeartbeatRunRuntimeStatus(run.id, {- companyId,- agentId,- ...(issueId !== undefined ? { issueId } : {}),- })- : null;-- return {- ...run,- currentStatusMessage: currentStatus?.message ?? null,- currentStatusUpdatedAt: currentStatus?.updatedAt ?? null,- currentToolName: currentStatus?.currentToolName ?? null,- lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null,- lastEventAt: currentStatus?.lastEventAt ?? null,- };-}--function publishHeartbeatRunRuntimeProgress(status: {- companyId: string;- runId: string;- agentId: string;- issueId: string | null;- phase: HeartbeatRunStatusPhase;- message: string;- updatedAt: Date;- currentToolName?: string | null;- lastAssistantSnippet?: string | null;- lastEventAt?: Date | null;-}) {- publishLiveEvent({- companyId: status.companyId,- type: "heartbeat.run.progress",- payload: {- runId: status.runId,- agentId: status.agentId,- issueId: status.issueId,- phase: status.phase,- message: status.message,- currentToolName: status.currentToolName ?? null,- lastAssistantSnippet: status.lastAssistantSnippet ?? null,- lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(),- updatedAt: status.updatedAt.toISOString(),- },- });-}--function recordHeartbeatRunRuntimeProgress(- run: Pick<- typeof heartbeatRuns.$inferSelect,- "id" | "companyId" | "agentId" | "status" | "contextSnapshot"- >,- update: RuntimeStatusUpdate,- issueId: string | null,-) {- if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null;- const status = setHeartbeatRunRuntimeStatus({- companyId: run.companyId,- issueId,- agentId: run.agentId,- runId: run.id,- phase: update.phase as HeartbeatRunStatusPhase,- message: update.message,- currentToolName: readNonEmptyString(update.currentToolName) ?? null,- lastAssistantSnippet:- readNonEmptyString(update.lastAssistantSnippet) ?? null,- lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(),- });- if (!status) return null;-- publishHeartbeatRunRuntimeProgress(status);- return status;-}--function sanitizeLiveRunProgressText(- value: string,- maxChars: number,-): string | null {- const normalized = value.replace(/\s+/g, " ").trim();- if (!normalized) return null;- const redacted = redactSensitiveText(normalized);- if (redacted.length <= maxChars) return redacted;- return `${redacted.slice(0, maxChars - 3)}...`;-}--function readLiveRunProgressString(- value: unknown,- maxChars: number,-): string | null {- return typeof value === "string"- ? sanitizeLiveRunProgressText(value, maxChars)- : null;-}--function readFirstLiveRunProgressString(- maxChars: number,- values: unknown[],-): string | null {- for (const value of values) {- const text = readLiveRunProgressString(value, maxChars);- if (text) return text;- }- return null;-}--function readLiveRunToolName(- payload: Record<string, unknown> | null,- eventType: string,-): string | null {- const toolCall =- parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall);- const message = parseObject(payload?.message);- const direct = readFirstLiveRunProgressString(- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,- [- payload?.toolName,- payload?.tool_name,- payload?.tool,- payload?.name,- payload?.title,- toolCall?.name,- toolCall?.toolName,- message?.name,- message?.toolName,- ],- );- if (direct) return direct;-- const normalizedEventType = eventType.toLowerCase();- if (!normalizedEventType.includes("tool")) return null;- return readLiveRunProgressString(- eventType.replace(/[._-]+/g, " "),- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,- );-}--function readLiveRunAssistantSnippet(- payload: Record<string, unknown> | null,- eventType: string,- message: string | null,-): string | null {- const normalizedEventType = eventType.toLowerCase();- const messagePayload = parseObject(payload?.message);- const direct = readFirstLiveRunProgressString(- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,- [- payload?.text,- payload?.delta,- payload?.text_delta,- payload?.content,- payload?.summary,- messagePayload?.text,- messagePayload?.content,- ],- );- if (direct) return direct;-- if (- normalizedEventType.includes("assistant") ||- normalizedEventType.includes("text_delta") ||- normalizedEventType.includes("message.delta") ||- normalizedEventType.includes("message_delta")- ) {- return message- ? sanitizeLiveRunProgressText(- message,- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,- )- : null;- }-- return null;-}--function buildRunEventRuntimeProgress(input: {- eventType: string;- message: string | null;- payload: Record<string, unknown> | null;- at: Date;-}) {- const normalizedEventType = input.eventType.toLowerCase();- if (- normalizedEventType === "lifecycle" ||- normalizedEventType === "adapter.invoke"- ) {- return null;- }-- const currentToolName = readLiveRunToolName(input.payload, input.eventType);- const lastAssistantSnippet = readLiveRunAssistantSnippet(- input.payload,- input.eventType,- input.message,- );- const fallbackMessage =- readLiveRunProgressString(- input.message,- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,- ) ??- sanitizeLiveRunProgressText(- input.eventType.replace(/[._-]+/g, " "),- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,- );- const message = currentToolName- ? `Using ${currentToolName}`- : (lastAssistantSnippet ?? fallbackMessage);-- if (!message) return null;- return {- phase: "run_activity" as const,- message,- currentToolName,- lastAssistantSnippet,- lastEventAt: input.at,- };-}--export async function persistHeartbeatRunProcessMetadata(- db: Db,- runId: string,- meta: { pid: number; processGroupId: number | null; startedAt: string },-) {- const observedStartedAt = await readProcessStartedAt(meta.pid).catch(- () => null,- );- const startedAt = new Date(observedStartedAt ?? meta.startedAt);- return db.transaction(async tx => {- const run = await tx- .update(heartbeatRuns)- .set({- processPid: meta.pid,- processGroupId: meta.processGroupId,- processStartedAt: Number.isNaN(startedAt.getTime())- ? new Date()- : startedAt,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, runId))- .returning()- .then((rows) => rows[0] ?? null);- if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, {- companyId: run.companyId, runId, agentId: run.agentId,- eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info",- message: "Process identity recorded; prior stop evidence no longer applies.",- });- return run;- });-}-async function terminateHeartbeatRunProcess(input: {pid: number | null | undefined;processGroupId: number | null | undefined;@@ -1715,41 +1310,6 @@ async function terminateHeartbeatRunProcess(input: {);}-type SkillTestHeartbeatCompletion = {- outcome: "failed" | "cancelled";- error: string | null;- heartbeatOutcome: RunSessionOutcome;-};--export function resolveSkillTestRunCompletionForHeartbeatOutcome(- outcome: RunSessionOutcome,- error: string | null | undefined,-): SkillTestHeartbeatCompletion | null {- if (outcome === "cancelled") {- return {- outcome: "cancelled",- error: error ?? "Harness run was cancelled",- heartbeatOutcome: outcome,- };- }- if (outcome === "timed_out") {- return {- outcome: "failed",- error: error ?? "Timed out",- heartbeatOutcome: outcome,- };- }- if (outcome === "failed") {- return {- outcome: "failed",- error: error ?? "Adapter failed",- heartbeatOutcome: outcome,- };- }- return null;-}--export type HeartbeatEnvironmentRuntime = ReturnType<typeof environmentRuntimeService>;@@ -2421,6 +1981,86 @@ export function heartbeatService(return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);}+ const runLifecycle = createHeartbeatLifecycle(db, {+ issuesSvc,+ recovery,+ companySkills,+ getRun,+ emitTerminalAgentTaskRun,+ getIssueExecutionContext,+ budgets,+ enqueueWakeup,+ getCurrentUserRedactionOptions,+ treeControlSvc,+ getAgentInvokability,+ resolveSessionBeforeForWakeup,+ resolveResponsibleUserIdForRunContext,+ ensureRuntimeState,+ budgetHooks,+ });++ const {+ persistRunProcessMetadata,+ recordCurrentHeartbeatRunRuntimeProgress,+ completeSkillTestRunForHeartbeatOutcome,+ refreshContinuationSummaryForRun,+ findRunIssueComment,+ findLatestCompletedFinalAgentMessage,+ parseMaxTurnContinuationPolicy,+ finalizeIssueCommentPolicy,+ handleIssueReviewPathDisposition,+ handleRunLivenessContinuation,+ handleSuccessfulRunHandoff,+ updateRuntimeState,+ terminalizeRunOnLeaseRelease,+ clearDetachedRunWarning,+ } = runLifecycle;++ // Earlier factories capture these callbacks before lifecycle construction.+ function appendRunEvent(...args: Parameters<typeof runLifecycle.appendRunEvent>) {+ return runLifecycle.appendRunEvent(...args);+ }++ function escalatePlanApprovalResumeFailureNeedsAttention(...args: Parameters<typeof runLifecycle.escalatePlanApprovalResumeFailureNeedsAttention>) {+ return runLifecycle.escalatePlanApprovalResumeFailureNeedsAttention(...args);+ }++ function recordPlanApprovalResumeFailureRetry(...args: Parameters<typeof runLifecycle.recordPlanApprovalResumeFailureRetry>) {+ return runLifecycle.recordPlanApprovalResumeFailureRetry(...args);+ }++ function setRunStatusIfRunning(...args: Parameters<typeof runLifecycle.setRunStatusIfRunning>) {+ return runLifecycle.setRunStatusIfRunning(...args);+ }++ function setWakeupStatus(...args: Parameters<typeof runLifecycle.setWakeupStatus>) {+ return runLifecycle.setWakeupStatus(...args);+ }++ function mergeRunStopMetadataForAgent(...args: Parameters<typeof runLifecycle.mergeRunStopMetadataForAgent>) {+ return runLifecycle.mergeRunStopMetadataForAgent(...args);+ }++ function classifyAndPersistRunLiveness(...args: Parameters<typeof runLifecycle.classifyAndPersistRunLiveness>) {+ return runLifecycle.classifyAndPersistRunLiveness(...args);+ }++ function setRunStatusFromLive(...args: Parameters<typeof runLifecycle.setRunStatusFromLive>) {+ return runLifecycle.setRunStatusFromLive(...args);+ }++ function setRunStatus(...args: Parameters<typeof runLifecycle.setRunStatus>) {+ return runLifecycle.setRunStatus(...args);+ }++ function publishRunLifecyclePluginEventData(...args: Parameters<typeof runLifecycle.publishRunLifecyclePluginEventData>) {+ return runLifecycle.publishRunLifecyclePluginEventData(...args);+ }++ function publishRunLifecyclePluginEvent(...args: Parameters<typeof runLifecycle.publishRunLifecyclePluginEvent>) {+ return runLifecycle.publishRunLifecyclePluginEvent(...args);+ }+// Applies the post-commit effects a wake-queue release returns, exactly as// the original release function did before its writes moved into that// module: publish + dispatch a promoted or recovery run, log a reopened@@ -2482,369 +2122,29 @@ export function heartbeatService(}}- function isPlanApprovalConfirmationPayload(payload: unknown) {- const target = parseObject(parseObject(payload).target);- return (- readNonEmptyString(target.type) === "issue_document" &&- readNonEmptyString(target.key) === "plan"- );- }-- async function getAcceptedPlanApprovalInteractionForRun(- run: typeof heartbeatRuns.$inferSelect,- issueId: string | null,- ) {- const context = parseObject(run.contextSnapshot);- const interactionId = readNonEmptyString(context.interactionId);- if (!issueId || !interactionId) return null;-- const interaction = await db- .select({- id: issueThreadInteractions.id,- kind: issueThreadInteractions.kind,- status: issueThreadInteractions.status,- payload: issueThreadInteractions.payload,- result: issueThreadInteractions.result,- })- .from(issueThreadInteractions)- .where(- and(- eq(issueThreadInteractions.companyId, run.companyId),- eq(issueThreadInteractions.issueId, issueId),- eq(issueThreadInteractions.id, interactionId),- ),- )- .then((rows) => rows[0] ?? null);-- if (!interaction) return null;- if (- interaction.kind !== "request_confirmation" ||- interaction.status !== "accepted"- )- return null;- return isPlanApprovalConfirmationPayload(interaction.payload)- ? interaction- : null;- }-- function planApprovalResumeFailureErrorCode(- run: typeof heartbeatRuns.$inferSelect,- ) {- return readNonEmptyString(run.errorCode) ?? "unknown_error";- }-- function buildPlanApprovalResumeFailureComment(input: {- run: typeof heartbeatRuns.$inferSelect;- status: "retrying" | "needs_attention";- attempt: number;- maxAttempts: number;- }) {- const errorCode = planApprovalResumeFailureErrorCode(input.run);- if (input.status === "retrying") {- return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`;- }- return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`;- }-- function buildPlanApprovalResumeFailureResult(input: {- run: typeof heartbeatRuns.$inferSelect;- status: "retrying" | "needs_attention";- attempt: number;- maxAttempts: number;- retryRunId?: string | null;- recoveryActionId?: string | null;- }): NonNullable<RequestConfirmationResult["resumeFailure"]> {- return {- status: input.status,- errorCode: planApprovalResumeFailureErrorCode(input.run),- attempt: input.attempt,- maxAttempts: input.maxAttempts,- runId: input.run.id,- retryRunId: input.retryRunId ?? null,- recoveryActionId: input.recoveryActionId ?? null,- updatedAt: new Date().toISOString(),+ const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });+ async function releaseEnvironmentLeasesForRun(input: {+ runId: string;+ companyId: string;+ agentId: string;+ status: string | null | undefined;+ failureReason?: string | null;+ providerResourceDisposition?: ProviderResourceDisposition;+ nativeLifecycleTelemetry?: {+ provider: string;+ harness: string;+ lifecycleMode: "per_turn" | "warm";+ sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";};- }-- async function updatePlanApprovalInteractionResumeFailure(input: {- interaction: NonNullable<- Awaited<ReturnType<typeof getAcceptedPlanApprovalInteractionForRun>>- >;- failure: NonNullable<RequestConfirmationResult["resumeFailure"]>;- }) {- const result = parseObject(input.interaction.result);- const nextResult = {- ...result,- version: 1 as const,- outcome: "accepted" as const,- resumeFailure: input.failure,- } satisfies RequestConfirmationResult;-- await db- .update(issueThreadInteractions)- .set({- result: nextResult,- updatedAt: new Date(),- })- .where(eq(issueThreadInteractions.id, input.interaction.id));- }-- async function addPlanApprovalResumeFailureCommentOnce(input: {- issueId: string;- run: typeof heartbeatRuns.$inferSelect;- body: string;- }) {- const existing = await db- .select({ id: issueComments.id })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, input.run.companyId),- eq(issueComments.issueId, input.issueId),- or(- eq(issueComments.body, input.body),- sql`${issueComments.body} like ${`${input.body}\n%`}`,- ),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- if (existing) return null;- return issuesSvc.addComment(- input.issueId,- input.body,- { runId: input.run.id },- { authorType: "system" },- );- }-- async function getActiveRecoveryActionId(- companyId: string,- sourceIssueId: string,- ) {- return db- .select({ id: issueRecoveryActions.id })- .from(issueRecoveryActions)- .where(- and(- eq(issueRecoveryActions.companyId, companyId),- eq(issueRecoveryActions.sourceIssueId, sourceIssueId),- inArray(issueRecoveryActions.status, ["active", "escalated"]),- ),- )- .orderBy(desc(issueRecoveryActions.updatedAt))- .limit(1)- .then((rows) => rows[0]?.id ?? null);- }-- async function recordPlanApprovalResumeFailureRetry(input: {- run: typeof heartbeatRuns.$inferSelect;- issueId: string | null;- retryRunId: string | null;- attempt: number;- maxAttempts: number;- }) {- const interaction = await getAcceptedPlanApprovalInteractionForRun(- input.run,- input.issueId,- );- if (!interaction || !input.issueId) return null;-- const body = buildPlanApprovalResumeFailureComment({- run: input.run,- status: "retrying",- attempt: input.attempt,- maxAttempts: input.maxAttempts,- });- await addPlanApprovalResumeFailureCommentOnce({- issueId: input.issueId,- run: input.run,- body,- });- await updatePlanApprovalInteractionResumeFailure({- interaction,- failure: buildPlanApprovalResumeFailureResult({- run: input.run,- status: "retrying",- attempt: input.attempt,- maxAttempts: input.maxAttempts,- retryRunId: input.retryRunId,- }),- });- return interaction.id;- }-- async function escalatePlanApprovalResumeFailureNeedsAttention(input: {- run: typeof heartbeatRuns.$inferSelect;- issueId: string | null;- attempt: number;- maxAttempts: number;}) {- const interaction = await getAcceptedPlanApprovalInteractionForRun(- input.run,- input.issueId,- );- if (!interaction || !input.issueId) return null;-- const issue = await db- .select()- .from(issues)- .where(- and(- eq(issues.companyId, input.run.companyId),- eq(issues.id, input.issueId),- ),- )- .then((rows) => rows[0] ?? null);- if (!issue) return null;- if (- issue.status !== "todo" &&- issue.status !== "in_progress" &&- issue.status !== "in_review"- )- return null;-- const body = buildPlanApprovalResumeFailureComment({- run: input.run,- status: "needs_attention",- attempt: input.attempt,- maxAttempts: input.maxAttempts,- });- await recovery.escalateStrandedAssignedIssue({- issue,- previousStatus: issue.status,- latestRun: input.run,- comment: body,- });- await addPlanApprovalResumeFailureCommentOnce({- issueId: issue.id,- run: input.run,- body,- });-- const recoveryActionId = await getActiveRecoveryActionId(- issue.companyId,- issue.id,- );- await updatePlanApprovalInteractionResumeFailure({- interaction,- failure: buildPlanApprovalResumeFailureResult({- run: input.run,- status: "needs_attention",- attempt: input.attempt,- maxAttempts: input.maxAttempts,- recoveryActionId,- }),- });- return interaction.id;- }-- const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });- async function completeSkillTestRunForHeartbeatOutcome(input: {- run: typeof heartbeatRuns.$inferSelect;- issueId: string | null;- issueWorkMode?: string | null;- outcome: RunSessionOutcome;- error: string | null;- }) {- const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(- input.outcome,- input.error,- );- if (!completion || !input.issueId) return null;-- let isSkillTestIssue = input.issueWorkMode === "skill_test";- if (!isSkillTestIssue && input.issueWorkMode === undefined) {- const issueRow = await db- .select({- workMode: issues.workMode,- harnessKind: issues.harnessKind,- })- .from(issues)- .where(- and(- eq(issues.companyId, input.run.companyId),- eq(issues.id, input.issueId),- ),- )- .then((rows) => rows[0] ?? null);- isSkillTestIssue =- issueRow?.workMode === "skill_test" ||- issueRow?.harnessKind === "skill_test";- }- if (!isSkillTestIssue) return null;-- const existingRun = await db- .select({- id: companySkillTestRuns.id,- status: companySkillTestRuns.status,- })- .from(companySkillTestRuns)- .where(- and(- eq(companySkillTestRuns.companyId, input.run.companyId),- eq(companySkillTestRuns.issueId, input.issueId),- ),- )- .then((rows) => rows[0] ?? null);- if (- !existingRun ||- ["succeeded", "failed", "cancelled"].includes(existingRun.status)- )- return null;-- const completedRun = await companySkills.completeTestRunForIssue({- companyId: input.run.companyId,- issueId: input.issueId,- outcome: completion.outcome,- error: completion.error,- });- if (!completedRun) return null;-- await logActivity(db, {- companyId: input.run.companyId,- actorType: "system",- actorId: "heartbeat_finalize",- agentId: input.run.agentId,- runId: input.run.id,- action: "company.skill_test_run_completed",- entityType: "company_skill_test_run",- entityId: completedRun.id,- issueId: input.issueId,- details: {- issueId: input.issueId,- status: completedRun.status,- outputDocumentKey: completedRun.outputDocumentKey,- heartbeatOutcome: completion.heartbeatOutcome,- source: "heartbeat.run_finalized",- },- });-- return completedRun;- }-- async function releaseEnvironmentLeasesForRun(input: {- runId: string;- companyId: string;- agentId: string;- status: string | null | undefined;- failureReason?: string | null;- providerResourceDisposition?: ProviderResourceDisposition;- nativeLifecycleTelemetry?: {- provider: string;- harness: string;- lifecycleMode: "per_turn" | "warm";- sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";- };- }) {- const leaseOwnerRun = await getRun(input.runId);- if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;- // Recovery can finish workspace copy-back outside the executor's finally.- // Successful copy-back does not earn warm retention for a failed turn.- const status = leaseOwnerRun?.status ?? input.status;- const providerResourceDisposition = providerResourceDispositionForTerminalRun(- input.providerResourceDisposition,- status,+ const leaseOwnerRun = await getRun(input.runId);+ if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;+ // Recovery can finish workspace copy-back outside the executor's finally.+ // Successful copy-back does not earn warm retention for a failed turn.+ const status = leaseOwnerRun?.status ?? input.status;+ const providerResourceDisposition = providerResourceDispositionForTerminalRun(+ input.providerResourceDisposition,+ status,);if (providerResourceDisposition === "destroy") {const closeResult = await (@@ -3245,28 +2545,6 @@ export function heartbeatService(return evaluateAgentInvokabilityFromDb(db, agent);}- async function recordCurrentHeartbeatRunRuntimeProgress(- run: Pick<- typeof heartbeatRuns.$inferSelect,- "id" | "companyId" | "agentId" | "status" | "contextSnapshot"- >,- update: RuntimeStatusUpdate,- issueId: string | null,- ) {- if (!isHeartbeatRunRuntimeStatusActive(run.status)) {- clearHeartbeatRunRuntimeStatus(run.id);- return null;- }-- const currentRun = await getRun(run.id);- if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) {- clearHeartbeatRunRuntimeStatus(run.id);- return null;- }-- return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId);- }-const issueMonitorDispatchColumns = {id: issues.id,companyId: issues.companyId,@@ -4157,2020 +3435,83 @@ export function heartbeatService(void reportRunFailure(db, updated, failureReport);}- async function setRunStatus(- runId: string,- status: string,- patch?: Partial<typeof heartbeatRuns.$inferInsert>,- ) {- const previousStatus = await db- .select()- .from(heartbeatRuns)- .where(eq(heartbeatRuns.id, runId))- .then((rows) => rows[0] ?? null);-- if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,- resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),- };-- // Preserve the receipt-source fence when finalization enriches usage. A- // late spool replay must still be able to complete an unfinished receipt.- if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };-- // Cancelling a queued run that never acquired provider execution is- // positive bootstrap evidence. It must not hold unrelated queued messages.- if (previousStatus && (status === "cancelled" || status === "interrupted")) {- patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };- }- if (- status === "cancelled" &&- previousStatus?.status === "queued" &&- previousStatus.runtimeMode !== "native" &&- !previousStatus.startedAt &&- !previousStatus.processPid- ) {- patch = {- ...patch,- resultJson: {- ...previousStatus.resultJson,- ...patch?.resultJson,- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },- },- };- }- const updated =- previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)- ? await terminalizeLegacyExecution({- db,- run: previousStatus,- status,- patch,- reconcileIfNeeded: true,- })- : await db- .update(heartbeatRuns)- .set({- status,- ...patch,- executionStatusDeliveryId: randomUUID(),- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, runId))- .returning()- .then((rows) => rows[0] ?? null);-- if (updated) {- publishLiveEvent({- companyId: updated.companyId,- type: "heartbeat.run.status",- payload: buildHeartbeatRunStatusLiveEventPayload(updated),- });- publishRunLifecyclePluginEvent(updated);- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);- }-- return updated;- }-- async function setRunStatusIfRunning(- runId: string,- status: string,- patch?: Partial<typeof heartbeatRuns.$inferInsert>,- failureReport?: RunFailureReportOptions,- ) {- return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);+ function truncateAgentErrorReason(+ reason: string | null | undefined,+ ): string | null {+ if (!reason) return null;+ const trimmed = reason.trim();+ if (!trimmed) return null;+ return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;}- // Move a run to a new status only when its current status is one of- // `fromStatuses`. The compare-and-set is a single conditional update, so a- // concurrent path can win the race. When this update matches nothing, the- // function reads the current row and reports updated=false, so the caller can- // keep the terminal outcome that another path already wrote.- async function setRunStatusFromLive(- runId: string,- status: string,- fromStatuses: string[],- patch?: Partial<typeof heartbeatRuns.$inferInsert>,- failureReport?: RunFailureReportOptions,- cancellationCondition?: ReturnType<typeof nativeRetryCancellationCommitCondition>,+ async function finalizeAgentStatus(+ agentId: string,+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",+ failureReason?: string | null,+ options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },) {- // fromStatuses can name a terminal status as its own source (for example,- // an idempotent "still failed" patch), so the write below is not always a- // genuine transition. Read the pre-write status to tell the two apart.- const previousStatus = await db- .select()- .from(heartbeatRuns)- .where(eq(heartbeatRuns.id, runId))- .then((rows) => rows[0] ?? null);+ const existing = await getAgent(agentId);+ if (!existing) return;- if (previousStatus && (status === "cancelled" || status === "interrupted")) {- patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };+ if (existing.status === "paused" || existing.status === "terminated") {+ return;}- if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,- resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),- };+ const isFirstHeartbeat =+ options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;- // Preserve the receipt-source fence when finalization enriches usage. A- // late spool replay must still be able to complete an unfinished receipt.- if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };+ const runningCount = await countRunningRunsForAgent(agentId);+ const nextStatus =+ runningCount > 0+ ? "running"+ : outcome === "succeeded" ||+ outcome === "interrupted" ||+ outcome === "cancelled" ||+ (outcome === "failed" && options?.keepIdleOnFailure)+ ? "idle"+ : "error";- // Cancelling a queued run that never acquired provider execution is- // positive bootstrap evidence. It must not hold unrelated queued messages.- if (- status === "cancelled" &&- previousStatus?.status === "queued" &&- previousStatus.runtimeMode !== "native" &&- !previousStatus.startedAt &&- !previousStatus.processPid- ) {- patch = {- ...patch,- resultJson: {- ...previousStatus.resultJson,- ...patch?.resultJson,- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },- },- };+ const updated = await db+ .update(agents)+ .set({+ status: nextStatus,+ // Persist a human-readable reason on the agent record when it enters+ // error so operators see it on the agent page without digging into run+ // events; clear it whenever the agent leaves error.+ errorReason:+ nextStatus === "error"+ ? truncateAgentErrorReason(failureReason)+ : null,+ lastHeartbeatAt: new Date(),+ updatedAt: new Date(),+ })+ .where(eq(agents.id, agentId))+ .returning()+ .then((rows) => rows[0] ?? null);++ if (isFirstHeartbeat && updated) {+ const tc = getTelemetryClient();+ if (tc)+ trackAgentFirstHeartbeat(tc, {+ agentRole: updated.role,+ agentId: updated.id,+ });}- const updated =- previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)- ? await terminalizeLegacyExecution({- db,- run: previousStatus,- status,- patch,- fromStatuses,- reconcileIfNeeded: true,- writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])],- })- : await db- .update(heartbeatRuns)- .set({- status,- ...patch,- executionStatusDeliveryId: randomUUID(),- updatedAt: new Date(),- })- .where(- and(- eq(heartbeatRuns.id, runId),- inArray(heartbeatRuns.status, fromStatuses),- ...(cancellationCondition ? [cancellationCondition] : []),- ...(isHeartbeatRunTerminalStatus(status)- ? [nativeRunnerOwnershipNotHeldCondition()]- : []),- ),- )- .returning()- .then((rows) => rows[0] ?? null)- .catch((error: unknown) => {- if (cancellationCondition) rethrowNativeCancellationLockConflict(error);- throw error;- });if (updated) {publishLiveEvent({companyId: updated.companyId,- type: "heartbeat.run.status",- payload: buildHeartbeatRunStatusLiveEventPayload(updated),- });- publishRunLifecyclePluginEvent(updated);- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);- return { run: updated, updated: true as const };- }-- const current = await db- .select()- .from(heartbeatRuns)- .where(eq(heartbeatRuns.id, runId))- .then((rows) => rows[0] ?? null);-- return { run: current, updated: false as const };- }-- // Invariant: when a run releases its environment lease, the run row must be- // terminal. The finalizer writes the terminal status in a step that is- // separate from the agent status=done PATCH. If the sandbox or the run- // process stops between the two steps, heartbeat_runs.status stays "running".- // The UI reads liveness from that row, so a finished task shows "Live"- // forever. This function closes the gap in the run teardown: when the run is- // still running or queued, it forces a terminal status before the lease is- // released. It never overwrites a status that another path already made- // terminal.- async function terminalizeRunOnLeaseRelease(- run: typeof heartbeatRuns.$inferSelect,- ): Promise<typeof heartbeatRuns.$inferSelect> {- if (isNativeRunnerOwnershipHeld(run)) return run;- if (isHeartbeatRunTerminalStatus(run.status)) return run;- if (run.status !== "running" && run.status !== "queued") return run;-- // Choose the terminal status that reflects the true outcome. When the issue- // already reached a terminal status, the run reached its goal, so use the- // matching terminal run status. Otherwise the teardown cut the run short,- // so use "interrupted".- const issueId = readNonEmptyString(- parseObject(run.contextSnapshot).issueId,- );- let terminalStatus: "succeeded" | "cancelled" | "interrupted" =- "interrupted";- if (issueId) {- const issueStatus = await db- .select({ status: issues.status })- .from(issues)- .where(eq(issues.id, issueId))- .then((rows) => rows[0]?.status ?? null);- if (issueStatus === "done") terminalStatus = "succeeded";- else if (issueStatus === "cancelled") terminalStatus = "cancelled";- }-- // Teardown can beat the cancellation finalizer. Preserve the acknowledged- // user intent instead of reporting an infrastructure interruption.- if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled";-- const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`;- // Match both "running" and "queued". A queued run has released its lease but- // never reached "running", so a running-only update would miss it and leave- // a phantom live run behind.- const write = await setRunStatusFromLive(- run.id,- terminalStatus,- ["running", "queued"],- {- finishedAt: run.finishedAt ?? new Date(),- error: run.error ?? (terminalStatus === "interrupted" ? message : null),- errorCode:- run.errorCode ??- (terminalStatus === "interrupted"- ? "lease_released_before_terminal"- : null),- },- );- if (!write.updated) {- // Another path already finalized the run. Keep that terminal outcome.- return write.run ?? run;- }-- const terminalRun = write.run;- if (terminalRun) {- await appendRunEvent(terminalRun, {- eventType: "lifecycle",- stream: "system",- level: terminalStatus === "interrupted" ? "warn" : "info",- message,+ type: "agent.status",payload: {- previousStatus: run.status,- terminalStatus,- reason: "environment_lease_release",- ...(issueId ? { issueId } : {}),+ agentId: updated.id,+ status: updated.status,+ lastHeartbeatAt: updated.lastHeartbeatAt+ ? new Date(updated.lastHeartbeatAt).toISOString()+ : null,+ outcome,},- }).catch((eventErr) => {- logger.warn(- { err: eventErr, runId: run.id },- "failed to append run event for lease-release terminalization",- );});}- return terminalRun ?? run;- }-- function publishRunLifecyclePluginEvent(- run: typeof heartbeatRuns.$inferSelect,- ) {- publishRunLifecyclePluginEventData({- companyId: run.companyId,- runId: run.id,- agentId: run.agentId,- status: run.status,- invocationSource: run.invocationSource,- triggerDetail: run.triggerDetail,- error: run.error,- errorCode: run.errorCode,- issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),- startedAt: run.startedAt,- finishedAt: run.finishedAt,- });- }-- function publishRunLifecyclePluginEventData(run: {- companyId: string;- runId: string;- agentId: string;- status: string;- invocationSource: string;- triggerDetail: string | null;- error: string | null;- errorCode: string | null;- issueId: string | null;- startedAt: Date | null;- finishedAt: Date | null;- }) {- const eventType =- run.status === "running"- ? "agent.run.started"- : run.status === "succeeded"- ? "agent.run.finished"- : run.status === "failed" || run.status === "timed_out"- ? "agent.run.failed"- : run.status === "cancelled"- ? "agent.run.cancelled"- : null;- if (!eventType) return;- publishPluginDomainEvent({- eventId: randomUUID(),- eventType,- occurredAt: new Date().toISOString(),- actorId: run.agentId,- actorType: "agent",- entityId: run.runId,- entityType: "heartbeat_run",- companyId: run.companyId,- payload: {- runId: run.runId,- agentId: run.agentId,- status: run.status,- invocationSource: run.invocationSource,- triggerDetail: run.triggerDetail,- error: run.error ?? null,- errorCode: run.errorCode ?? null,- issueId: run.issueId,- startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null,- finishedAt: run.finishedAt- ? new Date(run.finishedAt).toISOString()- : null,- },- });- }-- async function setWakeupStatus(- wakeupRequestId: string | null | undefined,- status: string,- patch?: Partial<typeof agentWakeupRequests.$inferInsert>,- ) {- if (!wakeupRequestId) return;- await db- .update(agentWakeupRequests)- .set({ status, ...patch, updatedAt: new Date() })- // Reassignment revokes requests transactionally. Late execution settlement- // must not turn a revoked request back into authority for a later retry.- .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));- }-- async function addContinuationExhaustedCommentOnce(input: {- run: typeof heartbeatRuns.$inferSelect;- issueId: string;- comment: string;- }) {- const existing = await db- .select({ id: issueComments.id })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, input.run.companyId),- eq(issueComments.issueId, input.issueId),- eq(issueComments.createdByRunId, input.run.id),- sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- if (existing) return;- await issuesSvc.addComment(input.issueId, input.comment, {- agentId: input.run.agentId,- runId: input.run.id,- });- }-- async function handleRunLivenessContinuation(- run: typeof heartbeatRuns.$inferSelect,- ) {- const context = parseObject(run.contextSnapshot);- if (- readNonEmptyString(context.goalControlRequestId) ||- context.resumeSessionGoalHeartbeat === true- )- return;- const livenessState = run.livenessState as RunLivenessState | null;- if (livenessState !== "plan_only" && livenessState !== "empty_response")- return;-- const issueId = readNonEmptyString(context.issueId);- if (!issueId) return;- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;-- const [issue, agent] = await Promise.all([- db- .select({- id: issues.id,- companyId: issues.companyId,- identifier: issues.identifier,- title: issues.title,- status: issues.status,- assigneeAgentId: issues.assigneeAgentId,- executionState: issues.executionState,- projectId: issues.projectId,- })- .from(issues)- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))- .then((rows) => rows[0] ?? null),- db- .select({- id: agents.id,- companyId: agents.companyId,- status: agents.status,- })- .from(agents)- .where(eq(agents.id, run.agentId))- .then((rows) => rows[0] ?? null),- ]);-- const budgetBlock =- issue && agent- ? await budgets.getInvocationBlock(issue.companyId, agent.id, {- issueId: issue.id,- projectId: issue.projectId,- })- : null;-- const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;- const idempotencyKey = issue- ? buildRunLivenessContinuationIdempotencyKey({- issueId: issue.id,- sourceRunId: run.id,- livenessState,- nextAttempt,- })- : null;- const existingWake = idempotencyKey- ? await findExistingRunLivenessContinuationWake(db, {- companyId: run.companyId,- idempotencyKey,- })- : null;-- const decision = decideRunLivenessContinuation({- run,- issue,- agent,- livenessState,- livenessReason: run.livenessReason,- nextAction: run.nextAction,- budgetBlocked: Boolean(budgetBlock),- idempotentWakeExists: Boolean(existingWake),- });-- if (decision.kind === "exhausted") {- await setRunStatus(run.id, run.status, {- livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`,- });- await addContinuationExhaustedCommentOnce({- run,- issueId,- comment: decision.comment,- });- return;- }-- if (decision.kind !== "enqueue") return;-- const continuationRun = await enqueueWakeup(run.agentId, {- source: "automation",- triggerDetail: "system",- reason: RUN_LIVENESS_CONTINUATION_REASON,- payload: decision.payload,- contextSnapshot: {- ...decision.contextSnapshot,- originIdentityContextId: null,- parentRunId: run.id,- },- idempotencyKey: decision.idempotencyKey,- requestedByActorType: "system",- requestedByActorId: "heartbeat",- });-- if (continuationRun) {- await db- .update(heartbeatRuns)- .set({- continuationAttempt: decision.nextAttempt,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, run.id));- }- }-- function issueUiLink(- issue: Pick<typeof issues.$inferSelect, "id" | "identifier">,- ) {- const label = issue.identifier ?? issue.id;- const prefix = issue.identifier?.split("-")[0] || "PAP";- return `[${label}](/${prefix}/issues/${label})`;- }-- function hasUnmanagedBackgroundTaskEvidence(- resultJson: Record<string, unknown> | null | undefined,- ) {- const evidence = parseObject(resultJson?.unmanagedBackgroundTask);- return (- evidence.stopped === true &&- (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON ||- evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON)- );- }-- function withUnmanagedBackgroundTaskStopReason(- resultJson: Record<string, unknown> | null | undefined,- ) {- return {- ...(resultJson ?? {}),- stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,- };- }-- function buildDetectedSuccessfulRunProgressSummary(- run: typeof heartbeatRuns.$inferSelect,- currentUserRedactionOptions: CurrentUserRedactionOptions,- ) {- const resultJson = parseObject(run.resultJson);- const candidates = [- hasUnmanagedBackgroundTaskEvidence(resultJson)- ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON- : null,- readNonEmptyString(run.nextAction)- ? `Next action noted: ${readNonEmptyString(run.nextAction)}`- : null,- readNonEmptyString(run.livenessReason),- readNonEmptyString(resultJson.summary),- readNonEmptyString(resultJson.result),- readNonEmptyString(resultJson.message),- ].filter((value): value is string => Boolean(value));- const summary = candidates[0];- if (!summary) return null;- return redactDetectedSuccessfulRunProgressSummaryForBoard(- summary,- currentUserRedactionOptions,- );- }-- async function addSuccessfulRunHandoffCommentOnce(input: {- issue: Pick<- typeof issues.$inferSelect,- "id" | "identifier" | "title" | "status"- >;- run: typeof heartbeatRuns.$inferSelect;- agent: Pick<typeof agents.$inferSelect, "id" | "name">;- detectedProgressSummary: string;- }) {- const existing = await db- .select({ id: issueComments.id })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, input.run.companyId),- eq(issueComments.issueId, input.issue.id),- eq(issueComments.createdByRunId, input.run.id),- sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);- if (existing) return null;- const notice = buildSuccessfulRunHandoffRequiredNotice(input);- return issuesSvc.addComment(- input.issue.id,- notice.body,- { runId: input.run.id },- {- authorType: "system",- presentation: notice.presentation,- metadata: notice.metadata,- },- );- }-- async function handleSuccessfulRunHandoff(- run: typeof heartbeatRuns.$inferSelect,- agent: typeof agents.$inferSelect,- ) {- if (run.status !== "succeeded") return;- const context = parseObject(run.contextSnapshot);- const issueId =- readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);- if (!issueId) return;- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;- if (- readNonEmptyString(context.goalControlRequestId) ||- context.resumeSessionGoalHeartbeat === true- ) {- const goalProjection = await runnerGoalService(db).projection(- run.companyId,- issueId,- run.agentId,- );- if (goalProjection?.goal?.status !== "complete") return;- }-- const issue = await db- .select({- id: issues.id,- companyId: issues.companyId,- identifier: issues.identifier,- title: issues.title,- description: issues.description,- status: issues.status,- assigneeAgentId: issues.assigneeAgentId,- assigneeUserId: issues.assigneeUserId,- executionState: issues.executionState,- monitorNextCheckAt: issues.monitorNextCheckAt,- projectId: issues.projectId,- originKind: issues.originKind,- })- .from(issues)- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))- .then((rows) => rows[0] ?? null);- const idempotencyKey = issue- ? buildFinishSuccessfulRunHandoffIdempotencyKey({- issueId: issue.id,- sourceRunId: run.id,- })- : null;- const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null);- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();- const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary(- run,- currentUserRedactionOptions,- );- const resultJson = parseObject(run.resultJson);- const finalReport = redactSuccessfulRunHandoffEvidence(- [- readNonEmptyString(resultJson.summary),- readNonEmptyString(resultJson.result),- readNonEmptyString(resultJson.message),- ].find((value): value is string => Boolean(value)) ?? null,- currentUserRedactionOptions,- );- const nextAction = redactSuccessfulRunHandoffEvidence(- readNonEmptyString(run.nextAction),- currentUserRedactionOptions,- );-- const [- activeExecutionPath,- queuedWake,- pendingInteraction,- pendingApproval,- explicitBlocker,- openRecoveryIssue,- existingWake,- budgetBlock,- pauseHold,- activeRoutineContinuation,- ] = await Promise.all([- issue- ? db- .select({ id: heartbeatRuns.id })- .from(heartbeatRuns)- .where(- and(- eq(heartbeatRuns.companyId, issue.companyId),- eq(heartbeatRuns.agentId, run.agentId),- inArray(heartbeatRuns.status, [- ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,- ]),- sql`(- ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}- or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id}- )`,- sql`${heartbeatRuns.id} <> ${run.id}`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- issue- ? db- .select({ id: agentWakeupRequests.id })- .from(agentWakeupRequests)- .where(- and(- eq(agentWakeupRequests.companyId, issue.companyId),- eq(agentWakeupRequests.agentId, run.agentId),- inArray(agentWakeupRequests.status, [- "queued",- "deferred_issue_execution",- "claimed",- ]),- sql`(- ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}- or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id}- or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id}- or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id}- )`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- issue- ? db- .select({ id: issueThreadInteractions.id })- .from(issueThreadInteractions)- .where(- and(- eq(issueThreadInteractions.companyId, issue.companyId),- eq(issueThreadInteractions.issueId, issue.id),- eq(issueThreadInteractions.status, "pending"),- activeIssueInteractionCondition(),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- issue- ? db- .select({ id: issueApprovals.approvalId })- .from(issueApprovals)- .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))- .where(- and(- eq(issueApprovals.companyId, issue.companyId),- eq(issueApprovals.issueId, issue.id),- inArray(approvals.status, ["pending", "revision_requested"]),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- issue- ? db- .select({ id: issueRelations.issueId })- .from(issueRelations)- .where(- and(- eq(issueRelations.companyId, issue.companyId),- eq(issueRelations.relatedIssueId, issue.id),- eq(issueRelations.type, "blocks"),- sql`exists (- select 1- from issues blocker- where blocker.id = ${issueRelations.issueId}- and blocker.company_id = ${issue.companyId}- and blocker.status not in ('done', 'cancelled')- and blocker.hidden_at is null- )`,- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- issue- ? db- .select({ id: issues.id })- .from(issues)- .where(- and(- eq(issues.companyId, issue.companyId),- inArray(issues.originKind, [- RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,- RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,- ]),- eq(issues.originId, issue.id),- visibleIssueCondition(),- notInArray(issues.status, ["done", "cancelled"]),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- idempotencyKey- ? findExistingFinishSuccessfulRunHandoffWake(db, {- companyId: run.companyId,- idempotencyKey,- })- : Promise.resolve(null),- issue- ? budgets.getInvocationBlock(issue.companyId, run.agentId, {- issueId: issue.id,- projectId: issue.projectId,- })- : Promise.resolve(null),- issue- ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id)- : Promise.resolve(null),- issue- ? db- .select({ id: routines.id })- .from(routines)- .where(- and(- eq(routines.companyId, issue.companyId),- eq(routines.parentIssueId, issue.id),- eq(routines.status, "active"),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null)- : Promise.resolve(null),- ]);-- const decision = decideSuccessfulRunHandoff({- run,- issue,- agent,- livenessState: run.livenessState as RunLivenessState | null,- detectedProgressSummary,- finalReport,- nextAction,- taskKey,- hasActiveExecutionPath: Boolean(activeExecutionPath),- hasQueuedWake: Boolean(queuedWake),- hasPendingInteractionOrApproval: Boolean(- pendingInteraction || pendingApproval,- ),- hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt),- hasExplicitBlockerPath: Boolean(explicitBlocker),- hasOpenRecoveryIssue: Boolean(openRecoveryIssue),- hasPauseHold: Boolean(pauseHold),- hasActiveRoutineContinuation: Boolean(activeRoutineContinuation),- budgetBlocked: Boolean(budgetBlock),- idempotentWakeExists: Boolean(existingWake),- });-- if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) {- await resolveRequiredSuccessfulRunHandoffOnValidPath(db, {- companyId: issue.companyId,- issueId: issue.id,- issueIdentifier: issue.identifier,- agentId: run.agentId,- runId: run.id,- skipReason: decision.reason,- });- }-- if (decision.kind !== "enqueue" || !issue) return;-- if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) {- await db- .update(heartbeatRuns)- .set({- livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||- ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, run.id));- }-- const handoffRun = await enqueueWakeup(decision.targetAgentId, {- source: "automation",- triggerDetail: "system",- reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,- payload: decision.payload,- contextSnapshot: {- ...decision.contextSnapshot,- originIdentityContextId: null,- parentRunId: run.id,- },- idempotencyKey: decision.idempotencyKey,- requestedByActorType: "system",- requestedByActorId: "heartbeat",- });- if (!handoffRun) return;-- await addSuccessfulRunHandoffCommentOnce({- issue,- run,- agent,- detectedProgressSummary:- detectedProgressSummary ??- "The run reported progress, but did not choose a next step.",- });- await logActivity(db, {- companyId: issue.companyId,- actorType: "system",- actorId: "heartbeat",- agentId: run.agentId,- runId: run.id,- action: "issue.successful_run_handoff_required",- entityType: "issue",- entityId: issue.id,- details: {- label: "Successful run missing issue disposition",- sourceRunId: run.id,- correctiveRunId: handoffRun.id,- handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON,- missingDisposition: "clear_next_step",- detectedProgressSummary,- issue: issueUiLink(issue),- },- });- }-- async function handleIssueReviewPathDisposition(- run: typeof heartbeatRuns.$inferSelect,- ) {- const contextSnapshot = parseObject(run.contextSnapshot);- if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return;- const issueId =- readNonEmptyString(contextSnapshot.issueId) ??- readNonEmptyString(contextSnapshot.taskId);- if (!issueId) return;- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;-- const issue = await db- .select({- id: issues.id,- companyId: issues.companyId,- identifier: issues.identifier,- status: issues.status,- assigneeAgentId: issues.assigneeAgentId,- })- .from(issues)- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))- .then((rows) => rows[0] ?? null);- if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId)- return;-- const reviewAttention = await issuesSvc- .listReviewAttention(issue.companyId, [issue])- .then(- (map) =>- map.get(issue.id) ?? {- state: "none" as const,- paths: [],- reason: null,- },- );- if (reviewAttention.state !== "stalled") return;-- const consumedPathRef = reviewPathConsumedRefFromRun({- runId: run.id,- issueId: issue.id,- contextSnapshot,- });- const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({- issueId: issue.id,- consumedPathRef,- });- const existingWake = await db- .select({ id: agentWakeupRequests.id })- .from(agentWakeupRequests)- .where(- and(- eq(agentWakeupRequests.companyId, issue.companyId),- eq(agentWakeupRequests.idempotencyKey, idempotencyKey),- notInArray(agentWakeupRequests.status, ["skipped"]),- ),- )- .limit(1)- .then((rows) => rows[0] ?? null);-- const decision = decideIssueReviewPathRecovery({- issueId: issue.id,- sourceRunId: run.id,- assigneeAgentId: issue.assigneeAgentId,- contextSnapshot,- reviewAttention,- existingWake: Boolean(existingWake),- });- if (decision.kind !== "enqueue") return;-- const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, {- source: "automation",- triggerDetail: "system",- reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON,- idempotencyKey: decision.idempotencyKey,- payload: decision.payload,- contextSnapshot: decision.contextSnapshot,- requestedByActorType: "system",- requestedByActorId: "heartbeat",- }).catch((error: unknown) => {- if (isReviewPathRecoveryIdempotencyConflict(error)) return null;- throw error;- });- if (!recoveryRun) return;-- await logActivity(db, {- companyId: issue.companyId,- actorType: "system",- actorId: "heartbeat",- agentId: issue.assigneeAgentId,- runId: run.id,- action: "issue.review_path_recovery_queued",- entityType: "issue",- entityId: issue.id,- details: {- sourceRunId: run.id,- recoveryRunId: recoveryRun.id,- consumedPathRef,- recoveryAttempt: 1,- maxRecoveryAttempts: 1,- },- });- }-- async function appendRunEvent(- run: typeof heartbeatRuns.$inferSelect,- event: {- eventType: string;- stream?: "system" | "stdout" | "stderr";- level?: "info" | "warn" | "error";- color?: string;- message?: string;- payload?: Record<string, unknown>;- retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"];- },- ) {- const eventAt = new Date();- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();- const sanitizedMessage = event.message- ? redactSensitiveText(- redactCurrentUserText(event.message, currentUserRedactionOptions),- )- : event.message;- const boundedPayload = event.payload- ? boundHeartbeatRunEventPayloadForStorage(event.payload)- : event.payload;- const secretSanitizedPayload = boundedPayload- ? redactEventPayload(boundedPayload)- : boundedPayload;- const sanitizedPayload = secretSanitizedPayload- ? redactCurrentUserValue(- secretSanitizedPayload,- currentUserRedactionOptions,- )- : secretSanitizedPayload;- const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null;- const progress = buildRunEventRuntimeProgress({- eventType: event.eventType,- message: sanitizedMessage ?? null,- payload: sanitizedPayload ?? null,- at: eventAt,- });- const persistedEvent = await appendHeartbeatRunEvent(db, {- companyId: run.companyId,- runId: run.id,- agentId: run.agentId,- eventType: event.eventType,- stream: event.stream,- level: event.level,- color: event.color,- message: sanitizedMessage,- payload: sanitizedPayload,- retryExhaustion: event.retryExhaustion,- });- if (persistedEvent.disposition === "duplicate") return;- const seq = persistedEvent.row.seq;-- publishLiveEvent({- companyId: run.companyId,- type: "heartbeat.run.event",- payload: {- runId: run.id,- agentId: run.agentId,- issueId,- seq,- eventType: event.eventType,- stream: event.stream ?? null,- level: event.level ?? null,- color: event.color ?? null,- message: sanitizedMessage ?? null,- currentToolName: progress?.currentToolName ?? null,- lastAssistantSnippet: progress?.lastAssistantSnippet ?? null,- lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(),- payload: sanitizedPayload ?? null,- },- });- if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) {- const status = setHeartbeatRunRuntimeStatus({- companyId: run.companyId,- issueId,- agentId: run.agentId,- runId: run.id,- phase: progress.phase,- message: progress.message,- updatedAt: eventAt,- currentToolName: progress.currentToolName,- lastAssistantSnippet: progress.lastAssistantSnippet,- lastEventAt: progress.lastEventAt,- });- if (status) publishHeartbeatRunRuntimeProgress(status);- }- }-- async function persistRunProcessMetadata(- runId: string,- meta: { pid: number; processGroupId: number | null; startedAt: string },- ) {- return persistHeartbeatRunProcessMetadata(db, runId, meta);- }-- async function clearDetachedRunWarning(runId: string) {- const updated = await db- .update(heartbeatRuns)- .set({- error: null,- errorCode: null,- updatedAt: new Date(),- })- .where(- and(- eq(heartbeatRuns.id, runId),- eq(heartbeatRuns.status, "running"),- eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE),- ),- )- .returning()- .then((rows) => rows[0] ?? null);- if (!updated) return null;-- await appendRunEvent(updated, {- eventType: "lifecycle",- stream: "system",- level: "info",- message:- "Detached child process reported activity; cleared detached warning",- });- return updated;- }-- async function patchRunIssueCommentStatus(- runId: string,- patch: Partial<- Pick<- typeof heartbeatRuns.$inferInsert,- | "issueCommentStatus"- | "issueCommentSatisfiedByCommentId"- | "issueCommentRetryQueuedAt"- >- >,- ) {- return db- .update(heartbeatRuns)- .set({ ...patch, updatedAt: new Date() })- .where(eq(heartbeatRuns.id, runId))- .returning()- .then((rows) => rows[0] ?? null);- }-- async function findRunIssueComment(- runId: string,- companyId: string,- issueId: string,- resultJson?: Record<string, unknown> | null,- ) {- const comments = await db- .select({- id: issueComments.id,- body: issueComments.body,- })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, companyId),- eq(issueComments.issueId, issueId),- eq(issueComments.createdByRunId, runId),- ),- )- .orderBy(desc(issueComments.createdAt), desc(issueComments.id));- return findHeartbeatRunCompletionComment(comments, resultJson);- }-- async function findLatestCompletedFinalAgentMessage(- runId: string,- companyId: string,- ) {- const rows = await db- .select({- seq: heartbeatRunEvents.seq,- payload: heartbeatRunEvents.payload,- })- .from(heartbeatRunEvents)- .where(- and(- eq(heartbeatRunEvents.companyId, companyId),- eq(heartbeatRunEvents.runId, runId),- eq(heartbeatRunEvents.eventType, "item.completed"),- ),- )- .orderBy(desc(heartbeatRunEvents.seq))- .limit(200);- const candidates: Array<{- seq: number;- text: string;- sourceEventId: string | null;- channel: "final" | "unknown";- }> = [];- for (const row of rows) {- const prpEvent = parseObject(parseObject(row.payload).prpEvent);- const candidate = readCompletedAssistantMessageCandidate({- seq: row.seq,- prpEvent,- });- if (candidate) candidates.push(candidate);- }- const recoveryBoundary = await db- .select({- seq: heartbeatRunEvents.seq,- payload: heartbeatRunEvents.payload,- })- .from(heartbeatRunEvents)- .where(- and(- eq(heartbeatRunEvents.companyId, companyId),- eq(heartbeatRunEvents.runId, runId),- eq(heartbeatRunEvents.eventType, "lifecycle"),- ),- )- .orderBy(heartbeatRunEvents.seq)- .limit(200)- .then(- (lifecycleRows) =>- lifecycleRows.find(- (row) =>- parseObject(row.payload).retryReasonCode ===- "semantic_result_missing",- )?.seq ?? null,- );- return selectHeartbeatRunFinalAgentMessage({- candidates,- semanticResultRecoveryAfterSeq: recoveryBoundary,- });- }-- async function refreshContinuationSummaryForRun(- run: typeof heartbeatRuns.$inferSelect,- agent: typeof agents.$inferSelect,- ) {- const contextSnapshot = parseObject(run.contextSnapshot);- const issueId = readNonEmptyString(contextSnapshot.issueId);- if (!issueId) return null;- try {- return await refreshIssueContinuationSummary({- db,- issueId,- run: {- id: run.id,- status: run.status,- error: run.error,- errorCode: run.errorCode,- resultJson: run.resultJson as Record<string, unknown> | null,- stdoutExcerpt: run.stdoutExcerpt,- stderrExcerpt: run.stderrExcerpt,- finishedAt: run.finishedAt,- },- agent: {- id: agent.id,- name: agent.name,- adapterType: agent.adapterType,- },- });- } catch (err) {- logger.warn(- {- err,- runId: run.id,- issueId,- agentId: agent.id,- },- "failed to refresh issue continuation summary",- );- return null;- }- }-- async function enqueueMissingIssueCommentRetry(- run: typeof heartbeatRuns.$inferSelect,- agent: typeof agents.$inferSelect,- issueId: string,- ) {- const invokability = await getAgentInvokability(agent);- if (!invokability.invokable) {- await appendRunEvent(run, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message:- "Missing-comment retry suppressed because the agent is not invokable",- payload: {- reason: invokability.reason,- invalidOrgChain: invokability.invalidOrgChain,- ...invokability.details,- },- });- return null;- }-- const contextSnapshot = parseObject(run.contextSnapshot);- const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);- const retryContextSnapshot = withRecoveryContext(- {- ...contextSnapshot,- retryOfRunId: run.id,- wakeReason: "missing_issue_comment",- retryReason: "missing_issue_comment",- missingIssueCommentForRunId: run.id,- },- "status_only",- );- const responsibleUserId = await resolveResponsibleUserIdForRunContext(- run,- retryContextSnapshot,- );- const now = new Date();-- const retryRun = await db.transaction(async (tx) => {- await tx.execute(- sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`,- );-- const issue = await tx- .select({ id: issues.id })- .from(issues)- .where(- and(- eq(issues.companyId, run.companyId),- eq(issues.executionRunId, run.id),- ),- )- .then((rows) => rows[0] ?? null);- if (!issue) return null;-- const wakeupRequest = await tx- .insert(agentWakeupRequests)- .values({- companyId: run.companyId,- agentId: run.agentId,- source: "automation",- triggerDetail: "system",- reason: "missing_issue_comment",- payload: withRecoveryContext(- {- issueId,- retryOfRunId: run.id,- retryReason: "missing_issue_comment",- },- "status_only",- ),- status: "queued",- requestedByActorType: "system",- requestedByActorId: null,- updatedAt: now,- })- .returning()- .then((rows) => rows[0]);-- const queuedRun = await tx- .insert(heartbeatRuns)- .values({- companyId: run.companyId,- agentId: run.agentId,- scopeKind: "issue",- issueId,- invocationSource: "automation",- triggerDetail: "system",- status: "queued",- wakeupRequestId: wakeupRequest.id,- contextSnapshot: retryContextSnapshot,- responsibleUserId,- sessionIdBefore: sessionBefore,- retryOfRunId: run.id,- issueCommentStatus: "not_applicable",- updatedAt: now,- })- .returning()- .then((rows) => rows[0]);-- await tx- .update(agentWakeupRequests)- .set({- runId: queuedRun.id,- updatedAt: now,- })- .where(eq(agentWakeupRequests.id, wakeupRequest.id));-- await tx- .update(issues)- .set({- executionRunId: queuedRun.id,- executionAgentNameKey: normalizeAgentNameKey(agent.name),- executionLockedAt: now,- updatedAt: now,- })- .where(eq(issues.id, issue.id));-- await tx- .update(heartbeatRuns)- .set({- issueCommentStatus: "retry_queued",- issueCommentRetryQueuedAt: now,- updatedAt: now,- })- .where(eq(heartbeatRuns.id, run.id));-- return queuedRun;- });-- if (!retryRun) return null;-- publishLiveEvent({- companyId: retryRun.companyId,- type: "heartbeat.run.queued",- payload: {- runId: retryRun.id,- agentId: retryRun.agentId,- invocationSource: retryRun.invocationSource,- triggerDetail: retryRun.triggerDetail,- wakeupRequestId: retryRun.wakeupRequestId,- },- });-- return retryRun;- }-- async function hasDeferredIssueCommentWake(- companyId: string,- issueId: string,- agentId: string,- ) {- const deferredPayloads = await db- .select({ payload: agentWakeupRequests.payload })- .from(agentWakeupRequests)- .where(- and(- eq(agentWakeupRequests.companyId, companyId),- eq(agentWakeupRequests.agentId, agentId),- eq(agentWakeupRequests.status, "deferred_issue_execution"),- sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,- ),- );-- return deferredPayloads.some(({ payload }) => {- const parsedPayload = parseObject(payload);- const deferredContext = parseObject(- parsedPayload[DEFERRED_WAKE_CONTEXT_KEY],- );- return Boolean(deriveCommentId(deferredContext, parsedPayload));- });- }-- async function finalizeIssueCommentPolicy(- run: typeof heartbeatRuns.$inferSelect,- agent: typeof agents.$inferSelect,- presentationDecision?: RunPresentationDecision | null,- ) {- const contextSnapshot = parseObject(run.contextSnapshot);- // The explicit receipt admitted one turn. A prose-only follow-up cannot- // reuse it or renew it under the separate transient-failure retry policy.- if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {- if (run.issueCommentStatus !== "not_applicable") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- }- return { outcome: "not_applicable" as const, queuedRun: null };- }- const issueId = readNonEmptyString(contextSnapshot.issueId);- if (!issueId) {- if (run.issueCommentStatus !== "not_applicable") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- }- return { outcome: "not_applicable" as const, queuedRun: null };- }-- // A failed/timed-out/cancelled run may immediately enqueue the normal- // assignment or continuation recovery below. Do not let the lower-priority- // status-only missing-comment wake claim the issue first. The dedicated- // missing-comment retry still records exhaustion if that retry itself- // fails, but an original failed execution remains on the established- // direct-adapter recovery path.- if (- run.status !== "succeeded" &&- readNonEmptyString(contextSnapshot.retryReason) !==- "missing_issue_comment"- ) {- if (run.issueCommentStatus !== "not_applicable") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- }- return { outcome: "not_applicable" as const, queuedRun: null };- }-- // A settled run may legitimately have no user-facing prose. The response- // resolver owns that decision; do not wake the agent again merely to force- // an artificial comment into the issue thread.- if (- presentationDecision?.chosenSource === "none" &&- (hasAcceptedSemanticResult(parseObject(run.resultJson)) ||- presentationDecision.reasonCodes.includes(- "legacy_adapter_summary_ambiguous",- ))- ) {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- return { outcome: "not_applicable" as const, queuedRun: null };- }-- // A pre-dispatch setup failure means the adapter process never started (for- // example an unresolved workspace base ref). No agent could run, so no agent- // could post an issue comment. A missing-comment retry cannot help and would- // loop the identical pre-adapter failure, so mark the policy not_applicable- // and queue nothing.- if (- run.errorCode != null &&- PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode)- ) {- if (run.issueCommentStatus !== "not_applicable") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- }- return { outcome: "not_applicable" as const, queuedRun: null };- }-- const postedComment = await findRunIssueComment(- run.id,- run.companyId,- issueId,- parseObject(run.resultJson),- );- if (postedComment) {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "satisfied",- issueCommentSatisfiedByCommentId: postedComment.id,- issueCommentRetryQueuedAt: null,- });- return { outcome: "satisfied" as const, queuedRun: null };- }-- // Missing-comment recovery is a legacy compatibility path for otherwise- // successful runs. A failed, timed-out, or cancelled run is already owned- // by lifecycle recovery and its terminal system presentation. Queuing a- // prose-only retry here can seize the issue execution lock before the- // authoritative continuation is materialized, replacing real recovery- // with a cheap status-only turn.- if (run.status !== "succeeded") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- return { outcome: "not_applicable" as const, queuedRun: null };- }-- if (- readNonEmptyString(contextSnapshot.retryReason) ===- "missing_issue_comment"- ) {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "retry_exhausted",- issueCommentSatisfiedByCommentId: null,- });- await appendRunEvent(run, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message:- "Run ended without an issue comment after one retry; no further comment wake will be queued",- });- return { outcome: "retry_exhausted" as const, queuedRun: null };- }-- if (!shouldRequireIssueCommentForWake(contextSnapshot)) {- if (run.issueCommentStatus !== "not_applicable") {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- }- return { outcome: "not_applicable" as const, queuedRun: null };- }-- if (- await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)- ) {- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "not_applicable",- issueCommentSatisfiedByCommentId: null,- issueCommentRetryQueuedAt: null,- });- await appendRunEvent(run, {- eventType: "lifecycle",- stream: "system",- level: "info",- message:- "Run ended without an issue comment; a deferred comment wake already exists for this issue",- });- return { outcome: "not_applicable" as const, queuedRun: null };- }-- const queuedRun = await enqueueMissingIssueCommentRetry(- run,- agent,- issueId,- );- if (queuedRun) {- await appendRunEvent(run, {- eventType: "lifecycle",- stream: "system",- level: "warn",- message:- "Run ended without an issue comment; queued one follow-up wake to require a comment",- });- return { outcome: "retry_queued" as const, queuedRun };- }-- await patchRunIssueCommentStatus(run.id, {- issueCommentStatus: "retry_exhausted",- issueCommentSatisfiedByCommentId: null,- });- return { outcome: "retry_exhausted" as const, queuedRun: null };- }-- function parseMaxTurnContinuationPolicy(- agent: typeof agents.$inferSelect,- ): MaxTurnContinuationPolicy {- const runtimeConfig = parseObject(agent.runtimeConfig);- const heartbeat = parseObject(runtimeConfig.heartbeat);- const configured = parseObject(heartbeat.maxTurnContinuation);- const rawMaxAttempts = Math.floor(- asNumber(- configured.maxAttempts,- MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,- ),- );- const rawDelayMs = Math.floor(- asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),- );-- return {- enabled: asBoolean(configured.enabled, true),- maxAttempts: Math.max(- 0,- Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),- ),- delayMs: Math.max(- 0,- Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),- ),- };- }-- function truncateAgentErrorReason(- reason: string | null | undefined,- ): string | null {- if (!reason) return null;- const trimmed = reason.trim();- if (!trimmed) return null;- return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;- }-- async function finalizeAgentStatus(- agentId: string,- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",- failureReason?: string | null,- options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },- ) {- const existing = await getAgent(agentId);- if (!existing) return;-- if (existing.status === "paused" || existing.status === "terminated") {- return;- }-- const isFirstHeartbeat =- options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;-- const runningCount = await countRunningRunsForAgent(agentId);- const nextStatus =- runningCount > 0- ? "running"- : outcome === "succeeded" ||- outcome === "interrupted" ||- outcome === "cancelled" ||- (outcome === "failed" && options?.keepIdleOnFailure)- ? "idle"- : "error";-- const updated = await db- .update(agents)- .set({- status: nextStatus,- // Persist a human-readable reason on the agent record when it enters- // error so operators see it on the agent page without digging into run- // events; clear it whenever the agent leaves error.- errorReason:- nextStatus === "error"- ? truncateAgentErrorReason(failureReason)- : null,- lastHeartbeatAt: new Date(),- updatedAt: new Date(),- })- .where(eq(agents.id, agentId))- .returning()- .then((rows) => rows[0] ?? null);-- if (isFirstHeartbeat && updated) {- const tc = getTelemetryClient();- if (tc)- trackAgentFirstHeartbeat(tc, {- agentRole: updated.role,- agentId: updated.id,- });- }-- if (updated) {- publishLiveEvent({- companyId: updated.companyId,- type: "agent.status",- payload: {- agentId: updated.id,- status: updated.status,- lastHeartbeatAt: updated.lastHeartbeatAt- ? new Date(updated.lastHeartbeatAt).toISOString()- : null,- outcome,- },- });- }- }-- function mergeRunStopMetadataForAgent(- agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",- options?: {- resultJson?: Record<string, unknown> | null;- conversationContinuationEligible?: boolean;- errorCode?: string | null;- errorMessage?: string | null;- },- ) {- const stopMetadata = buildHeartbeatRunStopMetadata({- adapterType: agent.adapterType,- adapterConfig: parseObject(agent.adapterConfig),- outcome,- errorCode: options?.errorCode ?? null,- errorMessage: options?.errorMessage ?? null,- });- const result = mergeHeartbeatRunStopMetadata(- options?.resultJson ?? null,- stopMetadata,- );- const cancellationAcknowledged =- parseObject(result?.executionCancellation).state === "acknowledged";- return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&- (outcome !== "cancelled" || cancellationAcknowledged) &&- isConversationAdapter(agent.adapterType)- ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }- : result;- }-- function countValue(value: unknown) {- const parsed = Number(value ?? 0);- return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;- }-- function dateValue(value: unknown) {- if (value instanceof Date)- return Number.isNaN(value.getTime()) ? null : value;- if (typeof value === "string" || typeof value === "number") {- const parsed = new Date(value);- return Number.isNaN(parsed.getTime()) ? null : parsed;- }- return null;- }-- function latestDate(...values: unknown[]) {- let latest: Date | null = null;- for (const value of values) {- const parsed = dateValue(value);- if (!parsed) continue;- if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;- }- return latest;- }-- async function buildRunLivenessInput(- run: typeof heartbeatRuns.$inferSelect,- resultJson: Record<string, unknown> | null | undefined,- ): Promise<RunLivenessClassificationInput> {- const context = parseObject(run.contextSnapshot);- const contextIssueId = readNonEmptyString(context.issueId);- const continuationAttempt = asNumber(- context.continuationAttempt,- run.continuationAttempt ?? 0,- );-- const issue = contextIssueId- ? await db- .select({- status: issues.status,- title: issues.title,- description: issues.description,- workMode: issues.workMode,- })- .from(issues)- .where(- and(- eq(issues.companyId, run.companyId),- eq(issues.id, contextIssueId),- ),- )- .then((rows) => rows[0] ?? null)- : null;-- const [commentStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${issueComments.createdAt})`,- })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, run.companyId),- eq(issueComments.issueId, contextIssueId),- eq(issueComments.createdByRunId, run.id),- isNull(issueComments.deletedAt),- ),- )- : [{ count: 0, latestAt: null }];-- const issueCommentBodies = contextIssueId- ? await db- .select({ body: issueComments.body })- .from(issueComments)- .where(- and(- eq(issueComments.companyId, run.companyId),- eq(issueComments.issueId, contextIssueId),- eq(issueComments.createdByRunId, run.id),- ),- )- .orderBy(desc(issueComments.createdAt), desc(issueComments.id))- .limit(5)- .then((rows) => rows.reverse().map((row) => row.body))- : [];-- const continuationSummary = contextIssueId- ? await getIssueContinuationSummaryDocument(db, contextIssueId)- : null;-- const [documentStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,- latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,- })- .from(documentRevisions)- .innerJoin(- issueDocuments,- eq(documentRevisions.documentId, issueDocuments.documentId),- )- .where(- and(- eq(documentRevisions.companyId, run.companyId),- eq(documentRevisions.createdByRunId, run.id),- eq(issueDocuments.companyId, run.companyId),- eq(issueDocuments.issueId, contextIssueId),- sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,- ),- )- : [{ count: 0, planCount: 0, latestAt: null }];-- const [workProductStats] = contextIssueId- ? await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,- })- .from(issueWorkProducts)- .where(- and(- eq(issueWorkProducts.companyId, run.companyId),- eq(issueWorkProducts.issueId, contextIssueId),- eq(issueWorkProducts.createdByRunId, run.id),- ),- )- : [{ count: 0, latestAt: null }];-- const [workspaceOperationStats] = await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,- })- .from(workspaceOperations)- .where(- and(- eq(workspaceOperations.companyId, run.companyId),- eq(workspaceOperations.heartbeatRunId, run.id),- ),- );-- const [activityStats] = await db- .select({- count: sql<number>`count(*)::int`,- latestAt: sql<Date | null>`max(${activityLog.createdAt})`,- })- .from(activityLog)- .where(- and(- eq(activityLog.companyId, run.companyId),- eq(activityLog.runId, run.id),- notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),- ),- );-- const [eventStats] = await db- .select({- count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,- latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,- })- .from(heartbeatRunEvents)- .where(- and(- eq(heartbeatRunEvents.companyId, run.companyId),- eq(heartbeatRunEvents.runId, run.id),- ),- );-- return {- runStatus: run.status,- issue,- resultJson: resultJson ?? run.resultJson ?? null,- issueCommentBodies,- continuationSummaryBody: continuationSummary?.body ?? null,- stdoutExcerpt: run.stdoutExcerpt ?? null,- stderrExcerpt: run.stderrExcerpt ?? null,- error: run.error ?? null,- errorCode: run.errorCode ?? null,- continuationAttempt,- evidence: {- issueCommentsCreated: countValue(commentStats?.count),- documentRevisionsCreated: countValue(documentStats?.count),- planDocumentRevisionsCreated: countValue(documentStats?.planCount),- workProductsCreated: countValue(workProductStats?.count),- workspaceOperationsCreated: countValue(workspaceOperationStats?.count),- activityEventsCreated: countValue(activityStats?.count),- toolOrActionEventsCreated: countValue(eventStats?.count),- latestEvidenceAt: latestDate(- commentStats?.latestAt,- documentStats?.latestAt,- workProductStats?.latestAt,- workspaceOperationStats?.latestAt,- activityStats?.latestAt,- eventStats?.latestAt,- ),- },- };- }-- async function classifyAndPersistRunLiveness(- run: typeof heartbeatRuns.$inferSelect,- resultJson?: Record<string, unknown> | null,- ) {- const authRepair = run.status === "failed"- ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {- logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");- return null;- })- : null;- const classification = classifyRunLiveness({- ...await buildRunLivenessInput(run, resultJson),- authenticationRepairRequested: Boolean(authRepair?.interactionId),- });- return db- .update(heartbeatRuns)- .set({- livenessState: classification.livenessState,- livenessReason: classification.livenessReason,- continuationAttempt: classification.continuationAttempt,- lastUsefulActionAt: classification.lastUsefulActionAt,- nextAction: classification.nextAction,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, run.id))- .returning()- .then((rows) => rows[0] ?? null);}async function recoverActiveSessionGoals() {@@ -6406,29 +3747,6 @@ export function heartbeatService(return recovery.reconcileResolvedDependencyWakeBackstop(opts);}- async function updateRuntimeState(- agent: typeof agents.$inferSelect,- run: typeof heartbeatRuns.$inferSelect,- result: AdapterExecutionResult,- session: { legacySessionId: string | null },- normalizedUsage?: UsageTotals | null,- ) {- await ensureRuntimeState(agent);- await accountRunCost(db, run.id, budgetHooks);- await db- .update(agentRuntimeState)- .set({- adapterType: agent.adapterType,- sessionId: session.legacySessionId,- lastRunId: run.id,- lastRunStatus: run.status,- lastError: run.error ?? null,- updatedAt: new Date(),- })- .where(eq(agentRuntimeState.agentId, agent.id));-- }-// Await every background heartbeat execution that is currently in flight. A// draining run can, in its finally block, promote and dispatch the next queued// run for the same agent — that follow-up execution is registered in the setdiff --git a/server/src/services/heartbeat/run-lifecycle.test.ts b/server/src/services/heartbeat/run-lifecycle.test.tsnew file mode 100644index 0000000000..414315fbae--- /dev/null+++ b/server/src/services/heartbeat/run-lifecycle.test.ts@@ -0,0 +1,196 @@+import type { Db } from "@paperclipai/db";+import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";+import { eq, sql } from "drizzle-orm";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import type { CurrentUserRedactionOptions } from "../../log-redaction.js";+import { publishLiveEvent } from "../live-events.js";+import { getHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";+import { createHeartbeatRunState } from "./run-state.js";+import { createHeartbeatRunPreparation } from "./run-preparation.js";+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";++vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));++type Run = typeof heartbeatRuns.$inferSelect;+function callbacks(db: Db) {+ const state = createHeartbeatRunState(db);+ return {+ ...state,+ ...createHeartbeatRunPreparation(db),+ getRun: vi.fn(state.getRun),+ issuesSvc: {+ addComment: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["addComment"]>(),+ listReviewAttention: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["listReviewAttention"]>(),+ },+ recovery: { escalateStrandedAssignedIssue: vi.fn<HeartbeatLifecycleDependencies["recovery"]["escalateStrandedAssignedIssue"]>() },+ companySkills: { completeTestRunForIssue: vi.fn<HeartbeatLifecycleDependencies["companySkills"]["completeTestRunForIssue"]>() },+ budgets: { getInvocationBlock: vi.fn(async () => null) },+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },+ enqueueWakeup: vi.fn<HeartbeatLifecycleDependencies["enqueueWakeup"]>(),+ getCurrentUserRedactionOptions: vi.fn(async (): Promise<CurrentUserRedactionOptions> => ({})),+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),+ emitTerminalAgentTaskRun: vi.fn<HeartbeatLifecycleDependencies["emitTerminalAgentTaskRun"]>(),+ budgetHooks: {},+ } satisfies HeartbeatLifecycleDependencies;+}+function guardedDatabase() {+ const access = vi.fn(() => { throw new Error("Unexpected lifecycle database access"); });+ return { db: new Proxy({}, { get: access }) as Db, access };+}++describe("heartbeat lifecycle module boundary", () => {+ it("constructs without database work or reporting effects", () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ createHeartbeatLifecycle(database.db, deps);+ expect(database.access).not.toHaveBeenCalled();+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();+ });++ it("ignores absent wake requests without accessing the database", async () => {+ const database = guardedDatabase();+ const lifecycle = createHeartbeatLifecycle(database.db, callbacks(database.db));+ await lifecycle.setWakeupStatus(null, "completed");+ await lifecycle.setWakeupStatus(undefined, "completed");+ expect(database.access).not.toHaveBeenCalled();+ });++ it.each([+ { livenessState: "empty_response", contextSnapshot: { goalControlRequestId: "goal" } },+ { livenessState: "plan_only", contextSnapshot: { resumeSessionGoalHeartbeat: true } },+ { livenessState: "productive", contextSnapshot: { issueId: "issue" } },+ { livenessState: "empty_response", contextSnapshot: {} },+ ])("keeps ineligible continuations out of wake admission (%j)", async (input) => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ await createHeartbeatLifecycle(database.db, deps).handleRunLivenessContinuation(input as unknown as Run);+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();+ expect(deps.budgets.getInvocationBlock).not.toHaveBeenCalled();+ });++ it("preserves retained native ownership before lease-release teardown reads or writes", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run;+ expect(await createHeartbeatLifecycle(database.db, deps).terminalizeRunOnLeaseRelease(run)).toBe(run);+ expect(database.access).not.toHaveBeenCalled();+ expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled();+ });++ it("rejects late progress after re-reading a terminal run through the service callback", async () => {+ const database = guardedDatabase();+ const deps = callbacks(database.db);+ const run = { id: "late-progress", companyId: "company", agentId: "agent", status: "running", contextSnapshot: {} } as Run;+ deps.getRun.mockResolvedValue({ ...run, status: "succeeded" });+ expect(await createHeartbeatLifecycle(database.db, deps).recordCurrentHeartbeatRunRuntimeProgress(run, { phase: "finalize", message: "late update" }, null)).toBeNull();+ expect(deps.getRun).toHaveBeenCalledWith(run.id);+ expect(getHeartbeatRunRuntimeStatus(run.id)).toBeNull();+ expect(database.access).not.toHaveBeenCalled();+ });+});++const support = await getEmbeddedPostgresTestSupport();+if (!support.supported) console.warn(`Skipping lifecycle module database tests: ${support.reason}`);+describe.skipIf(!support.supported)("heartbeat lifecycle database wiring", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: Db;+ beforeAll(async () => {+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-lifecycle-module");+ db = createDb(database.connectionString);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => {+ await db?.$client.end();+ await database?.cleanup();+ });+ beforeEach(async () => {+ vi.mocked(publishLiveEvent).mockClear();+ await db.execute(sql`truncate table companies restart identity cascade`);+ });++ async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {+ const [company] = await db.insert(companies).values({ name: "Lifecycle module", issuePrefix: "LM" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Lifecycle agent" }).returning();+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "native", ...patch }).returning();+ return { company, agent, wake, run };+ }++ it("keeps one terminal outcome and reports only the winner of competing writes", async () => {+ const { run } = await fixture();+ const deps = callbacks(db);+ const lifecycle = createHeartbeatLifecycle(db, deps);+ const results = await Promise.all([+ lifecycle.setRunStatusIfRunning(run.id, "succeeded", { finishedAt: new Date() }),+ lifecycle.setRunStatusIfRunning(run.id, "failed", { errorCode: "late_failure", finishedAt: new Date() }),+ ]);+ const winner = results.find(result => result.updated)!;+ expect(results.filter(result => result.updated)).toHaveLength(1);+ const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ expect(persisted.status).toBe(winner.run?.status);+ expect(results.every(result => result.run?.status === persisted.status)).toBe(true);+ expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledTimes(1);+ expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledWith(winner.run, "running", undefined);+ expect(vi.mocked(publishLiveEvent).mock.calls.filter(([event]) => event.type === "heartbeat.run.status")).toHaveLength(1);+ });++ it("rechecks native ownership atomically before a terminal write", async () => {+ const { run } = await fixture({ errorCode: "native_execution_ownership_unverified", nativePhase: "terminal_failure" });+ const deps = callbacks(db);+ const result = await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(run.id, "failed");+ expect(result).toMatchObject({ updated: false, run: { status: "running", errorCode: run.errorCode } });+ expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled();+ expect(publishLiveEvent).not.toHaveBeenCalled();+ });++ it.each(["setRunStatus", "setRunStatusIfRunning"] as const)("preserves accounting metadata while %s enriches usage", async (operation) => {+ const { run } = await fixture({ usageJson: { accountingReceiptReady: false, usageSource: "per_run", inputTokens: 5 } });+ await createHeartbeatLifecycle(db, callbacks(db))[operation](run.id, "succeeded", { usageJson: { inputTokens: 8, outputTokens: 3 } });+ const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ expect(persisted.usageJson).toEqual({ accountingReceiptReady: false, usageSource: "per_run", inputTokens: 8, outputTokens: 3 });+ });++ it.each(["claimed", "cancelled"] as const)("settles a %s wake without reviving a revoked request", async (status) => {+ const { wake } = await fixture();+ await db.update(agentWakeupRequests).set({ status }).where(eq(agentWakeupRequests.id, wake.id));+ await createHeartbeatLifecycle(db, callbacks(db)).setWakeupStatus(wake.id, "completed", { reason: "settled" });+ const [persisted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));+ expect(persisted.status).toBe(status === "cancelled" ? "cancelled" : "completed");+ expect(persisted.reason).toBe(status === "cancelled" ? wake.reason : "settled");+ });++ it("publishes the persisted sequence for concurrent event writes", async () => {+ const { run } = await fixture();+ const lifecycle = createHeartbeatLifecycle(db, callbacks(db));+ await Promise.all(["first", "second"].map(message => lifecycle.appendRunEvent(run, { eventType: "lifecycle", message })));+ const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq);+ expect(rows.map(row => row.seq)).toEqual([1, 2]);+ const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event");+ expect(published).toHaveLength(2);+ for (const row of rows) expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ runId: run.id, message: row.message });+ });++ it("reads current redaction settings for each event and persists the published sanitized payload", async () => {+ const { run } = await fixture();+ const deps = callbacks(db);+ deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["alice"], homeDirs: ["/Users/alice"] });+ deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["bob"], homeDirs: ["/Users/bob"] });+ const lifecycle = createHeartbeatLifecycle(db, deps);+ for (const name of ["alice", "bob"]) await lifecycle.appendRunEvent(run, {+ eventType: "lifecycle", message: `Opening /Users/${name}/project`, payload: { path: `/Users/${name}/project`, token: "test-credential-material" },+ });+ const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq);+ expect(deps.getCurrentUserRedactionOptions).toHaveBeenCalledTimes(2);+ expect(rows).toHaveLength(2);+ const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event");+ expect(published).toHaveLength(2);+ for (const row of rows) {+ const text = JSON.stringify({ message: row.message, payload: row.payload });+ expect(text).not.toContain("alice");+ expect(text).not.toContain("bob");+ expect(text).not.toContain("test-credential-material");+ expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ message: row.message, payload: row.payload });+ }+ });+});diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.tsnew file mode 100644index 0000000000..55215e6301--- /dev/null+++ b/server/src/services/heartbeat/run-lifecycle.ts@@ -0,0 +1,2982 @@+import {+ DEFERRED_WAKE_CONTEXT_KEY,+ EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,+ shouldRequireIssueCommentForWake,+} from "./queue.js";+import {+ HEARTBEAT_RUN_TERMINAL_STATUSES,+ DETACHED_PROCESS_ERROR_CODE,+} from "./recovery.js";+import { normalizeAgentNameKey } from "./retries.js";+import {+ type RunSessionOutcome,+ deriveTaskKeyWithHeartbeatFallback,+ type UsageTotals,+} from "./run-state.js";+import { CONFIGURATION_INCOMPLETE_FAILURE_CODE } from "./run-preparation.js";+import { WORKSPACE_VALIDATION_FAILURE_CODE } from "./workspaces.js";+import { boundHeartbeatRunEventPayloadForStorage } from "./run-log.js";+import { preserveWorkspaceRestoreRecoveryMetadata } from "../workspace-restore-recovery-state.js";+import { activeIssueInteractionCondition } from "../issue-question-context.js";+import {+ nativeRetryCancellationCommitCondition,+ rethrowNativeCancellationLockConflict,+} from "../native-runtime/native-cancellation-request.js";+import { cancellationResultJson } from "../run-cancellation.js";+import { accountRunCost } from "../run-cost-accounting.js";+import { isWaitingConversation } from "../agent-conversations.js";+import { PROCESS_IDENTITY_RECORDED } from "../native-local-process-stop.js";+import {+ hasAcknowledgedNativeReassignmentStopIntent,+ hasAcknowledgedNativeStopIntent,+} from "../acknowledged-native-stop.js";+import { connectionIntentService } from "../connection-intents.js";+import {+ CONVERSATION_CONTINUATION_POLICY,+ isConversationAdapter,+} from "../conversation-continuation.js";+import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js";+import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js";+import { randomUUID } from "node:crypto";+import {+ and,+ desc,+ eq,+ inArray,+ isNull,+ ne,+ notInArray,+ or,+ sql,+} from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import {+ ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,+ type HeartbeatRunStatusPhase,+ type RequestConfirmationResult,+ type RunLivenessState,+} from "@paperclipai/shared";+import {+ agents,+ agentRuntimeState,+ agentWakeupRequests,+ activityLog,+ approvals,+ companySkillTestRuns,+ documentRevisions,+ issueDocuments,+ heartbeatRunEvents,+ heartbeatRuns,+ issueApprovals,+ issueComments,+ issueRecoveryActions,+ issueRelations,+ issueThreadInteractions,+ issues,+ issueWorkProducts,+ routines,+ workspaceOperations,+} from "@paperclipai/db";+import { logger } from "../../middleware/logger.js";+import { publishLiveEvent } from "../live-events.js";+import {+ appendHeartbeatRunEvent,+ type AppendHeartbeatRunEventInput,+} from "../heartbeat-run-events.js";+import type { AdapterExecutionResult } from "../../adapters/index.js";+import {+ parseObject,+ asBoolean,+ asNumber,+} from "../../adapters/utils.js";+import {+ isNativeRunnerOwnershipHeld,+ nativeRunnerOwnershipNotHeldCondition,+} from "../native-runtime/native-runner-ownership.js";+import {+ type RunFailureReportOptions,+} from "../run-failure-diagnostics.js";+import {+ findHeartbeatRunCompletionComment,+ hasAcceptedSemanticResult,+ readCompletedAssistantMessageCandidate,+ selectHeartbeatRunFinalAgentMessage,+ type RunPresentationDecision,+} from "../heartbeat-run-summary.js";+import {+ buildHeartbeatRunStopMetadata,+ mergeHeartbeatRunStopMetadata,+} from "../heartbeat-stop-metadata.js";+import { CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";+import {+ classifyRunLiveness,+ type RunLivenessClassificationInput,+} from "../run-liveness.js";+import {+ logActivity,+ publishPluginDomainEvent,+} from "../activity-log.js";+import { runnerGoalService } from "../runner-goals.js";+import { visibleIssueCondition } from "../issue-visibility.js";+import {+ getIssueContinuationSummaryDocument,+ refreshIssueContinuationSummary,+} from "../issue-continuation-summary.js";+import {+ RECOVERY_ORIGIN_KINDS,+ FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,+ SUCCESSFUL_RUN_MISSING_STATE_REASON,+ RUN_LIVENESS_CONTINUATION_REASON,+ buildRunLivenessContinuationIdempotencyKey,+ buildFinishSuccessfulRunHandoffIdempotencyKey,+ buildSuccessfulRunHandoffRequiredNotice,+ decideRunLivenessContinuation,+ decideSuccessfulRunHandoff,+ findExistingFinishSuccessfulRunHandoffWake,+ findExistingRunLivenessContinuationWake,+ isSuccessfulRunHandoffValidPathSkip,+ SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,+ readContinuationAttempt,+} from "../recovery/index.js";+import { withRecoveryContext } from "../recovery/status-only-context.js";+import { deriveCommentId } from "../../modules/run-dispatch/index.js";+import {+ buildIssueReviewPathLostIdempotencyKey,+ decideIssueReviewPathRecovery,+ ISSUE_REVIEW_PATH_LOST_WAKE_REASON,+ isReviewPathRecoveryIdempotencyConflict,+ reviewPathConsumedRefFromRun,+} from "../recovery/review-path-recovery.js";+import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "../successful-run-handoff-state.js";+import {+ redactCurrentUserText,+ redactCurrentUserValue,+ type CurrentUserRedactionOptions,+} from "../../log-redaction.js";+import {+ redactEventPayload,+ redactSensitiveText,+} from "../../redaction.js";+import {+ type RuntimeStatusUpdate,+} from "@paperclipai/adapter-utils";+import {+ UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,+ UNMANAGED_BACKGROUND_TASK_STOP_REASON,+} from "@paperclipai/adapter-utils/server-utils";+import {+ clearHeartbeatRunRuntimeStatus,+ getHeartbeatRunRuntimeStatus,+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,+ setHeartbeatRunRuntimeStatus,+} from "../heartbeat-run-runtime-status.js";+import { readProcessStartedAt } from "../hot-restart.js";++import type { createHeartbeatRunState } from "./run-state.js";+import type { createHeartbeatRunPreparation } from "./run-preparation.js";+import type { createHeartbeatQueue } from "./queue.js";+import type { HeartbeatRetryDependencies } from "./retries.js";+import type { issueService } from "../issues.js";+import type { recoveryService } from "../recovery/index.js";+import type { companySkillService } from "../company-skills.js";+import type { budgetService, BudgetServiceHooks } from "../budgets.js";+import type { issueTreeControlService } from "../issue-tree-control.js";++type HeartbeatRun = typeof heartbeatRuns.$inferSelect;++/** Lifecycle owns run writes; the service supplies reporting and admission effects. */+export interface HeartbeatLifecycleDependencies extends Pick<ReturnType<typeof createHeartbeatRunState>,+ "getRun" | "resolveSessionBeforeForWakeup" | "ensureRuntimeState">,+ Pick<ReturnType<typeof createHeartbeatRunPreparation>,+ "getIssueExecutionContext" | "resolveResponsibleUserIdForRunContext">,+ Pick<HeartbeatRetryDependencies, "getAgentInvokability"> {+ issuesSvc: Pick<ReturnType<typeof issueService>, "addComment" | "listReviewAttention">;+ recovery: Pick<ReturnType<typeof recoveryService>, "escalateStrandedAssignedIssue">;+ companySkills: Pick<ReturnType<typeof companySkillService>, "completeTestRunForIssue">;+ budgets: Pick<ReturnType<typeof budgetService>, "getInvocationBlock">;+ treeControlSvc: Pick<ReturnType<typeof issueTreeControlService>, "getActivePauseHoldGate">;+ enqueueWakeup: ReturnType<typeof createHeartbeatQueue>["enqueueWakeup"];+ getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;+ budgetHooks: BudgetServiceHooks;+ emitTerminalAgentTaskRun: (run: HeartbeatRun, previousStatus: string | null, failureReport?: RunFailureReportOptions) => void;+}++export function redactDetectedSuccessfulRunProgressSummaryForBoard(+ summary: string,+ currentUserRedactionOptions?: CurrentUserRedactionOptions,+) {+ const normalized = summary.replace(/\s+/g, " ").trim();+ const redacted = redactSensitiveText(+ redactCurrentUserText(normalized, currentUserRedactionOptions),+ );+ return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`;+}++export function redactSuccessfulRunHandoffEvidence(+ value: string | null,+ currentUserRedactionOptions?: CurrentUserRedactionOptions,+) {+ if (!value) return null;+ return redactSensitiveText(+ redactCurrentUserText(value, currentUserRedactionOptions),+ );+}++const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [+ "environment.lease_acquired",+ "environment.lease_released",+ "cost.reported",+];++export const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set<string>([+ "low_trust_isolation_unavailable",+ "low_trust_requires_isolated_workspace",+ "low_trust_boundary_mismatch",+ "low_trust_requires_sandbox_environment",+ "low_trust_runtime_services_denied",+ "chat_failed_run_retry_not_authorized",+ CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,+]);++// Error codes that mark a pre-dispatch setup failure. The adapter process never+// started, so no agent could post an issue comment. The setup catch writes one+// of these codes when a failure happens before `adapter.execute`.+const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([+ "setup_failed",+ CONFIGURATION_INCOMPLETE_FAILURE_CODE,+ WORKSPACE_VALIDATION_FAILURE_CODE,+ ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,+]);++const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;++const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;++const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;++const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;++interface MaxTurnContinuationPolicy {+ enabled: boolean;+ maxAttempts: number;+ delayMs: number;+}++export function isHeartbeatRunTerminalStatus(+ status: string | null | undefined,+): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {+ return HEARTBEAT_RUN_TERMINAL_STATUSES.includes(+ status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number],+ );+}++export function isHeartbeatRunRuntimeStatusActive(+ status: string | null | undefined,+): boolean {+ return status === "queued" || status === "running";+}++type HeartbeatRunRuntimeStatusRunLike = {+ id: string;+ status?: string | null;+ companyId?: string | null;+ agentId?: string | null;+ issueId?: string | null;+ contextSnapshot?: Record<string, unknown> | null;+};++function readRuntimeStatusIssueIdCandidate(+ run: HeartbeatRunRuntimeStatusRunLike,+): string | null | undefined {+ if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null;+ if ("contextSnapshot" in run) {+ return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null;+ }+ return undefined;+}++export function decorateHeartbeatRunRuntimeStatus<+ T extends HeartbeatRunRuntimeStatusRunLike,+>(+ run: T,+ expected: {+ companyId?: string | null;+ issueId?: string | null;+ agentId?: string | null;+ } = {},+): T & {+ currentStatusMessage: string | null;+ currentStatusUpdatedAt: Date | null;+ currentToolName: string | null;+ lastAssistantSnippet: string | null;+ lastEventAt: Date | null;+} {+ if (isHeartbeatRunTerminalStatus(run.status)) {+ clearHeartbeatRunRuntimeStatus(run.id);+ }++ const companyId = expected.companyId ?? run.companyId ?? null;+ const agentId = expected.agentId ?? run.agentId ?? null;+ const issueId =+ expected.issueId !== undefined+ ? expected.issueId+ : readRuntimeStatusIssueIdCandidate(run);+ const currentStatus =+ isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId+ ? getHeartbeatRunRuntimeStatus(run.id, {+ companyId,+ agentId,+ ...(issueId !== undefined ? { issueId } : {}),+ })+ : null;++ return {+ ...run,+ currentStatusMessage: currentStatus?.message ?? null,+ currentStatusUpdatedAt: currentStatus?.updatedAt ?? null,+ currentToolName: currentStatus?.currentToolName ?? null,+ lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null,+ lastEventAt: currentStatus?.lastEventAt ?? null,+ };+}++export function publishHeartbeatRunRuntimeProgress(status: {+ companyId: string;+ runId: string;+ agentId: string;+ issueId: string | null;+ phase: HeartbeatRunStatusPhase;+ message: string;+ updatedAt: Date;+ currentToolName?: string | null;+ lastAssistantSnippet?: string | null;+ lastEventAt?: Date | null;+}) {+ publishLiveEvent({+ companyId: status.companyId,+ type: "heartbeat.run.progress",+ payload: {+ runId: status.runId,+ agentId: status.agentId,+ issueId: status.issueId,+ phase: status.phase,+ message: status.message,+ currentToolName: status.currentToolName ?? null,+ lastAssistantSnippet: status.lastAssistantSnippet ?? null,+ lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(),+ updatedAt: status.updatedAt.toISOString(),+ },+ });+}++function recordHeartbeatRunRuntimeProgress(+ run: Pick<+ typeof heartbeatRuns.$inferSelect,+ "id" | "companyId" | "agentId" | "status" | "contextSnapshot"+ >,+ update: RuntimeStatusUpdate,+ issueId: string | null,+) {+ if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null;+ const status = setHeartbeatRunRuntimeStatus({+ companyId: run.companyId,+ issueId,+ agentId: run.agentId,+ runId: run.id,+ phase: update.phase as HeartbeatRunStatusPhase,+ message: update.message,+ currentToolName: readNonEmptyString(update.currentToolName) ?? null,+ lastAssistantSnippet:+ readNonEmptyString(update.lastAssistantSnippet) ?? null,+ lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(),+ });+ if (!status) return null;++ publishHeartbeatRunRuntimeProgress(status);+ return status;+}++function sanitizeLiveRunProgressText(+ value: string,+ maxChars: number,+): string | null {+ const normalized = value.replace(/\s+/g, " ").trim();+ if (!normalized) return null;+ const redacted = redactSensitiveText(normalized);+ if (redacted.length <= maxChars) return redacted;+ return `${redacted.slice(0, maxChars - 3)}...`;+}++function readLiveRunProgressString(+ value: unknown,+ maxChars: number,+): string | null {+ return typeof value === "string"+ ? sanitizeLiveRunProgressText(value, maxChars)+ : null;+}++function readFirstLiveRunProgressString(+ maxChars: number,+ values: unknown[],+): string | null {+ for (const value of values) {+ const text = readLiveRunProgressString(value, maxChars);+ if (text) return text;+ }+ return null;+}++function readLiveRunToolName(+ payload: Record<string, unknown> | null,+ eventType: string,+): string | null {+ const toolCall =+ parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall);+ const message = parseObject(payload?.message);+ const direct = readFirstLiveRunProgressString(+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,+ [+ payload?.toolName,+ payload?.tool_name,+ payload?.tool,+ payload?.name,+ payload?.title,+ toolCall?.name,+ toolCall?.toolName,+ message?.name,+ message?.toolName,+ ],+ );+ if (direct) return direct;++ const normalizedEventType = eventType.toLowerCase();+ if (!normalizedEventType.includes("tool")) return null;+ return readLiveRunProgressString(+ eventType.replace(/[._-]+/g, " "),+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,+ );+}++function readLiveRunAssistantSnippet(+ payload: Record<string, unknown> | null,+ eventType: string,+ message: string | null,+): string | null {+ const normalizedEventType = eventType.toLowerCase();+ const messagePayload = parseObject(payload?.message);+ const direct = readFirstLiveRunProgressString(+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,+ [+ payload?.text,+ payload?.delta,+ payload?.text_delta,+ payload?.content,+ payload?.summary,+ messagePayload?.text,+ messagePayload?.content,+ ],+ );+ if (direct) return direct;++ if (+ normalizedEventType.includes("assistant") ||+ normalizedEventType.includes("text_delta") ||+ normalizedEventType.includes("message.delta") ||+ normalizedEventType.includes("message_delta")+ ) {+ return message+ ? sanitizeLiveRunProgressText(+ message,+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,+ )+ : null;+ }++ return null;+}++function buildRunEventRuntimeProgress(input: {+ eventType: string;+ message: string | null;+ payload: Record<string, unknown> | null;+ at: Date;+}) {+ const normalizedEventType = input.eventType.toLowerCase();+ if (+ normalizedEventType === "lifecycle" ||+ normalizedEventType === "adapter.invoke"+ ) {+ return null;+ }++ const currentToolName = readLiveRunToolName(input.payload, input.eventType);+ const lastAssistantSnippet = readLiveRunAssistantSnippet(+ input.payload,+ input.eventType,+ input.message,+ );+ const fallbackMessage =+ readLiveRunProgressString(+ input.message,+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,+ ) ??+ sanitizeLiveRunProgressText(+ input.eventType.replace(/[._-]+/g, " "),+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,+ );+ const message = currentToolName+ ? `Using ${currentToolName}`+ : (lastAssistantSnippet ?? fallbackMessage);++ if (!message) return null;+ return {+ phase: "run_activity" as const,+ message,+ currentToolName,+ lastAssistantSnippet,+ lastEventAt: input.at,+ };+}++export async function persistHeartbeatRunProcessMetadata(+ db: Db,+ runId: string,+ meta: { pid: number; processGroupId: number | null; startedAt: string },+) {+ const observedStartedAt = await readProcessStartedAt(meta.pid).catch(+ () => null,+ );+ const startedAt = new Date(observedStartedAt ?? meta.startedAt);+ return db.transaction(async tx => {+ const run = await tx+ .update(heartbeatRuns)+ .set({+ processPid: meta.pid,+ processGroupId: meta.processGroupId,+ processStartedAt: Number.isNaN(startedAt.getTime())+ ? new Date()+ : startedAt,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, runId))+ .returning()+ .then((rows) => rows[0] ?? null);+ if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, {+ companyId: run.companyId, runId, agentId: run.agentId,+ eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info",+ message: "Process identity recorded; prior stop evidence no longer applies.",+ });+ return run;+ });+}++type SkillTestHeartbeatCompletion = {+ outcome: "failed" | "cancelled";+ error: string | null;+ heartbeatOutcome: RunSessionOutcome;+};++export function resolveSkillTestRunCompletionForHeartbeatOutcome(+ outcome: RunSessionOutcome,+ error: string | null | undefined,+): SkillTestHeartbeatCompletion | null {+ if (outcome === "cancelled") {+ return {+ outcome: "cancelled",+ error: error ?? "Harness run was cancelled",+ heartbeatOutcome: outcome,+ };+ }+ if (outcome === "timed_out") {+ return {+ outcome: "failed",+ error: error ?? "Timed out",+ heartbeatOutcome: outcome,+ };+ }+ if (outcome === "failed") {+ return {+ outcome: "failed",+ error: error ?? "Adapter failed",+ heartbeatOutcome: outcome,+ };+ }+ return null;+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++/** Run lifecycle binds status, progress, liveness, and completion operations. */+export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycleDependencies) {+ const {+ issuesSvc,+ recovery,+ companySkills,+ getRun,+ emitTerminalAgentTaskRun,+ getIssueExecutionContext,+ budgets,+ enqueueWakeup,+ getCurrentUserRedactionOptions,+ treeControlSvc,+ getAgentInvokability,+ resolveSessionBeforeForWakeup,+ resolveResponsibleUserIdForRunContext,+ ensureRuntimeState,+ budgetHooks,+ } = dependencies;++ function isPlanApprovalConfirmationPayload(payload: unknown) {+ const target = parseObject(parseObject(payload).target);+ return (+ readNonEmptyString(target.type) === "issue_document" &&+ readNonEmptyString(target.key) === "plan"+ );+ }++ async function getAcceptedPlanApprovalInteractionForRun(+ run: typeof heartbeatRuns.$inferSelect,+ issueId: string | null,+ ) {+ const context = parseObject(run.contextSnapshot);+ const interactionId = readNonEmptyString(context.interactionId);+ if (!issueId || !interactionId) return null;++ const interaction = await db+ .select({+ id: issueThreadInteractions.id,+ kind: issueThreadInteractions.kind,+ status: issueThreadInteractions.status,+ payload: issueThreadInteractions.payload,+ result: issueThreadInteractions.result,+ })+ .from(issueThreadInteractions)+ .where(+ and(+ eq(issueThreadInteractions.companyId, run.companyId),+ eq(issueThreadInteractions.issueId, issueId),+ eq(issueThreadInteractions.id, interactionId),+ ),+ )+ .then((rows) => rows[0] ?? null);++ if (!interaction) return null;+ if (+ interaction.kind !== "request_confirmation" ||+ interaction.status !== "accepted"+ )+ return null;+ return isPlanApprovalConfirmationPayload(interaction.payload)+ ? interaction+ : null;+ }++ function planApprovalResumeFailureErrorCode(+ run: typeof heartbeatRuns.$inferSelect,+ ) {+ return readNonEmptyString(run.errorCode) ?? "unknown_error";+ }++ function buildPlanApprovalResumeFailureComment(input: {+ run: typeof heartbeatRuns.$inferSelect;+ status: "retrying" | "needs_attention";+ attempt: number;+ maxAttempts: number;+ }) {+ const errorCode = planApprovalResumeFailureErrorCode(input.run);+ if (input.status === "retrying") {+ return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`;+ }+ return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`;+ }++ function buildPlanApprovalResumeFailureResult(input: {+ run: typeof heartbeatRuns.$inferSelect;+ status: "retrying" | "needs_attention";+ attempt: number;+ maxAttempts: number;+ retryRunId?: string | null;+ recoveryActionId?: string | null;+ }): NonNullable<RequestConfirmationResult["resumeFailure"]> {+ return {+ status: input.status,+ errorCode: planApprovalResumeFailureErrorCode(input.run),+ attempt: input.attempt,+ maxAttempts: input.maxAttempts,+ runId: input.run.id,+ retryRunId: input.retryRunId ?? null,+ recoveryActionId: input.recoveryActionId ?? null,+ updatedAt: new Date().toISOString(),+ };+ }++ async function updatePlanApprovalInteractionResumeFailure(input: {+ interaction: NonNullable<+ Awaited<ReturnType<typeof getAcceptedPlanApprovalInteractionForRun>>+ >;+ failure: NonNullable<RequestConfirmationResult["resumeFailure"]>;+ }) {+ const result = parseObject(input.interaction.result);+ const nextResult = {+ ...result,+ version: 1 as const,+ outcome: "accepted" as const,+ resumeFailure: input.failure,+ } satisfies RequestConfirmationResult;++ await db+ .update(issueThreadInteractions)+ .set({+ result: nextResult,+ updatedAt: new Date(),+ })+ .where(eq(issueThreadInteractions.id, input.interaction.id));+ }++ async function addPlanApprovalResumeFailureCommentOnce(input: {+ issueId: string;+ run: typeof heartbeatRuns.$inferSelect;+ body: string;+ }) {+ const existing = await db+ .select({ id: issueComments.id })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, input.run.companyId),+ eq(issueComments.issueId, input.issueId),+ or(+ eq(issueComments.body, input.body),+ sql`${issueComments.body} like ${`${input.body}\n%`}`,+ ),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (existing) return null;+ return issuesSvc.addComment(+ input.issueId,+ input.body,+ { runId: input.run.id },+ { authorType: "system" },+ );+ }++ async function getActiveRecoveryActionId(+ companyId: string,+ sourceIssueId: string,+ ) {+ return db+ .select({ id: issueRecoveryActions.id })+ .from(issueRecoveryActions)+ .where(+ and(+ eq(issueRecoveryActions.companyId, companyId),+ eq(issueRecoveryActions.sourceIssueId, sourceIssueId),+ inArray(issueRecoveryActions.status, ["active", "escalated"]),+ ),+ )+ .orderBy(desc(issueRecoveryActions.updatedAt))+ .limit(1)+ .then((rows) => rows[0]?.id ?? null);+ }++ async function recordPlanApprovalResumeFailureRetry(input: {+ run: typeof heartbeatRuns.$inferSelect;+ issueId: string | null;+ retryRunId: string | null;+ attempt: number;+ maxAttempts: number;+ }) {+ const interaction = await getAcceptedPlanApprovalInteractionForRun(+ input.run,+ input.issueId,+ );+ if (!interaction || !input.issueId) return null;++ const body = buildPlanApprovalResumeFailureComment({+ run: input.run,+ status: "retrying",+ attempt: input.attempt,+ maxAttempts: input.maxAttempts,+ });+ await addPlanApprovalResumeFailureCommentOnce({+ issueId: input.issueId,+ run: input.run,+ body,+ });+ await updatePlanApprovalInteractionResumeFailure({+ interaction,+ failure: buildPlanApprovalResumeFailureResult({+ run: input.run,+ status: "retrying",+ attempt: input.attempt,+ maxAttempts: input.maxAttempts,+ retryRunId: input.retryRunId,+ }),+ });+ return interaction.id;+ }++ async function escalatePlanApprovalResumeFailureNeedsAttention(input: {+ run: typeof heartbeatRuns.$inferSelect;+ issueId: string | null;+ attempt: number;+ maxAttempts: number;+ }) {+ const interaction = await getAcceptedPlanApprovalInteractionForRun(+ input.run,+ input.issueId,+ );+ if (!interaction || !input.issueId) return null;++ const issue = await db+ .select()+ .from(issues)+ .where(+ and(+ eq(issues.companyId, input.run.companyId),+ eq(issues.id, input.issueId),+ ),+ )+ .then((rows) => rows[0] ?? null);+ if (!issue) return null;+ if (+ issue.status !== "todo" &&+ issue.status !== "in_progress" &&+ issue.status !== "in_review"+ )+ return null;++ const body = buildPlanApprovalResumeFailureComment({+ run: input.run,+ status: "needs_attention",+ attempt: input.attempt,+ maxAttempts: input.maxAttempts,+ });+ await recovery.escalateStrandedAssignedIssue({+ issue,+ previousStatus: issue.status,+ latestRun: input.run,+ comment: body,+ });+ await addPlanApprovalResumeFailureCommentOnce({+ issueId: issue.id,+ run: input.run,+ body,+ });++ const recoveryActionId = await getActiveRecoveryActionId(+ issue.companyId,+ issue.id,+ );+ await updatePlanApprovalInteractionResumeFailure({+ interaction,+ failure: buildPlanApprovalResumeFailureResult({+ run: input.run,+ status: "needs_attention",+ attempt: input.attempt,+ maxAttempts: input.maxAttempts,+ recoveryActionId,+ }),+ });+ return interaction.id;+ }++ async function completeSkillTestRunForHeartbeatOutcome(input: {+ run: typeof heartbeatRuns.$inferSelect;+ issueId: string | null;+ issueWorkMode?: string | null;+ outcome: RunSessionOutcome;+ error: string | null;+ }) {+ const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(+ input.outcome,+ input.error,+ );+ if (!completion || !input.issueId) return null;++ let isSkillTestIssue = input.issueWorkMode === "skill_test";+ if (!isSkillTestIssue && input.issueWorkMode === undefined) {+ const issueRow = await db+ .select({+ workMode: issues.workMode,+ harnessKind: issues.harnessKind,+ })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, input.run.companyId),+ eq(issues.id, input.issueId),+ ),+ )+ .then((rows) => rows[0] ?? null);+ isSkillTestIssue =+ issueRow?.workMode === "skill_test" ||+ issueRow?.harnessKind === "skill_test";+ }+ if (!isSkillTestIssue) return null;++ const existingRun = await db+ .select({+ id: companySkillTestRuns.id,+ status: companySkillTestRuns.status,+ })+ .from(companySkillTestRuns)+ .where(+ and(+ eq(companySkillTestRuns.companyId, input.run.companyId),+ eq(companySkillTestRuns.issueId, input.issueId),+ ),+ )+ .then((rows) => rows[0] ?? null);+ if (+ !existingRun ||+ ["succeeded", "failed", "cancelled"].includes(existingRun.status)+ )+ return null;++ const completedRun = await companySkills.completeTestRunForIssue({+ companyId: input.run.companyId,+ issueId: input.issueId,+ outcome: completion.outcome,+ error: completion.error,+ });+ if (!completedRun) return null;++ await logActivity(db, {+ companyId: input.run.companyId,+ actorType: "system",+ actorId: "heartbeat_finalize",+ agentId: input.run.agentId,+ runId: input.run.id,+ action: "company.skill_test_run_completed",+ entityType: "company_skill_test_run",+ entityId: completedRun.id,+ issueId: input.issueId,+ details: {+ issueId: input.issueId,+ status: completedRun.status,+ outputDocumentKey: completedRun.outputDocumentKey,+ heartbeatOutcome: completion.heartbeatOutcome,+ source: "heartbeat.run_finalized",+ },+ });++ return completedRun;+ }++ async function recordCurrentHeartbeatRunRuntimeProgress(+ run: Pick<+ typeof heartbeatRuns.$inferSelect,+ "id" | "companyId" | "agentId" | "status" | "contextSnapshot"+ >,+ update: RuntimeStatusUpdate,+ issueId: string | null,+ ) {+ if (!isHeartbeatRunRuntimeStatusActive(run.status)) {+ clearHeartbeatRunRuntimeStatus(run.id);+ return null;+ }++ const currentRun = await getRun(run.id);+ if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) {+ clearHeartbeatRunRuntimeStatus(run.id);+ return null;+ }++ return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId);+ }++ async function setRunStatus(+ runId: string,+ status: string,+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,+ ) {+ const previousStatus = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0] ?? null);++ if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,+ resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),+ };++ // Preserve the receipt-source fence when finalization enriches usage. A+ // late spool replay must still be able to complete an unfinished receipt.+ if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };++ // Cancelling a queued run that never acquired provider execution is+ // positive bootstrap evidence. It must not hold unrelated queued messages.+ if (previousStatus && (status === "cancelled" || status === "interrupted")) {+ patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };+ }+ if (+ status === "cancelled" &&+ previousStatus?.status === "queued" &&+ previousStatus.runtimeMode !== "native" &&+ !previousStatus.startedAt &&+ !previousStatus.processPid+ ) {+ patch = {+ ...patch,+ resultJson: {+ ...previousStatus.resultJson,+ ...patch?.resultJson,+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },+ },+ };+ }+ const updated =+ previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)+ ? await terminalizeLegacyExecution({+ db,+ run: previousStatus,+ status,+ patch,+ reconcileIfNeeded: true,+ })+ : await db+ .update(heartbeatRuns)+ .set({+ status,+ ...patch,+ executionStatusDeliveryId: randomUUID(),+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, runId))+ .returning()+ .then((rows) => rows[0] ?? null);++ if (updated) {+ publishLiveEvent({+ companyId: updated.companyId,+ type: "heartbeat.run.status",+ payload: buildHeartbeatRunStatusLiveEventPayload(updated),+ });+ publishRunLifecyclePluginEvent(updated);+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);+ }++ return updated;+ }++ async function setRunStatusIfRunning(+ runId: string,+ status: string,+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,+ failureReport?: RunFailureReportOptions,+ ) {+ return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);+ }++ // Move a run to a new status only when its current status is one of+ // `fromStatuses`. The compare-and-set is a single conditional update, so a+ // concurrent path can win the race. When this update matches nothing, the+ // function reads the current row and reports updated=false, so the caller can+ // keep the terminal outcome that another path already wrote.+ async function setRunStatusFromLive(+ runId: string,+ status: string,+ fromStatuses: string[],+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,+ failureReport?: RunFailureReportOptions,+ cancellationCondition?: ReturnType<typeof nativeRetryCancellationCommitCondition>,+ ) {+ // fromStatuses can name a terminal status as its own source (for example,+ // an idempotent "still failed" patch), so the write below is not always a+ // genuine transition. Read the pre-write status to tell the two apart.+ const previousStatus = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0] ?? null);++ if (previousStatus && (status === "cancelled" || status === "interrupted")) {+ patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };+ }++ if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,+ resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),+ };++ // Preserve the receipt-source fence when finalization enriches usage. A+ // late spool replay must still be able to complete an unfinished receipt.+ if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };++ // Cancelling a queued run that never acquired provider execution is+ // positive bootstrap evidence. It must not hold unrelated queued messages.+ if (+ status === "cancelled" &&+ previousStatus?.status === "queued" &&+ previousStatus.runtimeMode !== "native" &&+ !previousStatus.startedAt &&+ !previousStatus.processPid+ ) {+ patch = {+ ...patch,+ resultJson: {+ ...previousStatus.resultJson,+ ...patch?.resultJson,+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },+ },+ };+ }+ const updated =+ previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)+ ? await terminalizeLegacyExecution({+ db,+ run: previousStatus,+ status,+ patch,+ fromStatuses,+ reconcileIfNeeded: true,+ writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])],+ })+ : await db+ .update(heartbeatRuns)+ .set({+ status,+ ...patch,+ executionStatusDeliveryId: randomUUID(),+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(heartbeatRuns.id, runId),+ inArray(heartbeatRuns.status, fromStatuses),+ ...(cancellationCondition ? [cancellationCondition] : []),+ ...(isHeartbeatRunTerminalStatus(status)+ ? [nativeRunnerOwnershipNotHeldCondition()]+ : []),+ ),+ )+ .returning()+ .then((rows) => rows[0] ?? null)+ .catch((error: unknown) => {+ if (cancellationCondition) rethrowNativeCancellationLockConflict(error);+ throw error;+ });++ if (updated) {+ publishLiveEvent({+ companyId: updated.companyId,+ type: "heartbeat.run.status",+ payload: buildHeartbeatRunStatusLiveEventPayload(updated),+ });+ publishRunLifecyclePluginEvent(updated);+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);+ return { run: updated, updated: true as const };+ }++ const current = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0] ?? null);++ return { run: current, updated: false as const };+ }++ // Invariant: when a run releases its environment lease, the run row must be+ // terminal. The finalizer writes the terminal status in a step that is+ // separate from the agent status=done PATCH. If the sandbox or the run+ // process stops between the two steps, heartbeat_runs.status stays "running".+ // The UI reads liveness from that row, so a finished task shows "Live"+ // forever. This function closes the gap in the run teardown: when the run is+ // still running or queued, it forces a terminal status before the lease is+ // released. It never overwrites a status that another path already made+ // terminal.+ async function terminalizeRunOnLeaseRelease(+ run: typeof heartbeatRuns.$inferSelect,+ ): Promise<typeof heartbeatRuns.$inferSelect> {+ if (isNativeRunnerOwnershipHeld(run)) return run;+ if (isHeartbeatRunTerminalStatus(run.status)) return run;+ if (run.status !== "running" && run.status !== "queued") return run;++ // Choose the terminal status that reflects the true outcome. When the issue+ // already reached a terminal status, the run reached its goal, so use the+ // matching terminal run status. Otherwise the teardown cut the run short,+ // so use "interrupted".+ const issueId = readNonEmptyString(+ parseObject(run.contextSnapshot).issueId,+ );+ let terminalStatus: "succeeded" | "cancelled" | "interrupted" =+ "interrupted";+ if (issueId) {+ const issueStatus = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0]?.status ?? null);+ if (issueStatus === "done") terminalStatus = "succeeded";+ else if (issueStatus === "cancelled") terminalStatus = "cancelled";+ }++ // Teardown can beat the cancellation finalizer. Preserve the acknowledged+ // user intent instead of reporting an infrastructure interruption.+ if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled";++ const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`;+ // Match both "running" and "queued". A queued run has released its lease but+ // never reached "running", so a running-only update would miss it and leave+ // a phantom live run behind.+ const write = await setRunStatusFromLive(+ run.id,+ terminalStatus,+ ["running", "queued"],+ {+ finishedAt: run.finishedAt ?? new Date(),+ error: run.error ?? (terminalStatus === "interrupted" ? message : null),+ errorCode:+ run.errorCode ??+ (terminalStatus === "interrupted"+ ? "lease_released_before_terminal"+ : null),+ },+ );+ if (!write.updated) {+ // Another path already finalized the run. Keep that terminal outcome.+ return write.run ?? run;+ }++ const terminalRun = write.run;+ if (terminalRun) {+ await appendRunEvent(terminalRun, {+ eventType: "lifecycle",+ stream: "system",+ level: terminalStatus === "interrupted" ? "warn" : "info",+ message,+ payload: {+ previousStatus: run.status,+ terminalStatus,+ reason: "environment_lease_release",+ ...(issueId ? { issueId } : {}),+ },+ }).catch((eventErr) => {+ logger.warn(+ { err: eventErr, runId: run.id },+ "failed to append run event for lease-release terminalization",+ );+ });+ }+ return terminalRun ?? run;+ }++ function publishRunLifecyclePluginEvent(+ run: typeof heartbeatRuns.$inferSelect,+ ) {+ publishRunLifecyclePluginEventData({+ companyId: run.companyId,+ runId: run.id,+ agentId: run.agentId,+ status: run.status,+ invocationSource: run.invocationSource,+ triggerDetail: run.triggerDetail,+ error: run.error,+ errorCode: run.errorCode,+ issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),+ startedAt: run.startedAt,+ finishedAt: run.finishedAt,+ });+ }++ function publishRunLifecyclePluginEventData(run: {+ companyId: string;+ runId: string;+ agentId: string;+ status: string;+ invocationSource: string;+ triggerDetail: string | null;+ error: string | null;+ errorCode: string | null;+ issueId: string | null;+ startedAt: Date | null;+ finishedAt: Date | null;+ }) {+ const eventType =+ run.status === "running"+ ? "agent.run.started"+ : run.status === "succeeded"+ ? "agent.run.finished"+ : run.status === "failed" || run.status === "timed_out"+ ? "agent.run.failed"+ : run.status === "cancelled"+ ? "agent.run.cancelled"+ : null;+ if (!eventType) return;+ publishPluginDomainEvent({+ eventId: randomUUID(),+ eventType,+ occurredAt: new Date().toISOString(),+ actorId: run.agentId,+ actorType: "agent",+ entityId: run.runId,+ entityType: "heartbeat_run",+ companyId: run.companyId,+ payload: {+ runId: run.runId,+ agentId: run.agentId,+ status: run.status,+ invocationSource: run.invocationSource,+ triggerDetail: run.triggerDetail,+ error: run.error ?? null,+ errorCode: run.errorCode ?? null,+ issueId: run.issueId,+ startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null,+ finishedAt: run.finishedAt+ ? new Date(run.finishedAt).toISOString()+ : null,+ },+ });+ }++ async function setWakeupStatus(+ wakeupRequestId: string | null | undefined,+ status: string,+ patch?: Partial<typeof agentWakeupRequests.$inferInsert>,+ ) {+ if (!wakeupRequestId) return;+ await db+ .update(agentWakeupRequests)+ .set({ status, ...patch, updatedAt: new Date() })+ // Reassignment revokes requests transactionally. Late execution settlement+ // must not turn a revoked request back into authority for a later retry.+ .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));+ }++ async function addContinuationExhaustedCommentOnce(input: {+ run: typeof heartbeatRuns.$inferSelect;+ issueId: string;+ comment: string;+ }) {+ const existing = await db+ .select({ id: issueComments.id })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, input.run.companyId),+ eq(issueComments.issueId, input.issueId),+ eq(issueComments.createdByRunId, input.run.id),+ sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`,+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (existing) return;+ await issuesSvc.addComment(input.issueId, input.comment, {+ agentId: input.run.agentId,+ runId: input.run.id,+ });+ }++ async function handleRunLivenessContinuation(+ run: typeof heartbeatRuns.$inferSelect,+ ) {+ const context = parseObject(run.contextSnapshot);+ if (+ readNonEmptyString(context.goalControlRequestId) ||+ context.resumeSessionGoalHeartbeat === true+ )+ return;+ const livenessState = run.livenessState as RunLivenessState | null;+ if (livenessState !== "plan_only" && livenessState !== "empty_response")+ return;++ const issueId = readNonEmptyString(context.issueId);+ if (!issueId) return;+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;++ const [issue, agent] = await Promise.all([+ db+ .select({+ id: issues.id,+ companyId: issues.companyId,+ identifier: issues.identifier,+ title: issues.title,+ status: issues.status,+ assigneeAgentId: issues.assigneeAgentId,+ executionState: issues.executionState,+ projectId: issues.projectId,+ })+ .from(issues)+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))+ .then((rows) => rows[0] ?? null),+ db+ .select({+ id: agents.id,+ companyId: agents.companyId,+ status: agents.status,+ })+ .from(agents)+ .where(eq(agents.id, run.agentId))+ .then((rows) => rows[0] ?? null),+ ]);++ const budgetBlock =+ issue && agent+ ? await budgets.getInvocationBlock(issue.companyId, agent.id, {+ issueId: issue.id,+ projectId: issue.projectId,+ })+ : null;++ const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;+ const idempotencyKey = issue+ ? buildRunLivenessContinuationIdempotencyKey({+ issueId: issue.id,+ sourceRunId: run.id,+ livenessState,+ nextAttempt,+ })+ : null;+ const existingWake = idempotencyKey+ ? await findExistingRunLivenessContinuationWake(db, {+ companyId: run.companyId,+ idempotencyKey,+ })+ : null;++ const decision = decideRunLivenessContinuation({+ run,+ issue,+ agent,+ livenessState,+ livenessReason: run.livenessReason,+ nextAction: run.nextAction,+ budgetBlocked: Boolean(budgetBlock),+ idempotentWakeExists: Boolean(existingWake),+ });++ if (decision.kind === "exhausted") {+ await setRunStatus(run.id, run.status, {+ livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`,+ });+ await addContinuationExhaustedCommentOnce({+ run,+ issueId,+ comment: decision.comment,+ });+ return;+ }++ if (decision.kind !== "enqueue") return;++ const continuationRun = await enqueueWakeup(run.agentId, {+ source: "automation",+ triggerDetail: "system",+ reason: RUN_LIVENESS_CONTINUATION_REASON,+ payload: decision.payload,+ contextSnapshot: {+ ...decision.contextSnapshot,+ originIdentityContextId: null,+ parentRunId: run.id,+ },+ idempotencyKey: decision.idempotencyKey,+ requestedByActorType: "system",+ requestedByActorId: "heartbeat",+ });++ if (continuationRun) {+ await db+ .update(heartbeatRuns)+ .set({+ continuationAttempt: decision.nextAttempt,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, run.id));+ }+ }++ function issueUiLink(+ issue: Pick<typeof issues.$inferSelect, "id" | "identifier">,+ ) {+ const label = issue.identifier ?? issue.id;+ const prefix = issue.identifier?.split("-")[0] || "PAP";+ return `[${label}](/${prefix}/issues/${label})`;+ }++ function hasUnmanagedBackgroundTaskEvidence(+ resultJson: Record<string, unknown> | null | undefined,+ ) {+ const evidence = parseObject(resultJson?.unmanagedBackgroundTask);+ return (+ evidence.stopped === true &&+ (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON ||+ evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON)+ );+ }++ function withUnmanagedBackgroundTaskStopReason(+ resultJson: Record<string, unknown> | null | undefined,+ ) {+ return {+ ...(resultJson ?? {}),+ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,+ };+ }++ function buildDetectedSuccessfulRunProgressSummary(+ run: typeof heartbeatRuns.$inferSelect,+ currentUserRedactionOptions: CurrentUserRedactionOptions,+ ) {+ const resultJson = parseObject(run.resultJson);+ const candidates = [+ hasUnmanagedBackgroundTaskEvidence(resultJson)+ ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON+ : null,+ readNonEmptyString(run.nextAction)+ ? `Next action noted: ${readNonEmptyString(run.nextAction)}`+ : null,+ readNonEmptyString(run.livenessReason),+ readNonEmptyString(resultJson.summary),+ readNonEmptyString(resultJson.result),+ readNonEmptyString(resultJson.message),+ ].filter((value): value is string => Boolean(value));+ const summary = candidates[0];+ if (!summary) return null;+ return redactDetectedSuccessfulRunProgressSummaryForBoard(+ summary,+ currentUserRedactionOptions,+ );+ }++ async function addSuccessfulRunHandoffCommentOnce(input: {+ issue: Pick<+ typeof issues.$inferSelect,+ "id" | "identifier" | "title" | "status"+ >;+ run: typeof heartbeatRuns.$inferSelect;+ agent: Pick<typeof agents.$inferSelect, "id" | "name">;+ detectedProgressSummary: string;+ }) {+ const existing = await db+ .select({ id: issueComments.id })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, input.run.companyId),+ eq(issueComments.issueId, input.issue.id),+ eq(issueComments.createdByRunId, input.run.id),+ sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`,+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (existing) return null;+ const notice = buildSuccessfulRunHandoffRequiredNotice(input);+ return issuesSvc.addComment(+ input.issue.id,+ notice.body,+ { runId: input.run.id },+ {+ authorType: "system",+ presentation: notice.presentation,+ metadata: notice.metadata,+ },+ );+ }++ async function handleSuccessfulRunHandoff(+ run: typeof heartbeatRuns.$inferSelect,+ agent: typeof agents.$inferSelect,+ ) {+ if (run.status !== "succeeded") return;+ const context = parseObject(run.contextSnapshot);+ const issueId =+ readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);+ if (!issueId) return;+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;+ if (+ readNonEmptyString(context.goalControlRequestId) ||+ context.resumeSessionGoalHeartbeat === true+ ) {+ const goalProjection = await runnerGoalService(db).projection(+ run.companyId,+ issueId,+ run.agentId,+ );+ if (goalProjection?.goal?.status !== "complete") return;+ }++ const issue = await db+ .select({+ id: issues.id,+ companyId: issues.companyId,+ identifier: issues.identifier,+ title: issues.title,+ description: issues.description,+ status: issues.status,+ assigneeAgentId: issues.assigneeAgentId,+ assigneeUserId: issues.assigneeUserId,+ executionState: issues.executionState,+ monitorNextCheckAt: issues.monitorNextCheckAt,+ projectId: issues.projectId,+ originKind: issues.originKind,+ })+ .from(issues)+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))+ .then((rows) => rows[0] ?? null);+ const idempotencyKey = issue+ ? buildFinishSuccessfulRunHandoffIdempotencyKey({+ issueId: issue.id,+ sourceRunId: run.id,+ })+ : null;+ const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null);+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();+ const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary(+ run,+ currentUserRedactionOptions,+ );+ const resultJson = parseObject(run.resultJson);+ const finalReport = redactSuccessfulRunHandoffEvidence(+ [+ readNonEmptyString(resultJson.summary),+ readNonEmptyString(resultJson.result),+ readNonEmptyString(resultJson.message),+ ].find((value): value is string => Boolean(value)) ?? null,+ currentUserRedactionOptions,+ );+ const nextAction = redactSuccessfulRunHandoffEvidence(+ readNonEmptyString(run.nextAction),+ currentUserRedactionOptions,+ );++ const [+ activeExecutionPath,+ queuedWake,+ pendingInteraction,+ pendingApproval,+ explicitBlocker,+ openRecoveryIssue,+ existingWake,+ budgetBlock,+ pauseHold,+ activeRoutineContinuation,+ ] = await Promise.all([+ issue+ ? db+ .select({ id: heartbeatRuns.id })+ .from(heartbeatRuns)+ .where(+ and(+ eq(heartbeatRuns.companyId, issue.companyId),+ eq(heartbeatRuns.agentId, run.agentId),+ inArray(heartbeatRuns.status, [+ ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,+ ]),+ sql`(+ ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}+ or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id}+ )`,+ sql`${heartbeatRuns.id} <> ${run.id}`,+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: agentWakeupRequests.id })+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, issue.companyId),+ eq(agentWakeupRequests.agentId, run.agentId),+ inArray(agentWakeupRequests.status, [+ "queued",+ "deferred_issue_execution",+ "claimed",+ ]),+ sql`(+ ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}+ or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id}+ or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id}+ or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id}+ )`,+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: issueThreadInteractions.id })+ .from(issueThreadInteractions)+ .where(+ and(+ eq(issueThreadInteractions.companyId, issue.companyId),+ eq(issueThreadInteractions.issueId, issue.id),+ eq(issueThreadInteractions.status, "pending"),+ activeIssueInteractionCondition(),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: issueApprovals.approvalId })+ .from(issueApprovals)+ .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))+ .where(+ and(+ eq(issueApprovals.companyId, issue.companyId),+ eq(issueApprovals.issueId, issue.id),+ inArray(approvals.status, ["pending", "revision_requested"]),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: issueRelations.issueId })+ .from(issueRelations)+ .where(+ and(+ eq(issueRelations.companyId, issue.companyId),+ eq(issueRelations.relatedIssueId, issue.id),+ eq(issueRelations.type, "blocks"),+ sql`exists (+ select 1+ from issues blocker+ where blocker.id = ${issueRelations.issueId}+ and blocker.company_id = ${issue.companyId}+ and blocker.status not in ('done', 'cancelled')+ and blocker.hidden_at is null+ )`,+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: issues.id })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, issue.companyId),+ inArray(issues.originKind, [+ RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,+ RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,+ ]),+ eq(issues.originId, issue.id),+ visibleIssueCondition(),+ notInArray(issues.status, ["done", "cancelled"]),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ idempotencyKey+ ? findExistingFinishSuccessfulRunHandoffWake(db, {+ companyId: run.companyId,+ idempotencyKey,+ })+ : Promise.resolve(null),+ issue+ ? budgets.getInvocationBlock(issue.companyId, run.agentId, {+ issueId: issue.id,+ projectId: issue.projectId,+ })+ : Promise.resolve(null),+ issue+ ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id)+ : Promise.resolve(null),+ issue+ ? db+ .select({ id: routines.id })+ .from(routines)+ .where(+ and(+ eq(routines.companyId, issue.companyId),+ eq(routines.parentIssueId, issue.id),+ eq(routines.status, "active"),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null)+ : Promise.resolve(null),+ ]);++ const decision = decideSuccessfulRunHandoff({+ run,+ issue,+ agent,+ livenessState: run.livenessState as RunLivenessState | null,+ detectedProgressSummary,+ finalReport,+ nextAction,+ taskKey,+ hasActiveExecutionPath: Boolean(activeExecutionPath),+ hasQueuedWake: Boolean(queuedWake),+ hasPendingInteractionOrApproval: Boolean(+ pendingInteraction || pendingApproval,+ ),+ hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt),+ hasExplicitBlockerPath: Boolean(explicitBlocker),+ hasOpenRecoveryIssue: Boolean(openRecoveryIssue),+ hasPauseHold: Boolean(pauseHold),+ hasActiveRoutineContinuation: Boolean(activeRoutineContinuation),+ budgetBlocked: Boolean(budgetBlock),+ idempotentWakeExists: Boolean(existingWake),+ });++ if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) {+ await resolveRequiredSuccessfulRunHandoffOnValidPath(db, {+ companyId: issue.companyId,+ issueId: issue.id,+ issueIdentifier: issue.identifier,+ agentId: run.agentId,+ runId: run.id,+ skipReason: decision.reason,+ });+ }++ if (decision.kind !== "enqueue" || !issue) return;++ if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) {+ await db+ .update(heartbeatRuns)+ .set({+ livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||+ ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, run.id));+ }++ const handoffRun = await enqueueWakeup(decision.targetAgentId, {+ source: "automation",+ triggerDetail: "system",+ reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,+ payload: decision.payload,+ contextSnapshot: {+ ...decision.contextSnapshot,+ originIdentityContextId: null,+ parentRunId: run.id,+ },+ idempotencyKey: decision.idempotencyKey,+ requestedByActorType: "system",+ requestedByActorId: "heartbeat",+ });+ if (!handoffRun) return;++ await addSuccessfulRunHandoffCommentOnce({+ issue,+ run,+ agent,+ detectedProgressSummary:+ detectedProgressSummary ??+ "The run reported progress, but did not choose a next step.",+ });+ await logActivity(db, {+ companyId: issue.companyId,+ actorType: "system",+ actorId: "heartbeat",+ agentId: run.agentId,+ runId: run.id,+ action: "issue.successful_run_handoff_required",+ entityType: "issue",+ entityId: issue.id,+ details: {+ label: "Successful run missing issue disposition",+ sourceRunId: run.id,+ correctiveRunId: handoffRun.id,+ handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON,+ missingDisposition: "clear_next_step",+ detectedProgressSummary,+ issue: issueUiLink(issue),+ },+ });+ }++ async function handleIssueReviewPathDisposition(+ run: typeof heartbeatRuns.$inferSelect,+ ) {+ const contextSnapshot = parseObject(run.contextSnapshot);+ if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return;+ const issueId =+ readNonEmptyString(contextSnapshot.issueId) ??+ readNonEmptyString(contextSnapshot.taskId);+ if (!issueId) return;+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;++ const issue = await db+ .select({+ id: issues.id,+ companyId: issues.companyId,+ identifier: issues.identifier,+ status: issues.status,+ assigneeAgentId: issues.assigneeAgentId,+ })+ .from(issues)+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))+ .then((rows) => rows[0] ?? null);+ if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId)+ return;++ const reviewAttention = await issuesSvc+ .listReviewAttention(issue.companyId, [issue])+ .then(+ (map) =>+ map.get(issue.id) ?? {+ state: "none" as const,+ paths: [],+ reason: null,+ },+ );+ if (reviewAttention.state !== "stalled") return;++ const consumedPathRef = reviewPathConsumedRefFromRun({+ runId: run.id,+ issueId: issue.id,+ contextSnapshot,+ });+ const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({+ issueId: issue.id,+ consumedPathRef,+ });+ const existingWake = await db+ .select({ id: agentWakeupRequests.id })+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, issue.companyId),+ eq(agentWakeupRequests.idempotencyKey, idempotencyKey),+ notInArray(agentWakeupRequests.status, ["skipped"]),+ ),+ )+ .limit(1)+ .then((rows) => rows[0] ?? null);++ const decision = decideIssueReviewPathRecovery({+ issueId: issue.id,+ sourceRunId: run.id,+ assigneeAgentId: issue.assigneeAgentId,+ contextSnapshot,+ reviewAttention,+ existingWake: Boolean(existingWake),+ });+ if (decision.kind !== "enqueue") return;++ const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, {+ source: "automation",+ triggerDetail: "system",+ reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON,+ idempotencyKey: decision.idempotencyKey,+ payload: decision.payload,+ contextSnapshot: decision.contextSnapshot,+ requestedByActorType: "system",+ requestedByActorId: "heartbeat",+ }).catch((error: unknown) => {+ if (isReviewPathRecoveryIdempotencyConflict(error)) return null;+ throw error;+ });+ if (!recoveryRun) return;++ await logActivity(db, {+ companyId: issue.companyId,+ actorType: "system",+ actorId: "heartbeat",+ agentId: issue.assigneeAgentId,+ runId: run.id,+ action: "issue.review_path_recovery_queued",+ entityType: "issue",+ entityId: issue.id,+ details: {+ sourceRunId: run.id,+ recoveryRunId: recoveryRun.id,+ consumedPathRef,+ recoveryAttempt: 1,+ maxRecoveryAttempts: 1,+ },+ });+ }++ async function appendRunEvent(+ run: typeof heartbeatRuns.$inferSelect,+ event: {+ eventType: string;+ stream?: "system" | "stdout" | "stderr";+ level?: "info" | "warn" | "error";+ color?: string;+ message?: string;+ payload?: Record<string, unknown>;+ retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"];+ },+ ) {+ const eventAt = new Date();+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();+ const sanitizedMessage = event.message+ ? redactSensitiveText(+ redactCurrentUserText(event.message, currentUserRedactionOptions),+ )+ : event.message;+ const boundedPayload = event.payload+ ? boundHeartbeatRunEventPayloadForStorage(event.payload)+ : event.payload;+ const secretSanitizedPayload = boundedPayload+ ? redactEventPayload(boundedPayload)+ : boundedPayload;+ const sanitizedPayload = secretSanitizedPayload+ ? redactCurrentUserValue(+ secretSanitizedPayload,+ currentUserRedactionOptions,+ )+ : secretSanitizedPayload;+ const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null;+ const progress = buildRunEventRuntimeProgress({+ eventType: event.eventType,+ message: sanitizedMessage ?? null,+ payload: sanitizedPayload ?? null,+ at: eventAt,+ });+ const persistedEvent = await appendHeartbeatRunEvent(db, {+ companyId: run.companyId,+ runId: run.id,+ agentId: run.agentId,+ eventType: event.eventType,+ stream: event.stream,+ level: event.level,+ color: event.color,+ message: sanitizedMessage,+ payload: sanitizedPayload,+ retryExhaustion: event.retryExhaustion,+ });+ if (persistedEvent.disposition === "duplicate") return;+ const seq = persistedEvent.row.seq;++ publishLiveEvent({+ companyId: run.companyId,+ type: "heartbeat.run.event",+ payload: {+ runId: run.id,+ agentId: run.agentId,+ issueId,+ seq,+ eventType: event.eventType,+ stream: event.stream ?? null,+ level: event.level ?? null,+ color: event.color ?? null,+ message: sanitizedMessage ?? null,+ currentToolName: progress?.currentToolName ?? null,+ lastAssistantSnippet: progress?.lastAssistantSnippet ?? null,+ lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(),+ payload: sanitizedPayload ?? null,+ },+ });+ if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) {+ const status = setHeartbeatRunRuntimeStatus({+ companyId: run.companyId,+ issueId,+ agentId: run.agentId,+ runId: run.id,+ phase: progress.phase,+ message: progress.message,+ updatedAt: eventAt,+ currentToolName: progress.currentToolName,+ lastAssistantSnippet: progress.lastAssistantSnippet,+ lastEventAt: progress.lastEventAt,+ });+ if (status) publishHeartbeatRunRuntimeProgress(status);+ }+ }++ async function persistRunProcessMetadata(+ runId: string,+ meta: { pid: number; processGroupId: number | null; startedAt: string },+ ) {+ return persistHeartbeatRunProcessMetadata(db, runId, meta);+ }++ async function clearDetachedRunWarning(runId: string) {+ const updated = await db+ .update(heartbeatRuns)+ .set({+ error: null,+ errorCode: null,+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(heartbeatRuns.id, runId),+ eq(heartbeatRuns.status, "running"),+ eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE),+ ),+ )+ .returning()+ .then((rows) => rows[0] ?? null);+ if (!updated) return null;++ await appendRunEvent(updated, {+ eventType: "lifecycle",+ stream: "system",+ level: "info",+ message:+ "Detached child process reported activity; cleared detached warning",+ });+ return updated;+ }++ async function patchRunIssueCommentStatus(+ runId: string,+ patch: Partial<+ Pick<+ typeof heartbeatRuns.$inferInsert,+ | "issueCommentStatus"+ | "issueCommentSatisfiedByCommentId"+ | "issueCommentRetryQueuedAt"+ >+ >,+ ) {+ return db+ .update(heartbeatRuns)+ .set({ ...patch, updatedAt: new Date() })+ .where(eq(heartbeatRuns.id, runId))+ .returning()+ .then((rows) => rows[0] ?? null);+ }++ async function findRunIssueComment(+ runId: string,+ companyId: string,+ issueId: string,+ resultJson?: Record<string, unknown> | null,+ ) {+ const comments = await db+ .select({+ id: issueComments.id,+ body: issueComments.body,+ })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, companyId),+ eq(issueComments.issueId, issueId),+ eq(issueComments.createdByRunId, runId),+ ),+ )+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id));+ return findHeartbeatRunCompletionComment(comments, resultJson);+ }++ async function findLatestCompletedFinalAgentMessage(+ runId: string,+ companyId: string,+ ) {+ const rows = await db+ .select({+ seq: heartbeatRunEvents.seq,+ payload: heartbeatRunEvents.payload,+ })+ .from(heartbeatRunEvents)+ .where(+ and(+ eq(heartbeatRunEvents.companyId, companyId),+ eq(heartbeatRunEvents.runId, runId),+ eq(heartbeatRunEvents.eventType, "item.completed"),+ ),+ )+ .orderBy(desc(heartbeatRunEvents.seq))+ .limit(200);+ const candidates: Array<{+ seq: number;+ text: string;+ sourceEventId: string | null;+ channel: "final" | "unknown";+ }> = [];+ for (const row of rows) {+ const prpEvent = parseObject(parseObject(row.payload).prpEvent);+ const candidate = readCompletedAssistantMessageCandidate({+ seq: row.seq,+ prpEvent,+ });+ if (candidate) candidates.push(candidate);+ }+ const recoveryBoundary = await db+ .select({+ seq: heartbeatRunEvents.seq,+ payload: heartbeatRunEvents.payload,+ })+ .from(heartbeatRunEvents)+ .where(+ and(+ eq(heartbeatRunEvents.companyId, companyId),+ eq(heartbeatRunEvents.runId, runId),+ eq(heartbeatRunEvents.eventType, "lifecycle"),+ ),+ )+ .orderBy(heartbeatRunEvents.seq)+ .limit(200)+ .then(+ (lifecycleRows) =>+ lifecycleRows.find(+ (row) =>+ parseObject(row.payload).retryReasonCode ===+ "semantic_result_missing",+ )?.seq ?? null,+ );+ return selectHeartbeatRunFinalAgentMessage({+ candidates,+ semanticResultRecoveryAfterSeq: recoveryBoundary,+ });+ }++ async function refreshContinuationSummaryForRun(+ run: typeof heartbeatRuns.$inferSelect,+ agent: typeof agents.$inferSelect,+ ) {+ const contextSnapshot = parseObject(run.contextSnapshot);+ const issueId = readNonEmptyString(contextSnapshot.issueId);+ if (!issueId) return null;+ try {+ return await refreshIssueContinuationSummary({+ db,+ issueId,+ run: {+ id: run.id,+ status: run.status,+ error: run.error,+ errorCode: run.errorCode,+ resultJson: run.resultJson as Record<string, unknown> | null,+ stdoutExcerpt: run.stdoutExcerpt,+ stderrExcerpt: run.stderrExcerpt,+ finishedAt: run.finishedAt,+ },+ agent: {+ id: agent.id,+ name: agent.name,+ adapterType: agent.adapterType,+ },+ });+ } catch (err) {+ logger.warn(+ {+ err,+ runId: run.id,+ issueId,+ agentId: agent.id,+ },+ "failed to refresh issue continuation summary",+ );+ return null;+ }+ }++ async function enqueueMissingIssueCommentRetry(+ run: typeof heartbeatRuns.$inferSelect,+ agent: typeof agents.$inferSelect,+ issueId: string,+ ) {+ const invokability = await getAgentInvokability(agent);+ if (!invokability.invokable) {+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message:+ "Missing-comment retry suppressed because the agent is not invokable",+ payload: {+ reason: invokability.reason,+ invalidOrgChain: invokability.invalidOrgChain,+ ...invokability.details,+ },+ });+ return null;+ }++ const contextSnapshot = parseObject(run.contextSnapshot);+ const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);+ const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);+ const retryContextSnapshot = withRecoveryContext(+ {+ ...contextSnapshot,+ retryOfRunId: run.id,+ wakeReason: "missing_issue_comment",+ retryReason: "missing_issue_comment",+ missingIssueCommentForRunId: run.id,+ },+ "status_only",+ );+ const responsibleUserId = await resolveResponsibleUserIdForRunContext(+ run,+ retryContextSnapshot,+ );+ const now = new Date();++ const retryRun = await db.transaction(async (tx) => {+ await tx.execute(+ sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`,+ );++ const issue = await tx+ .select({ id: issues.id })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, run.companyId),+ eq(issues.executionRunId, run.id),+ ),+ )+ .then((rows) => rows[0] ?? null);+ if (!issue) return null;++ const wakeupRequest = await tx+ .insert(agentWakeupRequests)+ .values({+ companyId: run.companyId,+ agentId: run.agentId,+ source: "automation",+ triggerDetail: "system",+ reason: "missing_issue_comment",+ payload: withRecoveryContext(+ {+ issueId,+ retryOfRunId: run.id,+ retryReason: "missing_issue_comment",+ },+ "status_only",+ ),+ status: "queued",+ requestedByActorType: "system",+ requestedByActorId: null,+ updatedAt: now,+ })+ .returning()+ .then((rows) => rows[0]);++ const queuedRun = await tx+ .insert(heartbeatRuns)+ .values({+ companyId: run.companyId,+ agentId: run.agentId,+ scopeKind: "issue",+ issueId,+ invocationSource: "automation",+ triggerDetail: "system",+ status: "queued",+ wakeupRequestId: wakeupRequest.id,+ contextSnapshot: retryContextSnapshot,+ responsibleUserId,+ sessionIdBefore: sessionBefore,+ retryOfRunId: run.id,+ issueCommentStatus: "not_applicable",+ updatedAt: now,+ })+ .returning()+ .then((rows) => rows[0]);++ await tx+ .update(agentWakeupRequests)+ .set({+ runId: queuedRun.id,+ updatedAt: now,+ })+ .where(eq(agentWakeupRequests.id, wakeupRequest.id));++ await tx+ .update(issues)+ .set({+ executionRunId: queuedRun.id,+ executionAgentNameKey: normalizeAgentNameKey(agent.name),+ executionLockedAt: now,+ updatedAt: now,+ })+ .where(eq(issues.id, issue.id));++ await tx+ .update(heartbeatRuns)+ .set({+ issueCommentStatus: "retry_queued",+ issueCommentRetryQueuedAt: now,+ updatedAt: now,+ })+ .where(eq(heartbeatRuns.id, run.id));++ return queuedRun;+ });++ if (!retryRun) return null;++ publishLiveEvent({+ companyId: retryRun.companyId,+ type: "heartbeat.run.queued",+ payload: {+ runId: retryRun.id,+ agentId: retryRun.agentId,+ invocationSource: retryRun.invocationSource,+ triggerDetail: retryRun.triggerDetail,+ wakeupRequestId: retryRun.wakeupRequestId,+ },+ });++ return retryRun;+ }++ async function hasDeferredIssueCommentWake(+ companyId: string,+ issueId: string,+ agentId: string,+ ) {+ const deferredPayloads = await db+ .select({ payload: agentWakeupRequests.payload })+ .from(agentWakeupRequests)+ .where(+ and(+ eq(agentWakeupRequests.companyId, companyId),+ eq(agentWakeupRequests.agentId, agentId),+ eq(agentWakeupRequests.status, "deferred_issue_execution"),+ sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,+ ),+ );++ return deferredPayloads.some(({ payload }) => {+ const parsedPayload = parseObject(payload);+ const deferredContext = parseObject(+ parsedPayload[DEFERRED_WAKE_CONTEXT_KEY],+ );+ return Boolean(deriveCommentId(deferredContext, parsedPayload));+ });+ }++ async function finalizeIssueCommentPolicy(+ run: typeof heartbeatRuns.$inferSelect,+ agent: typeof agents.$inferSelect,+ presentationDecision?: RunPresentationDecision | null,+ ) {+ const contextSnapshot = parseObject(run.contextSnapshot);+ // The explicit receipt admitted one turn. A prose-only follow-up cannot+ // reuse it or renew it under the separate transient-failure retry policy.+ if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {+ if (run.issueCommentStatus !== "not_applicable") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ }+ return { outcome: "not_applicable" as const, queuedRun: null };+ }+ const issueId = readNonEmptyString(contextSnapshot.issueId);+ if (!issueId) {+ if (run.issueCommentStatus !== "not_applicable") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ }+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ // A failed/timed-out/cancelled run may immediately enqueue the normal+ // assignment or continuation recovery below. Do not let the lower-priority+ // status-only missing-comment wake claim the issue first. The dedicated+ // missing-comment retry still records exhaustion if that retry itself+ // fails, but an original failed execution remains on the established+ // direct-adapter recovery path.+ if (+ run.status !== "succeeded" &&+ readNonEmptyString(contextSnapshot.retryReason) !==+ "missing_issue_comment"+ ) {+ if (run.issueCommentStatus !== "not_applicable") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ }+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ // A settled run may legitimately have no user-facing prose. The response+ // resolver owns that decision; do not wake the agent again merely to force+ // an artificial comment into the issue thread.+ if (+ presentationDecision?.chosenSource === "none" &&+ (hasAcceptedSemanticResult(parseObject(run.resultJson)) ||+ presentationDecision.reasonCodes.includes(+ "legacy_adapter_summary_ambiguous",+ ))+ ) {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ // A pre-dispatch setup failure means the adapter process never started (for+ // example an unresolved workspace base ref). No agent could run, so no agent+ // could post an issue comment. A missing-comment retry cannot help and would+ // loop the identical pre-adapter failure, so mark the policy not_applicable+ // and queue nothing.+ if (+ run.errorCode != null &&+ PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode)+ ) {+ if (run.issueCommentStatus !== "not_applicable") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ }+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ const postedComment = await findRunIssueComment(+ run.id,+ run.companyId,+ issueId,+ parseObject(run.resultJson),+ );+ if (postedComment) {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "satisfied",+ issueCommentSatisfiedByCommentId: postedComment.id,+ issueCommentRetryQueuedAt: null,+ });+ return { outcome: "satisfied" as const, queuedRun: null };+ }++ // Missing-comment recovery is a legacy compatibility path for otherwise+ // successful runs. A failed, timed-out, or cancelled run is already owned+ // by lifecycle recovery and its terminal system presentation. Queuing a+ // prose-only retry here can seize the issue execution lock before the+ // authoritative continuation is materialized, replacing real recovery+ // with a cheap status-only turn.+ if (run.status !== "succeeded") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ if (+ readNonEmptyString(contextSnapshot.retryReason) ===+ "missing_issue_comment"+ ) {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "retry_exhausted",+ issueCommentSatisfiedByCommentId: null,+ });+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message:+ "Run ended without an issue comment after one retry; no further comment wake will be queued",+ });+ return { outcome: "retry_exhausted" as const, queuedRun: null };+ }++ if (!shouldRequireIssueCommentForWake(contextSnapshot)) {+ if (run.issueCommentStatus !== "not_applicable") {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ }+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ if (+ await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)+ ) {+ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "not_applicable",+ issueCommentSatisfiedByCommentId: null,+ issueCommentRetryQueuedAt: null,+ });+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: "info",+ message:+ "Run ended without an issue comment; a deferred comment wake already exists for this issue",+ });+ return { outcome: "not_applicable" as const, queuedRun: null };+ }++ const queuedRun = await enqueueMissingIssueCommentRetry(+ run,+ agent,+ issueId,+ );+ if (queuedRun) {+ await appendRunEvent(run, {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message:+ "Run ended without an issue comment; queued one follow-up wake to require a comment",+ });+ return { outcome: "retry_queued" as const, queuedRun };+ }++ await patchRunIssueCommentStatus(run.id, {+ issueCommentStatus: "retry_exhausted",+ issueCommentSatisfiedByCommentId: null,+ });+ return { outcome: "retry_exhausted" as const, queuedRun: null };+ }++ function parseMaxTurnContinuationPolicy(+ agent: typeof agents.$inferSelect,+ ): MaxTurnContinuationPolicy {+ const runtimeConfig = parseObject(agent.runtimeConfig);+ const heartbeat = parseObject(runtimeConfig.heartbeat);+ const configured = parseObject(heartbeat.maxTurnContinuation);+ const rawMaxAttempts = Math.floor(+ asNumber(+ configured.maxAttempts,+ MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,+ ),+ );+ const rawDelayMs = Math.floor(+ asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),+ );++ return {+ enabled: asBoolean(configured.enabled, true),+ maxAttempts: Math.max(+ 0,+ Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),+ ),+ delayMs: Math.max(+ 0,+ Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),+ ),+ };+ }++ function mergeRunStopMetadataForAgent(+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",+ options?: {+ resultJson?: Record<string, unknown> | null;+ conversationContinuationEligible?: boolean;+ errorCode?: string | null;+ errorMessage?: string | null;+ },+ ) {+ const stopMetadata = buildHeartbeatRunStopMetadata({+ adapterType: agent.adapterType,+ adapterConfig: parseObject(agent.adapterConfig),+ outcome,+ errorCode: options?.errorCode ?? null,+ errorMessage: options?.errorMessage ?? null,+ });+ const result = mergeHeartbeatRunStopMetadata(+ options?.resultJson ?? null,+ stopMetadata,+ );+ const cancellationAcknowledged =+ parseObject(result?.executionCancellation).state === "acknowledged";+ return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&+ (outcome !== "cancelled" || cancellationAcknowledged) &&+ isConversationAdapter(agent.adapterType)+ ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }+ : result;+ }++ function countValue(value: unknown) {+ const parsed = Number(value ?? 0);+ return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;+ }++ function dateValue(value: unknown) {+ if (value instanceof Date)+ return Number.isNaN(value.getTime()) ? null : value;+ if (typeof value === "string" || typeof value === "number") {+ const parsed = new Date(value);+ return Number.isNaN(parsed.getTime()) ? null : parsed;+ }+ return null;+ }++ function latestDate(...values: unknown[]) {+ let latest: Date | null = null;+ for (const value of values) {+ const parsed = dateValue(value);+ if (!parsed) continue;+ if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;+ }+ return latest;+ }++ async function buildRunLivenessInput(+ run: typeof heartbeatRuns.$inferSelect,+ resultJson: Record<string, unknown> | null | undefined,+ ): Promise<RunLivenessClassificationInput> {+ const context = parseObject(run.contextSnapshot);+ const contextIssueId = readNonEmptyString(context.issueId);+ const continuationAttempt = asNumber(+ context.continuationAttempt,+ run.continuationAttempt ?? 0,+ );++ const issue = contextIssueId+ ? await db+ .select({+ status: issues.status,+ title: issues.title,+ description: issues.description,+ workMode: issues.workMode,+ })+ .from(issues)+ .where(+ and(+ eq(issues.companyId, run.companyId),+ eq(issues.id, contextIssueId),+ ),+ )+ .then((rows) => rows[0] ?? null)+ : null;++ const [commentStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${issueComments.createdAt})`,+ })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, run.companyId),+ eq(issueComments.issueId, contextIssueId),+ eq(issueComments.createdByRunId, run.id),+ isNull(issueComments.deletedAt),+ ),+ )+ : [{ count: 0, latestAt: null }];++ const issueCommentBodies = contextIssueId+ ? await db+ .select({ body: issueComments.body })+ .from(issueComments)+ .where(+ and(+ eq(issueComments.companyId, run.companyId),+ eq(issueComments.issueId, contextIssueId),+ eq(issueComments.createdByRunId, run.id),+ ),+ )+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id))+ .limit(5)+ .then((rows) => rows.reverse().map((row) => row.body))+ : [];++ const continuationSummary = contextIssueId+ ? await getIssueContinuationSummaryDocument(db, contextIssueId)+ : null;++ const [documentStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,+ latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,+ })+ .from(documentRevisions)+ .innerJoin(+ issueDocuments,+ eq(documentRevisions.documentId, issueDocuments.documentId),+ )+ .where(+ and(+ eq(documentRevisions.companyId, run.companyId),+ eq(documentRevisions.createdByRunId, run.id),+ eq(issueDocuments.companyId, run.companyId),+ eq(issueDocuments.issueId, contextIssueId),+ sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,+ ),+ )+ : [{ count: 0, planCount: 0, latestAt: null }];++ const [workProductStats] = contextIssueId+ ? await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,+ })+ .from(issueWorkProducts)+ .where(+ and(+ eq(issueWorkProducts.companyId, run.companyId),+ eq(issueWorkProducts.issueId, contextIssueId),+ eq(issueWorkProducts.createdByRunId, run.id),+ ),+ )+ : [{ count: 0, latestAt: null }];++ const [workspaceOperationStats] = await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,+ })+ .from(workspaceOperations)+ .where(+ and(+ eq(workspaceOperations.companyId, run.companyId),+ eq(workspaceOperations.heartbeatRunId, run.id),+ ),+ );++ const [activityStats] = await db+ .select({+ count: sql<number>`count(*)::int`,+ latestAt: sql<Date | null>`max(${activityLog.createdAt})`,+ })+ .from(activityLog)+ .where(+ and(+ eq(activityLog.companyId, run.companyId),+ eq(activityLog.runId, run.id),+ notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),+ ),+ );++ const [eventStats] = await db+ .select({+ count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,+ latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,+ })+ .from(heartbeatRunEvents)+ .where(+ and(+ eq(heartbeatRunEvents.companyId, run.companyId),+ eq(heartbeatRunEvents.runId, run.id),+ ),+ );++ return {+ runStatus: run.status,+ issue,+ resultJson: resultJson ?? run.resultJson ?? null,+ issueCommentBodies,+ continuationSummaryBody: continuationSummary?.body ?? null,+ stdoutExcerpt: run.stdoutExcerpt ?? null,+ stderrExcerpt: run.stderrExcerpt ?? null,+ error: run.error ?? null,+ errorCode: run.errorCode ?? null,+ continuationAttempt,+ evidence: {+ issueCommentsCreated: countValue(commentStats?.count),+ documentRevisionsCreated: countValue(documentStats?.count),+ planDocumentRevisionsCreated: countValue(documentStats?.planCount),+ workProductsCreated: countValue(workProductStats?.count),+ workspaceOperationsCreated: countValue(workspaceOperationStats?.count),+ activityEventsCreated: countValue(activityStats?.count),+ toolOrActionEventsCreated: countValue(eventStats?.count),+ latestEvidenceAt: latestDate(+ commentStats?.latestAt,+ documentStats?.latestAt,+ workProductStats?.latestAt,+ workspaceOperationStats?.latestAt,+ activityStats?.latestAt,+ eventStats?.latestAt,+ ),+ },+ };+ }++ async function classifyAndPersistRunLiveness(+ run: typeof heartbeatRuns.$inferSelect,+ resultJson?: Record<string, unknown> | null,+ ) {+ const authRepair = run.status === "failed"+ ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {+ logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");+ return null;+ })+ : null;+ const classification = classifyRunLiveness({+ ...await buildRunLivenessInput(run, resultJson),+ authenticationRepairRequested: Boolean(authRepair?.interactionId),+ });+ return db+ .update(heartbeatRuns)+ .set({+ livenessState: classification.livenessState,+ livenessReason: classification.livenessReason,+ continuationAttempt: classification.continuationAttempt,+ lastUsefulActionAt: classification.lastUsefulActionAt,+ nextAction: classification.nextAction,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, run.id))+ .returning()+ .then((rows) => rows[0] ?? null);+ }++ async function updateRuntimeState(+ agent: typeof agents.$inferSelect,+ run: typeof heartbeatRuns.$inferSelect,+ result: AdapterExecutionResult,+ session: { legacySessionId: string | null },+ normalizedUsage?: UsageTotals | null,+ ) {+ await ensureRuntimeState(agent);+ await accountRunCost(db, run.id, budgetHooks);+ await db+ .update(agentRuntimeState)+ .set({+ adapterType: agent.adapterType,+ sessionId: session.legacySessionId,+ lastRunId: run.id,+ lastRunStatus: run.status,+ lastError: run.error ?? null,+ updatedAt: new Date(),+ })+ .where(eq(agentRuntimeState.agentId, agent.id));++ }++ return {+ appendRunEvent,+ escalatePlanApprovalResumeFailureNeedsAttention,+ recordPlanApprovalResumeFailureRetry,+ setRunStatusIfRunning,+ setWakeupStatus,+ mergeRunStopMetadataForAgent,+ classifyAndPersistRunLiveness,+ setRunStatusFromLive,+ setRunStatus,+ publishRunLifecyclePluginEventData,+ publishRunLifecyclePluginEvent,+ persistRunProcessMetadata,+ recordCurrentHeartbeatRunRuntimeProgress,+ completeSkillTestRunForHeartbeatOutcome,+ refreshContinuationSummaryForRun,+ findRunIssueComment,+ findLatestCompletedFinalAgentMessage,+ parseMaxTurnContinuationPolicy,+ finalizeIssueCommentPolicy,+ handleIssueReviewPathDisposition,+ handleRunLivenessContinuation,+ handleSuccessfulRunHandoff,+ updateRuntimeState,+ terminalizeRunOnLeaseRelease,+ clearDetachedRunWarning,+ };+}