Skip to content

The open-source app everyone uses to manage agents at work

refactor(heartbeat): extract run lifecycle handling (#15672)

Dotta0b80ea17ef32
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index 318fe32102..3fda91d3f6 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -1250,9 +1250,9 @@ It owns bounded run projections, database encoding checks, task session reads an
writes, explicit resumes, session compaction, and usage/billing helpers.
`createHeartbeatRunState(db)` binds these operations without doing database work
during construction. The encoding-check cache belongs to each factory instance.
-`heartbeat.ts` keeps run execution, session-goal recovery, cost accounting writes,
-and status transitions, and re-exports the existing public helpers. Keep session
-policy changes separate from run orchestration changes.
+`heartbeat.ts` keeps run execution and session-goal recovery, and re-exports the
+existing public helpers. Keep session policy changes separate from run
+orchestration changes.
Retry scheduling is in `server/src/services/heartbeat/retries.ts`. It owns bounded
retry schedules, connection and workspace contention deferrals, shared-workspace
@@ -1281,6 +1281,15 @@ its process-wide execution and wakeup promise sets. Forwarding callbacks preserv
construction order for retry and recovery services. Keep queue policy changes
separate from this extraction and from adapter execution changes.
+Run lifecycle handling is in `server/src/services/heartbeat/run-lifecycle.ts`.
+It owns status transitions, run events and progress, liveness classification,
+completion handoffs, issue-comment finalization, and runtime/cost settlement.
+`createHeartbeatLifecycle` binds the database and explicit reporting, recovery,
+and wakeup callbacks without starting work. The service supplies terminal
+reporting and forwarding callbacks preserve construction order for the other
+heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`.
+Keep lifecycle policy changes separate from this extraction and adapter execution.
+
## Wake Context Delivery
Built-in adapters deliver wake context through the run prompt, including structured
diff --git a/server/src/services/chat-publication-reconciliation.test.ts b/server/src/services/chat-publication-reconciliation.test.ts
index 46c27ab8ce..3af863db2d 100644
--- a/server/src/services/chat-publication-reconciliation.test.ts
+++ b/server/src/services/chat-publication-reconciliation.test.ts
@@ -220,18 +220,18 @@ describe("createCoalescedAsyncTrigger", () => {
describe("chat publication commit signals", () => {
it("emits the accepted signals only after their durable source writes", () => {
- const heartbeatSource = readFileSync(
- new URL("./heartbeat.ts", import.meta.url),
+ const lifecycleSource = readFileSync(
+ new URL("./heartbeat/run-lifecycle.ts", import.meta.url),
"utf8",
);
- const appendRunEventStart = heartbeatSource.indexOf(
+ const appendRunEventStart = lifecycleSource.indexOf(
"async function appendRunEvent(",
);
- const persistedEvent = heartbeatSource.indexOf(
+ const persistedEvent = lifecycleSource.indexOf(
"await appendHeartbeatRunEvent",
appendRunEventStart,
);
- const emittedEvent = heartbeatSource.indexOf(
+ const emittedEvent = lifecycleSource.indexOf(
"publishLiveEvent({",
persistedEvent,
);
@@ -239,6 +239,10 @@ describe("chat publication commit signals", () => {
expect(persistedEvent).toBeGreaterThan(appendRunEventStart);
expect(emittedEvent).toBeGreaterThan(persistedEvent);
+ const heartbeatSource = readFileSync(
+ new URL("./heartbeat.ts", import.meta.url),
+ "utf8",
+ );
const presentationMarker = heartbeatSource.indexOf(
'eventType: "run.presentation.resolved"',
);
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 8d84f116e6..be77bf2f40 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -1,10 +1,22 @@
+import {
+ NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
+ isHeartbeatRunTerminalStatus,
+ isHeartbeatRunRuntimeStatusActive,
+ publishHeartbeatRunRuntimeProgress,
+ decorateHeartbeatRunRuntimeStatus,
+ createHeartbeatLifecycle,
+} from "./heartbeat/run-lifecycle.js";
+export {
+ redactDetectedSuccessfulRunProgressSummaryForBoard,
+ redactSuccessfulRunHandoffEvidence,
+ persistHeartbeatRunProcessMetadata,
+ resolveSkillTestRunCompletionForHeartbeatOutcome,
+} from "./heartbeat/run-lifecycle.js";
import {
ChatControlRecoveryUnresolvedError,
shouldDeferFollowupWakeForSameIssue,
shouldQueueFollowupForRunningIssueWake,
DEFERRED_WAKE_CONTEXT_KEY,
- EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
- shouldRequireIssueCommentForWake,
createHeartbeatQueue,
} from "./heartbeat/queue.js";
export {
@@ -12,15 +24,12 @@ export {
shouldQueueFollowupForRunningIssueWake,
} from "./heartbeat/queue.js";
import {
- HEARTBEAT_RUN_TERMINAL_STATUSES,
- DETACHED_PROCESS_ERROR_CODE,
isProcessAlive,
CANCELLABLE_HEARTBEAT_RUN_STATUSES,
createHeartbeatRecovery,
} from "./heartbeat/recovery.js";
import {
BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS,
- normalizeAgentNameKey,
WorkspaceBusyDeferral,
readTransientRecoveryContractFromRun,
readHeartbeatRunErrorFamily,
@@ -44,7 +53,6 @@ import {
type RunSessionOutcome,
EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
deriveTaskKeyWithHeartbeatFallback,
- type UsageTotals,
getAdapterSessionCodec,
shouldResetTaskSessionForWake,
normalizeSessionParams,
@@ -119,7 +127,6 @@ export {
buildPaperclipWakePayload,
} from "./heartbeat/run-preparation.js";
import {
- WORKSPACE_VALIDATION_FAILURE_CODE,
isWorkspaceValidationFailedRun,
resolveNativeRecoveryExecutionWorkspaceBinding,
resolveExecutionWorkspaceReuseRequestForIssue,
@@ -224,13 +231,13 @@ export {
import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";
export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js";
import { preserveWorkspaceRestoreRecoveryMetadataSql } from "./legacy-workspace-restore-recovery.js";
-import { preserveWorkspaceRestoreRecoveryMetadata } from "./workspace-restore-recovery-state.js";
+
import { recordLegacyWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
import {
configuredEnvironmentProjection,
} from "../vendor/paperclip-runner/index.js";
import { decisionModelService } from "./decision-models.js";
-import { activeIssueInteractionCondition, TASK_QUESTION_GUIDANCE } from "./issue-question-context.js";
+import { TASK_QUESTION_GUIDANCE } from "./issue-question-context.js";
import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";
import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-identity.js";
import { buildAgentIdentityEnv } from "@paperclipai/adapter-utils/server-utils";
@@ -241,7 +248,11 @@ import { externalObjectService } from "./external-objects.js";
import { resolvePaperclipInstanceRoot } from "../home-paths.js";
import { dotRunnerBroker } from "./dot-runner-broker.js";
import { isAiAuthenticationBlocked } from "./ai-auth-failure.js";
-import { nativeRetryCancellationCommitCondition, rethrowNativeCancellationLockConflict, claimCancellationRequest, startupCancellationFence } from "./native-runtime/native-cancellation-request.js";
+import {
+ nativeRetryCancellationCommitCondition,
+ claimCancellationRequest,
+ startupCancellationFence,
+} from "./native-runtime/native-cancellation-request.js";
import {
prepareChatCompletionTurn,
chatCompletionInstruction,
@@ -274,14 +285,18 @@ import { toolActionDeliveryService } from "./tool-action-delivery.js";
import { readQueuedInteractionResponse } from "./queued-interaction-response.js";
import {
isConversation,
- isWaitingConversation,
prepareConversationTurn,
settleConversationTurn,
} from "./agent-conversations.js";
import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase";
import { getConversationConfirmationContext } from "./conversation-confirmation-context.js";
-import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";
-import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";
+import {
+ recordNativeLocalProcessStop,
+} from "./native-local-process-stop.js";
+import {
+ isAcknowledgedNativeStop,
+ acknowledgedNativeStopExecutionHasStopped,
+} from "./acknowledged-native-stop.js";
import {
legacyControllerBootId,
renewLegacyControllerLease,
@@ -313,7 +328,12 @@ import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiCon
import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js";
import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js";
import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js";
-import { CONVERSATION_CONTINUATION_POLICY, claimedAdapterType, runUsedConversationAdapter, hasConversationContinuationPolicy, isConversationAdapter } from "./conversation-continuation.js";
+import {
+ CONVERSATION_CONTINUATION_POLICY,
+ claimedAdapterType,
+ runUsedConversationAdapter,
+ hasConversationContinuationPolicy,
+} from "./conversation-continuation.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
@@ -381,48 +401,33 @@ import {
import type { Db } from "@paperclipai/db";
import {
CHAT_PROVIDERS,
- ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
isEnvironmentDriverSupportedForAdapter,
type EnvironmentLeaseStatus,
type ExecutionWorkspace,
- type HeartbeatRunStatusPhase,
type IssueExecutionMonitorClearReason,
type IssueExecutionMonitorPolicy,
type IssueExecutionMonitorRecoveryPolicy,
- type RequestConfirmationResult,
- type RunLivenessState,
} from "@paperclipai/shared";
import {
agents,
agentConfigRevisions,
- agentRuntimeState,
agentSessionGoalActions,
agentTaskSessions,
agentWakeupRequests,
- activityLog,
- approvals,
chatActions,
- companySkillTestRuns,
companies,
completionContracts,
- documentRevisions,
environmentLeases,
- issueDocuments,
heartbeatRunEvents,
heartbeatRuns,
- issueApprovals,
issueComments,
issueRecoveryActions,
- issueRelations,
- issueThreadInteractions,
issues,
- issueWorkProducts,
nativeRunFinalizations,
projects,
projectWorkspaces,
- routines,
workspaceOperations,
} from "@paperclipai/db";
import { conflict, HttpError, notFound } from "../errors.js";
@@ -442,10 +447,7 @@ import {
// git-credentials module became its canonical home; existing importers keep working.
export { scrubGitCredentialText };
import { publishLiveEvent } from "./live-events.js";
-import {
- appendHeartbeatRunEvent,
- type AppendHeartbeatRunEventInput,
-} from "./heartbeat-run-events.js";
+
import {
queuedCommentIdsFromWakePayload,
queuedCommentIdsFromRunContext,
@@ -528,7 +530,6 @@ import { createLocalAgentJwt } from "../agent-auth-jwt.js";
import { createRuntimeToolsToken } from "../runtime-tools-token.js";
import {
parseObject,
- asBoolean,
asNumber,
} from "../adapters/utils.js";
@@ -539,7 +540,6 @@ import { materializeExternalChatQuestionResponseInput } from "./native-runtime/e
import {
NativeRunnerOwnershipUnverifiedError,
isNativeRunnerOwnershipHeld,
- nativeRunnerOwnershipNotHeldCondition,
NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE,
} from "./native-runtime/native-runner-ownership.js";
import {
@@ -570,32 +570,21 @@ import {
import {
buildHeartbeatRunIssueComment,
CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON,
- findHeartbeatRunCompletionComment,
- hasAcceptedSemanticResult,
isExternalChatPresentationContext,
mergeHeartbeatRunResultJson,
- readCompletedAssistantMessageCandidate,
resolveHeartbeatRunResponse,
- selectHeartbeatRunFinalAgentMessage,
type RunPresentationDecision,
} from "./heartbeat-run-summary.js";
import {
- buildHeartbeatRunStopMetadata,
- mergeHeartbeatRunStopMetadata,
normalizeMaxTurnStopReason,
} from "./heartbeat-stop-metadata.js";
import {
CHAT_CONTROL_RECOVERY_ADMISSION_KEY,
CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
} from "./chat-control-recovery-stop.js";
-import {
- classifyRunLiveness,
- type RunLivenessClassificationInput,
-} from "./run-liveness.js";
import {
logActivity,
- publishPluginDomainEvent,
type LogActivityInput,
} from "./activity-log.js";
import {
@@ -646,7 +635,6 @@ import {
import {
continuationSummaryParksExecutor,
getIssueContinuationSummaryDocument,
- refreshIssueContinuationSummary,
} from "./issue-continuation-summary.js";
import {
buildPlanReviewContext,
@@ -694,19 +682,6 @@ import {
} from "./execution-allowlist.js";
import {
RECOVERY_ORIGIN_KINDS,
- FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
- SUCCESSFUL_RUN_MISSING_STATE_REASON,
- RUN_LIVENESS_CONTINUATION_REASON,
- buildRunLivenessContinuationIdempotencyKey,
- buildFinishSuccessfulRunHandoffIdempotencyKey,
- buildSuccessfulRunHandoffRequiredNotice,
- decideRunLivenessContinuation,
- decideSuccessfulRunHandoff,
- findExistingFinishSuccessfulRunHandoffWake,
- findExistingRunLivenessContinuationWake,
- isSuccessfulRunHandoffValidPathSkip,
- SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,
- readContinuationAttempt,
} from "./recovery/index.js";
import {
buildConfigurationIncompleteRecoveryNoticeSeed,
@@ -741,14 +716,9 @@ import {
type RunSnapshot as WakeQueueRunSnapshot,
} from "../modules/wake-queue/index.js";
import {
- buildIssueReviewPathLostIdempotencyKey,
- decideIssueReviewPathRecovery,
- ISSUE_REVIEW_PATH_LOST_WAKE_REASON,
- isReviewPathRecoveryIdempotencyConflict,
REVIEW_PATH_RECOVERY_INSTRUCTION,
- reviewPathConsumedRefFromRun,
} from "./recovery/review-path-recovery.js";
-import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "./successful-run-handoff-state.js";
+
import { taskWatchdogService } from "./task-watchdogs.js";
import {
@@ -763,20 +733,15 @@ import {
} from "./source-trust.js";
import {
redactCurrentUserText,
- redactCurrentUserValue,
- type CurrentUserRedactionOptions,
} from "../log-redaction.js";
-import { redactEventPayload, redactSensitiveText } from "../redaction.js";
+
import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js";
import {
resolvePaperclipRunnerIdleTimeoutMs,
- type RuntimeStatusUpdate,
} from "@paperclipai/adapter-utils";
import {
readPaperclipSkillSyncPreference,
selectPaperclipTaskMarkdown,
- UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
- UNMANAGED_BACKGROUND_TASK_STOP_REASON,
} from "@paperclipai/adapter-utils/server-utils";
import {
isUuidLike,
@@ -794,16 +759,10 @@ import { environmentRunOrchestrator } from "./environment-run-orchestrator.js";
import {
clearHeartbeatRunRuntimeStatus,
- getHeartbeatRunRuntimeStatus,
- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
- setHeartbeatRunRuntimeStatus,
sweepExpiredHeartbeatRunRuntimeStatuses,
touchHeartbeatRunRuntimeStatus,
} from "./heartbeat-run-runtime-status.js";
-import {
- readProcessStartedAt,
-} from "./hot-restart.js";
+
import {
assertLowTrustRuntimeServicesAllowed,
} from "./low-trust-runtime-containment.js";
@@ -816,32 +775,6 @@ import { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain, readTa
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
-export function redactDetectedSuccessfulRunProgressSummaryForBoard(
- summary: string,
- currentUserRedactionOptions?: CurrentUserRedactionOptions,
-) {
- const normalized = summary.replace(/\s+/g, " ").trim();
- const redacted = redactSensitiveText(
- redactCurrentUserText(normalized, currentUserRedactionOptions),
- );
- return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`;
-}
-
-export function redactSuccessfulRunHandoffEvidence(
- value: string | null,
- currentUserRedactionOptions?: CurrentUserRedactionOptions,
-) {
- if (!value) return null;
- return redactSensitiveText(
- redactCurrentUserText(value, currentUserRedactionOptions),
- );
-}
-
-const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
- "environment.lease_acquired",
- "environment.lease_released",
- "cost.reported",
-];
const ACCEPTED_PLAN_CONVERSION_SKILL_KEY =
"paperclipai/paperclip/paperclip-converting-plans-to-tasks";
@@ -873,25 +806,6 @@ const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON =
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE =
"execution_review_participant_recovery";
-const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set<string>([
- "low_trust_isolation_unavailable",
- "low_trust_requires_isolated_workspace",
- "low_trust_boundary_mismatch",
- "low_trust_requires_sandbox_environment",
- "low_trust_runtime_services_denied",
- "chat_failed_run_retry_not_authorized",
- CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
-]);
-// Error codes that mark a pre-dispatch setup failure. The adapter process never
-// started, so no agent could post an issue comment. The setup catch writes one
-// of these codes when a failure happens before `adapter.execute`.
-const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([
- "setup_failed",
- CONFIGURATION_INCOMPLETE_FAILURE_CODE,
- WORKSPACE_VALIDATION_FAILURE_CODE,
- ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
-]);
-
function nonRetryablePreflightFailureCode(error: unknown): string | null {
if (error instanceof ChatControlRecoveryUnresolvedError)
return CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE;
@@ -908,19 +822,9 @@ function nonRetryablePreflightFailureCode(error: unknown): string | null {
}
export { MAX_TURN_CONTINUATION_RETRY_REASON };
export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
-const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
-const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
-const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;
-const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;
export { WORKSPACE_BUSY_RETRY_REASON };
-interface MaxTurnContinuationPolicy {
- enabled: boolean;
- maxAttempts: number;
- delayMs: number;
-}
-
// Build the configuration-incomplete result payload for a workspace base ref
// that never resolved to a commit. The setup catch maps this to errorCode
// `configuration_incomplete`, so the recovery path routes it to a human owner
@@ -1379,315 +1283,6 @@ function isSameTaskScope(left: string | null, right: string | null) {
return (left ?? null) === (right ?? null);
}
-function isHeartbeatRunTerminalStatus(
- status: string | null | undefined,
-): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {
- return HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
- status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
- );
-}
-
-function isHeartbeatRunRuntimeStatusActive(
- status: string | null | undefined,
-): boolean {
- return status === "queued" || status === "running";
-}
-
-type HeartbeatRunRuntimeStatusRunLike = {
- id: string;
- status?: string | null;
- companyId?: string | null;
- agentId?: string | null;
- issueId?: string | null;
- contextSnapshot?: Record<string, unknown> | null;
-};
-
-function readRuntimeStatusIssueIdCandidate(
- run: HeartbeatRunRuntimeStatusRunLike,
-): string | null | undefined {
- if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null;
- if ("contextSnapshot" in run) {
- return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null;
- }
- return undefined;
-}
-
-function decorateHeartbeatRunRuntimeStatus<
- T extends HeartbeatRunRuntimeStatusRunLike,
->(
- run: T,
- expected: {
- companyId?: string | null;
- issueId?: string | null;
- agentId?: string | null;
- } = {},
-): T & {
- currentStatusMessage: string | null;
- currentStatusUpdatedAt: Date | null;
- currentToolName: string | null;
- lastAssistantSnippet: string | null;
- lastEventAt: Date | null;
-} {
- if (isHeartbeatRunTerminalStatus(run.status)) {
- clearHeartbeatRunRuntimeStatus(run.id);
- }
-
- const companyId = expected.companyId ?? run.companyId ?? null;
- const agentId = expected.agentId ?? run.agentId ?? null;
- const issueId =
- expected.issueId !== undefined
- ? expected.issueId
- : readRuntimeStatusIssueIdCandidate(run);
- const currentStatus =
- isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId
- ? getHeartbeatRunRuntimeStatus(run.id, {
- companyId,
- agentId,
- ...(issueId !== undefined ? { issueId } : {}),
- })
- : null;
-
- return {
- ...run,
- currentStatusMessage: currentStatus?.message ?? null,
- currentStatusUpdatedAt: currentStatus?.updatedAt ?? null,
- currentToolName: currentStatus?.currentToolName ?? null,
- lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null,
- lastEventAt: currentStatus?.lastEventAt ?? null,
- };
-}
-
-function publishHeartbeatRunRuntimeProgress(status: {
- companyId: string;
- runId: string;
- agentId: string;
- issueId: string | null;
- phase: HeartbeatRunStatusPhase;
- message: string;
- updatedAt: Date;
- currentToolName?: string | null;
- lastAssistantSnippet?: string | null;
- lastEventAt?: Date | null;
-}) {
- publishLiveEvent({
- companyId: status.companyId,
- type: "heartbeat.run.progress",
- payload: {
- runId: status.runId,
- agentId: status.agentId,
- issueId: status.issueId,
- phase: status.phase,
- message: status.message,
- currentToolName: status.currentToolName ?? null,
- lastAssistantSnippet: status.lastAssistantSnippet ?? null,
- lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(),
- updatedAt: status.updatedAt.toISOString(),
- },
- });
-}
-
-function recordHeartbeatRunRuntimeProgress(
- run: Pick<
- typeof heartbeatRuns.$inferSelect,
- "id" | "companyId" | "agentId" | "status" | "contextSnapshot"
- >,
- update: RuntimeStatusUpdate,
- issueId: string | null,
-) {
- if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null;
- const status = setHeartbeatRunRuntimeStatus({
- companyId: run.companyId,
- issueId,
- agentId: run.agentId,
- runId: run.id,
- phase: update.phase as HeartbeatRunStatusPhase,
- message: update.message,
- currentToolName: readNonEmptyString(update.currentToolName) ?? null,
- lastAssistantSnippet:
- readNonEmptyString(update.lastAssistantSnippet) ?? null,
- lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(),
- });
- if (!status) return null;
-
- publishHeartbeatRunRuntimeProgress(status);
- return status;
-}
-
-function sanitizeLiveRunProgressText(
- value: string,
- maxChars: number,
-): string | null {
- const normalized = value.replace(/\s+/g, " ").trim();
- if (!normalized) return null;
- const redacted = redactSensitiveText(normalized);
- if (redacted.length <= maxChars) return redacted;
- return `${redacted.slice(0, maxChars - 3)}...`;
-}
-
-function readLiveRunProgressString(
- value: unknown,
- maxChars: number,
-): string | null {
- return typeof value === "string"
- ? sanitizeLiveRunProgressText(value, maxChars)
- : null;
-}
-
-function readFirstLiveRunProgressString(
- maxChars: number,
- values: unknown[],
-): string | null {
- for (const value of values) {
- const text = readLiveRunProgressString(value, maxChars);
- if (text) return text;
- }
- return null;
-}
-
-function readLiveRunToolName(
- payload: Record<string, unknown> | null,
- eventType: string,
-): string | null {
- const toolCall =
- parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall);
- const message = parseObject(payload?.message);
- const direct = readFirstLiveRunProgressString(
- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
- [
- payload?.toolName,
- payload?.tool_name,
- payload?.tool,
- payload?.name,
- payload?.title,
- toolCall?.name,
- toolCall?.toolName,
- message?.name,
- message?.toolName,
- ],
- );
- if (direct) return direct;
-
- const normalizedEventType = eventType.toLowerCase();
- if (!normalizedEventType.includes("tool")) return null;
- return readLiveRunProgressString(
- eventType.replace(/[._-]+/g, " "),
- MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
- );
-}
-
-function readLiveRunAssistantSnippet(
- payload: Record<string, unknown> | null,
- eventType: string,
- message: string | null,
-): string | null {
- const normalizedEventType = eventType.toLowerCase();
- const messagePayload = parseObject(payload?.message);
- const direct = readFirstLiveRunProgressString(
- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
- [
- payload?.text,
- payload?.delta,
- payload?.text_delta,
- payload?.content,
- payload?.summary,
- messagePayload?.text,
- messagePayload?.content,
- ],
- );
- if (direct) return direct;
-
- if (
- normalizedEventType.includes("assistant") ||
- normalizedEventType.includes("text_delta") ||
- normalizedEventType.includes("message.delta") ||
- normalizedEventType.includes("message_delta")
- ) {
- return message
- ? sanitizeLiveRunProgressText(
- message,
- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
- )
- : null;
- }
-
- return null;
-}
-
-function buildRunEventRuntimeProgress(input: {
- eventType: string;
- message: string | null;
- payload: Record<string, unknown> | null;
- at: Date;
-}) {
- const normalizedEventType = input.eventType.toLowerCase();
- if (
- normalizedEventType === "lifecycle" ||
- normalizedEventType === "adapter.invoke"
- ) {
- return null;
- }
-
- const currentToolName = readLiveRunToolName(input.payload, input.eventType);
- const lastAssistantSnippet = readLiveRunAssistantSnippet(
- input.payload,
- input.eventType,
- input.message,
- );
- const fallbackMessage =
- readLiveRunProgressString(
- input.message,
- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
- ) ??
- sanitizeLiveRunProgressText(
- input.eventType.replace(/[._-]+/g, " "),
- MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
- );
- const message = currentToolName
- ? `Using ${currentToolName}`
- : (lastAssistantSnippet ?? fallbackMessage);
-
- if (!message) return null;
- return {
- phase: "run_activity" as const,
- message,
- currentToolName,
- lastAssistantSnippet,
- lastEventAt: input.at,
- };
-}
-
-export async function persistHeartbeatRunProcessMetadata(
- db: Db,
- runId: string,
- meta: { pid: number; processGroupId: number | null; startedAt: string },
-) {
- const observedStartedAt = await readProcessStartedAt(meta.pid).catch(
- () => null,
- );
- const startedAt = new Date(observedStartedAt ?? meta.startedAt);
- return db.transaction(async tx => {
- const run = await tx
- .update(heartbeatRuns)
- .set({
- processPid: meta.pid,
- processGroupId: meta.processGroupId,
- processStartedAt: Number.isNaN(startedAt.getTime())
- ? new Date()
- : startedAt,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, runId))
- .returning()
- .then((rows) => rows[0] ?? null);
- if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, {
- companyId: run.companyId, runId, agentId: run.agentId,
- eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info",
- message: "Process identity recorded; prior stop evidence no longer applies.",
- });
- return run;
- });
-}
-
async function terminateHeartbeatRunProcess(input: {
pid: number | null | undefined;
processGroupId: number | null | undefined;
@@ -1715,41 +1310,6 @@ async function terminateHeartbeatRunProcess(input: {
);
}
-type SkillTestHeartbeatCompletion = {
- outcome: "failed" | "cancelled";
- error: string | null;
- heartbeatOutcome: RunSessionOutcome;
-};
-
-export function resolveSkillTestRunCompletionForHeartbeatOutcome(
- outcome: RunSessionOutcome,
- error: string | null | undefined,
-): SkillTestHeartbeatCompletion | null {
- if (outcome === "cancelled") {
- return {
- outcome: "cancelled",
- error: error ?? "Harness run was cancelled",
- heartbeatOutcome: outcome,
- };
- }
- if (outcome === "timed_out") {
- return {
- outcome: "failed",
- error: error ?? "Timed out",
- heartbeatOutcome: outcome,
- };
- }
- if (outcome === "failed") {
- return {
- outcome: "failed",
- error: error ?? "Adapter failed",
- heartbeatOutcome: outcome,
- };
- }
- return null;
-}
-
-
export type HeartbeatEnvironmentRuntime = ReturnType<
typeof environmentRuntimeService
>;
@@ -2421,6 +1981,86 @@ export function heartbeatService(
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
}
+ const runLifecycle = createHeartbeatLifecycle(db, {
+ issuesSvc,
+ recovery,
+ companySkills,
+ getRun,
+ emitTerminalAgentTaskRun,
+ getIssueExecutionContext,
+ budgets,
+ enqueueWakeup,
+ getCurrentUserRedactionOptions,
+ treeControlSvc,
+ getAgentInvokability,
+ resolveSessionBeforeForWakeup,
+ resolveResponsibleUserIdForRunContext,
+ ensureRuntimeState,
+ budgetHooks,
+ });
+
+ const {
+ persistRunProcessMetadata,
+ recordCurrentHeartbeatRunRuntimeProgress,
+ completeSkillTestRunForHeartbeatOutcome,
+ refreshContinuationSummaryForRun,
+ findRunIssueComment,
+ findLatestCompletedFinalAgentMessage,
+ parseMaxTurnContinuationPolicy,
+ finalizeIssueCommentPolicy,
+ handleIssueReviewPathDisposition,
+ handleRunLivenessContinuation,
+ handleSuccessfulRunHandoff,
+ updateRuntimeState,
+ terminalizeRunOnLeaseRelease,
+ clearDetachedRunWarning,
+ } = runLifecycle;
+
+ // Earlier factories capture these callbacks before lifecycle construction.
+ function appendRunEvent(...args: Parameters<typeof runLifecycle.appendRunEvent>) {
+ return runLifecycle.appendRunEvent(...args);
+ }
+
+ function escalatePlanApprovalResumeFailureNeedsAttention(...args: Parameters<typeof runLifecycle.escalatePlanApprovalResumeFailureNeedsAttention>) {
+ return runLifecycle.escalatePlanApprovalResumeFailureNeedsAttention(...args);
+ }
+
+ function recordPlanApprovalResumeFailureRetry(...args: Parameters<typeof runLifecycle.recordPlanApprovalResumeFailureRetry>) {
+ return runLifecycle.recordPlanApprovalResumeFailureRetry(...args);
+ }
+
+ function setRunStatusIfRunning(...args: Parameters<typeof runLifecycle.setRunStatusIfRunning>) {
+ return runLifecycle.setRunStatusIfRunning(...args);
+ }
+
+ function setWakeupStatus(...args: Parameters<typeof runLifecycle.setWakeupStatus>) {
+ return runLifecycle.setWakeupStatus(...args);
+ }
+
+ function mergeRunStopMetadataForAgent(...args: Parameters<typeof runLifecycle.mergeRunStopMetadataForAgent>) {
+ return runLifecycle.mergeRunStopMetadataForAgent(...args);
+ }
+
+ function classifyAndPersistRunLiveness(...args: Parameters<typeof runLifecycle.classifyAndPersistRunLiveness>) {
+ return runLifecycle.classifyAndPersistRunLiveness(...args);
+ }
+
+ function setRunStatusFromLive(...args: Parameters<typeof runLifecycle.setRunStatusFromLive>) {
+ return runLifecycle.setRunStatusFromLive(...args);
+ }
+
+ function setRunStatus(...args: Parameters<typeof runLifecycle.setRunStatus>) {
+ return runLifecycle.setRunStatus(...args);
+ }
+
+ function publishRunLifecyclePluginEventData(...args: Parameters<typeof runLifecycle.publishRunLifecyclePluginEventData>) {
+ return runLifecycle.publishRunLifecyclePluginEventData(...args);
+ }
+
+ function publishRunLifecyclePluginEvent(...args: Parameters<typeof runLifecycle.publishRunLifecyclePluginEvent>) {
+ return runLifecycle.publishRunLifecyclePluginEvent(...args);
+ }
+
// Applies the post-commit effects a wake-queue release returns, exactly as
// the original release function did before its writes moved into that
// module: publish + dispatch a promoted or recovery run, log a reopened
@@ -2482,369 +2122,29 @@ export function heartbeatService(
}
}
- function isPlanApprovalConfirmationPayload(payload: unknown) {
- const target = parseObject(parseObject(payload).target);
- return (
- readNonEmptyString(target.type) === "issue_document" &&
- readNonEmptyString(target.key) === "plan"
- );
- }
-
- async function getAcceptedPlanApprovalInteractionForRun(
- run: typeof heartbeatRuns.$inferSelect,
- issueId: string | null,
- ) {
- const context = parseObject(run.contextSnapshot);
- const interactionId = readNonEmptyString(context.interactionId);
- if (!issueId || !interactionId) return null;
-
- const interaction = await db
- .select({
- id: issueThreadInteractions.id,
- kind: issueThreadInteractions.kind,
- status: issueThreadInteractions.status,
- payload: issueThreadInteractions.payload,
- result: issueThreadInteractions.result,
- })
- .from(issueThreadInteractions)
- .where(
- and(
- eq(issueThreadInteractions.companyId, run.companyId),
- eq(issueThreadInteractions.issueId, issueId),
- eq(issueThreadInteractions.id, interactionId),
- ),
- )
- .then((rows) => rows[0] ?? null);
-
- if (!interaction) return null;
- if (
- interaction.kind !== "request_confirmation" ||
- interaction.status !== "accepted"
- )
- return null;
- return isPlanApprovalConfirmationPayload(interaction.payload)
- ? interaction
- : null;
- }
-
- function planApprovalResumeFailureErrorCode(
- run: typeof heartbeatRuns.$inferSelect,
- ) {
- return readNonEmptyString(run.errorCode) ?? "unknown_error";
- }
-
- function buildPlanApprovalResumeFailureComment(input: {
- run: typeof heartbeatRuns.$inferSelect;
- status: "retrying" | "needs_attention";
- attempt: number;
- maxAttempts: number;
- }) {
- const errorCode = planApprovalResumeFailureErrorCode(input.run);
- if (input.status === "retrying") {
- return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`;
- }
- return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`;
- }
-
- function buildPlanApprovalResumeFailureResult(input: {
- run: typeof heartbeatRuns.$inferSelect;
- status: "retrying" | "needs_attention";
- attempt: number;
- maxAttempts: number;
- retryRunId?: string | null;
- recoveryActionId?: string | null;
- }): NonNullable<RequestConfirmationResult["resumeFailure"]> {
- return {
- status: input.status,
- errorCode: planApprovalResumeFailureErrorCode(input.run),
- attempt: input.attempt,
- maxAttempts: input.maxAttempts,
- runId: input.run.id,
- retryRunId: input.retryRunId ?? null,
- recoveryActionId: input.recoveryActionId ?? null,
- updatedAt: new Date().toISOString(),
+ const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });
+ async function releaseEnvironmentLeasesForRun(input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ status: string | null | undefined;
+ failureReason?: string | null;
+ providerResourceDisposition?: ProviderResourceDisposition;
+ nativeLifecycleTelemetry?: {
+ provider: string;
+ harness: string;
+ lifecycleMode: "per_turn" | "warm";
+ sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
};
- }
-
- async function updatePlanApprovalInteractionResumeFailure(input: {
- interaction: NonNullable<
- Awaited<ReturnType<typeof getAcceptedPlanApprovalInteractionForRun>>
- >;
- failure: NonNullable<RequestConfirmationResult["resumeFailure"]>;
- }) {
- const result = parseObject(input.interaction.result);
- const nextResult = {
- ...result,
- version: 1 as const,
- outcome: "accepted" as const,
- resumeFailure: input.failure,
- } satisfies RequestConfirmationResult;
-
- await db
- .update(issueThreadInteractions)
- .set({
- result: nextResult,
- updatedAt: new Date(),
- })
- .where(eq(issueThreadInteractions.id, input.interaction.id));
- }
-
- async function addPlanApprovalResumeFailureCommentOnce(input: {
- issueId: string;
- run: typeof heartbeatRuns.$inferSelect;
- body: string;
- }) {
- const existing = await db
- .select({ id: issueComments.id })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, input.run.companyId),
- eq(issueComments.issueId, input.issueId),
- or(
- eq(issueComments.body, input.body),
- sql`${issueComments.body} like ${`${input.body}\n%`}`,
- ),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
- if (existing) return null;
- return issuesSvc.addComment(
- input.issueId,
- input.body,
- { runId: input.run.id },
- { authorType: "system" },
- );
- }
-
- async function getActiveRecoveryActionId(
- companyId: string,
- sourceIssueId: string,
- ) {
- return db
- .select({ id: issueRecoveryActions.id })
- .from(issueRecoveryActions)
- .where(
- and(
- eq(issueRecoveryActions.companyId, companyId),
- eq(issueRecoveryActions.sourceIssueId, sourceIssueId),
- inArray(issueRecoveryActions.status, ["active", "escalated"]),
- ),
- )
- .orderBy(desc(issueRecoveryActions.updatedAt))
- .limit(1)
- .then((rows) => rows[0]?.id ?? null);
- }
-
- async function recordPlanApprovalResumeFailureRetry(input: {
- run: typeof heartbeatRuns.$inferSelect;
- issueId: string | null;
- retryRunId: string | null;
- attempt: number;
- maxAttempts: number;
- }) {
- const interaction = await getAcceptedPlanApprovalInteractionForRun(
- input.run,
- input.issueId,
- );
- if (!interaction || !input.issueId) return null;
-
- const body = buildPlanApprovalResumeFailureComment({
- run: input.run,
- status: "retrying",
- attempt: input.attempt,
- maxAttempts: input.maxAttempts,
- });
- await addPlanApprovalResumeFailureCommentOnce({
- issueId: input.issueId,
- run: input.run,
- body,
- });
- await updatePlanApprovalInteractionResumeFailure({
- interaction,
- failure: buildPlanApprovalResumeFailureResult({
- run: input.run,
- status: "retrying",
- attempt: input.attempt,
- maxAttempts: input.maxAttempts,
- retryRunId: input.retryRunId,
- }),
- });
- return interaction.id;
- }
-
- async function escalatePlanApprovalResumeFailureNeedsAttention(input: {
- run: typeof heartbeatRuns.$inferSelect;
- issueId: string | null;
- attempt: number;
- maxAttempts: number;
}) {
- const interaction = await getAcceptedPlanApprovalInteractionForRun(
- input.run,
- input.issueId,
- );
- if (!interaction || !input.issueId) return null;
-
- const issue = await db
- .select()
- .from(issues)
- .where(
- and(
- eq(issues.companyId, input.run.companyId),
- eq(issues.id, input.issueId),
- ),
- )
- .then((rows) => rows[0] ?? null);
- if (!issue) return null;
- if (
- issue.status !== "todo" &&
- issue.status !== "in_progress" &&
- issue.status !== "in_review"
- )
- return null;
-
- const body = buildPlanApprovalResumeFailureComment({
- run: input.run,
- status: "needs_attention",
- attempt: input.attempt,
- maxAttempts: input.maxAttempts,
- });
- await recovery.escalateStrandedAssignedIssue({
- issue,
- previousStatus: issue.status,
- latestRun: input.run,
- comment: body,
- });
- await addPlanApprovalResumeFailureCommentOnce({
- issueId: issue.id,
- run: input.run,
- body,
- });
-
- const recoveryActionId = await getActiveRecoveryActionId(
- issue.companyId,
- issue.id,
- );
- await updatePlanApprovalInteractionResumeFailure({
- interaction,
- failure: buildPlanApprovalResumeFailureResult({
- run: input.run,
- status: "needs_attention",
- attempt: input.attempt,
- maxAttempts: input.maxAttempts,
- recoveryActionId,
- }),
- });
- return interaction.id;
- }
-
- const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });
- async function completeSkillTestRunForHeartbeatOutcome(input: {
- run: typeof heartbeatRuns.$inferSelect;
- issueId: string | null;
- issueWorkMode?: string | null;
- outcome: RunSessionOutcome;
- error: string | null;
- }) {
- const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(
- input.outcome,
- input.error,
- );
- if (!completion || !input.issueId) return null;
-
- let isSkillTestIssue = input.issueWorkMode === "skill_test";
- if (!isSkillTestIssue && input.issueWorkMode === undefined) {
- const issueRow = await db
- .select({
- workMode: issues.workMode,
- harnessKind: issues.harnessKind,
- })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, input.run.companyId),
- eq(issues.id, input.issueId),
- ),
- )
- .then((rows) => rows[0] ?? null);
- isSkillTestIssue =
- issueRow?.workMode === "skill_test" ||
- issueRow?.harnessKind === "skill_test";
- }
- if (!isSkillTestIssue) return null;
-
- const existingRun = await db
- .select({
- id: companySkillTestRuns.id,
- status: companySkillTestRuns.status,
- })
- .from(companySkillTestRuns)
- .where(
- and(
- eq(companySkillTestRuns.companyId, input.run.companyId),
- eq(companySkillTestRuns.issueId, input.issueId),
- ),
- )
- .then((rows) => rows[0] ?? null);
- if (
- !existingRun ||
- ["succeeded", "failed", "cancelled"].includes(existingRun.status)
- )
- return null;
-
- const completedRun = await companySkills.completeTestRunForIssue({
- companyId: input.run.companyId,
- issueId: input.issueId,
- outcome: completion.outcome,
- error: completion.error,
- });
- if (!completedRun) return null;
-
- await logActivity(db, {
- companyId: input.run.companyId,
- actorType: "system",
- actorId: "heartbeat_finalize",
- agentId: input.run.agentId,
- runId: input.run.id,
- action: "company.skill_test_run_completed",
- entityType: "company_skill_test_run",
- entityId: completedRun.id,
- issueId: input.issueId,
- details: {
- issueId: input.issueId,
- status: completedRun.status,
- outputDocumentKey: completedRun.outputDocumentKey,
- heartbeatOutcome: completion.heartbeatOutcome,
- source: "heartbeat.run_finalized",
- },
- });
-
- return completedRun;
- }
-
- async function releaseEnvironmentLeasesForRun(input: {
- runId: string;
- companyId: string;
- agentId: string;
- status: string | null | undefined;
- failureReason?: string | null;
- providerResourceDisposition?: ProviderResourceDisposition;
- nativeLifecycleTelemetry?: {
- provider: string;
- harness: string;
- lifecycleMode: "per_turn" | "warm";
- sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
- };
- }) {
- const leaseOwnerRun = await getRun(input.runId);
- if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;
- // Recovery can finish workspace copy-back outside the executor's finally.
- // Successful copy-back does not earn warm retention for a failed turn.
- const status = leaseOwnerRun?.status ?? input.status;
- const providerResourceDisposition = providerResourceDispositionForTerminalRun(
- input.providerResourceDisposition,
- status,
+ const leaseOwnerRun = await getRun(input.runId);
+ if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return;
+ // Recovery can finish workspace copy-back outside the executor's finally.
+ // Successful copy-back does not earn warm retention for a failed turn.
+ const status = leaseOwnerRun?.status ?? input.status;
+ const providerResourceDisposition = providerResourceDispositionForTerminalRun(
+ input.providerResourceDisposition,
+ status,
);
if (providerResourceDisposition === "destroy") {
const closeResult = await (
@@ -3245,28 +2545,6 @@ export function heartbeatService(
return evaluateAgentInvokabilityFromDb(db, agent);
}
- async function recordCurrentHeartbeatRunRuntimeProgress(
- run: Pick<
- typeof heartbeatRuns.$inferSelect,
- "id" | "companyId" | "agentId" | "status" | "contextSnapshot"
- >,
- update: RuntimeStatusUpdate,
- issueId: string | null,
- ) {
- if (!isHeartbeatRunRuntimeStatusActive(run.status)) {
- clearHeartbeatRunRuntimeStatus(run.id);
- return null;
- }
-
- const currentRun = await getRun(run.id);
- if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) {
- clearHeartbeatRunRuntimeStatus(run.id);
- return null;
- }
-
- return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId);
- }
-
const issueMonitorDispatchColumns = {
id: issues.id,
companyId: issues.companyId,
@@ -4157,2020 +3435,83 @@ export function heartbeatService(
void reportRunFailure(db, updated, failureReport);
}
- async function setRunStatus(
- runId: string,
- status: string,
- patch?: Partial<typeof heartbeatRuns.$inferInsert>,
- ) {
- const previousStatus = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, runId))
- .then((rows) => rows[0] ?? null);
-
- if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,
- resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),
- };
-
- // Preserve the receipt-source fence when finalization enriches usage. A
- // late spool replay must still be able to complete an unfinished receipt.
- if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
-
- // Cancelling a queued run that never acquired provider execution is
- // positive bootstrap evidence. It must not hold unrelated queued messages.
- if (previousStatus && (status === "cancelled" || status === "interrupted")) {
- patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
- }
- if (
- status === "cancelled" &&
- previousStatus?.status === "queued" &&
- previousStatus.runtimeMode !== "native" &&
- !previousStatus.startedAt &&
- !previousStatus.processPid
- ) {
- patch = {
- ...patch,
- resultJson: {
- ...previousStatus.resultJson,
- ...patch?.resultJson,
- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
- },
- };
- }
- const updated =
- previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
- ? await terminalizeLegacyExecution({
- db,
- run: previousStatus,
- status,
- patch,
- reconcileIfNeeded: true,
- })
- : await db
- .update(heartbeatRuns)
- .set({
- status,
- ...patch,
- executionStatusDeliveryId: randomUUID(),
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, runId))
- .returning()
- .then((rows) => rows[0] ?? null);
-
- if (updated) {
- publishLiveEvent({
- companyId: updated.companyId,
- type: "heartbeat.run.status",
- payload: buildHeartbeatRunStatusLiveEventPayload(updated),
- });
- publishRunLifecyclePluginEvent(updated);
- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);
- }
-
- return updated;
- }
-
- async function setRunStatusIfRunning(
- runId: string,
- status: string,
- patch?: Partial<typeof heartbeatRuns.$inferInsert>,
- failureReport?: RunFailureReportOptions,
- ) {
- return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);
+ function truncateAgentErrorReason(
+ reason: string | null | undefined,
+ ): string | null {
+ if (!reason) return null;
+ const trimmed = reason.trim();
+ if (!trimmed) return null;
+ return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;
}
- // Move a run to a new status only when its current status is one of
- // `fromStatuses`. The compare-and-set is a single conditional update, so a
- // concurrent path can win the race. When this update matches nothing, the
- // function reads the current row and reports updated=false, so the caller can
- // keep the terminal outcome that another path already wrote.
- async function setRunStatusFromLive(
- runId: string,
- status: string,
- fromStatuses: string[],
- patch?: Partial<typeof heartbeatRuns.$inferInsert>,
- failureReport?: RunFailureReportOptions,
- cancellationCondition?: ReturnType<typeof nativeRetryCancellationCommitCondition>,
+ async function finalizeAgentStatus(
+ agentId: string,
+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
+ failureReason?: string | null,
+ options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },
) {
- // fromStatuses can name a terminal status as its own source (for example,
- // an idempotent "still failed" patch), so the write below is not always a
- // genuine transition. Read the pre-write status to tell the two apart.
- const previousStatus = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, runId))
- .then((rows) => rows[0] ?? null);
+ const existing = await getAgent(agentId);
+ if (!existing) return;
- if (previousStatus && (status === "cancelled" || status === "interrupted")) {
- patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
+ if (existing.status === "paused" || existing.status === "terminated") {
+ return;
}
- if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,
- resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),
- };
+ const isFirstHeartbeat =
+ options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;
- // Preserve the receipt-source fence when finalization enriches usage. A
- // late spool replay must still be able to complete an unfinished receipt.
- if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
+ const runningCount = await countRunningRunsForAgent(agentId);
+ const nextStatus =
+ runningCount > 0
+ ? "running"
+ : outcome === "succeeded" ||
+ outcome === "interrupted" ||
+ outcome === "cancelled" ||
+ (outcome === "failed" && options?.keepIdleOnFailure)
+ ? "idle"
+ : "error";
- // Cancelling a queued run that never acquired provider execution is
- // positive bootstrap evidence. It must not hold unrelated queued messages.
- if (
- status === "cancelled" &&
- previousStatus?.status === "queued" &&
- previousStatus.runtimeMode !== "native" &&
- !previousStatus.startedAt &&
- !previousStatus.processPid
- ) {
- patch = {
- ...patch,
- resultJson: {
- ...previousStatus.resultJson,
- ...patch?.resultJson,
- executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
- },
- };
+ const updated = await db
+ .update(agents)
+ .set({
+ status: nextStatus,
+ // Persist a human-readable reason on the agent record when it enters
+ // error so operators see it on the agent page without digging into run
+ // events; clear it whenever the agent leaves error.
+ errorReason:
+ nextStatus === "error"
+ ? truncateAgentErrorReason(failureReason)
+ : null,
+ lastHeartbeatAt: new Date(),
+ updatedAt: new Date(),
+ })
+ .where(eq(agents.id, agentId))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+
+ if (isFirstHeartbeat && updated) {
+ const tc = getTelemetryClient();
+ if (tc)
+ trackAgentFirstHeartbeat(tc, {
+ agentRole: updated.role,
+ agentId: updated.id,
+ });
}
- const updated =
- previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
- ? await terminalizeLegacyExecution({
- db,
- run: previousStatus,
- status,
- patch,
- fromStatuses,
- reconcileIfNeeded: true,
- writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])],
- })
- : await db
- .update(heartbeatRuns)
- .set({
- status,
- ...patch,
- executionStatusDeliveryId: randomUUID(),
- updatedAt: new Date(),
- })
- .where(
- and(
- eq(heartbeatRuns.id, runId),
- inArray(heartbeatRuns.status, fromStatuses),
- ...(cancellationCondition ? [cancellationCondition] : []),
- ...(isHeartbeatRunTerminalStatus(status)
- ? [nativeRunnerOwnershipNotHeldCondition()]
- : []),
- ),
- )
- .returning()
- .then((rows) => rows[0] ?? null)
- .catch((error: unknown) => {
- if (cancellationCondition) rethrowNativeCancellationLockConflict(error);
- throw error;
- });
if (updated) {
publishLiveEvent({
companyId: updated.companyId,
- type: "heartbeat.run.status",
- payload: buildHeartbeatRunStatusLiveEventPayload(updated),
- });
- publishRunLifecyclePluginEvent(updated);
- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);
- return { run: updated, updated: true as const };
- }
-
- const current = await db
- .select()
- .from(heartbeatRuns)
- .where(eq(heartbeatRuns.id, runId))
- .then((rows) => rows[0] ?? null);
-
- return { run: current, updated: false as const };
- }
-
- // Invariant: when a run releases its environment lease, the run row must be
- // terminal. The finalizer writes the terminal status in a step that is
- // separate from the agent status=done PATCH. If the sandbox or the run
- // process stops between the two steps, heartbeat_runs.status stays "running".
- // The UI reads liveness from that row, so a finished task shows "Live"
- // forever. This function closes the gap in the run teardown: when the run is
- // still running or queued, it forces a terminal status before the lease is
- // released. It never overwrites a status that another path already made
- // terminal.
- async function terminalizeRunOnLeaseRelease(
- run: typeof heartbeatRuns.$inferSelect,
- ): Promise<typeof heartbeatRuns.$inferSelect> {
- if (isNativeRunnerOwnershipHeld(run)) return run;
- if (isHeartbeatRunTerminalStatus(run.status)) return run;
- if (run.status !== "running" && run.status !== "queued") return run;
-
- // Choose the terminal status that reflects the true outcome. When the issue
- // already reached a terminal status, the run reached its goal, so use the
- // matching terminal run status. Otherwise the teardown cut the run short,
- // so use "interrupted".
- const issueId = readNonEmptyString(
- parseObject(run.contextSnapshot).issueId,
- );
- let terminalStatus: "succeeded" | "cancelled" | "interrupted" =
- "interrupted";
- if (issueId) {
- const issueStatus = await db
- .select({ status: issues.status })
- .from(issues)
- .where(eq(issues.id, issueId))
- .then((rows) => rows[0]?.status ?? null);
- if (issueStatus === "done") terminalStatus = "succeeded";
- else if (issueStatus === "cancelled") terminalStatus = "cancelled";
- }
-
- // Teardown can beat the cancellation finalizer. Preserve the acknowledged
- // user intent instead of reporting an infrastructure interruption.
- if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled";
-
- const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`;
- // Match both "running" and "queued". A queued run has released its lease but
- // never reached "running", so a running-only update would miss it and leave
- // a phantom live run behind.
- const write = await setRunStatusFromLive(
- run.id,
- terminalStatus,
- ["running", "queued"],
- {
- finishedAt: run.finishedAt ?? new Date(),
- error: run.error ?? (terminalStatus === "interrupted" ? message : null),
- errorCode:
- run.errorCode ??
- (terminalStatus === "interrupted"
- ? "lease_released_before_terminal"
- : null),
- },
- );
- if (!write.updated) {
- // Another path already finalized the run. Keep that terminal outcome.
- return write.run ?? run;
- }
-
- const terminalRun = write.run;
- if (terminalRun) {
- await appendRunEvent(terminalRun, {
- eventType: "lifecycle",
- stream: "system",
- level: terminalStatus === "interrupted" ? "warn" : "info",
- message,
+ type: "agent.status",
payload: {
- previousStatus: run.status,
- terminalStatus,
- reason: "environment_lease_release",
- ...(issueId ? { issueId } : {}),
+ agentId: updated.id,
+ status: updated.status,
+ lastHeartbeatAt: updated.lastHeartbeatAt
+ ? new Date(updated.lastHeartbeatAt).toISOString()
+ : null,
+ outcome,
},
- }).catch((eventErr) => {
- logger.warn(
- { err: eventErr, runId: run.id },
- "failed to append run event for lease-release terminalization",
- );
});
}
- return terminalRun ?? run;
- }
-
- function publishRunLifecyclePluginEvent(
- run: typeof heartbeatRuns.$inferSelect,
- ) {
- publishRunLifecyclePluginEventData({
- companyId: run.companyId,
- runId: run.id,
- agentId: run.agentId,
- status: run.status,
- invocationSource: run.invocationSource,
- triggerDetail: run.triggerDetail,
- error: run.error,
- errorCode: run.errorCode,
- issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
- startedAt: run.startedAt,
- finishedAt: run.finishedAt,
- });
- }
-
- function publishRunLifecyclePluginEventData(run: {
- companyId: string;
- runId: string;
- agentId: string;
- status: string;
- invocationSource: string;
- triggerDetail: string | null;
- error: string | null;
- errorCode: string | null;
- issueId: string | null;
- startedAt: Date | null;
- finishedAt: Date | null;
- }) {
- const eventType =
- run.status === "running"
- ? "agent.run.started"
- : run.status === "succeeded"
- ? "agent.run.finished"
- : run.status === "failed" || run.status === "timed_out"
- ? "agent.run.failed"
- : run.status === "cancelled"
- ? "agent.run.cancelled"
- : null;
- if (!eventType) return;
- publishPluginDomainEvent({
- eventId: randomUUID(),
- eventType,
- occurredAt: new Date().toISOString(),
- actorId: run.agentId,
- actorType: "agent",
- entityId: run.runId,
- entityType: "heartbeat_run",
- companyId: run.companyId,
- payload: {
- runId: run.runId,
- agentId: run.agentId,
- status: run.status,
- invocationSource: run.invocationSource,
- triggerDetail: run.triggerDetail,
- error: run.error ?? null,
- errorCode: run.errorCode ?? null,
- issueId: run.issueId,
- startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null,
- finishedAt: run.finishedAt
- ? new Date(run.finishedAt).toISOString()
- : null,
- },
- });
- }
-
- async function setWakeupStatus(
- wakeupRequestId: string | null | undefined,
- status: string,
- patch?: Partial<typeof agentWakeupRequests.$inferInsert>,
- ) {
- if (!wakeupRequestId) return;
- await db
- .update(agentWakeupRequests)
- .set({ status, ...patch, updatedAt: new Date() })
- // Reassignment revokes requests transactionally. Late execution settlement
- // must not turn a revoked request back into authority for a later retry.
- .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));
- }
-
- async function addContinuationExhaustedCommentOnce(input: {
- run: typeof heartbeatRuns.$inferSelect;
- issueId: string;
- comment: string;
- }) {
- const existing = await db
- .select({ id: issueComments.id })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, input.run.companyId),
- eq(issueComments.issueId, input.issueId),
- eq(issueComments.createdByRunId, input.run.id),
- sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
- if (existing) return;
- await issuesSvc.addComment(input.issueId, input.comment, {
- agentId: input.run.agentId,
- runId: input.run.id,
- });
- }
-
- async function handleRunLivenessContinuation(
- run: typeof heartbeatRuns.$inferSelect,
- ) {
- const context = parseObject(run.contextSnapshot);
- if (
- readNonEmptyString(context.goalControlRequestId) ||
- context.resumeSessionGoalHeartbeat === true
- )
- return;
- const livenessState = run.livenessState as RunLivenessState | null;
- if (livenessState !== "plan_only" && livenessState !== "empty_response")
- return;
-
- const issueId = readNonEmptyString(context.issueId);
- if (!issueId) return;
- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
-
- const [issue, agent] = await Promise.all([
- db
- .select({
- id: issues.id,
- companyId: issues.companyId,
- identifier: issues.identifier,
- title: issues.title,
- status: issues.status,
- assigneeAgentId: issues.assigneeAgentId,
- executionState: issues.executionState,
- projectId: issues.projectId,
- })
- .from(issues)
- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
- .then((rows) => rows[0] ?? null),
- db
- .select({
- id: agents.id,
- companyId: agents.companyId,
- status: agents.status,
- })
- .from(agents)
- .where(eq(agents.id, run.agentId))
- .then((rows) => rows[0] ?? null),
- ]);
-
- const budgetBlock =
- issue && agent
- ? await budgets.getInvocationBlock(issue.companyId, agent.id, {
- issueId: issue.id,
- projectId: issue.projectId,
- })
- : null;
-
- const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
- const idempotencyKey = issue
- ? buildRunLivenessContinuationIdempotencyKey({
- issueId: issue.id,
- sourceRunId: run.id,
- livenessState,
- nextAttempt,
- })
- : null;
- const existingWake = idempotencyKey
- ? await findExistingRunLivenessContinuationWake(db, {
- companyId: run.companyId,
- idempotencyKey,
- })
- : null;
-
- const decision = decideRunLivenessContinuation({
- run,
- issue,
- agent,
- livenessState,
- livenessReason: run.livenessReason,
- nextAction: run.nextAction,
- budgetBlocked: Boolean(budgetBlock),
- idempotentWakeExists: Boolean(existingWake),
- });
-
- if (decision.kind === "exhausted") {
- await setRunStatus(run.id, run.status, {
- livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`,
- });
- await addContinuationExhaustedCommentOnce({
- run,
- issueId,
- comment: decision.comment,
- });
- return;
- }
-
- if (decision.kind !== "enqueue") return;
-
- const continuationRun = await enqueueWakeup(run.agentId, {
- source: "automation",
- triggerDetail: "system",
- reason: RUN_LIVENESS_CONTINUATION_REASON,
- payload: decision.payload,
- contextSnapshot: {
- ...decision.contextSnapshot,
- originIdentityContextId: null,
- parentRunId: run.id,
- },
- idempotencyKey: decision.idempotencyKey,
- requestedByActorType: "system",
- requestedByActorId: "heartbeat",
- });
-
- if (continuationRun) {
- await db
- .update(heartbeatRuns)
- .set({
- continuationAttempt: decision.nextAttempt,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, run.id));
- }
- }
-
- function issueUiLink(
- issue: Pick<typeof issues.$inferSelect, "id" | "identifier">,
- ) {
- const label = issue.identifier ?? issue.id;
- const prefix = issue.identifier?.split("-")[0] || "PAP";
- return `[${label}](/${prefix}/issues/${label})`;
- }
-
- function hasUnmanagedBackgroundTaskEvidence(
- resultJson: Record<string, unknown> | null | undefined,
- ) {
- const evidence = parseObject(resultJson?.unmanagedBackgroundTask);
- return (
- evidence.stopped === true &&
- (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON ||
- evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON)
- );
- }
-
- function withUnmanagedBackgroundTaskStopReason(
- resultJson: Record<string, unknown> | null | undefined,
- ) {
- return {
- ...(resultJson ?? {}),
- stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
- };
- }
-
- function buildDetectedSuccessfulRunProgressSummary(
- run: typeof heartbeatRuns.$inferSelect,
- currentUserRedactionOptions: CurrentUserRedactionOptions,
- ) {
- const resultJson = parseObject(run.resultJson);
- const candidates = [
- hasUnmanagedBackgroundTaskEvidence(resultJson)
- ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON
- : null,
- readNonEmptyString(run.nextAction)
- ? `Next action noted: ${readNonEmptyString(run.nextAction)}`
- : null,
- readNonEmptyString(run.livenessReason),
- readNonEmptyString(resultJson.summary),
- readNonEmptyString(resultJson.result),
- readNonEmptyString(resultJson.message),
- ].filter((value): value is string => Boolean(value));
- const summary = candidates[0];
- if (!summary) return null;
- return redactDetectedSuccessfulRunProgressSummaryForBoard(
- summary,
- currentUserRedactionOptions,
- );
- }
-
- async function addSuccessfulRunHandoffCommentOnce(input: {
- issue: Pick<
- typeof issues.$inferSelect,
- "id" | "identifier" | "title" | "status"
- >;
- run: typeof heartbeatRuns.$inferSelect;
- agent: Pick<typeof agents.$inferSelect, "id" | "name">;
- detectedProgressSummary: string;
- }) {
- const existing = await db
- .select({ id: issueComments.id })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, input.run.companyId),
- eq(issueComments.issueId, input.issue.id),
- eq(issueComments.createdByRunId, input.run.id),
- sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
- if (existing) return null;
- const notice = buildSuccessfulRunHandoffRequiredNotice(input);
- return issuesSvc.addComment(
- input.issue.id,
- notice.body,
- { runId: input.run.id },
- {
- authorType: "system",
- presentation: notice.presentation,
- metadata: notice.metadata,
- },
- );
- }
-
- async function handleSuccessfulRunHandoff(
- run: typeof heartbeatRuns.$inferSelect,
- agent: typeof agents.$inferSelect,
- ) {
- if (run.status !== "succeeded") return;
- const context = parseObject(run.contextSnapshot);
- const issueId =
- readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);
- if (!issueId) return;
- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
- if (
- readNonEmptyString(context.goalControlRequestId) ||
- context.resumeSessionGoalHeartbeat === true
- ) {
- const goalProjection = await runnerGoalService(db).projection(
- run.companyId,
- issueId,
- run.agentId,
- );
- if (goalProjection?.goal?.status !== "complete") return;
- }
-
- const issue = await db
- .select({
- id: issues.id,
- companyId: issues.companyId,
- identifier: issues.identifier,
- title: issues.title,
- description: issues.description,
- status: issues.status,
- assigneeAgentId: issues.assigneeAgentId,
- assigneeUserId: issues.assigneeUserId,
- executionState: issues.executionState,
- monitorNextCheckAt: issues.monitorNextCheckAt,
- projectId: issues.projectId,
- originKind: issues.originKind,
- })
- .from(issues)
- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
- .then((rows) => rows[0] ?? null);
- const idempotencyKey = issue
- ? buildFinishSuccessfulRunHandoffIdempotencyKey({
- issueId: issue.id,
- sourceRunId: run.id,
- })
- : null;
- const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null);
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
- const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary(
- run,
- currentUserRedactionOptions,
- );
- const resultJson = parseObject(run.resultJson);
- const finalReport = redactSuccessfulRunHandoffEvidence(
- [
- readNonEmptyString(resultJson.summary),
- readNonEmptyString(resultJson.result),
- readNonEmptyString(resultJson.message),
- ].find((value): value is string => Boolean(value)) ?? null,
- currentUserRedactionOptions,
- );
- const nextAction = redactSuccessfulRunHandoffEvidence(
- readNonEmptyString(run.nextAction),
- currentUserRedactionOptions,
- );
-
- const [
- activeExecutionPath,
- queuedWake,
- pendingInteraction,
- pendingApproval,
- explicitBlocker,
- openRecoveryIssue,
- existingWake,
- budgetBlock,
- pauseHold,
- activeRoutineContinuation,
- ] = await Promise.all([
- issue
- ? db
- .select({ id: heartbeatRuns.id })
- .from(heartbeatRuns)
- .where(
- and(
- eq(heartbeatRuns.companyId, issue.companyId),
- eq(heartbeatRuns.agentId, run.agentId),
- inArray(heartbeatRuns.status, [
- ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
- ]),
- sql`(
- ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}
- or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id}
- )`,
- sql`${heartbeatRuns.id} <> ${run.id}`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, issue.companyId),
- eq(agentWakeupRequests.agentId, run.agentId),
- inArray(agentWakeupRequests.status, [
- "queued",
- "deferred_issue_execution",
- "claimed",
- ]),
- sql`(
- ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}
- or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id}
- or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id}
- or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id}
- )`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: issueThreadInteractions.id })
- .from(issueThreadInteractions)
- .where(
- and(
- eq(issueThreadInteractions.companyId, issue.companyId),
- eq(issueThreadInteractions.issueId, issue.id),
- eq(issueThreadInteractions.status, "pending"),
- activeIssueInteractionCondition(),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: issueApprovals.approvalId })
- .from(issueApprovals)
- .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
- .where(
- and(
- eq(issueApprovals.companyId, issue.companyId),
- eq(issueApprovals.issueId, issue.id),
- inArray(approvals.status, ["pending", "revision_requested"]),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: issueRelations.issueId })
- .from(issueRelations)
- .where(
- and(
- eq(issueRelations.companyId, issue.companyId),
- eq(issueRelations.relatedIssueId, issue.id),
- eq(issueRelations.type, "blocks"),
- sql`exists (
- select 1
- from issues blocker
- where blocker.id = ${issueRelations.issueId}
- and blocker.company_id = ${issue.companyId}
- and blocker.status not in ('done', 'cancelled')
- and blocker.hidden_at is null
- )`,
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: issues.id })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, issue.companyId),
- inArray(issues.originKind, [
- RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
- RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
- ]),
- eq(issues.originId, issue.id),
- visibleIssueCondition(),
- notInArray(issues.status, ["done", "cancelled"]),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- idempotencyKey
- ? findExistingFinishSuccessfulRunHandoffWake(db, {
- companyId: run.companyId,
- idempotencyKey,
- })
- : Promise.resolve(null),
- issue
- ? budgets.getInvocationBlock(issue.companyId, run.agentId, {
- issueId: issue.id,
- projectId: issue.projectId,
- })
- : Promise.resolve(null),
- issue
- ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id)
- : Promise.resolve(null),
- issue
- ? db
- .select({ id: routines.id })
- .from(routines)
- .where(
- and(
- eq(routines.companyId, issue.companyId),
- eq(routines.parentIssueId, issue.id),
- eq(routines.status, "active"),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null)
- : Promise.resolve(null),
- ]);
-
- const decision = decideSuccessfulRunHandoff({
- run,
- issue,
- agent,
- livenessState: run.livenessState as RunLivenessState | null,
- detectedProgressSummary,
- finalReport,
- nextAction,
- taskKey,
- hasActiveExecutionPath: Boolean(activeExecutionPath),
- hasQueuedWake: Boolean(queuedWake),
- hasPendingInteractionOrApproval: Boolean(
- pendingInteraction || pendingApproval,
- ),
- hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt),
- hasExplicitBlockerPath: Boolean(explicitBlocker),
- hasOpenRecoveryIssue: Boolean(openRecoveryIssue),
- hasPauseHold: Boolean(pauseHold),
- hasActiveRoutineContinuation: Boolean(activeRoutineContinuation),
- budgetBlocked: Boolean(budgetBlock),
- idempotentWakeExists: Boolean(existingWake),
- });
-
- if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) {
- await resolveRequiredSuccessfulRunHandoffOnValidPath(db, {
- companyId: issue.companyId,
- issueId: issue.id,
- issueIdentifier: issue.identifier,
- agentId: run.agentId,
- runId: run.id,
- skipReason: decision.reason,
- });
- }
-
- if (decision.kind !== "enqueue" || !issue) return;
-
- if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) {
- await db
- .update(heartbeatRuns)
- .set({
- livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
- resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
- ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, run.id));
- }
-
- const handoffRun = await enqueueWakeup(decision.targetAgentId, {
- source: "automation",
- triggerDetail: "system",
- reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
- payload: decision.payload,
- contextSnapshot: {
- ...decision.contextSnapshot,
- originIdentityContextId: null,
- parentRunId: run.id,
- },
- idempotencyKey: decision.idempotencyKey,
- requestedByActorType: "system",
- requestedByActorId: "heartbeat",
- });
- if (!handoffRun) return;
-
- await addSuccessfulRunHandoffCommentOnce({
- issue,
- run,
- agent,
- detectedProgressSummary:
- detectedProgressSummary ??
- "The run reported progress, but did not choose a next step.",
- });
- await logActivity(db, {
- companyId: issue.companyId,
- actorType: "system",
- actorId: "heartbeat",
- agentId: run.agentId,
- runId: run.id,
- action: "issue.successful_run_handoff_required",
- entityType: "issue",
- entityId: issue.id,
- details: {
- label: "Successful run missing issue disposition",
- sourceRunId: run.id,
- correctiveRunId: handoffRun.id,
- handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON,
- missingDisposition: "clear_next_step",
- detectedProgressSummary,
- issue: issueUiLink(issue),
- },
- });
- }
-
- async function handleIssueReviewPathDisposition(
- run: typeof heartbeatRuns.$inferSelect,
- ) {
- const contextSnapshot = parseObject(run.contextSnapshot);
- if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return;
- const issueId =
- readNonEmptyString(contextSnapshot.issueId) ??
- readNonEmptyString(contextSnapshot.taskId);
- if (!issueId) return;
- const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
- if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
-
- const issue = await db
- .select({
- id: issues.id,
- companyId: issues.companyId,
- identifier: issues.identifier,
- status: issues.status,
- assigneeAgentId: issues.assigneeAgentId,
- })
- .from(issues)
- .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
- .then((rows) => rows[0] ?? null);
- if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId)
- return;
-
- const reviewAttention = await issuesSvc
- .listReviewAttention(issue.companyId, [issue])
- .then(
- (map) =>
- map.get(issue.id) ?? {
- state: "none" as const,
- paths: [],
- reason: null,
- },
- );
- if (reviewAttention.state !== "stalled") return;
-
- const consumedPathRef = reviewPathConsumedRefFromRun({
- runId: run.id,
- issueId: issue.id,
- contextSnapshot,
- });
- const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({
- issueId: issue.id,
- consumedPathRef,
- });
- const existingWake = await db
- .select({ id: agentWakeupRequests.id })
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, issue.companyId),
- eq(agentWakeupRequests.idempotencyKey, idempotencyKey),
- notInArray(agentWakeupRequests.status, ["skipped"]),
- ),
- )
- .limit(1)
- .then((rows) => rows[0] ?? null);
-
- const decision = decideIssueReviewPathRecovery({
- issueId: issue.id,
- sourceRunId: run.id,
- assigneeAgentId: issue.assigneeAgentId,
- contextSnapshot,
- reviewAttention,
- existingWake: Boolean(existingWake),
- });
- if (decision.kind !== "enqueue") return;
-
- const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, {
- source: "automation",
- triggerDetail: "system",
- reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON,
- idempotencyKey: decision.idempotencyKey,
- payload: decision.payload,
- contextSnapshot: decision.contextSnapshot,
- requestedByActorType: "system",
- requestedByActorId: "heartbeat",
- }).catch((error: unknown) => {
- if (isReviewPathRecoveryIdempotencyConflict(error)) return null;
- throw error;
- });
- if (!recoveryRun) return;
-
- await logActivity(db, {
- companyId: issue.companyId,
- actorType: "system",
- actorId: "heartbeat",
- agentId: issue.assigneeAgentId,
- runId: run.id,
- action: "issue.review_path_recovery_queued",
- entityType: "issue",
- entityId: issue.id,
- details: {
- sourceRunId: run.id,
- recoveryRunId: recoveryRun.id,
- consumedPathRef,
- recoveryAttempt: 1,
- maxRecoveryAttempts: 1,
- },
- });
- }
-
- async function appendRunEvent(
- run: typeof heartbeatRuns.$inferSelect,
- event: {
- eventType: string;
- stream?: "system" | "stdout" | "stderr";
- level?: "info" | "warn" | "error";
- color?: string;
- message?: string;
- payload?: Record<string, unknown>;
- retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"];
- },
- ) {
- const eventAt = new Date();
- const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
- const sanitizedMessage = event.message
- ? redactSensitiveText(
- redactCurrentUserText(event.message, currentUserRedactionOptions),
- )
- : event.message;
- const boundedPayload = event.payload
- ? boundHeartbeatRunEventPayloadForStorage(event.payload)
- : event.payload;
- const secretSanitizedPayload = boundedPayload
- ? redactEventPayload(boundedPayload)
- : boundedPayload;
- const sanitizedPayload = secretSanitizedPayload
- ? redactCurrentUserValue(
- secretSanitizedPayload,
- currentUserRedactionOptions,
- )
- : secretSanitizedPayload;
- const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null;
- const progress = buildRunEventRuntimeProgress({
- eventType: event.eventType,
- message: sanitizedMessage ?? null,
- payload: sanitizedPayload ?? null,
- at: eventAt,
- });
- const persistedEvent = await appendHeartbeatRunEvent(db, {
- companyId: run.companyId,
- runId: run.id,
- agentId: run.agentId,
- eventType: event.eventType,
- stream: event.stream,
- level: event.level,
- color: event.color,
- message: sanitizedMessage,
- payload: sanitizedPayload,
- retryExhaustion: event.retryExhaustion,
- });
- if (persistedEvent.disposition === "duplicate") return;
- const seq = persistedEvent.row.seq;
-
- publishLiveEvent({
- companyId: run.companyId,
- type: "heartbeat.run.event",
- payload: {
- runId: run.id,
- agentId: run.agentId,
- issueId,
- seq,
- eventType: event.eventType,
- stream: event.stream ?? null,
- level: event.level ?? null,
- color: event.color ?? null,
- message: sanitizedMessage ?? null,
- currentToolName: progress?.currentToolName ?? null,
- lastAssistantSnippet: progress?.lastAssistantSnippet ?? null,
- lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(),
- payload: sanitizedPayload ?? null,
- },
- });
- if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) {
- const status = setHeartbeatRunRuntimeStatus({
- companyId: run.companyId,
- issueId,
- agentId: run.agentId,
- runId: run.id,
- phase: progress.phase,
- message: progress.message,
- updatedAt: eventAt,
- currentToolName: progress.currentToolName,
- lastAssistantSnippet: progress.lastAssistantSnippet,
- lastEventAt: progress.lastEventAt,
- });
- if (status) publishHeartbeatRunRuntimeProgress(status);
- }
- }
-
- async function persistRunProcessMetadata(
- runId: string,
- meta: { pid: number; processGroupId: number | null; startedAt: string },
- ) {
- return persistHeartbeatRunProcessMetadata(db, runId, meta);
- }
-
- async function clearDetachedRunWarning(runId: string) {
- const updated = await db
- .update(heartbeatRuns)
- .set({
- error: null,
- errorCode: null,
- updatedAt: new Date(),
- })
- .where(
- and(
- eq(heartbeatRuns.id, runId),
- eq(heartbeatRuns.status, "running"),
- eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE),
- ),
- )
- .returning()
- .then((rows) => rows[0] ?? null);
- if (!updated) return null;
-
- await appendRunEvent(updated, {
- eventType: "lifecycle",
- stream: "system",
- level: "info",
- message:
- "Detached child process reported activity; cleared detached warning",
- });
- return updated;
- }
-
- async function patchRunIssueCommentStatus(
- runId: string,
- patch: Partial<
- Pick<
- typeof heartbeatRuns.$inferInsert,
- | "issueCommentStatus"
- | "issueCommentSatisfiedByCommentId"
- | "issueCommentRetryQueuedAt"
- >
- >,
- ) {
- return db
- .update(heartbeatRuns)
- .set({ ...patch, updatedAt: new Date() })
- .where(eq(heartbeatRuns.id, runId))
- .returning()
- .then((rows) => rows[0] ?? null);
- }
-
- async function findRunIssueComment(
- runId: string,
- companyId: string,
- issueId: string,
- resultJson?: Record<string, unknown> | null,
- ) {
- const comments = await db
- .select({
- id: issueComments.id,
- body: issueComments.body,
- })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, companyId),
- eq(issueComments.issueId, issueId),
- eq(issueComments.createdByRunId, runId),
- ),
- )
- .orderBy(desc(issueComments.createdAt), desc(issueComments.id));
- return findHeartbeatRunCompletionComment(comments, resultJson);
- }
-
- async function findLatestCompletedFinalAgentMessage(
- runId: string,
- companyId: string,
- ) {
- const rows = await db
- .select({
- seq: heartbeatRunEvents.seq,
- payload: heartbeatRunEvents.payload,
- })
- .from(heartbeatRunEvents)
- .where(
- and(
- eq(heartbeatRunEvents.companyId, companyId),
- eq(heartbeatRunEvents.runId, runId),
- eq(heartbeatRunEvents.eventType, "item.completed"),
- ),
- )
- .orderBy(desc(heartbeatRunEvents.seq))
- .limit(200);
- const candidates: Array<{
- seq: number;
- text: string;
- sourceEventId: string | null;
- channel: "final" | "unknown";
- }> = [];
- for (const row of rows) {
- const prpEvent = parseObject(parseObject(row.payload).prpEvent);
- const candidate = readCompletedAssistantMessageCandidate({
- seq: row.seq,
- prpEvent,
- });
- if (candidate) candidates.push(candidate);
- }
- const recoveryBoundary = await db
- .select({
- seq: heartbeatRunEvents.seq,
- payload: heartbeatRunEvents.payload,
- })
- .from(heartbeatRunEvents)
- .where(
- and(
- eq(heartbeatRunEvents.companyId, companyId),
- eq(heartbeatRunEvents.runId, runId),
- eq(heartbeatRunEvents.eventType, "lifecycle"),
- ),
- )
- .orderBy(heartbeatRunEvents.seq)
- .limit(200)
- .then(
- (lifecycleRows) =>
- lifecycleRows.find(
- (row) =>
- parseObject(row.payload).retryReasonCode ===
- "semantic_result_missing",
- )?.seq ?? null,
- );
- return selectHeartbeatRunFinalAgentMessage({
- candidates,
- semanticResultRecoveryAfterSeq: recoveryBoundary,
- });
- }
-
- async function refreshContinuationSummaryForRun(
- run: typeof heartbeatRuns.$inferSelect,
- agent: typeof agents.$inferSelect,
- ) {
- const contextSnapshot = parseObject(run.contextSnapshot);
- const issueId = readNonEmptyString(contextSnapshot.issueId);
- if (!issueId) return null;
- try {
- return await refreshIssueContinuationSummary({
- db,
- issueId,
- run: {
- id: run.id,
- status: run.status,
- error: run.error,
- errorCode: run.errorCode,
- resultJson: run.resultJson as Record<string, unknown> | null,
- stdoutExcerpt: run.stdoutExcerpt,
- stderrExcerpt: run.stderrExcerpt,
- finishedAt: run.finishedAt,
- },
- agent: {
- id: agent.id,
- name: agent.name,
- adapterType: agent.adapterType,
- },
- });
- } catch (err) {
- logger.warn(
- {
- err,
- runId: run.id,
- issueId,
- agentId: agent.id,
- },
- "failed to refresh issue continuation summary",
- );
- return null;
- }
- }
-
- async function enqueueMissingIssueCommentRetry(
- run: typeof heartbeatRuns.$inferSelect,
- agent: typeof agents.$inferSelect,
- issueId: string,
- ) {
- const invokability = await getAgentInvokability(agent);
- if (!invokability.invokable) {
- await appendRunEvent(run, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message:
- "Missing-comment retry suppressed because the agent is not invokable",
- payload: {
- reason: invokability.reason,
- invalidOrgChain: invokability.invalidOrgChain,
- ...invokability.details,
- },
- });
- return null;
- }
-
- const contextSnapshot = parseObject(run.contextSnapshot);
- const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
- const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
- const retryContextSnapshot = withRecoveryContext(
- {
- ...contextSnapshot,
- retryOfRunId: run.id,
- wakeReason: "missing_issue_comment",
- retryReason: "missing_issue_comment",
- missingIssueCommentForRunId: run.id,
- },
- "status_only",
- );
- const responsibleUserId = await resolveResponsibleUserIdForRunContext(
- run,
- retryContextSnapshot,
- );
- const now = new Date();
-
- const retryRun = await db.transaction(async (tx) => {
- await tx.execute(
- sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`,
- );
-
- const issue = await tx
- .select({ id: issues.id })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.executionRunId, run.id),
- ),
- )
- .then((rows) => rows[0] ?? null);
- if (!issue) return null;
-
- const wakeupRequest = await tx
- .insert(agentWakeupRequests)
- .values({
- companyId: run.companyId,
- agentId: run.agentId,
- source: "automation",
- triggerDetail: "system",
- reason: "missing_issue_comment",
- payload: withRecoveryContext(
- {
- issueId,
- retryOfRunId: run.id,
- retryReason: "missing_issue_comment",
- },
- "status_only",
- ),
- status: "queued",
- requestedByActorType: "system",
- requestedByActorId: null,
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- const queuedRun = await tx
- .insert(heartbeatRuns)
- .values({
- companyId: run.companyId,
- agentId: run.agentId,
- scopeKind: "issue",
- issueId,
- invocationSource: "automation",
- triggerDetail: "system",
- status: "queued",
- wakeupRequestId: wakeupRequest.id,
- contextSnapshot: retryContextSnapshot,
- responsibleUserId,
- sessionIdBefore: sessionBefore,
- retryOfRunId: run.id,
- issueCommentStatus: "not_applicable",
- updatedAt: now,
- })
- .returning()
- .then((rows) => rows[0]);
-
- await tx
- .update(agentWakeupRequests)
- .set({
- runId: queuedRun.id,
- updatedAt: now,
- })
- .where(eq(agentWakeupRequests.id, wakeupRequest.id));
-
- await tx
- .update(issues)
- .set({
- executionRunId: queuedRun.id,
- executionAgentNameKey: normalizeAgentNameKey(agent.name),
- executionLockedAt: now,
- updatedAt: now,
- })
- .where(eq(issues.id, issue.id));
-
- await tx
- .update(heartbeatRuns)
- .set({
- issueCommentStatus: "retry_queued",
- issueCommentRetryQueuedAt: now,
- updatedAt: now,
- })
- .where(eq(heartbeatRuns.id, run.id));
-
- return queuedRun;
- });
-
- if (!retryRun) return null;
-
- publishLiveEvent({
- companyId: retryRun.companyId,
- type: "heartbeat.run.queued",
- payload: {
- runId: retryRun.id,
- agentId: retryRun.agentId,
- invocationSource: retryRun.invocationSource,
- triggerDetail: retryRun.triggerDetail,
- wakeupRequestId: retryRun.wakeupRequestId,
- },
- });
-
- return retryRun;
- }
-
- async function hasDeferredIssueCommentWake(
- companyId: string,
- issueId: string,
- agentId: string,
- ) {
- const deferredPayloads = await db
- .select({ payload: agentWakeupRequests.payload })
- .from(agentWakeupRequests)
- .where(
- and(
- eq(agentWakeupRequests.companyId, companyId),
- eq(agentWakeupRequests.agentId, agentId),
- eq(agentWakeupRequests.status, "deferred_issue_execution"),
- sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
- ),
- );
-
- return deferredPayloads.some(({ payload }) => {
- const parsedPayload = parseObject(payload);
- const deferredContext = parseObject(
- parsedPayload[DEFERRED_WAKE_CONTEXT_KEY],
- );
- return Boolean(deriveCommentId(deferredContext, parsedPayload));
- });
- }
-
- async function finalizeIssueCommentPolicy(
- run: typeof heartbeatRuns.$inferSelect,
- agent: typeof agents.$inferSelect,
- presentationDecision?: RunPresentationDecision | null,
- ) {
- const contextSnapshot = parseObject(run.contextSnapshot);
- // The explicit receipt admitted one turn. A prose-only follow-up cannot
- // reuse it or renew it under the separate transient-failure retry policy.
- if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {
- if (run.issueCommentStatus !== "not_applicable") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- }
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
- const issueId = readNonEmptyString(contextSnapshot.issueId);
- if (!issueId) {
- if (run.issueCommentStatus !== "not_applicable") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- }
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- // A failed/timed-out/cancelled run may immediately enqueue the normal
- // assignment or continuation recovery below. Do not let the lower-priority
- // status-only missing-comment wake claim the issue first. The dedicated
- // missing-comment retry still records exhaustion if that retry itself
- // fails, but an original failed execution remains on the established
- // direct-adapter recovery path.
- if (
- run.status !== "succeeded" &&
- readNonEmptyString(contextSnapshot.retryReason) !==
- "missing_issue_comment"
- ) {
- if (run.issueCommentStatus !== "not_applicable") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- }
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- // A settled run may legitimately have no user-facing prose. The response
- // resolver owns that decision; do not wake the agent again merely to force
- // an artificial comment into the issue thread.
- if (
- presentationDecision?.chosenSource === "none" &&
- (hasAcceptedSemanticResult(parseObject(run.resultJson)) ||
- presentationDecision.reasonCodes.includes(
- "legacy_adapter_summary_ambiguous",
- ))
- ) {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- // A pre-dispatch setup failure means the adapter process never started (for
- // example an unresolved workspace base ref). No agent could run, so no agent
- // could post an issue comment. A missing-comment retry cannot help and would
- // loop the identical pre-adapter failure, so mark the policy not_applicable
- // and queue nothing.
- if (
- run.errorCode != null &&
- PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode)
- ) {
- if (run.issueCommentStatus !== "not_applicable") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- }
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- const postedComment = await findRunIssueComment(
- run.id,
- run.companyId,
- issueId,
- parseObject(run.resultJson),
- );
- if (postedComment) {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "satisfied",
- issueCommentSatisfiedByCommentId: postedComment.id,
- issueCommentRetryQueuedAt: null,
- });
- return { outcome: "satisfied" as const, queuedRun: null };
- }
-
- // Missing-comment recovery is a legacy compatibility path for otherwise
- // successful runs. A failed, timed-out, or cancelled run is already owned
- // by lifecycle recovery and its terminal system presentation. Queuing a
- // prose-only retry here can seize the issue execution lock before the
- // authoritative continuation is materialized, replacing real recovery
- // with a cheap status-only turn.
- if (run.status !== "succeeded") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- if (
- readNonEmptyString(contextSnapshot.retryReason) ===
- "missing_issue_comment"
- ) {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "retry_exhausted",
- issueCommentSatisfiedByCommentId: null,
- });
- await appendRunEvent(run, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message:
- "Run ended without an issue comment after one retry; no further comment wake will be queued",
- });
- return { outcome: "retry_exhausted" as const, queuedRun: null };
- }
-
- if (!shouldRequireIssueCommentForWake(contextSnapshot)) {
- if (run.issueCommentStatus !== "not_applicable") {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- }
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- if (
- await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)
- ) {
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "not_applicable",
- issueCommentSatisfiedByCommentId: null,
- issueCommentRetryQueuedAt: null,
- });
- await appendRunEvent(run, {
- eventType: "lifecycle",
- stream: "system",
- level: "info",
- message:
- "Run ended without an issue comment; a deferred comment wake already exists for this issue",
- });
- return { outcome: "not_applicable" as const, queuedRun: null };
- }
-
- const queuedRun = await enqueueMissingIssueCommentRetry(
- run,
- agent,
- issueId,
- );
- if (queuedRun) {
- await appendRunEvent(run, {
- eventType: "lifecycle",
- stream: "system",
- level: "warn",
- message:
- "Run ended without an issue comment; queued one follow-up wake to require a comment",
- });
- return { outcome: "retry_queued" as const, queuedRun };
- }
-
- await patchRunIssueCommentStatus(run.id, {
- issueCommentStatus: "retry_exhausted",
- issueCommentSatisfiedByCommentId: null,
- });
- return { outcome: "retry_exhausted" as const, queuedRun: null };
- }
-
- function parseMaxTurnContinuationPolicy(
- agent: typeof agents.$inferSelect,
- ): MaxTurnContinuationPolicy {
- const runtimeConfig = parseObject(agent.runtimeConfig);
- const heartbeat = parseObject(runtimeConfig.heartbeat);
- const configured = parseObject(heartbeat.maxTurnContinuation);
- const rawMaxAttempts = Math.floor(
- asNumber(
- configured.maxAttempts,
- MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,
- ),
- );
- const rawDelayMs = Math.floor(
- asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),
- );
-
- return {
- enabled: asBoolean(configured.enabled, true),
- maxAttempts: Math.max(
- 0,
- Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),
- ),
- delayMs: Math.max(
- 0,
- Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),
- ),
- };
- }
-
- function truncateAgentErrorReason(
- reason: string | null | undefined,
- ): string | null {
- if (!reason) return null;
- const trimmed = reason.trim();
- if (!trimmed) return null;
- return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed;
- }
-
- async function finalizeAgentStatus(
- agentId: string,
- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
- failureReason?: string | null,
- options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean },
- ) {
- const existing = await getAgent(agentId);
- if (!existing) return;
-
- if (existing.status === "paused" || existing.status === "terminated") {
- return;
- }
-
- const isFirstHeartbeat =
- options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt;
-
- const runningCount = await countRunningRunsForAgent(agentId);
- const nextStatus =
- runningCount > 0
- ? "running"
- : outcome === "succeeded" ||
- outcome === "interrupted" ||
- outcome === "cancelled" ||
- (outcome === "failed" && options?.keepIdleOnFailure)
- ? "idle"
- : "error";
-
- const updated = await db
- .update(agents)
- .set({
- status: nextStatus,
- // Persist a human-readable reason on the agent record when it enters
- // error so operators see it on the agent page without digging into run
- // events; clear it whenever the agent leaves error.
- errorReason:
- nextStatus === "error"
- ? truncateAgentErrorReason(failureReason)
- : null,
- lastHeartbeatAt: new Date(),
- updatedAt: new Date(),
- })
- .where(eq(agents.id, agentId))
- .returning()
- .then((rows) => rows[0] ?? null);
-
- if (isFirstHeartbeat && updated) {
- const tc = getTelemetryClient();
- if (tc)
- trackAgentFirstHeartbeat(tc, {
- agentRole: updated.role,
- agentId: updated.id,
- });
- }
-
- if (updated) {
- publishLiveEvent({
- companyId: updated.companyId,
- type: "agent.status",
- payload: {
- agentId: updated.id,
- status: updated.status,
- lastHeartbeatAt: updated.lastHeartbeatAt
- ? new Date(updated.lastHeartbeatAt).toISOString()
- : null,
- outcome,
- },
- });
- }
- }
-
- function mergeRunStopMetadataForAgent(
- agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
- outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
- options?: {
- resultJson?: Record<string, unknown> | null;
- conversationContinuationEligible?: boolean;
- errorCode?: string | null;
- errorMessage?: string | null;
- },
- ) {
- const stopMetadata = buildHeartbeatRunStopMetadata({
- adapterType: agent.adapterType,
- adapterConfig: parseObject(agent.adapterConfig),
- outcome,
- errorCode: options?.errorCode ?? null,
- errorMessage: options?.errorMessage ?? null,
- });
- const result = mergeHeartbeatRunStopMetadata(
- options?.resultJson ?? null,
- stopMetadata,
- );
- const cancellationAcknowledged =
- parseObject(result?.executionCancellation).state === "acknowledged";
- return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&
- (outcome !== "cancelled" || cancellationAcknowledged) &&
- isConversationAdapter(agent.adapterType)
- ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }
- : result;
- }
-
- function countValue(value: unknown) {
- const parsed = Number(value ?? 0);
- return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
- }
-
- function dateValue(value: unknown) {
- if (value instanceof Date)
- return Number.isNaN(value.getTime()) ? null : value;
- if (typeof value === "string" || typeof value === "number") {
- const parsed = new Date(value);
- return Number.isNaN(parsed.getTime()) ? null : parsed;
- }
- return null;
- }
-
- function latestDate(...values: unknown[]) {
- let latest: Date | null = null;
- for (const value of values) {
- const parsed = dateValue(value);
- if (!parsed) continue;
- if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
- }
- return latest;
- }
-
- async function buildRunLivenessInput(
- run: typeof heartbeatRuns.$inferSelect,
- resultJson: Record<string, unknown> | null | undefined,
- ): Promise<RunLivenessClassificationInput> {
- const context = parseObject(run.contextSnapshot);
- const contextIssueId = readNonEmptyString(context.issueId);
- const continuationAttempt = asNumber(
- context.continuationAttempt,
- run.continuationAttempt ?? 0,
- );
-
- const issue = contextIssueId
- ? await db
- .select({
- status: issues.status,
- title: issues.title,
- description: issues.description,
- workMode: issues.workMode,
- })
- .from(issues)
- .where(
- and(
- eq(issues.companyId, run.companyId),
- eq(issues.id, contextIssueId),
- ),
- )
- .then((rows) => rows[0] ?? null)
- : null;
-
- const [commentStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
- })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, run.companyId),
- eq(issueComments.issueId, contextIssueId),
- eq(issueComments.createdByRunId, run.id),
- isNull(issueComments.deletedAt),
- ),
- )
- : [{ count: 0, latestAt: null }];
-
- const issueCommentBodies = contextIssueId
- ? await db
- .select({ body: issueComments.body })
- .from(issueComments)
- .where(
- and(
- eq(issueComments.companyId, run.companyId),
- eq(issueComments.issueId, contextIssueId),
- eq(issueComments.createdByRunId, run.id),
- ),
- )
- .orderBy(desc(issueComments.createdAt), desc(issueComments.id))
- .limit(5)
- .then((rows) => rows.reverse().map((row) => row.body))
- : [];
-
- const continuationSummary = contextIssueId
- ? await getIssueContinuationSummaryDocument(db, contextIssueId)
- : null;
-
- const [documentStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
- latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
- })
- .from(documentRevisions)
- .innerJoin(
- issueDocuments,
- eq(documentRevisions.documentId, issueDocuments.documentId),
- )
- .where(
- and(
- eq(documentRevisions.companyId, run.companyId),
- eq(documentRevisions.createdByRunId, run.id),
- eq(issueDocuments.companyId, run.companyId),
- eq(issueDocuments.issueId, contextIssueId),
- sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
- ),
- )
- : [{ count: 0, planCount: 0, latestAt: null }];
-
- const [workProductStats] = contextIssueId
- ? await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
- })
- .from(issueWorkProducts)
- .where(
- and(
- eq(issueWorkProducts.companyId, run.companyId),
- eq(issueWorkProducts.issueId, contextIssueId),
- eq(issueWorkProducts.createdByRunId, run.id),
- ),
- )
- : [{ count: 0, latestAt: null }];
-
- const [workspaceOperationStats] = await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
- })
- .from(workspaceOperations)
- .where(
- and(
- eq(workspaceOperations.companyId, run.companyId),
- eq(workspaceOperations.heartbeatRunId, run.id),
- ),
- );
-
- const [activityStats] = await db
- .select({
- count: sql<number>`count(*)::int`,
- latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
- })
- .from(activityLog)
- .where(
- and(
- eq(activityLog.companyId, run.companyId),
- eq(activityLog.runId, run.id),
- notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),
- ),
- );
-
- const [eventStats] = await db
- .select({
- count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
- latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
- })
- .from(heartbeatRunEvents)
- .where(
- and(
- eq(heartbeatRunEvents.companyId, run.companyId),
- eq(heartbeatRunEvents.runId, run.id),
- ),
- );
-
- return {
- runStatus: run.status,
- issue,
- resultJson: resultJson ?? run.resultJson ?? null,
- issueCommentBodies,
- continuationSummaryBody: continuationSummary?.body ?? null,
- stdoutExcerpt: run.stdoutExcerpt ?? null,
- stderrExcerpt: run.stderrExcerpt ?? null,
- error: run.error ?? null,
- errorCode: run.errorCode ?? null,
- continuationAttempt,
- evidence: {
- issueCommentsCreated: countValue(commentStats?.count),
- documentRevisionsCreated: countValue(documentStats?.count),
- planDocumentRevisionsCreated: countValue(documentStats?.planCount),
- workProductsCreated: countValue(workProductStats?.count),
- workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
- activityEventsCreated: countValue(activityStats?.count),
- toolOrActionEventsCreated: countValue(eventStats?.count),
- latestEvidenceAt: latestDate(
- commentStats?.latestAt,
- documentStats?.latestAt,
- workProductStats?.latestAt,
- workspaceOperationStats?.latestAt,
- activityStats?.latestAt,
- eventStats?.latestAt,
- ),
- },
- };
- }
-
- async function classifyAndPersistRunLiveness(
- run: typeof heartbeatRuns.$inferSelect,
- resultJson?: Record<string, unknown> | null,
- ) {
- const authRepair = run.status === "failed"
- ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {
- logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");
- return null;
- })
- : null;
- const classification = classifyRunLiveness({
- ...await buildRunLivenessInput(run, resultJson),
- authenticationRepairRequested: Boolean(authRepair?.interactionId),
- });
- return db
- .update(heartbeatRuns)
- .set({
- livenessState: classification.livenessState,
- livenessReason: classification.livenessReason,
- continuationAttempt: classification.continuationAttempt,
- lastUsefulActionAt: classification.lastUsefulActionAt,
- nextAction: classification.nextAction,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, run.id))
- .returning()
- .then((rows) => rows[0] ?? null);
}
async function recoverActiveSessionGoals() {
@@ -6406,29 +3747,6 @@ export function heartbeatService(
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
}
- async function updateRuntimeState(
- agent: typeof agents.$inferSelect,
- run: typeof heartbeatRuns.$inferSelect,
- result: AdapterExecutionResult,
- session: { legacySessionId: string | null },
- normalizedUsage?: UsageTotals | null,
- ) {
- await ensureRuntimeState(agent);
- await accountRunCost(db, run.id, budgetHooks);
- await db
- .update(agentRuntimeState)
- .set({
- adapterType: agent.adapterType,
- sessionId: session.legacySessionId,
- lastRunId: run.id,
- lastRunStatus: run.status,
- lastError: run.error ?? null,
- updatedAt: new Date(),
- })
- .where(eq(agentRuntimeState.agentId, agent.id));
-
- }
-
// Await every background heartbeat execution that is currently in flight. A
// draining run can, in its finally block, promote and dispatch the next queued
// run for the same agent — that follow-up execution is registered in the set
diff --git a/server/src/services/heartbeat/run-lifecycle.test.ts b/server/src/services/heartbeat/run-lifecycle.test.ts
new file mode 100644
index 0000000000..414315fbae
--- /dev/null
+++ b/server/src/services/heartbeat/run-lifecycle.test.ts
@@ -0,0 +1,196 @@
+import type { Db } from "@paperclipai/db";
+import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import type { CurrentUserRedactionOptions } from "../../log-redaction.js";
+import { publishLiveEvent } from "../live-events.js";
+import { getHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";
+import { createHeartbeatRunState } from "./run-state.js";
+import { createHeartbeatRunPreparation } from "./run-preparation.js";
+import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";
+
+vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
+
+type Run = typeof heartbeatRuns.$inferSelect;
+function callbacks(db: Db) {
+ const state = createHeartbeatRunState(db);
+ return {
+ ...state,
+ ...createHeartbeatRunPreparation(db),
+ getRun: vi.fn(state.getRun),
+ issuesSvc: {
+ addComment: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["addComment"]>(),
+ listReviewAttention: vi.fn<HeartbeatLifecycleDependencies["issuesSvc"]["listReviewAttention"]>(),
+ },
+ recovery: { escalateStrandedAssignedIssue: vi.fn<HeartbeatLifecycleDependencies["recovery"]["escalateStrandedAssignedIssue"]>() },
+ companySkills: { completeTestRunForIssue: vi.fn<HeartbeatLifecycleDependencies["companySkills"]["completeTestRunForIssue"]>() },
+ budgets: { getInvocationBlock: vi.fn(async () => null) },
+ treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
+ enqueueWakeup: vi.fn<HeartbeatLifecycleDependencies["enqueueWakeup"]>(),
+ getCurrentUserRedactionOptions: vi.fn(async (): Promise<CurrentUserRedactionOptions> => ({})),
+ getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),
+ emitTerminalAgentTaskRun: vi.fn<HeartbeatLifecycleDependencies["emitTerminalAgentTaskRun"]>(),
+ budgetHooks: {},
+ } satisfies HeartbeatLifecycleDependencies;
+}
+function guardedDatabase() {
+ const access = vi.fn(() => { throw new Error("Unexpected lifecycle database access"); });
+ return { db: new Proxy({}, { get: access }) as Db, access };
+}
+
+describe("heartbeat lifecycle module boundary", () => {
+ it("constructs without database work or reporting effects", () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ createHeartbeatLifecycle(database.db, deps);
+ expect(database.access).not.toHaveBeenCalled();
+ for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();
+ });
+
+ it("ignores absent wake requests without accessing the database", async () => {
+ const database = guardedDatabase();
+ const lifecycle = createHeartbeatLifecycle(database.db, callbacks(database.db));
+ await lifecycle.setWakeupStatus(null, "completed");
+ await lifecycle.setWakeupStatus(undefined, "completed");
+ expect(database.access).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { livenessState: "empty_response", contextSnapshot: { goalControlRequestId: "goal" } },
+ { livenessState: "plan_only", contextSnapshot: { resumeSessionGoalHeartbeat: true } },
+ { livenessState: "productive", contextSnapshot: { issueId: "issue" } },
+ { livenessState: "empty_response", contextSnapshot: {} },
+ ])("keeps ineligible continuations out of wake admission (%j)", async (input) => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ await createHeartbeatLifecycle(database.db, deps).handleRunLivenessContinuation(input as unknown as Run);
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.enqueueWakeup).not.toHaveBeenCalled();
+ expect(deps.budgets.getInvocationBlock).not.toHaveBeenCalled();
+ });
+
+ it("preserves retained native ownership before lease-release teardown reads or writes", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run;
+ expect(await createHeartbeatLifecycle(database.db, deps).terminalizeRunOnLeaseRelease(run)).toBe(run);
+ expect(database.access).not.toHaveBeenCalled();
+ expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled();
+ });
+
+ it("rejects late progress after re-reading a terminal run through the service callback", async () => {
+ const database = guardedDatabase();
+ const deps = callbacks(database.db);
+ const run = { id: "late-progress", companyId: "company", agentId: "agent", status: "running", contextSnapshot: {} } as Run;
+ deps.getRun.mockResolvedValue({ ...run, status: "succeeded" });
+ expect(await createHeartbeatLifecycle(database.db, deps).recordCurrentHeartbeatRunRuntimeProgress(run, { phase: "finalize", message: "late update" }, null)).toBeNull();
+ expect(deps.getRun).toHaveBeenCalledWith(run.id);
+ expect(getHeartbeatRunRuntimeStatus(run.id)).toBeNull();
+ expect(database.access).not.toHaveBeenCalled();
+ });
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+if (!support.supported) console.warn(`Skipping lifecycle module database tests: ${support.reason}`);
+describe.skipIf(!support.supported)("heartbeat lifecycle database wiring", () => {
+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+ let db: Db;
+ beforeAll(async () => {
+ database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-lifecycle-module");
+ db = createDb(database.connectionString);
+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+ afterAll(async () => {
+ await db?.$client.end();
+ await database?.cleanup();
+ });
+ beforeEach(async () => {
+ vi.mocked(publishLiveEvent).mockClear();
+ await db.execute(sql`truncate table companies restart identity cascade`);
+ });
+
+ async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
+ const [company] = await db.insert(companies).values({ name: "Lifecycle module", issuePrefix: "LM" }).returning();
+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Lifecycle agent" }).returning();
+ const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }).returning();
+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "native", ...patch }).returning();
+ return { company, agent, wake, run };
+ }
+
+ it("keeps one terminal outcome and reports only the winner of competing writes", async () => {
+ const { run } = await fixture();
+ const deps = callbacks(db);
+ const lifecycle = createHeartbeatLifecycle(db, deps);
+ const results = await Promise.all([
+ lifecycle.setRunStatusIfRunning(run.id, "succeeded", { finishedAt: new Date() }),
+ lifecycle.setRunStatusIfRunning(run.id, "failed", { errorCode: "late_failure", finishedAt: new Date() }),
+ ]);
+ const winner = results.find(result => result.updated)!;
+ expect(results.filter(result => result.updated)).toHaveLength(1);
+ const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
+ expect(persisted.status).toBe(winner.run?.status);
+ expect(results.every(result => result.run?.status === persisted.status)).toBe(true);
+ expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledTimes(1);
+ expect(deps.emitTerminalAgentTaskRun).toHaveBeenCalledWith(winner.run, "running", undefined);
+ expect(vi.mocked(publishLiveEvent).mock.calls.filter(([event]) => event.type === "heartbeat.run.status")).toHaveLength(1);
+ });
+
+ it("rechecks native ownership atomically before a terminal write", async () => {
+ const { run } = await fixture({ errorCode: "native_execution_ownership_unverified", nativePhase: "terminal_failure" });
+ const deps = callbacks(db);
+ const result = await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(run.id, "failed");
+ expect(result).toMatchObject({ updated: false, run: { status: "running", errorCode: run.errorCode } });
+ expect(deps.emitTerminalAgentTaskRun).not.toHaveBeenCalled();
+ expect(publishLiveEvent).not.toHaveBeenCalled();
+ });
+
+ it.each(["setRunStatus", "setRunStatusIfRunning"] as const)("preserves accounting metadata while %s enriches usage", async (operation) => {
+ const { run } = await fixture({ usageJson: { accountingReceiptReady: false, usageSource: "per_run", inputTokens: 5 } });
+ await createHeartbeatLifecycle(db, callbacks(db))[operation](run.id, "succeeded", { usageJson: { inputTokens: 8, outputTokens: 3 } });
+ const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
+ expect(persisted.usageJson).toEqual({ accountingReceiptReady: false, usageSource: "per_run", inputTokens: 8, outputTokens: 3 });
+ });
+
+ it.each(["claimed", "cancelled"] as const)("settles a %s wake without reviving a revoked request", async (status) => {
+ const { wake } = await fixture();
+ await db.update(agentWakeupRequests).set({ status }).where(eq(agentWakeupRequests.id, wake.id));
+ await createHeartbeatLifecycle(db, callbacks(db)).setWakeupStatus(wake.id, "completed", { reason: "settled" });
+ const [persisted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
+ expect(persisted.status).toBe(status === "cancelled" ? "cancelled" : "completed");
+ expect(persisted.reason).toBe(status === "cancelled" ? wake.reason : "settled");
+ });
+
+ it("publishes the persisted sequence for concurrent event writes", async () => {
+ const { run } = await fixture();
+ const lifecycle = createHeartbeatLifecycle(db, callbacks(db));
+ await Promise.all(["first", "second"].map(message => lifecycle.appendRunEvent(run, { eventType: "lifecycle", message })));
+ const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq);
+ expect(rows.map(row => row.seq)).toEqual([1, 2]);
+ const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event");
+ expect(published).toHaveLength(2);
+ for (const row of rows) expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ runId: run.id, message: row.message });
+ });
+
+ it("reads current redaction settings for each event and persists the published sanitized payload", async () => {
+ const { run } = await fixture();
+ const deps = callbacks(db);
+ deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["alice"], homeDirs: ["/Users/alice"] });
+ deps.getCurrentUserRedactionOptions.mockResolvedValueOnce({ enabled: true, userNames: ["bob"], homeDirs: ["/Users/bob"] });
+ const lifecycle = createHeartbeatLifecycle(db, deps);
+ for (const name of ["alice", "bob"]) await lifecycle.appendRunEvent(run, {
+ eventType: "lifecycle", message: `Opening /Users/${name}/project`, payload: { path: `/Users/${name}/project`, token: "test-credential-material" },
+ });
+ const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)).orderBy(heartbeatRunEvents.seq);
+ expect(deps.getCurrentUserRedactionOptions).toHaveBeenCalledTimes(2);
+ expect(rows).toHaveLength(2);
+ const published = vi.mocked(publishLiveEvent).mock.calls.map(([event]) => event).filter(event => event.type === "heartbeat.run.event");
+ expect(published).toHaveLength(2);
+ for (const row of rows) {
+ const text = JSON.stringify({ message: row.message, payload: row.payload });
+ expect(text).not.toContain("alice");
+ expect(text).not.toContain("bob");
+ expect(text).not.toContain("test-credential-material");
+ expect(published.find(event => event.payload?.seq === row.seq)?.payload).toMatchObject({ message: row.message, payload: row.payload });
+ }
+ });
+});
diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts
new file mode 100644
index 0000000000..55215e6301
--- /dev/null
+++ b/server/src/services/heartbeat/run-lifecycle.ts
@@ -0,0 +1,2982 @@
+import {
+ DEFERRED_WAKE_CONTEXT_KEY,
+ EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
+ shouldRequireIssueCommentForWake,
+} from "./queue.js";
+import {
+ HEARTBEAT_RUN_TERMINAL_STATUSES,
+ DETACHED_PROCESS_ERROR_CODE,
+} from "./recovery.js";
+import { normalizeAgentNameKey } from "./retries.js";
+import {
+ type RunSessionOutcome,
+ deriveTaskKeyWithHeartbeatFallback,
+ type UsageTotals,
+} from "./run-state.js";
+import { CONFIGURATION_INCOMPLETE_FAILURE_CODE } from "./run-preparation.js";
+import { WORKSPACE_VALIDATION_FAILURE_CODE } from "./workspaces.js";
+import { boundHeartbeatRunEventPayloadForStorage } from "./run-log.js";
+import { preserveWorkspaceRestoreRecoveryMetadata } from "../workspace-restore-recovery-state.js";
+import { activeIssueInteractionCondition } from "../issue-question-context.js";
+import {
+ nativeRetryCancellationCommitCondition,
+ rethrowNativeCancellationLockConflict,
+} from "../native-runtime/native-cancellation-request.js";
+import { cancellationResultJson } from "../run-cancellation.js";
+import { accountRunCost } from "../run-cost-accounting.js";
+import { isWaitingConversation } from "../agent-conversations.js";
+import { PROCESS_IDENTITY_RECORDED } from "../native-local-process-stop.js";
+import {
+ hasAcknowledgedNativeReassignmentStopIntent,
+ hasAcknowledgedNativeStopIntent,
+} from "../acknowledged-native-stop.js";
+import { connectionIntentService } from "../connection-intents.js";
+import {
+ CONVERSATION_CONTINUATION_POLICY,
+ isConversationAdapter,
+} from "../conversation-continuation.js";
+import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
+import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js";
+import { randomUUID } from "node:crypto";
+import {
+ and,
+ desc,
+ eq,
+ inArray,
+ isNull,
+ ne,
+ notInArray,
+ or,
+ sql,
+} from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import {
+ ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY,
+ type HeartbeatRunStatusPhase,
+ type RequestConfirmationResult,
+ type RunLivenessState,
+} from "@paperclipai/shared";
+import {
+ agents,
+ agentRuntimeState,
+ agentWakeupRequests,
+ activityLog,
+ approvals,
+ companySkillTestRuns,
+ documentRevisions,
+ issueDocuments,
+ heartbeatRunEvents,
+ heartbeatRuns,
+ issueApprovals,
+ issueComments,
+ issueRecoveryActions,
+ issueRelations,
+ issueThreadInteractions,
+ issues,
+ issueWorkProducts,
+ routines,
+ workspaceOperations,
+} from "@paperclipai/db";
+import { logger } from "../../middleware/logger.js";
+import { publishLiveEvent } from "../live-events.js";
+import {
+ appendHeartbeatRunEvent,
+ type AppendHeartbeatRunEventInput,
+} from "../heartbeat-run-events.js";
+import type { AdapterExecutionResult } from "../../adapters/index.js";
+import {
+ parseObject,
+ asBoolean,
+ asNumber,
+} from "../../adapters/utils.js";
+import {
+ isNativeRunnerOwnershipHeld,
+ nativeRunnerOwnershipNotHeldCondition,
+} from "../native-runtime/native-runner-ownership.js";
+import {
+ type RunFailureReportOptions,
+} from "../run-failure-diagnostics.js";
+import {
+ findHeartbeatRunCompletionComment,
+ hasAcceptedSemanticResult,
+ readCompletedAssistantMessageCandidate,
+ selectHeartbeatRunFinalAgentMessage,
+ type RunPresentationDecision,
+} from "../heartbeat-run-summary.js";
+import {
+ buildHeartbeatRunStopMetadata,
+ mergeHeartbeatRunStopMetadata,
+} from "../heartbeat-stop-metadata.js";
+import { CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js";
+import {
+ classifyRunLiveness,
+ type RunLivenessClassificationInput,
+} from "../run-liveness.js";
+import {
+ logActivity,
+ publishPluginDomainEvent,
+} from "../activity-log.js";
+import { runnerGoalService } from "../runner-goals.js";
+import { visibleIssueCondition } from "../issue-visibility.js";
+import {
+ getIssueContinuationSummaryDocument,
+ refreshIssueContinuationSummary,
+} from "../issue-continuation-summary.js";
+import {
+ RECOVERY_ORIGIN_KINDS,
+ FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
+ SUCCESSFUL_RUN_MISSING_STATE_REASON,
+ RUN_LIVENESS_CONTINUATION_REASON,
+ buildRunLivenessContinuationIdempotencyKey,
+ buildFinishSuccessfulRunHandoffIdempotencyKey,
+ buildSuccessfulRunHandoffRequiredNotice,
+ decideRunLivenessContinuation,
+ decideSuccessfulRunHandoff,
+ findExistingFinishSuccessfulRunHandoffWake,
+ findExistingRunLivenessContinuationWake,
+ isSuccessfulRunHandoffValidPathSkip,
+ SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,
+ readContinuationAttempt,
+} from "../recovery/index.js";
+import { withRecoveryContext } from "../recovery/status-only-context.js";
+import { deriveCommentId } from "../../modules/run-dispatch/index.js";
+import {
+ buildIssueReviewPathLostIdempotencyKey,
+ decideIssueReviewPathRecovery,
+ ISSUE_REVIEW_PATH_LOST_WAKE_REASON,
+ isReviewPathRecoveryIdempotencyConflict,
+ reviewPathConsumedRefFromRun,
+} from "../recovery/review-path-recovery.js";
+import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "../successful-run-handoff-state.js";
+import {
+ redactCurrentUserText,
+ redactCurrentUserValue,
+ type CurrentUserRedactionOptions,
+} from "../../log-redaction.js";
+import {
+ redactEventPayload,
+ redactSensitiveText,
+} from "../../redaction.js";
+import {
+ type RuntimeStatusUpdate,
+} from "@paperclipai/adapter-utils";
+import {
+ UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
+ UNMANAGED_BACKGROUND_TASK_STOP_REASON,
+} from "@paperclipai/adapter-utils/server-utils";
+import {
+ clearHeartbeatRunRuntimeStatus,
+ getHeartbeatRunRuntimeStatus,
+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
+ setHeartbeatRunRuntimeStatus,
+} from "../heartbeat-run-runtime-status.js";
+import { readProcessStartedAt } from "../hot-restart.js";
+
+import type { createHeartbeatRunState } from "./run-state.js";
+import type { createHeartbeatRunPreparation } from "./run-preparation.js";
+import type { createHeartbeatQueue } from "./queue.js";
+import type { HeartbeatRetryDependencies } from "./retries.js";
+import type { issueService } from "../issues.js";
+import type { recoveryService } from "../recovery/index.js";
+import type { companySkillService } from "../company-skills.js";
+import type { budgetService, BudgetServiceHooks } from "../budgets.js";
+import type { issueTreeControlService } from "../issue-tree-control.js";
+
+type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
+
+/** Lifecycle owns run writes; the service supplies reporting and admission effects. */
+export interface HeartbeatLifecycleDependencies extends Pick<ReturnType<typeof createHeartbeatRunState>,
+ "getRun" | "resolveSessionBeforeForWakeup" | "ensureRuntimeState">,
+ Pick<ReturnType<typeof createHeartbeatRunPreparation>,
+ "getIssueExecutionContext" | "resolveResponsibleUserIdForRunContext">,
+ Pick<HeartbeatRetryDependencies, "getAgentInvokability"> {
+ issuesSvc: Pick<ReturnType<typeof issueService>, "addComment" | "listReviewAttention">;
+ recovery: Pick<ReturnType<typeof recoveryService>, "escalateStrandedAssignedIssue">;
+ companySkills: Pick<ReturnType<typeof companySkillService>, "completeTestRunForIssue">;
+ budgets: Pick<ReturnType<typeof budgetService>, "getInvocationBlock">;
+ treeControlSvc: Pick<ReturnType<typeof issueTreeControlService>, "getActivePauseHoldGate">;
+ enqueueWakeup: ReturnType<typeof createHeartbeatQueue>["enqueueWakeup"];
+ getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;
+ budgetHooks: BudgetServiceHooks;
+ emitTerminalAgentTaskRun: (run: HeartbeatRun, previousStatus: string | null, failureReport?: RunFailureReportOptions) => void;
+}
+
+export function redactDetectedSuccessfulRunProgressSummaryForBoard(
+ summary: string,
+ currentUserRedactionOptions?: CurrentUserRedactionOptions,
+) {
+ const normalized = summary.replace(/\s+/g, " ").trim();
+ const redacted = redactSensitiveText(
+ redactCurrentUserText(normalized, currentUserRedactionOptions),
+ );
+ return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`;
+}
+
+export function redactSuccessfulRunHandoffEvidence(
+ value: string | null,
+ currentUserRedactionOptions?: CurrentUserRedactionOptions,
+) {
+ if (!value) return null;
+ return redactSensitiveText(
+ redactCurrentUserText(value, currentUserRedactionOptions),
+ );
+}
+
+const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [
+ "environment.lease_acquired",
+ "environment.lease_released",
+ "cost.reported",
+];
+
+export const NON_RETRYABLE_PREFLIGHT_FAILURE_CODES = new Set<string>([
+ "low_trust_isolation_unavailable",
+ "low_trust_requires_isolated_workspace",
+ "low_trust_boundary_mismatch",
+ "low_trust_requires_sandbox_environment",
+ "low_trust_runtime_services_denied",
+ "chat_failed_run_retry_not_authorized",
+ CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE,
+]);
+
+// Error codes that mark a pre-dispatch setup failure. The adapter process never
+// started, so no agent could post an issue comment. The setup catch writes one
+// of these codes when a failure happens before `adapter.execute`.
+const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([
+ "setup_failed",
+ CONFIGURATION_INCOMPLETE_FAILURE_CODE,
+ WORKSPACE_VALIDATION_FAILURE_CODE,
+ ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
+]);
+
+const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
+
+const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
+
+const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000;
+
+const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000;
+
+interface MaxTurnContinuationPolicy {
+ enabled: boolean;
+ maxAttempts: number;
+ delayMs: number;
+}
+
+export function isHeartbeatRunTerminalStatus(
+ status: string | null | undefined,
+): status is (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number] {
+ return HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
+ status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
+ );
+}
+
+export function isHeartbeatRunRuntimeStatusActive(
+ status: string | null | undefined,
+): boolean {
+ return status === "queued" || status === "running";
+}
+
+type HeartbeatRunRuntimeStatusRunLike = {
+ id: string;
+ status?: string | null;
+ companyId?: string | null;
+ agentId?: string | null;
+ issueId?: string | null;
+ contextSnapshot?: Record<string, unknown> | null;
+};
+
+function readRuntimeStatusIssueIdCandidate(
+ run: HeartbeatRunRuntimeStatusRunLike,
+): string | null | undefined {
+ if ("issueId" in run) return readNonEmptyString(run.issueId) ?? null;
+ if ("contextSnapshot" in run) {
+ return readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? null;
+ }
+ return undefined;
+}
+
+export function decorateHeartbeatRunRuntimeStatus<
+ T extends HeartbeatRunRuntimeStatusRunLike,
+>(
+ run: T,
+ expected: {
+ companyId?: string | null;
+ issueId?: string | null;
+ agentId?: string | null;
+ } = {},
+): T & {
+ currentStatusMessage: string | null;
+ currentStatusUpdatedAt: Date | null;
+ currentToolName: string | null;
+ lastAssistantSnippet: string | null;
+ lastEventAt: Date | null;
+} {
+ if (isHeartbeatRunTerminalStatus(run.status)) {
+ clearHeartbeatRunRuntimeStatus(run.id);
+ }
+
+ const companyId = expected.companyId ?? run.companyId ?? null;
+ const agentId = expected.agentId ?? run.agentId ?? null;
+ const issueId =
+ expected.issueId !== undefined
+ ? expected.issueId
+ : readRuntimeStatusIssueIdCandidate(run);
+ const currentStatus =
+ isHeartbeatRunRuntimeStatusActive(run.status) && companyId && agentId
+ ? getHeartbeatRunRuntimeStatus(run.id, {
+ companyId,
+ agentId,
+ ...(issueId !== undefined ? { issueId } : {}),
+ })
+ : null;
+
+ return {
+ ...run,
+ currentStatusMessage: currentStatus?.message ?? null,
+ currentStatusUpdatedAt: currentStatus?.updatedAt ?? null,
+ currentToolName: currentStatus?.currentToolName ?? null,
+ lastAssistantSnippet: currentStatus?.lastAssistantSnippet ?? null,
+ lastEventAt: currentStatus?.lastEventAt ?? null,
+ };
+}
+
+export function publishHeartbeatRunRuntimeProgress(status: {
+ companyId: string;
+ runId: string;
+ agentId: string;
+ issueId: string | null;
+ phase: HeartbeatRunStatusPhase;
+ message: string;
+ updatedAt: Date;
+ currentToolName?: string | null;
+ lastAssistantSnippet?: string | null;
+ lastEventAt?: Date | null;
+}) {
+ publishLiveEvent({
+ companyId: status.companyId,
+ type: "heartbeat.run.progress",
+ payload: {
+ runId: status.runId,
+ agentId: status.agentId,
+ issueId: status.issueId,
+ phase: status.phase,
+ message: status.message,
+ currentToolName: status.currentToolName ?? null,
+ lastAssistantSnippet: status.lastAssistantSnippet ?? null,
+ lastEventAt: (status.lastEventAt ?? status.updatedAt).toISOString(),
+ updatedAt: status.updatedAt.toISOString(),
+ },
+ });
+}
+
+function recordHeartbeatRunRuntimeProgress(
+ run: Pick<
+ typeof heartbeatRuns.$inferSelect,
+ "id" | "companyId" | "agentId" | "status" | "contextSnapshot"
+ >,
+ update: RuntimeStatusUpdate,
+ issueId: string | null,
+) {
+ if (!isHeartbeatRunRuntimeStatusActive(run.status)) return null;
+ const status = setHeartbeatRunRuntimeStatus({
+ companyId: run.companyId,
+ issueId,
+ agentId: run.agentId,
+ runId: run.id,
+ phase: update.phase as HeartbeatRunStatusPhase,
+ message: update.message,
+ currentToolName: readNonEmptyString(update.currentToolName) ?? null,
+ lastAssistantSnippet:
+ readNonEmptyString(update.lastAssistantSnippet) ?? null,
+ lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(),
+ });
+ if (!status) return null;
+
+ publishHeartbeatRunRuntimeProgress(status);
+ return status;
+}
+
+function sanitizeLiveRunProgressText(
+ value: string,
+ maxChars: number,
+): string | null {
+ const normalized = value.replace(/\s+/g, " ").trim();
+ if (!normalized) return null;
+ const redacted = redactSensitiveText(normalized);
+ if (redacted.length <= maxChars) return redacted;
+ return `${redacted.slice(0, maxChars - 3)}...`;
+}
+
+function readLiveRunProgressString(
+ value: unknown,
+ maxChars: number,
+): string | null {
+ return typeof value === "string"
+ ? sanitizeLiveRunProgressText(value, maxChars)
+ : null;
+}
+
+function readFirstLiveRunProgressString(
+ maxChars: number,
+ values: unknown[],
+): string | null {
+ for (const value of values) {
+ const text = readLiveRunProgressString(value, maxChars);
+ if (text) return text;
+ }
+ return null;
+}
+
+function readLiveRunToolName(
+ payload: Record<string, unknown> | null,
+ eventType: string,
+): string | null {
+ const toolCall =
+ parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall);
+ const message = parseObject(payload?.message);
+ const direct = readFirstLiveRunProgressString(
+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
+ [
+ payload?.toolName,
+ payload?.tool_name,
+ payload?.tool,
+ payload?.name,
+ payload?.title,
+ toolCall?.name,
+ toolCall?.toolName,
+ message?.name,
+ message?.toolName,
+ ],
+ );
+ if (direct) return direct;
+
+ const normalizedEventType = eventType.toLowerCase();
+ if (!normalizedEventType.includes("tool")) return null;
+ return readLiveRunProgressString(
+ eventType.replace(/[._-]+/g, " "),
+ MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS,
+ );
+}
+
+function readLiveRunAssistantSnippet(
+ payload: Record<string, unknown> | null,
+ eventType: string,
+ message: string | null,
+): string | null {
+ const normalizedEventType = eventType.toLowerCase();
+ const messagePayload = parseObject(payload?.message);
+ const direct = readFirstLiveRunProgressString(
+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
+ [
+ payload?.text,
+ payload?.delta,
+ payload?.text_delta,
+ payload?.content,
+ payload?.summary,
+ messagePayload?.text,
+ messagePayload?.content,
+ ],
+ );
+ if (direct) return direct;
+
+ if (
+ normalizedEventType.includes("assistant") ||
+ normalizedEventType.includes("text_delta") ||
+ normalizedEventType.includes("message.delta") ||
+ normalizedEventType.includes("message_delta")
+ ) {
+ return message
+ ? sanitizeLiveRunProgressText(
+ message,
+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
+ )
+ : null;
+ }
+
+ return null;
+}
+
+function buildRunEventRuntimeProgress(input: {
+ eventType: string;
+ message: string | null;
+ payload: Record<string, unknown> | null;
+ at: Date;
+}) {
+ const normalizedEventType = input.eventType.toLowerCase();
+ if (
+ normalizedEventType === "lifecycle" ||
+ normalizedEventType === "adapter.invoke"
+ ) {
+ return null;
+ }
+
+ const currentToolName = readLiveRunToolName(input.payload, input.eventType);
+ const lastAssistantSnippet = readLiveRunAssistantSnippet(
+ input.payload,
+ input.eventType,
+ input.message,
+ );
+ const fallbackMessage =
+ readLiveRunProgressString(
+ input.message,
+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
+ ) ??
+ sanitizeLiveRunProgressText(
+ input.eventType.replace(/[._-]+/g, " "),
+ MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS,
+ );
+ const message = currentToolName
+ ? `Using ${currentToolName}`
+ : (lastAssistantSnippet ?? fallbackMessage);
+
+ if (!message) return null;
+ return {
+ phase: "run_activity" as const,
+ message,
+ currentToolName,
+ lastAssistantSnippet,
+ lastEventAt: input.at,
+ };
+}
+
+export async function persistHeartbeatRunProcessMetadata(
+ db: Db,
+ runId: string,
+ meta: { pid: number; processGroupId: number | null; startedAt: string },
+) {
+ const observedStartedAt = await readProcessStartedAt(meta.pid).catch(
+ () => null,
+ );
+ const startedAt = new Date(observedStartedAt ?? meta.startedAt);
+ return db.transaction(async tx => {
+ const run = await tx
+ .update(heartbeatRuns)
+ .set({
+ processPid: meta.pid,
+ processGroupId: meta.processGroupId,
+ processStartedAt: Number.isNaN(startedAt.getTime())
+ ? new Date()
+ : startedAt,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, runId))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ if (run?.runtimeMode === "native") await appendHeartbeatRunEvent(tx as unknown as Db, {
+ companyId: run.companyId, runId, agentId: run.agentId,
+ eventType: PROCESS_IDENTITY_RECORDED, stream: "system", level: "info",
+ message: "Process identity recorded; prior stop evidence no longer applies.",
+ });
+ return run;
+ });
+}
+
+type SkillTestHeartbeatCompletion = {
+ outcome: "failed" | "cancelled";
+ error: string | null;
+ heartbeatOutcome: RunSessionOutcome;
+};
+
+export function resolveSkillTestRunCompletionForHeartbeatOutcome(
+ outcome: RunSessionOutcome,
+ error: string | null | undefined,
+): SkillTestHeartbeatCompletion | null {
+ if (outcome === "cancelled") {
+ return {
+ outcome: "cancelled",
+ error: error ?? "Harness run was cancelled",
+ heartbeatOutcome: outcome,
+ };
+ }
+ if (outcome === "timed_out") {
+ return {
+ outcome: "failed",
+ error: error ?? "Timed out",
+ heartbeatOutcome: outcome,
+ };
+ }
+ if (outcome === "failed") {
+ return {
+ outcome: "failed",
+ error: error ?? "Adapter failed",
+ heartbeatOutcome: outcome,
+ };
+ }
+ return null;
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+/** Run lifecycle binds status, progress, liveness, and completion operations. */
+export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycleDependencies) {
+ const {
+ issuesSvc,
+ recovery,
+ companySkills,
+ getRun,
+ emitTerminalAgentTaskRun,
+ getIssueExecutionContext,
+ budgets,
+ enqueueWakeup,
+ getCurrentUserRedactionOptions,
+ treeControlSvc,
+ getAgentInvokability,
+ resolveSessionBeforeForWakeup,
+ resolveResponsibleUserIdForRunContext,
+ ensureRuntimeState,
+ budgetHooks,
+ } = dependencies;
+
+ function isPlanApprovalConfirmationPayload(payload: unknown) {
+ const target = parseObject(parseObject(payload).target);
+ return (
+ readNonEmptyString(target.type) === "issue_document" &&
+ readNonEmptyString(target.key) === "plan"
+ );
+ }
+
+ async function getAcceptedPlanApprovalInteractionForRun(
+ run: typeof heartbeatRuns.$inferSelect,
+ issueId: string | null,
+ ) {
+ const context = parseObject(run.contextSnapshot);
+ const interactionId = readNonEmptyString(context.interactionId);
+ if (!issueId || !interactionId) return null;
+
+ const interaction = await db
+ .select({
+ id: issueThreadInteractions.id,
+ kind: issueThreadInteractions.kind,
+ status: issueThreadInteractions.status,
+ payload: issueThreadInteractions.payload,
+ result: issueThreadInteractions.result,
+ })
+ .from(issueThreadInteractions)
+ .where(
+ and(
+ eq(issueThreadInteractions.companyId, run.companyId),
+ eq(issueThreadInteractions.issueId, issueId),
+ eq(issueThreadInteractions.id, interactionId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+
+ if (!interaction) return null;
+ if (
+ interaction.kind !== "request_confirmation" ||
+ interaction.status !== "accepted"
+ )
+ return null;
+ return isPlanApprovalConfirmationPayload(interaction.payload)
+ ? interaction
+ : null;
+ }
+
+ function planApprovalResumeFailureErrorCode(
+ run: typeof heartbeatRuns.$inferSelect,
+ ) {
+ return readNonEmptyString(run.errorCode) ?? "unknown_error";
+ }
+
+ function buildPlanApprovalResumeFailureComment(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ status: "retrying" | "needs_attention";
+ attempt: number;
+ maxAttempts: number;
+ }) {
+ const errorCode = planApprovalResumeFailureErrorCode(input.run);
+ if (input.status === "retrying") {
+ return `Agent failed to resume after approval: \`${errorCode}\` — retrying (attempt ${input.attempt}/${input.maxAttempts})`;
+ }
+ return `Agent failed to resume after approval: \`${errorCode}\` — needs attention`;
+ }
+
+ function buildPlanApprovalResumeFailureResult(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ status: "retrying" | "needs_attention";
+ attempt: number;
+ maxAttempts: number;
+ retryRunId?: string | null;
+ recoveryActionId?: string | null;
+ }): NonNullable<RequestConfirmationResult["resumeFailure"]> {
+ return {
+ status: input.status,
+ errorCode: planApprovalResumeFailureErrorCode(input.run),
+ attempt: input.attempt,
+ maxAttempts: input.maxAttempts,
+ runId: input.run.id,
+ retryRunId: input.retryRunId ?? null,
+ recoveryActionId: input.recoveryActionId ?? null,
+ updatedAt: new Date().toISOString(),
+ };
+ }
+
+ async function updatePlanApprovalInteractionResumeFailure(input: {
+ interaction: NonNullable<
+ Awaited<ReturnType<typeof getAcceptedPlanApprovalInteractionForRun>>
+ >;
+ failure: NonNullable<RequestConfirmationResult["resumeFailure"]>;
+ }) {
+ const result = parseObject(input.interaction.result);
+ const nextResult = {
+ ...result,
+ version: 1 as const,
+ outcome: "accepted" as const,
+ resumeFailure: input.failure,
+ } satisfies RequestConfirmationResult;
+
+ await db
+ .update(issueThreadInteractions)
+ .set({
+ result: nextResult,
+ updatedAt: new Date(),
+ })
+ .where(eq(issueThreadInteractions.id, input.interaction.id));
+ }
+
+ async function addPlanApprovalResumeFailureCommentOnce(input: {
+ issueId: string;
+ run: typeof heartbeatRuns.$inferSelect;
+ body: string;
+ }) {
+ const existing = await db
+ .select({ id: issueComments.id })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, input.run.companyId),
+ eq(issueComments.issueId, input.issueId),
+ or(
+ eq(issueComments.body, input.body),
+ sql`${issueComments.body} like ${`${input.body}\n%`}`,
+ ),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (existing) return null;
+ return issuesSvc.addComment(
+ input.issueId,
+ input.body,
+ { runId: input.run.id },
+ { authorType: "system" },
+ );
+ }
+
+ async function getActiveRecoveryActionId(
+ companyId: string,
+ sourceIssueId: string,
+ ) {
+ return db
+ .select({ id: issueRecoveryActions.id })
+ .from(issueRecoveryActions)
+ .where(
+ and(
+ eq(issueRecoveryActions.companyId, companyId),
+ eq(issueRecoveryActions.sourceIssueId, sourceIssueId),
+ inArray(issueRecoveryActions.status, ["active", "escalated"]),
+ ),
+ )
+ .orderBy(desc(issueRecoveryActions.updatedAt))
+ .limit(1)
+ .then((rows) => rows[0]?.id ?? null);
+ }
+
+ async function recordPlanApprovalResumeFailureRetry(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ issueId: string | null;
+ retryRunId: string | null;
+ attempt: number;
+ maxAttempts: number;
+ }) {
+ const interaction = await getAcceptedPlanApprovalInteractionForRun(
+ input.run,
+ input.issueId,
+ );
+ if (!interaction || !input.issueId) return null;
+
+ const body = buildPlanApprovalResumeFailureComment({
+ run: input.run,
+ status: "retrying",
+ attempt: input.attempt,
+ maxAttempts: input.maxAttempts,
+ });
+ await addPlanApprovalResumeFailureCommentOnce({
+ issueId: input.issueId,
+ run: input.run,
+ body,
+ });
+ await updatePlanApprovalInteractionResumeFailure({
+ interaction,
+ failure: buildPlanApprovalResumeFailureResult({
+ run: input.run,
+ status: "retrying",
+ attempt: input.attempt,
+ maxAttempts: input.maxAttempts,
+ retryRunId: input.retryRunId,
+ }),
+ });
+ return interaction.id;
+ }
+
+ async function escalatePlanApprovalResumeFailureNeedsAttention(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ issueId: string | null;
+ attempt: number;
+ maxAttempts: number;
+ }) {
+ const interaction = await getAcceptedPlanApprovalInteractionForRun(
+ input.run,
+ input.issueId,
+ );
+ if (!interaction || !input.issueId) return null;
+
+ const issue = await db
+ .select()
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, input.run.companyId),
+ eq(issues.id, input.issueId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ if (!issue) return null;
+ if (
+ issue.status !== "todo" &&
+ issue.status !== "in_progress" &&
+ issue.status !== "in_review"
+ )
+ return null;
+
+ const body = buildPlanApprovalResumeFailureComment({
+ run: input.run,
+ status: "needs_attention",
+ attempt: input.attempt,
+ maxAttempts: input.maxAttempts,
+ });
+ await recovery.escalateStrandedAssignedIssue({
+ issue,
+ previousStatus: issue.status,
+ latestRun: input.run,
+ comment: body,
+ });
+ await addPlanApprovalResumeFailureCommentOnce({
+ issueId: issue.id,
+ run: input.run,
+ body,
+ });
+
+ const recoveryActionId = await getActiveRecoveryActionId(
+ issue.companyId,
+ issue.id,
+ );
+ await updatePlanApprovalInteractionResumeFailure({
+ interaction,
+ failure: buildPlanApprovalResumeFailureResult({
+ run: input.run,
+ status: "needs_attention",
+ attempt: input.attempt,
+ maxAttempts: input.maxAttempts,
+ recoveryActionId,
+ }),
+ });
+ return interaction.id;
+ }
+
+ async function completeSkillTestRunForHeartbeatOutcome(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ issueId: string | null;
+ issueWorkMode?: string | null;
+ outcome: RunSessionOutcome;
+ error: string | null;
+ }) {
+ const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(
+ input.outcome,
+ input.error,
+ );
+ if (!completion || !input.issueId) return null;
+
+ let isSkillTestIssue = input.issueWorkMode === "skill_test";
+ if (!isSkillTestIssue && input.issueWorkMode === undefined) {
+ const issueRow = await db
+ .select({
+ workMode: issues.workMode,
+ harnessKind: issues.harnessKind,
+ })
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, input.run.companyId),
+ eq(issues.id, input.issueId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ isSkillTestIssue =
+ issueRow?.workMode === "skill_test" ||
+ issueRow?.harnessKind === "skill_test";
+ }
+ if (!isSkillTestIssue) return null;
+
+ const existingRun = await db
+ .select({
+ id: companySkillTestRuns.id,
+ status: companySkillTestRuns.status,
+ })
+ .from(companySkillTestRuns)
+ .where(
+ and(
+ eq(companySkillTestRuns.companyId, input.run.companyId),
+ eq(companySkillTestRuns.issueId, input.issueId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ if (
+ !existingRun ||
+ ["succeeded", "failed", "cancelled"].includes(existingRun.status)
+ )
+ return null;
+
+ const completedRun = await companySkills.completeTestRunForIssue({
+ companyId: input.run.companyId,
+ issueId: input.issueId,
+ outcome: completion.outcome,
+ error: completion.error,
+ });
+ if (!completedRun) return null;
+
+ await logActivity(db, {
+ companyId: input.run.companyId,
+ actorType: "system",
+ actorId: "heartbeat_finalize",
+ agentId: input.run.agentId,
+ runId: input.run.id,
+ action: "company.skill_test_run_completed",
+ entityType: "company_skill_test_run",
+ entityId: completedRun.id,
+ issueId: input.issueId,
+ details: {
+ issueId: input.issueId,
+ status: completedRun.status,
+ outputDocumentKey: completedRun.outputDocumentKey,
+ heartbeatOutcome: completion.heartbeatOutcome,
+ source: "heartbeat.run_finalized",
+ },
+ });
+
+ return completedRun;
+ }
+
+ async function recordCurrentHeartbeatRunRuntimeProgress(
+ run: Pick<
+ typeof heartbeatRuns.$inferSelect,
+ "id" | "companyId" | "agentId" | "status" | "contextSnapshot"
+ >,
+ update: RuntimeStatusUpdate,
+ issueId: string | null,
+ ) {
+ if (!isHeartbeatRunRuntimeStatusActive(run.status)) {
+ clearHeartbeatRunRuntimeStatus(run.id);
+ return null;
+ }
+
+ const currentRun = await getRun(run.id);
+ if (!currentRun || !isHeartbeatRunRuntimeStatusActive(currentRun.status)) {
+ clearHeartbeatRunRuntimeStatus(run.id);
+ return null;
+ }
+
+ return recordHeartbeatRunRuntimeProgress(currentRun, update, issueId);
+ }
+
+ async function setRunStatus(
+ runId: string,
+ status: string,
+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,
+ ) {
+ const previousStatus = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, runId))
+ .then((rows) => rows[0] ?? null);
+
+ if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,
+ resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),
+ };
+
+ // Preserve the receipt-source fence when finalization enriches usage. A
+ // late spool replay must still be able to complete an unfinished receipt.
+ if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
+
+ // Cancelling a queued run that never acquired provider execution is
+ // positive bootstrap evidence. It must not hold unrelated queued messages.
+ if (previousStatus && (status === "cancelled" || status === "interrupted")) {
+ patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
+ }
+ if (
+ status === "cancelled" &&
+ previousStatus?.status === "queued" &&
+ previousStatus.runtimeMode !== "native" &&
+ !previousStatus.startedAt &&
+ !previousStatus.processPid
+ ) {
+ patch = {
+ ...patch,
+ resultJson: {
+ ...previousStatus.resultJson,
+ ...patch?.resultJson,
+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
+ },
+ };
+ }
+ const updated =
+ previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
+ ? await terminalizeLegacyExecution({
+ db,
+ run: previousStatus,
+ status,
+ patch,
+ reconcileIfNeeded: true,
+ })
+ : await db
+ .update(heartbeatRuns)
+ .set({
+ status,
+ ...patch,
+ executionStatusDeliveryId: randomUUID(),
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, runId))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+
+ if (updated) {
+ publishLiveEvent({
+ companyId: updated.companyId,
+ type: "heartbeat.run.status",
+ payload: buildHeartbeatRunStatusLiveEventPayload(updated),
+ });
+ publishRunLifecyclePluginEvent(updated);
+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);
+ }
+
+ return updated;
+ }
+
+ async function setRunStatusIfRunning(
+ runId: string,
+ status: string,
+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,
+ failureReport?: RunFailureReportOptions,
+ ) {
+ return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);
+ }
+
+ // Move a run to a new status only when its current status is one of
+ // `fromStatuses`. The compare-and-set is a single conditional update, so a
+ // concurrent path can win the race. When this update matches nothing, the
+ // function reads the current row and reports updated=false, so the caller can
+ // keep the terminal outcome that another path already wrote.
+ async function setRunStatusFromLive(
+ runId: string,
+ status: string,
+ fromStatuses: string[],
+ patch?: Partial<typeof heartbeatRuns.$inferInsert>,
+ failureReport?: RunFailureReportOptions,
+ cancellationCondition?: ReturnType<typeof nativeRetryCancellationCommitCondition>,
+ ) {
+ // fromStatuses can name a terminal status as its own source (for example,
+ // an idempotent "still failed" patch), so the write below is not always a
+ // genuine transition. Read the pre-write status to tell the two apart.
+ const previousStatus = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, runId))
+ .then((rows) => rows[0] ?? null);
+
+ if (previousStatus && (status === "cancelled" || status === "interrupted")) {
+ patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) };
+ }
+
+ if (previousStatus && patch?.resultJson !== undefined) patch = { ...patch,
+ resultJson: preserveWorkspaceRestoreRecoveryMetadata(previousStatus.resultJson, patch.resultJson),
+ };
+
+ // Preserve the receipt-source fence when finalization enriches usage. A
+ // late spool replay must still be able to complete an unfinished receipt.
+ if (patch?.usageJson && previousStatus?.usageJson) patch = { ...patch, usageJson: { ...previousStatus.usageJson, ...patch.usageJson } };
+
+ // Cancelling a queued run that never acquired provider execution is
+ // positive bootstrap evidence. It must not hold unrelated queued messages.
+ if (
+ status === "cancelled" &&
+ previousStatus?.status === "queued" &&
+ previousStatus.runtimeMode !== "native" &&
+ !previousStatus.startedAt &&
+ !previousStatus.processPid
+ ) {
+ patch = {
+ ...patch,
+ resultJson: {
+ ...previousStatus.resultJson,
+ ...patch?.resultJson,
+ executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
+ },
+ };
+ }
+ const updated =
+ previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
+ ? await terminalizeLegacyExecution({
+ db,
+ run: previousStatus,
+ status,
+ patch,
+ fromStatuses,
+ reconcileIfNeeded: true,
+ writeConditions: [nativeRunnerOwnershipNotHeldCondition(), ...(cancellationCondition ? [cancellationCondition] : [])],
+ })
+ : await db
+ .update(heartbeatRuns)
+ .set({
+ status,
+ ...patch,
+ executionStatusDeliveryId: randomUUID(),
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, runId),
+ inArray(heartbeatRuns.status, fromStatuses),
+ ...(cancellationCondition ? [cancellationCondition] : []),
+ ...(isHeartbeatRunTerminalStatus(status)
+ ? [nativeRunnerOwnershipNotHeldCondition()]
+ : []),
+ ),
+ )
+ .returning()
+ .then((rows) => rows[0] ?? null)
+ .catch((error: unknown) => {
+ if (cancellationCondition) rethrowNativeCancellationLockConflict(error);
+ throw error;
+ });
+
+ if (updated) {
+ publishLiveEvent({
+ companyId: updated.companyId,
+ type: "heartbeat.run.status",
+ payload: buildHeartbeatRunStatusLiveEventPayload(updated),
+ });
+ publishRunLifecyclePluginEvent(updated);
+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);
+ return { run: updated, updated: true as const };
+ }
+
+ const current = await db
+ .select()
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, runId))
+ .then((rows) => rows[0] ?? null);
+
+ return { run: current, updated: false as const };
+ }
+
+ // Invariant: when a run releases its environment lease, the run row must be
+ // terminal. The finalizer writes the terminal status in a step that is
+ // separate from the agent status=done PATCH. If the sandbox or the run
+ // process stops between the two steps, heartbeat_runs.status stays "running".
+ // The UI reads liveness from that row, so a finished task shows "Live"
+ // forever. This function closes the gap in the run teardown: when the run is
+ // still running or queued, it forces a terminal status before the lease is
+ // released. It never overwrites a status that another path already made
+ // terminal.
+ async function terminalizeRunOnLeaseRelease(
+ run: typeof heartbeatRuns.$inferSelect,
+ ): Promise<typeof heartbeatRuns.$inferSelect> {
+ if (isNativeRunnerOwnershipHeld(run)) return run;
+ if (isHeartbeatRunTerminalStatus(run.status)) return run;
+ if (run.status !== "running" && run.status !== "queued") return run;
+
+ // Choose the terminal status that reflects the true outcome. When the issue
+ // already reached a terminal status, the run reached its goal, so use the
+ // matching terminal run status. Otherwise the teardown cut the run short,
+ // so use "interrupted".
+ const issueId = readNonEmptyString(
+ parseObject(run.contextSnapshot).issueId,
+ );
+ let terminalStatus: "succeeded" | "cancelled" | "interrupted" =
+ "interrupted";
+ if (issueId) {
+ const issueStatus = await db
+ .select({ status: issues.status })
+ .from(issues)
+ .where(eq(issues.id, issueId))
+ .then((rows) => rows[0]?.status ?? null);
+ if (issueStatus === "done") terminalStatus = "succeeded";
+ else if (issueStatus === "cancelled") terminalStatus = "cancelled";
+ }
+
+ // Teardown can beat the cancellation finalizer. Preserve the acknowledged
+ // user intent instead of reporting an infrastructure interruption.
+ if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled";
+
+ const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`;
+ // Match both "running" and "queued". A queued run has released its lease but
+ // never reached "running", so a running-only update would miss it and leave
+ // a phantom live run behind.
+ const write = await setRunStatusFromLive(
+ run.id,
+ terminalStatus,
+ ["running", "queued"],
+ {
+ finishedAt: run.finishedAt ?? new Date(),
+ error: run.error ?? (terminalStatus === "interrupted" ? message : null),
+ errorCode:
+ run.errorCode ??
+ (terminalStatus === "interrupted"
+ ? "lease_released_before_terminal"
+ : null),
+ },
+ );
+ if (!write.updated) {
+ // Another path already finalized the run. Keep that terminal outcome.
+ return write.run ?? run;
+ }
+
+ const terminalRun = write.run;
+ if (terminalRun) {
+ await appendRunEvent(terminalRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: terminalStatus === "interrupted" ? "warn" : "info",
+ message,
+ payload: {
+ previousStatus: run.status,
+ terminalStatus,
+ reason: "environment_lease_release",
+ ...(issueId ? { issueId } : {}),
+ },
+ }).catch((eventErr) => {
+ logger.warn(
+ { err: eventErr, runId: run.id },
+ "failed to append run event for lease-release terminalization",
+ );
+ });
+ }
+ return terminalRun ?? run;
+ }
+
+ function publishRunLifecyclePluginEvent(
+ run: typeof heartbeatRuns.$inferSelect,
+ ) {
+ publishRunLifecyclePluginEventData({
+ companyId: run.companyId,
+ runId: run.id,
+ agentId: run.agentId,
+ status: run.status,
+ invocationSource: run.invocationSource,
+ triggerDetail: run.triggerDetail,
+ error: run.error,
+ errorCode: run.errorCode,
+ issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
+ startedAt: run.startedAt,
+ finishedAt: run.finishedAt,
+ });
+ }
+
+ function publishRunLifecyclePluginEventData(run: {
+ companyId: string;
+ runId: string;
+ agentId: string;
+ status: string;
+ invocationSource: string;
+ triggerDetail: string | null;
+ error: string | null;
+ errorCode: string | null;
+ issueId: string | null;
+ startedAt: Date | null;
+ finishedAt: Date | null;
+ }) {
+ const eventType =
+ run.status === "running"
+ ? "agent.run.started"
+ : run.status === "succeeded"
+ ? "agent.run.finished"
+ : run.status === "failed" || run.status === "timed_out"
+ ? "agent.run.failed"
+ : run.status === "cancelled"
+ ? "agent.run.cancelled"
+ : null;
+ if (!eventType) return;
+ publishPluginDomainEvent({
+ eventId: randomUUID(),
+ eventType,
+ occurredAt: new Date().toISOString(),
+ actorId: run.agentId,
+ actorType: "agent",
+ entityId: run.runId,
+ entityType: "heartbeat_run",
+ companyId: run.companyId,
+ payload: {
+ runId: run.runId,
+ agentId: run.agentId,
+ status: run.status,
+ invocationSource: run.invocationSource,
+ triggerDetail: run.triggerDetail,
+ error: run.error ?? null,
+ errorCode: run.errorCode ?? null,
+ issueId: run.issueId,
+ startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null,
+ finishedAt: run.finishedAt
+ ? new Date(run.finishedAt).toISOString()
+ : null,
+ },
+ });
+ }
+
+ async function setWakeupStatus(
+ wakeupRequestId: string | null | undefined,
+ status: string,
+ patch?: Partial<typeof agentWakeupRequests.$inferInsert>,
+ ) {
+ if (!wakeupRequestId) return;
+ await db
+ .update(agentWakeupRequests)
+ .set({ status, ...patch, updatedAt: new Date() })
+ // Reassignment revokes requests transactionally. Late execution settlement
+ // must not turn a revoked request back into authority for a later retry.
+ .where(and(eq(agentWakeupRequests.id, wakeupRequestId), ne(agentWakeupRequests.status, "cancelled")));
+ }
+
+ async function addContinuationExhaustedCommentOnce(input: {
+ run: typeof heartbeatRuns.$inferSelect;
+ issueId: string;
+ comment: string;
+ }) {
+ const existing = await db
+ .select({ id: issueComments.id })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, input.run.companyId),
+ eq(issueComments.issueId, input.issueId),
+ eq(issueComments.createdByRunId, input.run.id),
+ sql`${issueComments.body} like 'Bounded liveness continuation exhausted%'`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (existing) return;
+ await issuesSvc.addComment(input.issueId, input.comment, {
+ agentId: input.run.agentId,
+ runId: input.run.id,
+ });
+ }
+
+ async function handleRunLivenessContinuation(
+ run: typeof heartbeatRuns.$inferSelect,
+ ) {
+ const context = parseObject(run.contextSnapshot);
+ if (
+ readNonEmptyString(context.goalControlRequestId) ||
+ context.resumeSessionGoalHeartbeat === true
+ )
+ return;
+ const livenessState = run.livenessState as RunLivenessState | null;
+ if (livenessState !== "plan_only" && livenessState !== "empty_response")
+ return;
+
+ const issueId = readNonEmptyString(context.issueId);
+ if (!issueId) return;
+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
+
+ const [issue, agent] = await Promise.all([
+ db
+ .select({
+ id: issues.id,
+ companyId: issues.companyId,
+ identifier: issues.identifier,
+ title: issues.title,
+ status: issues.status,
+ assigneeAgentId: issues.assigneeAgentId,
+ executionState: issues.executionState,
+ projectId: issues.projectId,
+ })
+ .from(issues)
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
+ .then((rows) => rows[0] ?? null),
+ db
+ .select({
+ id: agents.id,
+ companyId: agents.companyId,
+ status: agents.status,
+ })
+ .from(agents)
+ .where(eq(agents.id, run.agentId))
+ .then((rows) => rows[0] ?? null),
+ ]);
+
+ const budgetBlock =
+ issue && agent
+ ? await budgets.getInvocationBlock(issue.companyId, agent.id, {
+ issueId: issue.id,
+ projectId: issue.projectId,
+ })
+ : null;
+
+ const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
+ const idempotencyKey = issue
+ ? buildRunLivenessContinuationIdempotencyKey({
+ issueId: issue.id,
+ sourceRunId: run.id,
+ livenessState,
+ nextAttempt,
+ })
+ : null;
+ const existingWake = idempotencyKey
+ ? await findExistingRunLivenessContinuationWake(db, {
+ companyId: run.companyId,
+ idempotencyKey,
+ })
+ : null;
+
+ const decision = decideRunLivenessContinuation({
+ run,
+ issue,
+ agent,
+ livenessState,
+ livenessReason: run.livenessReason,
+ nextAction: run.nextAction,
+ budgetBlocked: Boolean(budgetBlock),
+ idempotentWakeExists: Boolean(existingWake),
+ });
+
+ if (decision.kind === "exhausted") {
+ await setRunStatus(run.id, run.status, {
+ livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation attempts exhausted`,
+ });
+ await addContinuationExhaustedCommentOnce({
+ run,
+ issueId,
+ comment: decision.comment,
+ });
+ return;
+ }
+
+ if (decision.kind !== "enqueue") return;
+
+ const continuationRun = await enqueueWakeup(run.agentId, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: RUN_LIVENESS_CONTINUATION_REASON,
+ payload: decision.payload,
+ contextSnapshot: {
+ ...decision.contextSnapshot,
+ originIdentityContextId: null,
+ parentRunId: run.id,
+ },
+ idempotencyKey: decision.idempotencyKey,
+ requestedByActorType: "system",
+ requestedByActorId: "heartbeat",
+ });
+
+ if (continuationRun) {
+ await db
+ .update(heartbeatRuns)
+ .set({
+ continuationAttempt: decision.nextAttempt,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, run.id));
+ }
+ }
+
+ function issueUiLink(
+ issue: Pick<typeof issues.$inferSelect, "id" | "identifier">,
+ ) {
+ const label = issue.identifier ?? issue.id;
+ const prefix = issue.identifier?.split("-")[0] || "PAP";
+ return `[${label}](/${prefix}/issues/${label})`;
+ }
+
+ function hasUnmanagedBackgroundTaskEvidence(
+ resultJson: Record<string, unknown> | null | undefined,
+ ) {
+ const evidence = parseObject(resultJson?.unmanagedBackgroundTask);
+ return (
+ evidence.stopped === true &&
+ (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON ||
+ evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON)
+ );
+ }
+
+ function withUnmanagedBackgroundTaskStopReason(
+ resultJson: Record<string, unknown> | null | undefined,
+ ) {
+ return {
+ ...(resultJson ?? {}),
+ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
+ };
+ }
+
+ function buildDetectedSuccessfulRunProgressSummary(
+ run: typeof heartbeatRuns.$inferSelect,
+ currentUserRedactionOptions: CurrentUserRedactionOptions,
+ ) {
+ const resultJson = parseObject(run.resultJson);
+ const candidates = [
+ hasUnmanagedBackgroundTaskEvidence(resultJson)
+ ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON
+ : null,
+ readNonEmptyString(run.nextAction)
+ ? `Next action noted: ${readNonEmptyString(run.nextAction)}`
+ : null,
+ readNonEmptyString(run.livenessReason),
+ readNonEmptyString(resultJson.summary),
+ readNonEmptyString(resultJson.result),
+ readNonEmptyString(resultJson.message),
+ ].filter((value): value is string => Boolean(value));
+ const summary = candidates[0];
+ if (!summary) return null;
+ return redactDetectedSuccessfulRunProgressSummaryForBoard(
+ summary,
+ currentUserRedactionOptions,
+ );
+ }
+
+ async function addSuccessfulRunHandoffCommentOnce(input: {
+ issue: Pick<
+ typeof issues.$inferSelect,
+ "id" | "identifier" | "title" | "status"
+ >;
+ run: typeof heartbeatRuns.$inferSelect;
+ agent: Pick<typeof agents.$inferSelect, "id" | "name">;
+ detectedProgressSummary: string;
+ }) {
+ const existing = await db
+ .select({ id: issueComments.id })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, input.run.companyId),
+ eq(issueComments.issueId, input.issue.id),
+ eq(issueComments.createdByRunId, input.run.id),
+ sql`(${issueComments.body} = ${SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY} or ${issueComments.body} like '## This issue still needs a next step%' or ${issueComments.body} like '## Successful run missing issue disposition%')`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+ if (existing) return null;
+ const notice = buildSuccessfulRunHandoffRequiredNotice(input);
+ return issuesSvc.addComment(
+ input.issue.id,
+ notice.body,
+ { runId: input.run.id },
+ {
+ authorType: "system",
+ presentation: notice.presentation,
+ metadata: notice.metadata,
+ },
+ );
+ }
+
+ async function handleSuccessfulRunHandoff(
+ run: typeof heartbeatRuns.$inferSelect,
+ agent: typeof agents.$inferSelect,
+ ) {
+ if (run.status !== "succeeded") return;
+ const context = parseObject(run.contextSnapshot);
+ const issueId =
+ readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);
+ if (!issueId) return;
+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
+ if (
+ readNonEmptyString(context.goalControlRequestId) ||
+ context.resumeSessionGoalHeartbeat === true
+ ) {
+ const goalProjection = await runnerGoalService(db).projection(
+ run.companyId,
+ issueId,
+ run.agentId,
+ );
+ if (goalProjection?.goal?.status !== "complete") return;
+ }
+
+ const issue = await db
+ .select({
+ id: issues.id,
+ companyId: issues.companyId,
+ identifier: issues.identifier,
+ title: issues.title,
+ description: issues.description,
+ status: issues.status,
+ assigneeAgentId: issues.assigneeAgentId,
+ assigneeUserId: issues.assigneeUserId,
+ executionState: issues.executionState,
+ monitorNextCheckAt: issues.monitorNextCheckAt,
+ projectId: issues.projectId,
+ originKind: issues.originKind,
+ })
+ .from(issues)
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
+ .then((rows) => rows[0] ?? null);
+ const idempotencyKey = issue
+ ? buildFinishSuccessfulRunHandoffIdempotencyKey({
+ issueId: issue.id,
+ sourceRunId: run.id,
+ })
+ : null;
+ const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null);
+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
+ const detectedProgressSummary = buildDetectedSuccessfulRunProgressSummary(
+ run,
+ currentUserRedactionOptions,
+ );
+ const resultJson = parseObject(run.resultJson);
+ const finalReport = redactSuccessfulRunHandoffEvidence(
+ [
+ readNonEmptyString(resultJson.summary),
+ readNonEmptyString(resultJson.result),
+ readNonEmptyString(resultJson.message),
+ ].find((value): value is string => Boolean(value)) ?? null,
+ currentUserRedactionOptions,
+ );
+ const nextAction = redactSuccessfulRunHandoffEvidence(
+ readNonEmptyString(run.nextAction),
+ currentUserRedactionOptions,
+ );
+
+ const [
+ activeExecutionPath,
+ queuedWake,
+ pendingInteraction,
+ pendingApproval,
+ explicitBlocker,
+ openRecoveryIssue,
+ existingWake,
+ budgetBlock,
+ pauseHold,
+ activeRoutineContinuation,
+ ] = await Promise.all([
+ issue
+ ? db
+ .select({ id: heartbeatRuns.id })
+ .from(heartbeatRuns)
+ .where(
+ and(
+ eq(heartbeatRuns.companyId, issue.companyId),
+ eq(heartbeatRuns.agentId, run.agentId),
+ inArray(heartbeatRuns.status, [
+ ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES,
+ ]),
+ sql`(
+ ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}
+ or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id}
+ )`,
+ sql`${heartbeatRuns.id} <> ${run.id}`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, issue.companyId),
+ eq(agentWakeupRequests.agentId, run.agentId),
+ inArray(agentWakeupRequests.status, [
+ "queued",
+ "deferred_issue_execution",
+ "claimed",
+ ]),
+ sql`(
+ ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}
+ or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id}
+ or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id}
+ or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id}
+ )`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: issueThreadInteractions.id })
+ .from(issueThreadInteractions)
+ .where(
+ and(
+ eq(issueThreadInteractions.companyId, issue.companyId),
+ eq(issueThreadInteractions.issueId, issue.id),
+ eq(issueThreadInteractions.status, "pending"),
+ activeIssueInteractionCondition(),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: issueApprovals.approvalId })
+ .from(issueApprovals)
+ .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
+ .where(
+ and(
+ eq(issueApprovals.companyId, issue.companyId),
+ eq(issueApprovals.issueId, issue.id),
+ inArray(approvals.status, ["pending", "revision_requested"]),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: issueRelations.issueId })
+ .from(issueRelations)
+ .where(
+ and(
+ eq(issueRelations.companyId, issue.companyId),
+ eq(issueRelations.relatedIssueId, issue.id),
+ eq(issueRelations.type, "blocks"),
+ sql`exists (
+ select 1
+ from issues blocker
+ where blocker.id = ${issueRelations.issueId}
+ and blocker.company_id = ${issue.companyId}
+ and blocker.status not in ('done', 'cancelled')
+ and blocker.hidden_at is null
+ )`,
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: issues.id })
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, issue.companyId),
+ inArray(issues.originKind, [
+ RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
+ RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
+ ]),
+ eq(issues.originId, issue.id),
+ visibleIssueCondition(),
+ notInArray(issues.status, ["done", "cancelled"]),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ idempotencyKey
+ ? findExistingFinishSuccessfulRunHandoffWake(db, {
+ companyId: run.companyId,
+ idempotencyKey,
+ })
+ : Promise.resolve(null),
+ issue
+ ? budgets.getInvocationBlock(issue.companyId, run.agentId, {
+ issueId: issue.id,
+ projectId: issue.projectId,
+ })
+ : Promise.resolve(null),
+ issue
+ ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id)
+ : Promise.resolve(null),
+ issue
+ ? db
+ .select({ id: routines.id })
+ .from(routines)
+ .where(
+ and(
+ eq(routines.companyId, issue.companyId),
+ eq(routines.parentIssueId, issue.id),
+ eq(routines.status, "active"),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null)
+ : Promise.resolve(null),
+ ]);
+
+ const decision = decideSuccessfulRunHandoff({
+ run,
+ issue,
+ agent,
+ livenessState: run.livenessState as RunLivenessState | null,
+ detectedProgressSummary,
+ finalReport,
+ nextAction,
+ taskKey,
+ hasActiveExecutionPath: Boolean(activeExecutionPath),
+ hasQueuedWake: Boolean(queuedWake),
+ hasPendingInteractionOrApproval: Boolean(
+ pendingInteraction || pendingApproval,
+ ),
+ hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt),
+ hasExplicitBlockerPath: Boolean(explicitBlocker),
+ hasOpenRecoveryIssue: Boolean(openRecoveryIssue),
+ hasPauseHold: Boolean(pauseHold),
+ hasActiveRoutineContinuation: Boolean(activeRoutineContinuation),
+ budgetBlocked: Boolean(budgetBlock),
+ idempotentWakeExists: Boolean(existingWake),
+ });
+
+ if (isSuccessfulRunHandoffValidPathSkip(decision) && issue) {
+ await resolveRequiredSuccessfulRunHandoffOnValidPath(db, {
+ companyId: issue.companyId,
+ issueId: issue.id,
+ issueIdentifier: issue.identifier,
+ agentId: run.agentId,
+ runId: run.id,
+ skipReason: decision.reason,
+ });
+ }
+
+ if (decision.kind !== "enqueue" || !issue) return;
+
+ if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) {
+ await db
+ .update(heartbeatRuns)
+ .set({
+ livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
+ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
+ ${JSON.stringify({ stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON })}::jsonb`,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, run.id));
+ }
+
+ const handoffRun = await enqueueWakeup(decision.targetAgentId, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: FINISH_SUCCESSFUL_RUN_HANDOFF_REASON,
+ payload: decision.payload,
+ contextSnapshot: {
+ ...decision.contextSnapshot,
+ originIdentityContextId: null,
+ parentRunId: run.id,
+ },
+ idempotencyKey: decision.idempotencyKey,
+ requestedByActorType: "system",
+ requestedByActorId: "heartbeat",
+ });
+ if (!handoffRun) return;
+
+ await addSuccessfulRunHandoffCommentOnce({
+ issue,
+ run,
+ agent,
+ detectedProgressSummary:
+ detectedProgressSummary ??
+ "The run reported progress, but did not choose a next step.",
+ });
+ await logActivity(db, {
+ companyId: issue.companyId,
+ actorType: "system",
+ actorId: "heartbeat",
+ agentId: run.agentId,
+ runId: run.id,
+ action: "issue.successful_run_handoff_required",
+ entityType: "issue",
+ entityId: issue.id,
+ details: {
+ label: "Successful run missing issue disposition",
+ sourceRunId: run.id,
+ correctiveRunId: handoffRun.id,
+ handoffReason: SUCCESSFUL_RUN_MISSING_STATE_REASON,
+ missingDisposition: "clear_next_step",
+ detectedProgressSummary,
+ issue: issueUiLink(issue),
+ },
+ });
+ }
+
+ async function handleIssueReviewPathDisposition(
+ run: typeof heartbeatRuns.$inferSelect,
+ ) {
+ const contextSnapshot = parseObject(run.contextSnapshot);
+ if (readNonEmptyString(contextSnapshot.goalControlRequestId)) return;
+ const issueId =
+ readNonEmptyString(contextSnapshot.issueId) ??
+ readNonEmptyString(contextSnapshot.taskId);
+ if (!issueId) return;
+ const waitingContext = await getIssueExecutionContext(run.companyId, issueId);
+ if (isWaitingConversation(waitingContext) || waitingContext?.externalConversationState === "waiting") return;
+
+ const issue = await db
+ .select({
+ id: issues.id,
+ companyId: issues.companyId,
+ identifier: issues.identifier,
+ status: issues.status,
+ assigneeAgentId: issues.assigneeAgentId,
+ })
+ .from(issues)
+ .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
+ .then((rows) => rows[0] ?? null);
+ if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId)
+ return;
+
+ const reviewAttention = await issuesSvc
+ .listReviewAttention(issue.companyId, [issue])
+ .then(
+ (map) =>
+ map.get(issue.id) ?? {
+ state: "none" as const,
+ paths: [],
+ reason: null,
+ },
+ );
+ if (reviewAttention.state !== "stalled") return;
+
+ const consumedPathRef = reviewPathConsumedRefFromRun({
+ runId: run.id,
+ issueId: issue.id,
+ contextSnapshot,
+ });
+ const idempotencyKey = buildIssueReviewPathLostIdempotencyKey({
+ issueId: issue.id,
+ consumedPathRef,
+ });
+ const existingWake = await db
+ .select({ id: agentWakeupRequests.id })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, issue.companyId),
+ eq(agentWakeupRequests.idempotencyKey, idempotencyKey),
+ notInArray(agentWakeupRequests.status, ["skipped"]),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows[0] ?? null);
+
+ const decision = decideIssueReviewPathRecovery({
+ issueId: issue.id,
+ sourceRunId: run.id,
+ assigneeAgentId: issue.assigneeAgentId,
+ contextSnapshot,
+ reviewAttention,
+ existingWake: Boolean(existingWake),
+ });
+ if (decision.kind !== "enqueue") return;
+
+ const recoveryRun = await enqueueWakeup(issue.assigneeAgentId, {
+ source: "automation",
+ triggerDetail: "system",
+ reason: ISSUE_REVIEW_PATH_LOST_WAKE_REASON,
+ idempotencyKey: decision.idempotencyKey,
+ payload: decision.payload,
+ contextSnapshot: decision.contextSnapshot,
+ requestedByActorType: "system",
+ requestedByActorId: "heartbeat",
+ }).catch((error: unknown) => {
+ if (isReviewPathRecoveryIdempotencyConflict(error)) return null;
+ throw error;
+ });
+ if (!recoveryRun) return;
+
+ await logActivity(db, {
+ companyId: issue.companyId,
+ actorType: "system",
+ actorId: "heartbeat",
+ agentId: issue.assigneeAgentId,
+ runId: run.id,
+ action: "issue.review_path_recovery_queued",
+ entityType: "issue",
+ entityId: issue.id,
+ details: {
+ sourceRunId: run.id,
+ recoveryRunId: recoveryRun.id,
+ consumedPathRef,
+ recoveryAttempt: 1,
+ maxRecoveryAttempts: 1,
+ },
+ });
+ }
+
+ async function appendRunEvent(
+ run: typeof heartbeatRuns.$inferSelect,
+ event: {
+ eventType: string;
+ stream?: "system" | "stdout" | "stderr";
+ level?: "info" | "warn" | "error";
+ color?: string;
+ message?: string;
+ payload?: Record<string, unknown>;
+ retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"];
+ },
+ ) {
+ const eventAt = new Date();
+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
+ const sanitizedMessage = event.message
+ ? redactSensitiveText(
+ redactCurrentUserText(event.message, currentUserRedactionOptions),
+ )
+ : event.message;
+ const boundedPayload = event.payload
+ ? boundHeartbeatRunEventPayloadForStorage(event.payload)
+ : event.payload;
+ const secretSanitizedPayload = boundedPayload
+ ? redactEventPayload(boundedPayload)
+ : boundedPayload;
+ const sanitizedPayload = secretSanitizedPayload
+ ? redactCurrentUserValue(
+ secretSanitizedPayload,
+ currentUserRedactionOptions,
+ )
+ : secretSanitizedPayload;
+ const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null;
+ const progress = buildRunEventRuntimeProgress({
+ eventType: event.eventType,
+ message: sanitizedMessage ?? null,
+ payload: sanitizedPayload ?? null,
+ at: eventAt,
+ });
+ const persistedEvent = await appendHeartbeatRunEvent(db, {
+ companyId: run.companyId,
+ runId: run.id,
+ agentId: run.agentId,
+ eventType: event.eventType,
+ stream: event.stream,
+ level: event.level,
+ color: event.color,
+ message: sanitizedMessage,
+ payload: sanitizedPayload,
+ retryExhaustion: event.retryExhaustion,
+ });
+ if (persistedEvent.disposition === "duplicate") return;
+ const seq = persistedEvent.row.seq;
+
+ publishLiveEvent({
+ companyId: run.companyId,
+ type: "heartbeat.run.event",
+ payload: {
+ runId: run.id,
+ agentId: run.agentId,
+ issueId,
+ seq,
+ eventType: event.eventType,
+ stream: event.stream ?? null,
+ level: event.level ?? null,
+ color: event.color ?? null,
+ message: sanitizedMessage ?? null,
+ currentToolName: progress?.currentToolName ?? null,
+ lastAssistantSnippet: progress?.lastAssistantSnippet ?? null,
+ lastEventAt: (progress?.lastEventAt ?? eventAt).toISOString(),
+ payload: sanitizedPayload ?? null,
+ },
+ });
+ if (progress && isHeartbeatRunRuntimeStatusActive(run.status)) {
+ const status = setHeartbeatRunRuntimeStatus({
+ companyId: run.companyId,
+ issueId,
+ agentId: run.agentId,
+ runId: run.id,
+ phase: progress.phase,
+ message: progress.message,
+ updatedAt: eventAt,
+ currentToolName: progress.currentToolName,
+ lastAssistantSnippet: progress.lastAssistantSnippet,
+ lastEventAt: progress.lastEventAt,
+ });
+ if (status) publishHeartbeatRunRuntimeProgress(status);
+ }
+ }
+
+ async function persistRunProcessMetadata(
+ runId: string,
+ meta: { pid: number; processGroupId: number | null; startedAt: string },
+ ) {
+ return persistHeartbeatRunProcessMetadata(db, runId, meta);
+ }
+
+ async function clearDetachedRunWarning(runId: string) {
+ const updated = await db
+ .update(heartbeatRuns)
+ .set({
+ error: null,
+ errorCode: null,
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(heartbeatRuns.id, runId),
+ eq(heartbeatRuns.status, "running"),
+ eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE),
+ ),
+ )
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ if (!updated) return null;
+
+ await appendRunEvent(updated, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "info",
+ message:
+ "Detached child process reported activity; cleared detached warning",
+ });
+ return updated;
+ }
+
+ async function patchRunIssueCommentStatus(
+ runId: string,
+ patch: Partial<
+ Pick<
+ typeof heartbeatRuns.$inferInsert,
+ | "issueCommentStatus"
+ | "issueCommentSatisfiedByCommentId"
+ | "issueCommentRetryQueuedAt"
+ >
+ >,
+ ) {
+ return db
+ .update(heartbeatRuns)
+ .set({ ...patch, updatedAt: new Date() })
+ .where(eq(heartbeatRuns.id, runId))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ }
+
+ async function findRunIssueComment(
+ runId: string,
+ companyId: string,
+ issueId: string,
+ resultJson?: Record<string, unknown> | null,
+ ) {
+ const comments = await db
+ .select({
+ id: issueComments.id,
+ body: issueComments.body,
+ })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, companyId),
+ eq(issueComments.issueId, issueId),
+ eq(issueComments.createdByRunId, runId),
+ ),
+ )
+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id));
+ return findHeartbeatRunCompletionComment(comments, resultJson);
+ }
+
+ async function findLatestCompletedFinalAgentMessage(
+ runId: string,
+ companyId: string,
+ ) {
+ const rows = await db
+ .select({
+ seq: heartbeatRunEvents.seq,
+ payload: heartbeatRunEvents.payload,
+ })
+ .from(heartbeatRunEvents)
+ .where(
+ and(
+ eq(heartbeatRunEvents.companyId, companyId),
+ eq(heartbeatRunEvents.runId, runId),
+ eq(heartbeatRunEvents.eventType, "item.completed"),
+ ),
+ )
+ .orderBy(desc(heartbeatRunEvents.seq))
+ .limit(200);
+ const candidates: Array<{
+ seq: number;
+ text: string;
+ sourceEventId: string | null;
+ channel: "final" | "unknown";
+ }> = [];
+ for (const row of rows) {
+ const prpEvent = parseObject(parseObject(row.payload).prpEvent);
+ const candidate = readCompletedAssistantMessageCandidate({
+ seq: row.seq,
+ prpEvent,
+ });
+ if (candidate) candidates.push(candidate);
+ }
+ const recoveryBoundary = await db
+ .select({
+ seq: heartbeatRunEvents.seq,
+ payload: heartbeatRunEvents.payload,
+ })
+ .from(heartbeatRunEvents)
+ .where(
+ and(
+ eq(heartbeatRunEvents.companyId, companyId),
+ eq(heartbeatRunEvents.runId, runId),
+ eq(heartbeatRunEvents.eventType, "lifecycle"),
+ ),
+ )
+ .orderBy(heartbeatRunEvents.seq)
+ .limit(200)
+ .then(
+ (lifecycleRows) =>
+ lifecycleRows.find(
+ (row) =>
+ parseObject(row.payload).retryReasonCode ===
+ "semantic_result_missing",
+ )?.seq ?? null,
+ );
+ return selectHeartbeatRunFinalAgentMessage({
+ candidates,
+ semanticResultRecoveryAfterSeq: recoveryBoundary,
+ });
+ }
+
+ async function refreshContinuationSummaryForRun(
+ run: typeof heartbeatRuns.$inferSelect,
+ agent: typeof agents.$inferSelect,
+ ) {
+ const contextSnapshot = parseObject(run.contextSnapshot);
+ const issueId = readNonEmptyString(contextSnapshot.issueId);
+ if (!issueId) return null;
+ try {
+ return await refreshIssueContinuationSummary({
+ db,
+ issueId,
+ run: {
+ id: run.id,
+ status: run.status,
+ error: run.error,
+ errorCode: run.errorCode,
+ resultJson: run.resultJson as Record<string, unknown> | null,
+ stdoutExcerpt: run.stdoutExcerpt,
+ stderrExcerpt: run.stderrExcerpt,
+ finishedAt: run.finishedAt,
+ },
+ agent: {
+ id: agent.id,
+ name: agent.name,
+ adapterType: agent.adapterType,
+ },
+ });
+ } catch (err) {
+ logger.warn(
+ {
+ err,
+ runId: run.id,
+ issueId,
+ agentId: agent.id,
+ },
+ "failed to refresh issue continuation summary",
+ );
+ return null;
+ }
+ }
+
+ async function enqueueMissingIssueCommentRetry(
+ run: typeof heartbeatRuns.$inferSelect,
+ agent: typeof agents.$inferSelect,
+ issueId: string,
+ ) {
+ const invokability = await getAgentInvokability(agent);
+ if (!invokability.invokable) {
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message:
+ "Missing-comment retry suppressed because the agent is not invokable",
+ payload: {
+ reason: invokability.reason,
+ invalidOrgChain: invokability.invalidOrgChain,
+ ...invokability.details,
+ },
+ });
+ return null;
+ }
+
+ const contextSnapshot = parseObject(run.contextSnapshot);
+ const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null);
+ const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey);
+ const retryContextSnapshot = withRecoveryContext(
+ {
+ ...contextSnapshot,
+ retryOfRunId: run.id,
+ wakeReason: "missing_issue_comment",
+ retryReason: "missing_issue_comment",
+ missingIssueCommentForRunId: run.id,
+ },
+ "status_only",
+ );
+ const responsibleUserId = await resolveResponsibleUserIdForRunContext(
+ run,
+ retryContextSnapshot,
+ );
+ const now = new Date();
+
+ const retryRun = await db.transaction(async (tx) => {
+ await tx.execute(
+ sql`select id from issues where company_id = ${run.companyId} and execution_run_id = ${run.id} for update`,
+ );
+
+ const issue = await tx
+ .select({ id: issues.id })
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, run.companyId),
+ eq(issues.executionRunId, run.id),
+ ),
+ )
+ .then((rows) => rows[0] ?? null);
+ if (!issue) return null;
+
+ const wakeupRequest = await tx
+ .insert(agentWakeupRequests)
+ .values({
+ companyId: run.companyId,
+ agentId: run.agentId,
+ source: "automation",
+ triggerDetail: "system",
+ reason: "missing_issue_comment",
+ payload: withRecoveryContext(
+ {
+ issueId,
+ retryOfRunId: run.id,
+ retryReason: "missing_issue_comment",
+ },
+ "status_only",
+ ),
+ status: "queued",
+ requestedByActorType: "system",
+ requestedByActorId: null,
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ const queuedRun = await tx
+ .insert(heartbeatRuns)
+ .values({
+ companyId: run.companyId,
+ agentId: run.agentId,
+ scopeKind: "issue",
+ issueId,
+ invocationSource: "automation",
+ triggerDetail: "system",
+ status: "queued",
+ wakeupRequestId: wakeupRequest.id,
+ contextSnapshot: retryContextSnapshot,
+ responsibleUserId,
+ sessionIdBefore: sessionBefore,
+ retryOfRunId: run.id,
+ issueCommentStatus: "not_applicable",
+ updatedAt: now,
+ })
+ .returning()
+ .then((rows) => rows[0]);
+
+ await tx
+ .update(agentWakeupRequests)
+ .set({
+ runId: queuedRun.id,
+ updatedAt: now,
+ })
+ .where(eq(agentWakeupRequests.id, wakeupRequest.id));
+
+ await tx
+ .update(issues)
+ .set({
+ executionRunId: queuedRun.id,
+ executionAgentNameKey: normalizeAgentNameKey(agent.name),
+ executionLockedAt: now,
+ updatedAt: now,
+ })
+ .where(eq(issues.id, issue.id));
+
+ await tx
+ .update(heartbeatRuns)
+ .set({
+ issueCommentStatus: "retry_queued",
+ issueCommentRetryQueuedAt: now,
+ updatedAt: now,
+ })
+ .where(eq(heartbeatRuns.id, run.id));
+
+ return queuedRun;
+ });
+
+ if (!retryRun) return null;
+
+ publishLiveEvent({
+ companyId: retryRun.companyId,
+ type: "heartbeat.run.queued",
+ payload: {
+ runId: retryRun.id,
+ agentId: retryRun.agentId,
+ invocationSource: retryRun.invocationSource,
+ triggerDetail: retryRun.triggerDetail,
+ wakeupRequestId: retryRun.wakeupRequestId,
+ },
+ });
+
+ return retryRun;
+ }
+
+ async function hasDeferredIssueCommentWake(
+ companyId: string,
+ issueId: string,
+ agentId: string,
+ ) {
+ const deferredPayloads = await db
+ .select({ payload: agentWakeupRequests.payload })
+ .from(agentWakeupRequests)
+ .where(
+ and(
+ eq(agentWakeupRequests.companyId, companyId),
+ eq(agentWakeupRequests.agentId, agentId),
+ eq(agentWakeupRequests.status, "deferred_issue_execution"),
+ sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
+ ),
+ );
+
+ return deferredPayloads.some(({ payload }) => {
+ const parsedPayload = parseObject(payload);
+ const deferredContext = parseObject(
+ parsedPayload[DEFERRED_WAKE_CONTEXT_KEY],
+ );
+ return Boolean(deriveCommentId(deferredContext, parsedPayload));
+ });
+ }
+
+ async function finalizeIssueCommentPolicy(
+ run: typeof heartbeatRuns.$inferSelect,
+ agent: typeof agents.$inferSelect,
+ presentationDecision?: RunPresentationDecision | null,
+ ) {
+ const contextSnapshot = parseObject(run.contextSnapshot);
+ // The explicit receipt admitted one turn. A prose-only follow-up cannot
+ // reuse it or renew it under the separate transient-failure retry policy.
+ if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {
+ if (run.issueCommentStatus !== "not_applicable") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ }
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+ const issueId = readNonEmptyString(contextSnapshot.issueId);
+ if (!issueId) {
+ if (run.issueCommentStatus !== "not_applicable") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ }
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ // A failed/timed-out/cancelled run may immediately enqueue the normal
+ // assignment or continuation recovery below. Do not let the lower-priority
+ // status-only missing-comment wake claim the issue first. The dedicated
+ // missing-comment retry still records exhaustion if that retry itself
+ // fails, but an original failed execution remains on the established
+ // direct-adapter recovery path.
+ if (
+ run.status !== "succeeded" &&
+ readNonEmptyString(contextSnapshot.retryReason) !==
+ "missing_issue_comment"
+ ) {
+ if (run.issueCommentStatus !== "not_applicable") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ }
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ // A settled run may legitimately have no user-facing prose. The response
+ // resolver owns that decision; do not wake the agent again merely to force
+ // an artificial comment into the issue thread.
+ if (
+ presentationDecision?.chosenSource === "none" &&
+ (hasAcceptedSemanticResult(parseObject(run.resultJson)) ||
+ presentationDecision.reasonCodes.includes(
+ "legacy_adapter_summary_ambiguous",
+ ))
+ ) {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ // A pre-dispatch setup failure means the adapter process never started (for
+ // example an unresolved workspace base ref). No agent could run, so no agent
+ // could post an issue comment. A missing-comment retry cannot help and would
+ // loop the identical pre-adapter failure, so mark the policy not_applicable
+ // and queue nothing.
+ if (
+ run.errorCode != null &&
+ PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode)
+ ) {
+ if (run.issueCommentStatus !== "not_applicable") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ }
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ const postedComment = await findRunIssueComment(
+ run.id,
+ run.companyId,
+ issueId,
+ parseObject(run.resultJson),
+ );
+ if (postedComment) {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "satisfied",
+ issueCommentSatisfiedByCommentId: postedComment.id,
+ issueCommentRetryQueuedAt: null,
+ });
+ return { outcome: "satisfied" as const, queuedRun: null };
+ }
+
+ // Missing-comment recovery is a legacy compatibility path for otherwise
+ // successful runs. A failed, timed-out, or cancelled run is already owned
+ // by lifecycle recovery and its terminal system presentation. Queuing a
+ // prose-only retry here can seize the issue execution lock before the
+ // authoritative continuation is materialized, replacing real recovery
+ // with a cheap status-only turn.
+ if (run.status !== "succeeded") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ if (
+ readNonEmptyString(contextSnapshot.retryReason) ===
+ "missing_issue_comment"
+ ) {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "retry_exhausted",
+ issueCommentSatisfiedByCommentId: null,
+ });
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message:
+ "Run ended without an issue comment after one retry; no further comment wake will be queued",
+ });
+ return { outcome: "retry_exhausted" as const, queuedRun: null };
+ }
+
+ if (!shouldRequireIssueCommentForWake(contextSnapshot)) {
+ if (run.issueCommentStatus !== "not_applicable") {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ }
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ if (
+ await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)
+ ) {
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "not_applicable",
+ issueCommentSatisfiedByCommentId: null,
+ issueCommentRetryQueuedAt: null,
+ });
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "info",
+ message:
+ "Run ended without an issue comment; a deferred comment wake already exists for this issue",
+ });
+ return { outcome: "not_applicable" as const, queuedRun: null };
+ }
+
+ const queuedRun = await enqueueMissingIssueCommentRetry(
+ run,
+ agent,
+ issueId,
+ );
+ if (queuedRun) {
+ await appendRunEvent(run, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message:
+ "Run ended without an issue comment; queued one follow-up wake to require a comment",
+ });
+ return { outcome: "retry_queued" as const, queuedRun };
+ }
+
+ await patchRunIssueCommentStatus(run.id, {
+ issueCommentStatus: "retry_exhausted",
+ issueCommentSatisfiedByCommentId: null,
+ });
+ return { outcome: "retry_exhausted" as const, queuedRun: null };
+ }
+
+ function parseMaxTurnContinuationPolicy(
+ agent: typeof agents.$inferSelect,
+ ): MaxTurnContinuationPolicy {
+ const runtimeConfig = parseObject(agent.runtimeConfig);
+ const heartbeat = parseObject(runtimeConfig.heartbeat);
+ const configured = parseObject(heartbeat.maxTurnContinuation);
+ const rawMaxAttempts = Math.floor(
+ asNumber(
+ configured.maxAttempts,
+ MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS,
+ ),
+ );
+ const rawDelayMs = Math.floor(
+ asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS),
+ );
+
+ return {
+ enabled: asBoolean(configured.enabled, true),
+ maxAttempts: Math.max(
+ 0,
+ Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts),
+ ),
+ delayMs: Math.max(
+ 0,
+ Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs),
+ ),
+ };
+ }
+
+ function mergeRunStopMetadataForAgent(
+ agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,
+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",
+ options?: {
+ resultJson?: Record<string, unknown> | null;
+ conversationContinuationEligible?: boolean;
+ errorCode?: string | null;
+ errorMessage?: string | null;
+ },
+ ) {
+ const stopMetadata = buildHeartbeatRunStopMetadata({
+ adapterType: agent.adapterType,
+ adapterConfig: parseObject(agent.adapterConfig),
+ outcome,
+ errorCode: options?.errorCode ?? null,
+ errorMessage: options?.errorMessage ?? null,
+ });
+ const result = mergeHeartbeatRunStopMetadata(
+ options?.resultJson ?? null,
+ stopMetadata,
+ );
+ const cancellationAcknowledged =
+ parseObject(result?.executionCancellation).state === "acknowledged";
+ return options?.conversationContinuationEligible !== false && outcome !== "succeeded" &&
+ (outcome !== "cancelled" || cancellationAcknowledged) &&
+ isConversationAdapter(agent.adapterType)
+ ? { ...result, conversationContinuation: CONVERSATION_CONTINUATION_POLICY }
+ : result;
+ }
+
+ function countValue(value: unknown) {
+ const parsed = Number(value ?? 0);
+ return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : 0;
+ }
+
+ function dateValue(value: unknown) {
+ if (value instanceof Date)
+ return Number.isNaN(value.getTime()) ? null : value;
+ if (typeof value === "string" || typeof value === "number") {
+ const parsed = new Date(value);
+ return Number.isNaN(parsed.getTime()) ? null : parsed;
+ }
+ return null;
+ }
+
+ function latestDate(...values: unknown[]) {
+ let latest: Date | null = null;
+ for (const value of values) {
+ const parsed = dateValue(value);
+ if (!parsed) continue;
+ if (!latest || parsed.getTime() > latest.getTime()) latest = parsed;
+ }
+ return latest;
+ }
+
+ async function buildRunLivenessInput(
+ run: typeof heartbeatRuns.$inferSelect,
+ resultJson: Record<string, unknown> | null | undefined,
+ ): Promise<RunLivenessClassificationInput> {
+ const context = parseObject(run.contextSnapshot);
+ const contextIssueId = readNonEmptyString(context.issueId);
+ const continuationAttempt = asNumber(
+ context.continuationAttempt,
+ run.continuationAttempt ?? 0,
+ );
+
+ const issue = contextIssueId
+ ? await db
+ .select({
+ status: issues.status,
+ title: issues.title,
+ description: issues.description,
+ workMode: issues.workMode,
+ })
+ .from(issues)
+ .where(
+ and(
+ eq(issues.companyId, run.companyId),
+ eq(issues.id, contextIssueId),
+ ),
+ )
+ .then((rows) => rows[0] ?? null)
+ : null;
+
+ const [commentStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${issueComments.createdAt})`,
+ })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, run.companyId),
+ eq(issueComments.issueId, contextIssueId),
+ eq(issueComments.createdByRunId, run.id),
+ isNull(issueComments.deletedAt),
+ ),
+ )
+ : [{ count: 0, latestAt: null }];
+
+ const issueCommentBodies = contextIssueId
+ ? await db
+ .select({ body: issueComments.body })
+ .from(issueComments)
+ .where(
+ and(
+ eq(issueComments.companyId, run.companyId),
+ eq(issueComments.issueId, contextIssueId),
+ eq(issueComments.createdByRunId, run.id),
+ ),
+ )
+ .orderBy(desc(issueComments.createdAt), desc(issueComments.id))
+ .limit(5)
+ .then((rows) => rows.reverse().map((row) => row.body))
+ : [];
+
+ const continuationSummary = contextIssueId
+ ? await getIssueContinuationSummaryDocument(db, contextIssueId)
+ : null;
+
+ const [documentStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ planCount: sql<number>`count(*) filter (where ${issueDocuments.key} = 'plan')::int`,
+ latestAt: sql<Date | null>`max(${documentRevisions.createdAt})`,
+ })
+ .from(documentRevisions)
+ .innerJoin(
+ issueDocuments,
+ eq(documentRevisions.documentId, issueDocuments.documentId),
+ )
+ .where(
+ and(
+ eq(documentRevisions.companyId, run.companyId),
+ eq(documentRevisions.createdByRunId, run.id),
+ eq(issueDocuments.companyId, run.companyId),
+ eq(issueDocuments.issueId, contextIssueId),
+ sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`,
+ ),
+ )
+ : [{ count: 0, planCount: 0, latestAt: null }];
+
+ const [workProductStats] = contextIssueId
+ ? await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${issueWorkProducts.createdAt})`,
+ })
+ .from(issueWorkProducts)
+ .where(
+ and(
+ eq(issueWorkProducts.companyId, run.companyId),
+ eq(issueWorkProducts.issueId, contextIssueId),
+ eq(issueWorkProducts.createdByRunId, run.id),
+ ),
+ )
+ : [{ count: 0, latestAt: null }];
+
+ const [workspaceOperationStats] = await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${workspaceOperations.startedAt})`,
+ })
+ .from(workspaceOperations)
+ .where(
+ and(
+ eq(workspaceOperations.companyId, run.companyId),
+ eq(workspaceOperations.heartbeatRunId, run.id),
+ ),
+ );
+
+ const [activityStats] = await db
+ .select({
+ count: sql<number>`count(*)::int`,
+ latestAt: sql<Date | null>`max(${activityLog.createdAt})`,
+ })
+ .from(activityLog)
+ .where(
+ and(
+ eq(activityLog.companyId, run.companyId),
+ eq(activityLog.runId, run.id),
+ notInArray(activityLog.action, LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS),
+ ),
+ );
+
+ const [eventStats] = await db
+ .select({
+ count: sql<number>`count(*) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))::int`,
+ latestAt: sql<Date | null>`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`,
+ })
+ .from(heartbeatRunEvents)
+ .where(
+ and(
+ eq(heartbeatRunEvents.companyId, run.companyId),
+ eq(heartbeatRunEvents.runId, run.id),
+ ),
+ );
+
+ return {
+ runStatus: run.status,
+ issue,
+ resultJson: resultJson ?? run.resultJson ?? null,
+ issueCommentBodies,
+ continuationSummaryBody: continuationSummary?.body ?? null,
+ stdoutExcerpt: run.stdoutExcerpt ?? null,
+ stderrExcerpt: run.stderrExcerpt ?? null,
+ error: run.error ?? null,
+ errorCode: run.errorCode ?? null,
+ continuationAttempt,
+ evidence: {
+ issueCommentsCreated: countValue(commentStats?.count),
+ documentRevisionsCreated: countValue(documentStats?.count),
+ planDocumentRevisionsCreated: countValue(documentStats?.planCount),
+ workProductsCreated: countValue(workProductStats?.count),
+ workspaceOperationsCreated: countValue(workspaceOperationStats?.count),
+ activityEventsCreated: countValue(activityStats?.count),
+ toolOrActionEventsCreated: countValue(eventStats?.count),
+ latestEvidenceAt: latestDate(
+ commentStats?.latestAt,
+ documentStats?.latestAt,
+ workProductStats?.latestAt,
+ workspaceOperationStats?.latestAt,
+ activityStats?.latestAt,
+ eventStats?.latestAt,
+ ),
+ },
+ };
+ }
+
+ async function classifyAndPersistRunLiveness(
+ run: typeof heartbeatRuns.$inferSelect,
+ resultJson?: Record<string, unknown> | null,
+ ) {
+ const authRepair = run.status === "failed"
+ ? await connectionIntentService(db).requestForRunAuthFailure(run.id).catch(() => {
+ logger.warn({ runId: run.id }, "Could not attach provider authentication repair; run failure remains available");
+ return null;
+ })
+ : null;
+ const classification = classifyRunLiveness({
+ ...await buildRunLivenessInput(run, resultJson),
+ authenticationRepairRequested: Boolean(authRepair?.interactionId),
+ });
+ return db
+ .update(heartbeatRuns)
+ .set({
+ livenessState: classification.livenessState,
+ livenessReason: classification.livenessReason,
+ continuationAttempt: classification.continuationAttempt,
+ lastUsefulActionAt: classification.lastUsefulActionAt,
+ nextAction: classification.nextAction,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, run.id))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ }
+
+ async function updateRuntimeState(
+ agent: typeof agents.$inferSelect,
+ run: typeof heartbeatRuns.$inferSelect,
+ result: AdapterExecutionResult,
+ session: { legacySessionId: string | null },
+ normalizedUsage?: UsageTotals | null,
+ ) {
+ await ensureRuntimeState(agent);
+ await accountRunCost(db, run.id, budgetHooks);
+ await db
+ .update(agentRuntimeState)
+ .set({
+ adapterType: agent.adapterType,
+ sessionId: session.legacySessionId,
+ lastRunId: run.id,
+ lastRunStatus: run.status,
+ lastError: run.error ?? null,
+ updatedAt: new Date(),
+ })
+ .where(eq(agentRuntimeState.agentId, agent.id));
+
+ }
+
+ return {
+ appendRunEvent,
+ escalatePlanApprovalResumeFailureNeedsAttention,
+ recordPlanApprovalResumeFailureRetry,
+ setRunStatusIfRunning,
+ setWakeupStatus,
+ mergeRunStopMetadataForAgent,
+ classifyAndPersistRunLiveness,
+ setRunStatusFromLive,
+ setRunStatus,
+ publishRunLifecyclePluginEventData,
+ publishRunLifecyclePluginEvent,
+ persistRunProcessMetadata,
+ recordCurrentHeartbeatRunRuntimeProgress,
+ completeSkillTestRunForHeartbeatOutcome,
+ refreshContinuationSummaryForRun,
+ findRunIssueComment,
+ findLatestCompletedFinalAgentMessage,
+ parseMaxTurnContinuationPolicy,
+ finalizeIssueCommentPolicy,
+ handleIssueReviewPathDisposition,
+ handleRunLivenessContinuation,
+ handleSuccessfulRunHandoff,
+ updateRuntimeState,
+ terminalizeRunOnLeaseRelease,
+ clearDetachedRunWarning,
+ };
+}