The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: keep pre-dispatch review waits out of execution recovery (#15024)
Dotta9ae3d8db3dc9
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex 72a0cb9832..46b9e87e7f 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -488,6 +488,8 @@ A new user message can continue a terminal native run whose process fields wereThe task thread exposes the guarded Retry action for failed or timed-out conversation runs and native preparation cancelled before provider startup when the never-started proof is verified. The server projects this eligibility on the recovery notice and rechecks it on Retry. Cleanup quarantine retains its inspection path, and non-conversation reconciliation gates remain enforced. Pending decisions, active execution, pause, budget, dependency, and ownership gates remain in force. A refused Retry reports its reason inline and remains available for another attempt.+A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed.+A valid recovery action must name:- the source issue and companydiff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 07bef2d6d8..d88dac7d32 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -6221,14 +6221,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", assigneeAgentId: agentId });});- it("converts a continuation parked for review into a dependency wait on its open sub-tasks", async () => {- const { companyId, agentId, issueId } = await seedStrandedIssueFixture({+ it.each([false, "receipt", "launch", "provider", "cleanup"])("converts a continuation parked for review into a dependency wait on its open sub-tasks (dispatch receipt: %s)", async dispatchReceipt => {+ const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({status: "in_progress",runStatus: "cancelled",retryReason: "issue_continuation_needed",runErrorCode: "issue_continuation_waiting_on_review",runError: "Continuation parked: issue is waiting on review/approval",});+ if (dispatchReceipt) {+ const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));+ await db.update(heartbeatRuns).set({ startedAt: null,+ resultJson: { stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate" },+ }).where(eq(heartbeatRuns.id, run.id));+ }+ if (dispatchReceipt === "launch" || dispatchReceipt === "provider") {+ await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1,+ eventType: dispatchReceipt === "launch" ? "native.process_start_requested" : "provider.event",+ stream: "system", level: "info", message: "Contradictory execution evidence" });+ } else if (dispatchReceipt === "cleanup") {+ await db.insert(environmentLeases).values({ companyId, heartbeatRunId: runId,+ provider: "local", status: "pending_cleanup", cleanupStatus: "failed" });+ }const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;const openChildTodoId = randomUUID();const openChildInProgressId = randomUUID();@@ -6270,6 +6284,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {const heartbeat = heartbeatService(db);const result = await heartbeat.reconcileStrandedAssignedIssues();+ if (dispatchReceipt && dispatchReceipt !== "receipt") {+ expect(result.waitingOnReviewResolved).toBe(0);+ expect(result.escalated).toBe(1);+ expect(await getExecutionBlocker(db, companyId, issueId)).toMatchObject({+ cause: "legacy_execution_requires_reconciliation", runId, canRetry: false,+ });+ return;+ }expect(result.waitingOnReviewResolved).toBe(1);expect(result.escalated).toBe(0);expect(result.issueIds).toEqual([issueId]);diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.tsindex 812cdec546..9bc674d5fa 100644--- a/server/src/modules/wake-queue/adapters/postgres.test.ts+++ b/server/src/modules/wake-queue/adapters/postgres.test.ts@@ -9,6 +9,8 @@ import {agents,companies,createDb,+ environmentLeases,+ heartbeatRunEvents,heartbeatRuns,issueComments,issueRecoveryActions,@@ -59,6 +61,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {afterEach(async () => {await db.delete(activityLog);await db.delete(issueComments);+ await db.delete(heartbeatRunEvents);+ await db.delete(environmentLeases);// `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`,// so the run row must go first.await db.delete(heartbeatRuns);@@ -416,6 +420,42 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {expect(wake.status).toBe("deferred_issue_execution");});+ it.each(["clean", "launch", "cleanup"])("promotes saved input only after a verified pre-dispatch review wait (%s)", async evidence => {+ const companyId = await seedCompany(), agentId = await seedAgent({ companyId });+ const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });+ const runId = await seedRun({ companyId, agentId, status: "cancelled", contextSnapshot: { issueId },+ errorCode: "issue_continuation_waiting_on_review" });+ await db.update(heartbeatRuns).set({ finishedAt: new Date(Date.now() - 1000), resultJson: {+ stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate",+ } }).where(eq(heartbeatRuns.id, runId));+ if (evidence === "launch") await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1,+ eventType: "native.process_start_requested", stream: "system", level: "info" });+ if (evidence === "cleanup") await db.insert(environmentLeases).values({ companyId, heartbeatRunId: runId,+ provider: "local", status: "pending_cleanup", cleanupStatus: "failed" });+ const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user",+ authorUserId: "responsible-user", body: "Continue with this new message." }).returning();+ const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user",+ payload: { commentId: comment.id, _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } } });+ const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, stubDeps),+ recovery: { escalateStrandedAssignedIssue: async () => {}, escalateStrandedRecoveryIssueInPlace: async () => {} } });+ const result = await release({ companyId, runId, now: new Date() });+ const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));+ const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId));+ if (evidence !== "clean") {+ expect(result.outcome.kind).toBe("released");+ expect(runs).toHaveLength(1);+ expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null });+ return;+ }+ expect(result.outcome.kind).toBe("promoted");+ expect(runs).toHaveLength(2);+ const successor = runs.find(run => run.status === "queued")!;+ expect(wake.runId).toBe(successor.id);+ expect(successor.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] });+ await release({ companyId, runId, now: new Date() });+ expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(2);+ });+it("leaves deferred work untouched until the effective execution hold clears", async () => {const companyId = await seedCompany();const agentId = await seedAgent({ companyId });diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.tsindex 60b0e359b7..57d256e70b 100644--- a/server/src/modules/wake-queue/adapters/postgres.ts+++ b/server/src/modules/wake-queue/adapters/postgres.ts@@ -19,7 +19,7 @@ import {nativeRunFinalizations,} from "@paperclipai/db";import { hasConversationContinuationPolicy } from "../../../services/conversation-continuation.js";-import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js";+import { legacyExecutionNeedsReconciliationWithEvidence } from "../../../services/legacy-execution-recovery.js";import {authorizeFailedChatRunRetryWake,FailedChatRunRetryAuthorizationError,@@ -1104,7 +1104,7 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdissueStatus: issueRow?.status ?? "",hasAssigneeUser: Boolean(issueRow?.assigneeUserId),assigneeAgentMatchesRunAgent: issueRow?.assigneeAgentId === run.agentId,- legacyExecutionNeedsReconciliation: legacyExecutionNeedsReconciliation(run),+ legacyExecutionNeedsReconciliation: await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, run),// An operator stop never promotes old queued work by itself. The// next explicit wake adopts those messages atomically when it// queues a run.diff --git a/server/src/services/cancelled-native-startup.ts b/server/src/services/cancelled-native-startup.tsindex 9bcadce31c..53a17527f4 100644--- a/server/src/services/cancelled-native-startup.ts+++ b/server/src/services/cancelled-native-startup.ts@@ -1,3 +1,4 @@+import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js";import { and, eq, inArray, isNotNull, or } from "drizzle-orm";import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";import { claimedAdapterType } from "./conversation-continuation.js";@@ -38,7 +39,8 @@ export async function isCancelledNativeStartup(db: Db, run: Run, coordinator: Co(run.resultJson === null || cancellation?.beforeNativeSelection === true) &&Boolean(run.controllerBootId && run.controllerBootId !== legacyControllerBootId &&run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt <= new Date());- const beforeSelection = historicalBeforeSelection || run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&+ const beforeReviewDispatch = isPreDispatchReviewWait(run) && !coordinator;+ const beforeSelection = beforeReviewDispatch || historicalBeforeSelection || run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&!run.nativeSessionId && !coordinator && claimedAdapterType(run) === "paperclip_runner" &&cancellation?.beforeNativeSelection === true;const neverClaimed = run.runtimeMode === "native" && coordinator &&@@ -47,7 +49,7 @@ export async function isCancelledNativeStartup(db: Db, run: Run, coordinator: Co!coordinator.controllerPid && !coordinator.leaseOwner && !coordinator.leaseExpiresAt &&!coordinator.resultId && !coordinator.failureDetail?.successorRunId;if (!beforeSelection && !neverClaimed) return false;- const settled = typeof run.resultJson?.startupPreparationSettledAt === "string";+ const settled = beforeReviewDispatch || typeof run.resultJson?.startupPreparationSettledAt === "string";// The old preparer can still be unwinding even though the run is terminal.if (!settled && run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) return false;const leases = await db.select().from(environmentLeases).where(and(diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.tsindex 2d70f70add..14c316b5a0 100644--- a/server/src/services/execution-blocker.ts+++ b/server/src/services/execution-blocker.ts@@ -1,3 +1,4 @@+import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js";import { and, desc, eq, gt, inArray, not, or, sql } from "drizzle-orm";import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";import { z } from "zod";@@ -79,13 +80,15 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: stor consumed.context_snapshot->'wakeCommentIds' @> jsonb_build_array(${issueComments.id}::text)))`,)) : [];const cancellation = run ? readRunCancellation(run.resultJson) : null;- const runError = cancellation?.reason ?? (run?.status === "cancelled"- ? "Execution was cancelled; its source was not recorded."- : run?.error);const eligibleContinuation = Boolean(run && conversation?.assigneeAgentId === run.agentId &&!["done", "cancelled"].includes(conversation.status) && canContinueCancelledRun(run));const canRetry = Boolean(run && conversation?.assigneeAgentId === run.agentId &&!["done", "cancelled"].includes(conversation.status) && await canRetryStoppedRun(db, run));+ const runError = run && canRetry && isPreDispatchReviewWait(run)+ ? "Waiting for review; this continuation never started."+ : cancellation?.reason ?? (run?.status === "cancelled"+ ? "Execution was cancelled; its source was not recorded."+ : run?.error);const [chatBinding] = eligibleContinuation || canRetry ? await db.select({state: chatConversations.state, endpointStatus: chatEndpoints.status,connectionStatus: toolConnections.status, connectionEnabled: toolConnections.enabled,@@ -107,7 +110,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st} else {nextAction = "Send a new chat message to continue this conversation.";}- } else if (run?.status === "cancelled" && !eligibleContinuation && action.cause === "legacy_execution_requires_reconciliation") {+ } else if (run?.status === "cancelled" && !eligibleContinuation && !canRetry && action.cause === "legacy_execution_requires_reconciliation") {nextAction += " Inspect the run before sending a new message to request continuation.";}diff --git a/server/src/services/explicit-native-continuation.test.ts b/server/src/services/explicit-native-continuation.test.tsindex 9ddb4a898b..cd0d3ac143 100644--- a/server/src/services/explicit-native-continuation.test.ts+++ b/server/src/services/explicit-native-continuation.test.ts@@ -613,6 +613,73 @@ const support = await getEmbeddedPostgresTestSupport();return f;}+ async function seedCancelledReviewWait() {+ const f = await seedHistoricalCancelledPreparation();+ await db.update(heartbeatRuns).set({ startedAt: null, executionStage: null,+ controllerBootId: null, controllerLeaseExpiresAt: null, runnerProfileJson: null,+ errorCode: "issue_continuation_waiting_on_review",+ resultJson: { stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate" },+ }).where(eq(heartbeatRuns.id, f.sourceRunId));+ return f;+ }++ it.each(["message", "retry"])("recovers a pre-dispatch review wait through an explicit %s", async kind => {+ const f = await seedCancelledReviewWait();+ const notice = await getExecutionBlocker(db, f.companyId, f.issueId);+ expect(notice).toMatchObject({ canRetry: true, runError: "Waiting for review; this continuation never started." });+ expect(notice?.nextAction).not.toContain("Inspect the run before sending a new message");+ await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" });+ const successor = await heartbeatService(db).wakeup(f.agentId, { source: kind === "retry" ? "on_demand" : "automation", triggerDetail: "manual",+ reason: kind === "retry" ? "retry_failed_run" : "issue_commented",+ ...(kind === "retry" ? { failedRunId: f.sourceRunId } : {}), requestedByActorType: "user", requestedByActorId: "board",+ payload: { issueId: f.issueId, ...(kind === "message" ? { commentId: f.commentId } : {}) },+ contextSnapshot: { issueId: f.issueId, ...(kind === "message" ? { wakeCommentId: f.commentId } : {}) },+ });+ expect(successor).toMatchObject({ status: "queued", contextSnapshot: { forceFreshSession: true,+ previousRunId: f.sourceRunId, explicitUserContinuation: { commentId: kind === "message" ? f.commentId : null } } });+ expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();+ expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId)))[0].status).toBe("cancelled");+ });++ it("reconsiders saved input after a pre-dispatch review wait exactly once", async () => {+ const f = await seedCancelledReviewWait();+ await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" });+ const leaseId = randomUUID();+ await db.insert(environmentLeases).values({ id: leaseId, companyId: f.companyId,+ heartbeatRunId: f.sourceRunId, provider: "local", status: "pending_cleanup", cleanupStatus: "failed" });+ await heartbeatService(db).wakeup(f.agentId, { source: "automation", reason: "issue_commented",+ requestedByActorType: "user", requestedByActorId: "board", payload: { issueId: f.issueId, commentId: f.commentId },+ contextSnapshot: { issueId: f.issueId, wakeCommentId: f.commentId } });+ const [waiting] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId));+ expect(waiting.status).toBe("deferred_issue_execution");+ await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "succeeded" })+ .where(eq(environmentLeases.id, leaseId));+ await db.update(agentWakeupRequests).set({ updatedAt: new Date(0) }).where(eq(agentWakeupRequests.id, waiting.id));+ await Promise.all([heartbeatService(db).resumeExecutionWaitComments(), heartbeatService(db).resumeExecutionWaitComments()]);+ const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));+ expect(successors).toHaveLength(1);+ expect(successors[0].contextSnapshot).toMatchObject({ wakeCommentIds: [f.commentId],+ explicitUserContinuation: { commentId: f.commentId }, forceFreshSession: true });+ expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, waiting.id)))[0])+ .toMatchObject({ status: "coalesced", runId: successors[0].id });+ expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();+ });++ it.each(["launch", "provider", "cleanup"])("retains a pre-dispatch review wait with contradictory %s evidence", async kind => {+ const f = await seedCancelledReviewWait();+ if (kind === "cleanup") await db.insert(environmentLeases).values({ companyId: f.companyId,+ heartbeatRunId: f.sourceRunId, provider: "local", status: "pending_cleanup", cleanupStatus: "failed" });+ else await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.sourceRunId,+ agentId: f.agentId, seq: 1, eventType: kind === "launch" ? PROCESS_START_REQUESTED : "provider.event",+ stream: "system", level: "info", message: "Contradictory execution evidence" });+ try {+ expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ canRetry: false });+ expect(await admit(f)).toBeNull();+ } finally {+ await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, f.sourceRunId));+ }+ });+it.each([false, true].flatMap(receipt => ["message", "retry"].map(kind => ({ receipt, kind }))))("recovers historical native preparation through an explicit $kind (receipt: $receipt)", async ({ receipt, kind }) => {const f = await seedHistoricalCancelledPreparation(receipt);diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 88f73ea4c6..9ca39eb35c 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -44,7 +44,7 @@ import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./import { claimQueuedNativeReviewRun } from "./native-runtime/native-review-dispatch.js";import { buildNativeReviewRequest } from "./native-runtime/native-review-prompt.js";import {- legacyExecutionNeedsReconciliation,+ legacyExecutionNeedsReconciliationWithEvidence,settleInterruptedNativeBootstrap,terminalizeLegacyExecution,} from "./legacy-execution-recovery.js";@@ -11771,7 +11771,7 @@ export function heartbeatService("Native execution recovery owns this provider failure; a quota monitor cannot start a replacement.",);}- if (legacyExecutionNeedsReconciliation(sourceRun)) {+ if (await legacyExecutionNeedsReconciliationWithEvidence(db, sourceRun)) {await terminalizeLegacyExecution({db,run: sourceRun,@@ -12912,7 +12912,7 @@ export function heartbeatService(}const updated =previousStatus &&- legacyExecutionNeedsReconciliation({+ await legacyExecutionNeedsReconciliationWithEvidence(db, {...previousStatus,status,...patch,@@ -13002,7 +13002,7 @@ export function heartbeatService(}const updated =previousStatus &&- legacyExecutionNeedsReconciliation({+ await legacyExecutionNeedsReconciliationWithEvidence(db, {...previousStatus,status,...patch,@@ -14519,7 +14519,7 @@ export function heartbeatService(// Native sessions have their own fenced same-run controller. Legacy// bootstrap recovery shares the durable delay and incident counter with// transient retries; process loss must not open a second retry budget.- if (run.runtimeMode === "native" || legacyExecutionNeedsReconciliation(run))+ if (run.runtimeMode === "native" || await legacyExecutionNeedsReconciliationWithEvidence(db, run))return null;const scheduled = await scheduleBoundedRetryForRun(run, agent, { now });return scheduled.outcome === "scheduled" ? scheduled.run : null;@@ -15494,7 +15494,7 @@ export function heartbeatService(};}- if (legacyExecutionNeedsReconciliation(run)) {+ if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) {return {outcome: "not_scheduled" as const,reason:@@ -25812,7 +25812,7 @@ export function heartbeatService(await scheduleBoundedRetryForRun(livenessRun, agent);} else if (outcome === "failed" &&- !legacyExecutionNeedsReconciliation(livenessRun)+ !(await legacyExecutionNeedsReconciliationWithEvidence(db, livenessRun))) {await scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun,diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.tsindex 52397bf506..b1563e07d1 100644--- a/server/src/services/legacy-execution-recovery.test.ts+++ b/server/src/services/legacy-execution-recovery.test.ts@@ -1,3 +1,4 @@+import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js";import { expect, it } from "vitest";import { legacyExecutionNeedsReconciliation } from "./legacy-execution-recovery.js";@@ -86,3 +87,23 @@ it("retains conversation retry eligibility for a transient restore lock timeout"workspaceRestoreFailure: "restore_lock_timeout", conversationContinuation: "continue_conversation_v1",} })).toBe(false);});+++it("recognizes only an unclaimed pre-dispatch review wait receipt", () => {+ const run = { runtimeMode: "legacy", status: "cancelled", errorCode: "issue_continuation_waiting_on_review",+ startedAt: null, runtimeModeResolvedAt: null, processPid: null, processGroupId: null,+ processStartedAt: null, nativeIssueId: null, nativeSessionId: null, sessionIdAfter: null,+ controllerBootId: null, controllerLeaseExpiresAt: null, executionStage: null,+ resultJson: { stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate" } };+ expect(isPreDispatchReviewWait(run)).toBe(true);+ expect(legacyExecutionNeedsReconciliation(run)).toBe(true);+ for (const patch of [+ { startedAt: new Date() }, { processPid: 123 }, { processGroupId: 123 },+ { processStartedAt: new Date() }, { nativeSessionId: "provider-session" }, { nativeIssueId: "native-task" },+ { sessionIdAfter: "provider-session" }, { runtimeModeResolvedAt: new Date() },+ { controllerBootId: "live-controller" }, { controllerLeaseExpiresAt: new Date() }, { executionStage: "preparing" },+ { errorCode: "operator_interrupted" }, { resultJson: {} },+ { resultJson: { ...run.resultJson, stopReason: "cancelled" } },+ { resultJson: { ...run.resultJson, workspaceRestoreFailure: "restore_unsafe_archive" } },+ ]) expect(isPreDispatchReviewWait({ ...run, ...patch })).toBe(false);+});diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.tsindex 98c4c03f7e..6ab86f7ee4 100644--- a/server/src/services/legacy-execution-recovery.ts+++ b/server/src/services/legacy-execution-recovery.ts@@ -1,3 +1,4 @@+import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js";import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js";import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js";@@ -53,6 +54,12 @@ export function legacyExecutionNeedsReconciliation();}+/** Review-wait receipts are only exempt after retained execution evidence agrees.+ * The synchronous classifier stays conservative for callers without a DB proof. */+export async function legacyExecutionNeedsReconciliationWithEvidence(db: Db, run: Run): Promise<boolean> {+ return legacyExecutionNeedsReconciliation(run) && !(await isPreDispatchReviewWaitVerified(db, run));+}+/** Persist the failed legacy run, owned lock release and operator decision together. */export async function terminalizeLegacyExecution(input: {db: Db;diff --git a/server/src/services/pre-dispatch-review-wait.ts b/server/src/services/pre-dispatch-review-wait.tsnew file mode 100644index 0000000000..08ea49b498--- /dev/null+++ b/server/src/services/pre-dispatch-review-wait.ts@@ -0,0 +1,51 @@+import { and, eq, inArray, isNotNull, or } from "drizzle-orm";+import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";+import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js";+import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";++type Run = typeof heartbeatRuns.$inferSelect;+type DispatchFields = "startedAt" | "runtimeModeResolvedAt" | "processPid" | "processGroupId" |+ "processStartedAt" | "nativeIssueId" | "nativeSessionId" | "sessionIdAfter" |+ "controllerBootId" | "controllerLeaseExpiresAt" | "executionStage";++/** The queued-run gate owns this receipt before execution authority is claimed.+ * A deliberate review wait has no provider actions to reconcile. The error code+ * alone is not proof; missing or conflicting dispatch evidence retains the hold. */+export function isPreDispatchReviewWait(+ run: Pick<Run, "runtimeMode" | "status" | "errorCode" | "resultJson"> & Partial<Pick<Run, DispatchFields>>,+): boolean {+ return run.runtimeMode === "legacy" && run.status === "cancelled" &&+ run.errorCode === "issue_continuation_waiting_on_review" &&+ run.resultJson?.stopReason === run.errorCode &&+ run.resultJson?.timeoutSource === "stale_queued_run_gate" &&+ run.resultJson?.workspaceRestoreFailure !== "restore_unsafe_archive" &&+ run.startedAt === null && run.runtimeModeResolvedAt === null &&+ run.processPid === null && run.processGroupId === null && run.processStartedAt === null &&+ run.nativeIssueId === null && run.nativeSessionId === null && run.sessionIdAfter === null &&+ run.controllerBootId === null && run.controllerLeaseExpiresAt === null && run.executionStage === null;+}++/** The row receipt is a candidate, not permission to discard conflicting+ * retained execution or cleanup evidence. Historical false holds must be+ * checked against the same evidence that gates explicit user admission. */+export async function isPreDispatchReviewWaitVerified(db: Db, run: Run): Promise<boolean> {+ if (!isPreDispatchReviewWait(run)) return false;+ const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(+ eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id),+ )).limit(1);+ if (coordinator) return false;+ const leases = await db.select().from(environmentLeases).where(and(+ eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),+ ));+ if (leases.some(lease => lease.provider === "local"+ ? !lease.releasedAt || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed"+ : !hasRemoteTerminationReceipt(lease))) return false;+ const [execution] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(+ eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),+ or(isNotNull(heartbeatRunEvents.sourceEventId),+ inArray(heartbeatRunEvents.eventType, ["adapter.invoke", PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED,+ "harness.ready", "session.started", "session.resumed", "session.updated", "turn.started",+ "provider.event", "provider.rpc_result", "tool.execution.started"])),+ )).limit(1);+ return !execution;+}diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex 113ea5ce1c..99ed604dee 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -89,7 +89,7 @@ import { budgetService } from "../budgets.js";import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js";import { issueRecoveryActionService } from "../issue-recovery-actions.js";import {- legacyExecutionNeedsReconciliation,+ legacyExecutionNeedsReconciliationWithEvidence,terminalizeLegacyExecution,} from "../legacy-execution-recovery.js";import { issueTreeControlService } from "../issue-tree-control.js";@@ -1923,7 +1923,7 @@ export function recoveryService(// Failure recovery shares the durable incident budget and delay. It// cannot fall through into the productive-work continuation queue.if (predecessor.runtimeMode === "native") return null;- if (legacyExecutionNeedsReconciliation(predecessor)) {+ if (await legacyExecutionNeedsReconciliationWithEvidence(db, predecessor)) {await terminalizeLegacyExecution({db,run: predecessor,@@ -4603,7 +4603,7 @@ export function recoveryService(eq(heartbeatRuns.id, executionRecoverySource.id),),);- if (source && legacyExecutionNeedsReconciliation(source)) {+ if (source && await legacyExecutionNeedsReconciliationWithEvidence(db, source)) {await terminalizeLegacyExecution({db,run: source,