The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runner): recover journaled completion proposals (#16962)
Dottad84061b92368
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex 94a47a3963..aea7fb9bd8 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -1625,6 +1625,14 @@ controller must recover it through the authenticated ownership checks. Thispreserves active work and queued messages without treating a server restart asuser cancellation.+Recovery restores a structured result proposal from the durable run events when+its append survived but the active turn's session checkpoint did not. Advancing+the replay cursor must preserve that proposal for the exact run, session, source,+and turn. Conflicting or invalid proposals fail closed. A proposal from another+turn cannot complete the current turn, and a later started turn prevents adoption+of the older proposal. The provider must still supply terminal evidence before+normal finalization; recovery must not repeat completed work to obtain its result.+When a native turn creates a governed approval wait, its checkpoint retains theexact committed triggering event and yielded disposition before interrupting theprovider. Shutdown gives these turns a shared, bounded 20-second window to finishdiff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.mdindex bd9283471a..0801a2bb06 100644--- a/maps/agent-access-map/recovery.md+++ b/maps/agent-access-map/recovery.md@@ -69,6 +69,14 @@ prove successful recovery.## `bounded-continuation`+Controller recovery can restore the exact active turn's saved completion proposal+before reconnecting its provider. It requires matching run/session/source/turn+identity and subsequent provider termination. This adds no agent retry, completion,+or recovery-resolution authority. The+[runtime regressions](../../packages/paperclip-runner/src/native-session-runtime.test.ts)+cover the driver boundary with simulated transport; live hosted recovery remains+separate qualification.+Review: `baseline`.API: `present` — representative operation IDs:diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.mdindex 4e12774c94..0e0a0bfd1c 100644--- a/maps/feature-map/recovery.md+++ b/maps/feature-map/recovery.md@@ -134,6 +134,14 @@ and rejected identity evidence. The Runner restart tests require positive exitevidence before replacement. This coverage does not prove recovery of ahistorical retained sandbox or behavior during a live provider outage.+For completion recovery, restart after a native completion proposal is appended+but before its session checkpoint. Verify that the original result is finalized+after provider termination, with no new provider turn or repeated external action.+The [runtime regressions](../../packages/paperclip-runner/src/native-session-runtime.test.ts)+exercise the real Codex driver with a simulated transport, including a still-active+provider and invalid, conflicting, duplicate, or stale proposals. This does not+qualify a hosted sandbox restart.+The same database suite pauses an original preparation while recovery claimsthe remote run. It then verifies that the old preparation yields without openinga provider or changing the recovery owner, for both live and expired leases.diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.tsindex 68ab9d278d..8bc9af6512 100644--- a/packages/paperclip-runner/src/native-session-runtime.test.ts+++ b/packages/paperclip-runner/src/native-session-runtime.test.ts@@ -1,6 +1,8 @@import { createHash } from "node:crypto";import { resolveQualifiedAcpxProfile } from "./drivers/acpx/qualified-profiles.js";import { describe, expect, it, vi } from "vitest";+import { createCodexNativeSessionBackend } from "./backends/codex-native-backend.js";+import { FakeCodexTransport, WORKSPACE } from "./drivers/codex/codex-app-server-driver.test-support.js";import type { ControlPlanePort } from "./contracts/control-plane-port.js";import type { NativeExecutionInputV1, NativeExecutionInputV5 } from "./contracts/native-execution.js";@@ -226,6 +228,117 @@ function highestContiguous(events: PrpEvent[]): number {}describe("executeNativeSession recovery", () => {+ it.each([+ "completed", "pending", "duplicate", "conflicting", "other turn", "newer turn",+ "wrong session", "invalid",+ ])("recovers a journaled completion proposal after restart: %s", async (scenario) => {+ const originalTransport = new FakeCodexTransport();+ const recoveredTransport = new FakeCodexTransport();+ const transports = [originalTransport, recoveredTransport];+ const executionInput = { ...input, workspace: { ...input.workspace, cwd: WORKSPACE } };+ const backend = createCodexNativeSessionBackend(executionInput, {+ runnerInstanceId: "runner-recovery",+ transportFactory: () => transports.shift()!,+ });+ const original = await backend.openSession({ identity, workingDirectory: WORKSPACE });+ const journal: PrpEvent[] = [];+ const iterator = original.events()[Symbol.asyncIterator]();+ const drainThrough = async (eventType: string) => {+ while (true) {+ const next = await iterator.next();+ if (next.done) throw new Error(`stream closed before ${eventType}`);+ journal.push(structuredClone(next.value));+ if (next.value.eventType === eventType) return;+ }+ };+ await original.startTurn({ message: { role: "user", text: "Finish the assigned review." } });+ originalTransport.push("turn/started", {+ threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" },+ });+ await drainThrough("turn.started");+ const checkpoint = await original.snapshot();+ expect(checkpoint.semanticResult).toBeNull();+ originalTransport.push("paperclip/runResult", {+ threadId: "thread-1", turnId: "turn-1", result,+ });+ await drainThrough("run.result.proposed");+ expect((await original.snapshot()).semanticResult).toEqual(result);+ const proposal = journal.at(-1)!;+ if (scenario === "other turn") proposal.turnId = "prior-turn";+ if (scenario === "wrong session") proposal.normalizedSessionId = "other-session";+ if (scenario === "invalid") proposal.payload = { reportedWorkDisposition: "done" };+ if (["duplicate", "conflicting", "newer turn"].includes(scenario)) {+ journal.push({+ ...proposal, sourceSeq: proposal.sourceSeq + 1,+ sourceEventId: `${proposal.sourceEventId}:next`,+ ...(scenario === "newer turn"+ ? { eventType: "turn.started" as const, turnId: "newer-turn", payload: {} }+ : { payload: scenario === "conflicting" ? { ...result, summary: "A different result." } : result }),+ });+ }+ const persistedProposalCount = journal.filter(event => event.eventType === "run.result.proposed").length;+ await original.close({ reason: "controller disconnected after persisting the proposal" });+ recoveredTransport.readResponse = { thread: {+ id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE,+ turns: [{ id: "turn-1", status: scenario === "pending" ? "inProgress" : "completed", items: [+ { id: "answer", type: "agentMessage", text: "The review was published." },+ ] }],+ } };+ const completeRun = vi.fn(async () => undefined);+ const checkpoints: PersistedNativeSession[] = [];+ const port: ControlPlanePort = {+ async openRun() {},+ async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async checkpointSession(value) { checkpoints.push(structuredClone(value)); },+ async appendEvent(event) {+ const existing = journal.find(value => value.sourceEventId === event.sourceEventId);+ if (existing) expect(event).toEqual(existing);+ else journal.push(structuredClone(event as PrpEvent));+ return { cursor: journal.length, highestContiguousSourceSeq: event.sourceSeq,+ disposition: existing ? "duplicate" : "committed" };+ },+ async replayEvents(query) {+ const events = journal.filter(event => event.sourceInstanceId === query.sourceInstanceId);+ return {+ events: structuredClone(events.filter(event => event.sourceSeq > query.afterSourceSeq).slice(0, query.limit)),+ highestContiguousSourceSeq: highestContiguous(events),+ };+ },+ completeRun,+ };+ const execution = executeNativeSession({+ input: executionInput, backend, controlPlane: port,+ runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery",+ });+ if (["conflicting", "wrong session", "invalid"].includes(scenario)) {+ await expect(execution).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError);+ expect(completeRun).not.toHaveBeenCalled();+ expect(recoveredTransport.calls).toEqual([]);+ return;+ }+ if (["other turn", "newer turn"].includes(scenario)) {+ await expect(execution).rejects.toThrow("native_finalization_missing");+ expect(completeRun).not.toHaveBeenCalled();+ expect(recoveredTransport.calls.filter(call => call.method === "turn/start")).toEqual([]);+ return;+ }+ if (scenario === "pending") {+ await vi.waitFor(() => expect(checkpoints[0]?.semanticResult).toEqual(result));+ expect(completeRun).not.toHaveBeenCalled();+ recoveredTransport.push("turn/completed", {+ threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] },+ });+ }+ await expect(execution).resolves.toMatchObject({ result, turnId: "turn-1", terminal });+ expect(completeRun).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ result, turnId: "turn-1" }),+ expect.anything(),+ );+ expect(recoveredTransport.calls.filter(call => call.method === "turn/start")).toEqual([]);+ expect(journal.filter(event => event.eventType === "run.result.proposed")).toHaveLength(persistedProposalCount);+ expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, activeTurnId: null });+ });+it.each([{ label: "continues a restart-lost Codex turn", allowed: true, error: { code: "provider_turn_lost_on_restore", recoverable: true } },{ label: "requires the control plane restart claim", allowed: false, error: { code: "provider_turn_lost_on_restore", recoverable: true } },diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.tsindex c11c446aa6..e7875ec63a 100644--- a/packages/paperclip-runner/src/native-session-runtime.ts+++ b/packages/paperclip-runner/src/native-session-runtime.ts@@ -1566,7 +1566,7 @@ function checkpointCursor(cursor: string | null | undefined): number {return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0;}-async function reconcileRecoveryCursor(input: {+async function reconcileRecoveryCheckpoint(input: {controlPlane: ControlPlanePort;checkpoint: PersistedNativeSession;runId: string;@@ -1576,6 +1576,8 @@ async function reconcileRecoveryCursor(input: {const checkpointHighWater = checkpointCursor(input.checkpoint.cursor);let afterSourceSeq = checkpointHighWater;let persistedHighWater = checkpointHighWater;+ let proposalTurnId = input.checkpoint.activeTurnId ?? null;+ let recoveredResult: PrpStructuredRunResult | null = null;while (true) {const replay = await input.controlPlane.replayEvents({@@ -1588,6 +1590,34 @@ async function reconcileRecoveryCursor(input: {);input.signal.throwIfAborted();if (replay.events.length === 0) break;+ for (const event of replay.events) {+ if (event.eventType === "turn.started" && event.turnId !== proposalTurnId) {+ proposalTurnId = null;+ recoveredResult = null;+ }+ if (+ proposalTurnId &&+ event.turnId === proposalTurnId &&+ event.eventType === "run.result.proposed"+ ) {+ const validation = validatePrpStructuredRunResult(event.payload);+ if (+ !validation.ok ||+ event.runId !== input.runId ||+ event.normalizedSessionId !== input.checkpoint.identity.sessionId ||+ event.sourceInstanceId !== input.sourceInstanceId+ ) {+ throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict");+ }+ const previous = recoveredResult ?? input.checkpoint.semanticResult;+ const normalize = (result: PrpStructuredRunResult) =>+ canonicalJson(input.controlPlane.normalizeReplayPayload?.(result) ?? result);+ if (previous && normalize(previous) !== normalize(validation.result)) {+ throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict");+ }+ recoveredResult = validation.result;+ }+ }const pageHighWater = replay.events.reduce((highest, event) => Math.max(highest, event.sourceSeq),afterSourceSeq,@@ -1600,11 +1630,16 @@ async function reconcileRecoveryCursor(input: {}if (persistedHighWater === checkpointHighWater &&- input.checkpoint.cursor === String(checkpointHighWater)+ input.checkpoint.cursor === String(checkpointHighWater) &&+ recoveredResult === null) {return input.checkpoint;}- return { ...input.checkpoint, cursor: String(persistedHighWater) };+ return {+ ...input.checkpoint,+ cursor: String(persistedHighWater),+ ...(recoveredResult === null ? {} : { semanticResult: recoveredResult }),+ };}async function replayCheckpointedTurnTerminal(input: {@@ -2036,7 +2071,7 @@ export async function executeNativeSession(timeoutMs: recoveryTimeoutMs,timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`,operation: (signal) =>- reconcileRecoveryCursor({+ reconcileRecoveryCheckpoint({controlPlane: options.controlPlane,checkpoint: recoveryCheckpoint,runId: input.binding.runId,