The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(runner): add Codex ACPX harness driver (#12405)
Dotta74aabb7ea6c5
diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.tsnew file mode 100644index 0000000000..1806909fcf--- /dev/null+++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts@@ -0,0 +1,1483 @@+import { describe, expect, it, vi } from "vitest";++import type { AcpRuntimeEvent } from "acpx/runtime";++import {+ PRP_BLOCK_TOOL_NAME,+ PRP_COMPLETION_TOOL_NAME,+} from "../../contracts/completion-result.js";+import type { PrpEvent } from "../../protocol/replay-contract.js";+import { validatePrpEvent } from "../../protocol/replay-contract.js";+import {+ CodexAcpxDriver,+ type CodexAcpxDriverDependencies,+ type CodexAcpxDriverOptions,+} from "./codex-acpx-driver.js";+import type {+ AcpxRuntimeTurn,+ OpenAcpxRuntimeHostOptions,+} from "./runtime-host.js";++describe("Codex ACPX harness driver", () => {+ it("rejects a pre-aborted open before starting host admission", async () => {+ const fixture = driverFixture();+ const controller = new AbortController();+ const cancellation = new Error("session open cancelled");+ controller.abort(cancellation);++ await expect(+ fixture.driver.openSession({+ runId: "run-pre-abort",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ signal: controller.signal,+ }),+ ).rejects.toBe(cancellation);++ expect(fixture.openHost).not.toHaveBeenCalled();+ });++ it("forwards the exact open signal to host admission", async () => {+ const fixture = driverFixture();+ const controller = new AbortController();+ const session = await fixture.driver.openSession({+ runId: "run-open-signal",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ signal: controller.signal,+ });++ expect(fixture.hostOptions?.signal).toBe(controller.signal);+ await session.close({ reason: "signal forwarding verified" });+ });++ it("closes and quarantines a host that resolves after open is aborted", async () => {+ const hostAdmission = deferred<ReturnType<typeof fakeHost>>();+ const fixture = driverFixture(+ {},+ {+ openHost: () => hostAdmission.promise,+ closeSettlementTimeoutMs: 5,+ },+ );+ fixture.host.close.mockRejectedValueOnce(+ new Error("late host close failed once"),+ );+ const controller = new AbortController();+ const cancellation = new Error("host admission cancelled");+ const opening = fixture.driver.openSession({+ runId: "run-late-host",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ signal: controller.signal,+ });+ await vi.waitFor(() => expect(fixture.openHost).toHaveBeenCalledOnce());++ controller.abort(cancellation);+ await expect(opening).rejects.toBe(cancellation);+ hostAdmission.resolve(fixture.host);++ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));+ expect(fixture.host.close).toHaveBeenNthCalledWith(1, {+ reason: "Codex ACPX host resolved after admission was aborted",+ });+ expect(fixture.host.close).toHaveBeenNthCalledWith(2, {+ reason: expect.stringContaining("quarantined cleanup recovery"),+ });+ });++ it("quarantines failed cleanup after session construction fails", async () => {+ const quarantineRecovery = deferred<void>();+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const identity = fixture.host.identity();+ fixture.host.identity = vi+ .fn()+ .mockReturnValueOnce({+ ...identity,+ normalizedSessionId: "different-session",+ })+ .mockReturnValue(identity);+ fixture.host.close+ .mockRejectedValueOnce(new Error("initial cleanup failed"))+ .mockImplementationOnce(() => quarantineRecovery.promise)+ .mockResolvedValue(undefined);++ await expect(+ fixture.driver.openSession({+ runId: "run-construction-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ }),+ ).rejects.toThrow("Codex ACPX host returned a different session identity");+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));++ const nextAdmission = fixture.driver.openSession({+ runId: "run-after-construction-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ await Promise.resolve();+ expect(fixture.openHost).toHaveBeenCalledOnce();++ quarantineRecovery.resolve();+ const session = await nextAdmission;+ expect(fixture.openHost).toHaveBeenCalledTimes(2);+ expect(fixture.host.close).toHaveBeenNthCalledWith(2, {+ reason: expect.stringContaining("quarantined cleanup recovery"),+ });+ await session.close({ reason: "construction cleanup recovered" });+ });++ it("advertises only the implemented Codex production surface", async () => {+ const fixture = driverFixture();+ const descriptor = await fixture.driver.descriptor();++ expect(descriptor).toMatchObject({+ kind: "acpx_runtime",+ displayName: "Codex via ACPX",+ capabilities: {+ resume: false,+ interruption: true,+ dynamicTools: true,+ runtimeRequestResolution: false,+ },+ runtimeContextCapabilities: {+ instructions: "native",+ skills: "unsupported",+ mcp: "native",+ },+ });+ await expect(+ fixture.driver.validateConfig({+ agent: "claude",+ model: "claude-sonnet-5",+ permissionMode: "approve-reads",+ }),+ ).resolves.toMatchObject({+ ok: false,+ issues: [{ code: "unsupported_agent" }],+ });+ });++ it("maps one turn, dispatches tools, and commits one semantic result", async () => {+ const dynamicToolHandler = vi.fn(async () => ({ title: "Document" }));+ const fixture = driverFixture({ dynamicToolHandler });+ const session = await fixture.driver.openSession({+ runId: "run-1",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const terminalEvents = collectUntil(session.events(), "turn.completed");++ const { turnId } = await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });+ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;+ await expect(+ bridgeHandler({+ tool: "documents.read",+ callId: "tool-1",+ arguments: { id: "doc-1" },+ signal: new AbortController().signal,+ }),+ ).resolves.toEqual({ title: "Document" });+ expect(dynamicToolHandler).toHaveBeenCalledWith({+ tool: "documents.read",+ callId: "tool-1",+ providerSessionId: "agent-1",+ turnId,+ arguments: { id: "doc-1" },+ signal: expect.any(AbortSignal),+ });++ await expect(+ bridgeHandler({+ tool: PRP_COMPLETION_TOOL_NAME,+ callId: "finish-1",+ arguments: completedResult(),+ signal: new AbortController().signal,+ }),+ ).resolves.toEqual({ accepted: true });+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });++ const events = await terminalEvents;+ expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true);+ expect(events.map((event) => event.eventType)).toEqual(+ expect.arrayContaining([+ "turn.submitted",+ "turn.accepted",+ "turn.started",+ "item.delta",+ "tool.execution.started",+ "run.result.proposed",+ "item.completed",+ "turn.completed",+ ]),+ );+ expect(+ events.findIndex((event) => event.eventType === "run.result.proposed"),+ ).toBeLessThan(+ events.findIndex((event) => event.eventType === "turn.completed"),+ );+ await expect(session.snapshot()).resolves.toMatchObject({+ driverKind: "acpx_runtime",+ activeTurnId: null,+ providerIdentity: {+ kind: "acpx",+ agentSessionId: "agent-1",+ },+ semanticResult: {+ callId: "finish-1",+ turnId,+ result: { reportedWorkDisposition: "done" },+ },+ });+ await session.close({ reason: "complete" });+ await session.close({ reason: "idempotent close" });+ expect(fixture.host.close).toHaveBeenCalledOnce();+ });++ it("rejects terminal disposition drift and bounds interruption", async () => {+ const fixture = driverFixture();+ const session = await fixture.driver.openSession({+ runId: "run-2",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const { turnId } = await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });+ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;++ await expect(+ bridgeHandler({+ tool: PRP_BLOCK_TOOL_NAME,+ callId: "block-1",+ arguments: completedResult(),+ signal: new AbortController().signal,+ }),+ ).rejects.toThrow("does not match");+ await session.interrupt({ turnId, reason: "user cancelled" });+ expect(fixture.host.interruptActiveTurn).toHaveBeenCalledWith(+ "user cancelled",+ );+ await expect(session.interrupt({ turnId: "stale-turn" })).rejects.toThrow(+ "is not the active turn",+ );+ await session.close({ reason: "cancelled" });+ });++ it("emits an interrupted terminal before closing an active stream", async () => {+ const fixture = driverFixture();+ const session = await fixture.driver.openSession({+ runId: "run-close",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");+ await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });++ await Promise.all([+ session.close({ reason: "operator shutdown" }),+ session.close({ reason: "duplicate shutdown" }),+ ]);++ await expect(terminalEvents).resolves.toEqual(+ expect.arrayContaining([+ expect.objectContaining({ eventType: "turn.interrupted" }),+ ]),+ );+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: null,+ terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })],+ });+ expect(fixture.host.close).toHaveBeenCalledOnce();+ });++ it("classifies a pump rejection during close as interrupted", async () => {+ const runtimeEventFailure = deferred<never>();+ const fixture = driverFixture(+ {},+ { runtimeEventFailure: runtimeEventFailure.promise },+ );+ const session = await fixture.driver.openSession({+ runId: "run-close-pump-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");+ await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });++ const closing = session.close({ reason: "operator shutdown" });+ runtimeEventFailure.reject(+ new Error("provider stream closed during shutdown"),+ );+ await expect(closing).resolves.toBeUndefined();++ const emitted = await terminalEvents;+ expect(+ emitted.filter((event) => event.eventType === "turn.interrupted"),+ ).toHaveLength(1);+ expect(emitted.some((event) => event.eventType === "turn.failed")).toBe(+ false,+ );+ });++ it("reports a host close timeout while retaining the exact cleanup", async () => {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const hostClose = deferred<void>();+ fixture.host.close.mockImplementation(() => hostClose.promise);+ const session = await fixture.driver.openSession({+ runId: "run-close-timeout",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");+ await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });++ await expect(+ session.close({ reason: "runtime close stalled" }),+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");+ expect(fixture.host.close).toHaveBeenCalledOnce();+ await expect(terminalEvents).resolves.toEqual(+ expect.arrayContaining([+ expect.objectContaining({ eventType: "turn.interrupted" }),+ ]),+ );+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: null,+ terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })],+ });++ fixture.finishTurn({ status: "cancelled", stopReason: "session_closed" });+ hostClose.resolve();+ await new Promise<void>((resolve) => setTimeout(resolve, 10));+ expect(fixture.host.close).toHaveBeenCalledOnce();+ });++ it("reconciles the retained close when it settles after the wait bound", async () => {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const retainedClose = deferred<void>();+ fixture.host.close.mockImplementation(() => retainedClose.promise);+ const session = await fixture.driver.openSession({+ runId: "run-close-recovery",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ await expect(+ session.close({ reason: "runtime close stalled" }),+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");+ retainedClose.resolve();+ await expect(+ session.close({ reason: "observe retained completion" }),+ ).resolves.toBeUndefined();+ expect(fixture.host.close).toHaveBeenCalledOnce();+ });++ it("does not spin while the exact host cleanup remains pending", async () => {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const stalledClose = deferred<void>();+ fixture.host.close.mockImplementation(() => stalledClose.promise);+ const session = await fixture.driver.openSession({+ runId: "run-close-stalled-recovery",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ await expect(+ session.close({ reason: "runtime close never settles" }),+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");+ expect(fixture.host.close).toHaveBeenCalledOnce();+ await new Promise<void>((resolve) => setTimeout(resolve, 10));+ expect(fixture.host.close).toHaveBeenCalledOnce();+ });++ it("blocks admission while the first retained host close is still pending", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const stalledClose = deferred<void>();+ fixture.host.close.mockImplementation(() => stalledClose.promise);+ const session = await fixture.driver.openSession({+ runId: "run-close-pending-admission",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "runtime close remains pending" }),+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ const admission = fixture.driver.openSession({+ runId: "run-before-pending-close-settles",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const blocked = expect(admission).rejects.toThrow(+ "exceeded the admission grace",+ );+ // Drive the implementation's complete admission grace instead of+ // duplicating its derived shutdown bound in this test. Later transport+ // layers may add another bounded cleanup phase without weakening the+ // invariant exercised here: no replacement host opens while the exact+ // retained close is still pending.+ await vi.runAllTimersAsync();+ await blocked;+ expect(fixture.openHost).toHaveBeenCalledOnce();+ stalledClose.resolve();+ await vi.runAllTimersAsync();+ } finally {+ vi.useRealTimers();+ }+ });++ it("retains autonomous host cleanup recovery through repeated failure", async () => {+ const fixture = driverFixture(+ {},+ {+ closeSettlementTimeoutMs: 1,+ },+ );+ fixture.host.close+ .mockRejectedValueOnce(new Error("transient cleanup failure"))+ .mockRejectedValueOnce(new Error("second cleanup failure"))+ .mockResolvedValueOnce(undefined);+ const session = await fixture.driver.openSession({+ runId: "run-close-permanent-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const diagnosticEvents = collectUntil(+ session.events(),+ "harness.diagnostic",+ );++ await expect(+ session.close({ reason: "runtime close initially failed" }),+ ).rejects.toThrow("transient cleanup failure");+ await expect(diagnosticEvents).resolves.toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ eventType: "harness.diagnostic",+ payload: expect.objectContaining({+ code: "acpx_host_cleanup_deferred",+ }),+ }),+ ]),+ );+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(3));+ expect(fixture.host.close).toHaveBeenLastCalledWith({+ reason: "runtime close initially failed (automatic cleanup recovery 2)",+ });+ await expect(+ session.close({ reason: "observe recovered cleanup" }),+ ).resolves.toBeUndefined();+ expect(fixture.host.close).toHaveBeenCalledTimes(3);+ });++ it("keeps quarantined cleanup scheduled and lets admission await its exact owner", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockRejectedValue(+ new Error("persistent cleanup failure"),+ );+ const session = await fixture.driver.openSession({+ runId: "run-close-retry-bound",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "runtime close persistently failed" }),+ ).rejects.toThrow("persistent cleanup failure");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ await vi.advanceTimersByTimeAsync(10);+ expect(fixture.host.close).toHaveBeenCalledTimes(7);+ await vi.advanceTimersByTimeAsync(59_000);+ expect(fixture.host.close).toHaveBeenCalledTimes(7);++ fixture.host.close.mockImplementation(({ reason }) =>+ reason.includes("scheduled quarantined cleanup recovery")+ ? // Exercise the complete production host bound: two seconds for+ // active-turn cancellation plus six seconds for protocol/TERM/KILL.+ new Promise<void>((resolve) => setTimeout(resolve, 8_500))+ : Promise.resolve(),+ );+ await vi.advanceTimersToNextTimerAsync();+ let admissionSettled = false;+ const admission = fixture.driver.openSession({+ runId: "run-after-recovery",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ void admission.then(+ () => {+ admissionSettled = true;+ },+ () => {+ admissionSettled = true;+ },+ );+ expect(fixture.host.close).toHaveBeenCalledTimes(8);+ expect(fixture.host.close).toHaveBeenLastCalledWith({+ reason:+ "runtime close persistently failed (scheduled quarantined cleanup recovery)",+ });+ await vi.advanceTimersByTimeAsync(8_499);+ expect(admissionSettled).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ await expect(admission).resolves.toBeDefined();+ expect(fixture.host.close).toHaveBeenCalledTimes(8);+ } finally {+ vi.useRealTimers();+ }+ });++ it("runs a full admission batch after an inherited scheduled attempt fails", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockRejectedValue(+ new Error("persistent cleanup failure"),+ );+ const session = await fixture.driver.openSession({+ runId: "run-scheduled-owner-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "exhaust cleanup before scheduled recovery" }),+ ).rejects.toThrow("persistent cleanup failure");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ await vi.advanceTimersByTimeAsync(10);++ let scheduledAttempt = 0;+ let admissionAttempt = 0;+ fixture.host.close.mockImplementation(({ reason }) => {+ if (reason.includes("scheduled quarantined cleanup recovery")) {+ scheduledAttempt += 1;+ return new Promise<void>((_resolve, reject) => {+ setTimeout(+ () => reject(new Error("scheduled cleanup failed")),+ 8_000,+ );+ });+ }+ if (reason.includes("quarantined cleanup admission recovery")) {+ admissionAttempt += 1;+ const attempt = admissionAttempt;+ return new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (attempt < 3) reject(new Error("admission cleanup failed"));+ else resolve();+ }, 8_000);+ });+ }+ return Promise.reject(new Error("unexpected cleanup reason"));+ });++ await vi.advanceTimersToNextTimerAsync();+ const admission = fixture.driver.openSession({+ runId: "run-after-scheduled-owner-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ let admissionSettled = false;+ void admission+ .finally(() => {+ admissionSettled = true;+ })+ .catch(() => undefined);+ // The inherited attempt plus the replacement three-attempt batch takes+ // just over 32 seconds with this fixture. The production grace must cover+ // that complete bounded owner chain rather than expiring at 27 seconds.+ await vi.advanceTimersByTimeAsync(32_001);+ expect(admissionSettled).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ const admitted = await admission;++ expect(scheduledAttempt).toBe(1);+ expect(admissionAttempt).toBe(3);+ expect(fixture.openHost).toHaveBeenCalledTimes(2);+ fixture.host.close.mockResolvedValue(undefined);+ await admitted.close({+ reason: "complete after scheduled owner recovery",+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("lets admission observe a complete bounded multi-attempt recovery", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ let quarantineAttempt = 0;+ fixture.host.close.mockImplementation(({ reason }) => {+ if (!reason.includes("quarantined cleanup recovery")) {+ return Promise.reject(new Error("persistent cleanup failure"));+ }+ quarantineAttempt += 1;+ const attempt = quarantineAttempt;+ return new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (attempt < 3) reject(new Error("transient quarantine failure"));+ else resolve();+ }, 8_000);+ });+ });+ const session = await fixture.driver.openSession({+ runId: "run-multi-attempt-recovery",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "runtime close persistently failed" }),+ ).rejects.toThrow("persistent cleanup failure");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ await vi.advanceTimersByTimeAsync(4);+ expect(fixture.host.close).toHaveBeenCalledTimes(5);++ let admissionSettled = false;+ const admission = fixture.driver.openSession({+ runId: "run-after-multi-attempt-recovery",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ void admission.then(+ () => {+ admissionSettled = true;+ },+ () => {+ admissionSettled = true;+ },+ );+ await vi.advanceTimersByTimeAsync(23_000);+ expect(admissionSettled).toBe(false);+ expect(fixture.host.close).toHaveBeenCalledTimes(7);+ await vi.advanceTimersByTimeAsync(2_000);+ await expect(admission).resolves.toBeDefined();+ expect(quarantineAttempt).toBe(3);+ } finally {+ vi.useRealTimers();+ }+ });++ it("awaits quarantine recovery installed by an exhausted retained owner", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockImplementation(({ reason }) => {+ if (reason.includes("quarantined cleanup recovery")) {+ return new Promise<void>((resolve) => setTimeout(resolve, 2));+ }+ return Promise.reject(new Error("bounded host recovery failed"));+ });+ const session = await fixture.driver.openSession({+ runId: "run-replacement-cleanup-owner",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const closing = expect(+ session.close({ reason: "host cleanup must be replaced" }),+ ).rejects.toThrow("bounded host recovery failed");+ await vi.advanceTimersByTimeAsync(1);+ await closing;++ const admission = fixture.driver.openSession({+ runId: "run-after-replacement-cleanup",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ await vi.advanceTimersByTimeAsync(10);+ const admitted = await admission;+ expect(fixture.openHost).toHaveBeenCalledTimes(2);+ fixture.host.close.mockResolvedValue(undefined);+ await admitted.close({ reason: "complete after replacement cleanup" });+ } finally {+ vi.useRealTimers();+ }+ });++ it("gives replacement quarantine recovery a fresh admission grace", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockImplementation(+ ({ reason }) =>+ new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (reason.includes("quarantined cleanup recovery")) resolve();+ else reject(new Error("bounded autonomous cleanup failed"));+ }, 8_000);+ }),+ );+ const session = await fixture.driver.openSession({+ runId: "run-owner-phase-grace",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "exhaust cleanup near admission deadline" }),+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");+ await vi.advanceTimersByTimeAsync(1);+ await closing;++ let admissionSettled = false;+ const admission = fixture.driver.openSession({+ runId: "run-after-owner-phase-grace",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ void admission+ .finally(() => {+ admissionSettled = true;+ })+ .catch(() => undefined);++ // The inherited autonomous owner consumes about 32 seconds before it+ // installs a distinct bounded quarantine recovery. That replacement+ // owner must not inherit the nearly exhausted admission deadline.+ await vi.advanceTimersByTimeAsync(35_001);+ expect(admissionSettled).toBe(false);+ expect(fixture.openHost).toHaveBeenCalledOnce();++ await vi.advanceTimersByTimeAsync(5_002);+ const admitted = await admission;+ expect(fixture.openHost).toHaveBeenCalledTimes(2);+ fixture.host.close.mockResolvedValue(undefined);+ await admitted.close({ reason: "complete after replacement owner" });+ } finally {+ vi.useRealTimers();+ }+ });++ it("reschedules host cleanup after an admission batch is exhausted", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockRejectedValue(+ new Error("persistent cleanup failure"),+ );+ const session = await fixture.driver.openSession({+ runId: "run-transient-admission-cleanup",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "exhaust cleanup before admission" }),+ ).rejects.toThrow("persistent cleanup failure");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ await vi.advanceTimersByTimeAsync(10);++ let admissionAttempt = 0;+ let scheduledAttempt = 0;+ fixture.host.close.mockImplementation(({ reason }) => {+ if (reason.includes("quarantined cleanup admission recovery")) {+ admissionAttempt += 1;+ return Promise.reject(new Error("admission cleanup failed"));+ }+ if (reason.includes("scheduled quarantined cleanup recovery")) {+ scheduledAttempt += 1;+ return Promise.resolve();+ }+ return Promise.reject(new Error("unexpected cleanup reason"));+ });++ const admission = expect(+ fixture.driver.openSession({+ runId: "run-after-transient-admission-cleanup",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ }),+ ).rejects.toThrow("quarantined host cleanup remains incomplete");+ await vi.advanceTimersByTimeAsync(3);+ await admission;++ expect(admissionAttempt).toBe(3);+ expect(scheduledAttempt).toBe(0);+ await vi.advanceTimersByTimeAsync(60_000);+ expect(scheduledAttempt).toBe(1);++ const admitted = await fixture.driver.openSession({+ runId: "run-after-autonomous-cleanup",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ expect(fixture.openHost).toHaveBeenCalledTimes(2);+ fixture.host.close.mockResolvedValue(undefined);+ await admitted.close({ reason: "complete after admission retry" });+ } finally {+ vi.useRealTimers();+ }+ });++ it("restores another host's retry timer when admission recovery times out", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ const hostA = {+ ...fixture.host,+ close: vi.fn(async () => {+ throw new Error("host A cleanup failed");+ }),+ };+ const hostB = {+ ...fixture.host,+ close: vi.fn(async () => {+ throw new Error("host B cleanup failed");+ }),+ };+ fixture.openHost+ .mockReset()+ .mockResolvedValueOnce(hostB)+ .mockResolvedValueOnce(hostA)+ .mockResolvedValue(fixture.host);++ const sessionB = await fixture.driver.openSession({+ runId: "run-quarantined-host-b",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const sessionA = await fixture.driver.openSession({+ runId: "run-quarantined-host-a",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closingB = expect(+ sessionB.close({ reason: "quarantine host B" }),+ ).rejects.toThrow("host B cleanup failed");+ await vi.advanceTimersByTimeAsync(1);+ await closingB;+ await vi.advanceTimersByTimeAsync(10);+ expect(hostB.close).toHaveBeenCalledTimes(7);++ // Offset the autonomous retry timers so host B can own an inherited+ // scheduled attempt while host A still has a dormant retry timer.+ await vi.advanceTimersByTimeAsync(100);+ const closingA = expect(+ sessionA.close({ reason: "quarantine host A" }),+ ).rejects.toThrow("host A cleanup failed");+ await vi.advanceTimersByTimeAsync(1);+ await closingA;+ await vi.advanceTimersByTimeAsync(10);+ expect(hostA.close).toHaveBeenCalledTimes(7);++ let hostAAdmissionAttempts = 0;+ let hostAScheduledAttempts = 0;+ hostA.close.mockImplementation(({ reason }) => {+ if (reason.includes("quarantined cleanup admission recovery")) {+ hostAAdmissionAttempts += 1;+ return Promise.reject(new Error("host A admission cleanup failed"));+ }+ if (reason.includes("scheduled quarantined cleanup recovery")) {+ hostAScheduledAttempts += 1;+ return Promise.resolve();+ }+ return Promise.reject(new Error("unexpected host A cleanup reason"));+ });+ let hostBAdmissionAttempts = 0;+ hostB.close.mockImplementation(({ reason }) => {+ if (reason.includes("scheduled quarantined cleanup recovery")) {+ return new Promise<void>((_resolve, reject) => {+ setTimeout(+ () => reject(new Error("host B scheduled cleanup failed")),+ 1,+ );+ });+ }+ if (reason.includes("quarantined cleanup admission recovery")) {+ hostBAdmissionAttempts += 1;+ return new Promise<void>(() => undefined);+ }+ return Promise.reject(new Error("unexpected host B cleanup reason"));+ });++ await vi.advanceTimersToNextTimerAsync();+ expect(hostB.close).toHaveBeenLastCalledWith({+ reason: expect.stringContaining(+ "scheduled quarantined cleanup recovery",+ ),+ });+ expect(hostAScheduledAttempts).toBe(0);++ const admission = expect(+ fixture.driver.openSession({+ runId: "run-after-multi-host-quarantine",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ }),+ ).rejects.toThrow("exceeded the admission grace");+ await vi.advanceTimersByTimeAsync(40_000);+ await admission;++ expect(hostAAdmissionAttempts).toBe(3);+ expect(hostBAdmissionAttempts).toBe(1);+ expect(hostAScheduledAttempts).toBe(0);++ await vi.advanceTimersByTimeAsync(60_000);+ expect(hostAScheduledAttempts).toBe(1);+ } finally {+ vi.useRealTimers();+ }+ });++ it("fails admission within a finite grace when quarantined cleanup never settles", async () => {+ vi.useFakeTimers();+ try {+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });+ fixture.host.close.mockRejectedValue(+ new Error("persistent cleanup failure"),+ );+ const session = await fixture.driver.openSession({+ runId: "run-close-never-settles",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });++ const closing = expect(+ session.close({ reason: "runtime close persistently failed" }),+ ).rejects.toThrow("persistent cleanup failure");+ await vi.advanceTimersByTimeAsync(1);+ await closing;+ await vi.advanceTimersByTimeAsync(10);+ expect(fixture.host.close).toHaveBeenCalledTimes(7);++ fixture.host.close.mockImplementation(() => new Promise<void>(() => {}));+ let admissionSettled = false;+ const admission = fixture.driver.openSession({+ runId: "run-after-stalled-quarantine",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ void admission+ .finally(() => {+ admissionSettled = true;+ })+ .catch(() => undefined);+ await vi.advanceTimersByTimeAsync(34_999);+ expect(admissionSettled).toBe(false);+ await vi.advanceTimersByTimeAsync(2);+ await expect(admission).rejects.toThrow("exceeded the admission grace");+ expect(fixture.host.close).toHaveBeenCalledTimes(8);+ } finally {+ vi.useRealTimers();+ }+ });++ it("bounds lagging streams without introducing source sequence gaps", async () => {+ const fixture = driverFixture(+ {},+ {+ runtimeEvents: Array.from({ length: 1_100 }, (_, index) => ({+ type: "text_delta" as const,+ text: `chunk-${index}`,+ stream: "output" as const,+ })),+ },+ );+ const session = await fixture.driver.openSession({+ runId: "run-bounds",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ await session.startTurn({+ message: { role: "user", text: "Produce many events." },+ });+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });++ await vi.waitFor(async () => {+ const snapshot = await session.snapshot();+ expect(snapshot.terminalTurns).toHaveLength(1);+ });+ await vi.waitFor(async () => {+ const snapshot = await session.snapshot();+ expect(snapshot.terminalTurns).toHaveLength(1);+ const transcript = await session.transcript!();+ expect(transcript.eventCount).toBeGreaterThan(1_024);+ expect(transcript.complete).toBe(false);+ expect(transcript.events.length).toBeLessThanOrEqual(1_024);+ expect(transcript.omissionReason).toBe("retention_limit");+ });+ await expect(+ session.startTurn({+ message: {+ role: "user",+ text: "Do not overtake the lagging consumer.",+ },+ }),+ ).rejects.toThrow("event consumer must drain");++ await session.close({ reason: "bounds verified" });+ const iterator = session.events()[Symbol.asyncIterator]();+ const retained: PrpEvent[] = [];+ for (;;) {+ const next = await iterator.next();+ if (next.done) break;+ retained.push(next.value);+ }+ expect(retained.length).toBeLessThanOrEqual(512);+ expect(retained).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ eventType: "harness.diagnostic",+ payload: expect.objectContaining({+ code: "event_stream_retention_limit",+ }),+ }),+ ]),+ );+ expect(+ retained.filter((event) => event.eventType === "turn.completed"),+ ).toHaveLength(1);+ expect(retained.map((event) => event.sourceSeq)).toEqual(+ Array.from({ length: retained.length }, (_, index) => index + 1),+ );+ });++ it("retains a committed semantic proposal under terminal queue pressure", async () => {+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });+ const session = await fixture.driver.openSession({+ runId: "run-semantic-bounds",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const { turnId } = await session.startTurn({+ message: { role: "user", text: "Complete the task." },+ });+ await expect(+ fixture.hostOptions!.semanticTools!.handler({+ tool: PRP_COMPLETION_TOOL_NAME,+ callId: "finish-bounded",+ arguments: completedResult(),+ signal: new AbortController().signal,+ }),+ ).resolves.toEqual({ accepted: true });+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });++ await vi.waitFor(async () => {+ await expect(session.snapshot()).resolves.toMatchObject({+ terminalTurns: [expect.objectContaining({ turnId })],+ });+ });+ await session.close({ reason: "bounded result verified" });+ const retained: PrpEvent[] = [];+ for await (const event of session.events()) retained.push(event);++ const proposalIndex = retained.findIndex(+ (event) => event.eventType === "run.result.proposed",+ );+ const terminalIndex = retained.findIndex(+ (event) => event.eventType === "turn.completed",+ );+ expect(proposalIndex).toBeGreaterThanOrEqual(0);+ expect(terminalIndex).toBeGreaterThan(proposalIndex);+ expect(retained).toHaveLength(6);+ });++ it("keeps a full-queue terminal pending until it can be streamed", async () => {+ const dynamicToolHandler = vi.fn(async () => ({ title: "late result" }));+ const fixture = driverFixture(+ { dynamicToolHandler },+ {+ closeSettlementTimeoutMs: 1,+ maxBufferedEvents: 6,+ terminalEventReserve: 0,+ runtimeEvents: Array.from({ length: 8 }, (_, index) => ({+ type: "text_delta" as const,+ text: `pressure-${index}`,+ stream: "output" as const,+ })),+ },+ );+ const session = await fixture.driver.openSession({+ runId: "run-terminal-capacity-failure",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const { turnId } = await session.startTurn({+ message: { role: "user", text: "Fill the complete event queue." },+ });+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });++ await vi.waitFor(async () => {+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: turnId,+ terminalTurns: [],+ });+ const transcript = await session.transcript!();+ expect(transcript.complete).toBe(false);+ expect(transcript.eventCount).toBe(14);+ });++ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;+ await expect(+ bridgeHandler({+ tool: PRP_COMPLETION_TOOL_NAME,+ callId: "late-completion",+ arguments: completedResult(),+ signal: new AbortController().signal,+ }),+ ).rejects.toThrow("awaiting canonical terminal-event retention");+ await expect(+ bridgeHandler({+ tool: "documents.read",+ callId: "late-dynamic-tool",+ arguments: { id: "doc-after-terminal" },+ signal: new AbortController().signal,+ }),+ ).rejects.toThrow("awaiting canonical terminal-event retention");+ expect(dynamicToolHandler).not.toHaveBeenCalled();+ await expect(+ session.interrupt({ turnId, reason: "too late" }),+ ).rejects.toThrow("awaiting canonical terminal-event retention");+ expect(fixture.host.interruptActiveTurn).not.toHaveBeenCalled();+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: turnId,+ semanticResult: null,+ terminalTurns: [],+ });+ fixture.host.close+ .mockRejectedValueOnce(new Error("initial host cleanup failed"))+ .mockResolvedValueOnce(undefined);+ await expect(+ session.close({ reason: "queue is still full" }),+ ).rejects.toThrow("before its turn.completed event is retained");+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: turnId,+ semanticResult: null,+ terminalTurns: [],+ });+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));+ expect(fixture.host.close).toHaveBeenLastCalledWith({+ reason: "queue is still full (automatic cleanup recovery 1)",+ });++ const iterator = session.events()[Symbol.asyncIterator]();+ const pressureEvents: PrpEvent[] = [];+ for (let index = 0; index < 6; index += 1) {+ const next = await iterator.next();+ expect(next.done).toBe(false);+ if (!next.done) pressureEvents.push(next.value);+ }+ expect(pressureEvents).toHaveLength(6);+ expect(isCanonicalTurnTerminal(pressureEvents, turnId)).toHaveLength(0);++ await session.close({ reason: "capacity became available" });+ const settlementEvents: PrpEvent[] = [];+ for (;;) {+ const next = await iterator.next();+ if (next.done) break;+ settlementEvents.push(next.value);+ }+ expect(isCanonicalTurnTerminal(settlementEvents, turnId)).toEqual([+ expect.objectContaining({+ eventType: "turn.completed",+ turnId,+ }),+ ]);+ await expect(session.snapshot()).resolves.toMatchObject({+ activeTurnId: null,+ terminalTurns: [expect.objectContaining({ turnId })],+ });+ });++ it("rejects turn admission when only terminal reserve capacity remains", async () => {+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });+ const session = await fixture.driver.openSession({+ runId: "run-admission-bounds",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ await session.startTurn({+ message: { role: "user", text: "Fill the regular event capacity." },+ });+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });+ await vi.waitFor(async () => {+ await expect(session.snapshot()).resolves.toMatchObject({+ terminalTurns: [+ expect.objectContaining({ turnId: expect.any(String) }),+ ],+ });+ });+ const iterator = session.events()[Symbol.asyncIterator]();+ await iterator.next();+ await iterator.next();++ await expect(+ session.startTurn({+ message: { role: "user", text: "Must wait for the remaining events." },+ }),+ ).rejects.toThrow("event consumer must drain");+ expect(fixture.host.startTurn).toHaveBeenCalledOnce();+ await session.close({ reason: "admission bound verified" });+ });++ it("redacts a complete Authorization credential from events and transcripts", async () => {+ const fixture = driverFixture();+ const session = await fixture.driver.openSession({+ runId: "run-redaction",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ fixture.host.startTurn.mockImplementationOnce(() => {+ throw new Error(+ "Authorization: Bearer first-secret second-secret, code=denied",+ );+ });+ const terminalEvents = collectUntil(session.events(), "turn.failed");++ await expect(+ session.startTurn({+ message: { role: "user", text: "Do not expose provider credentials." },+ }),+ ).rejects.toThrow("Authorization");++ const serializedEvents = JSON.stringify(await terminalEvents);+ const serializedTranscript = JSON.stringify(await session.transcript!());+ for (const serialized of [serializedEvents, serializedTranscript]) {+ expect(serialized).toContain("[REDACTED]");+ expect(serialized).not.toContain("Bearer");+ expect(serialized).not.toContain("first-secret");+ expect(serialized).not.toContain("second-secret");+ }+ await session.close({ reason: "redaction verified" });+ });++ it("preserves every terminal when the bounded consumer keeps draining", async () => {+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });+ const session = await fixture.driver.openSession({+ runId: "run-critical-bounds",+ normalizedSessionId: "session-1",+ workingDirectory: "/workspace",+ });+ const terminalTurnIds: string[] = [];+ for (let index = 0; index < 6; index += 1) {+ const terminalEvents = collectUntil(session.events(), "turn.completed");+ const { turnId } = await session.startTurn({+ message: { role: "user", text: `Complete turn ${index}.` },+ });+ terminalTurnIds.push(turnId);+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });+ const emitted = await terminalEvents;+ expect(emitted.at(-1)).toMatchObject({+ eventType: "turn.completed",+ turnId,+ });+ }+ await expect(+ session.startTurn({+ message: { role: "user", text: "Exceed the bounded turn limit." },+ }),+ ).rejects.toThrow("bounded session turn limit");++ await session.close({ reason: "critical bounds verified" });+ await expect(session.snapshot()).resolves.toMatchObject({+ terminalTurns: terminalTurnIds.map((turnId) =>+ expect.objectContaining({ turnId }),+ ),+ });+ });+});++function driverFixture(+ overrides: Partial<CodexAcpxDriverOptions> = {},+ fixtureOptions: {+ runtimeEvents?: readonly AcpRuntimeEvent[];+ runtimeEventFailure?: Promise<never>;+ closeSettlementTimeoutMs?: number;+ maxBufferedEvents?: number;+ terminalEventReserve?: number;+ openHost?: NonNullable<CodexAcpxDriverDependencies["openHost"]>;+ } = {},+): {+ driver: CodexAcpxDriver;+ host: ReturnType<typeof fakeHost>;+ openHost: ReturnType<typeof vi.fn>;+ hostOptions: OpenAcpxRuntimeHostOptions | null;+ finishTurn(result: Awaited<AcpxRuntimeTurn["result"]>): void;+} {+ const result = deferred<Awaited<AcpxRuntimeTurn["result"]>>();+ const turn: AcpxRuntimeTurn = {+ requestId: "provider-turn-1",+ promptStarted: Promise.resolve(),+ events: {+ async *[Symbol.asyncIterator]() {+ yield* fixtureOptions.runtimeEvents ?? [+ {+ type: "text_delta" as const,+ text: "Task complete.",+ stream: "output" as const,+ },+ {+ type: "tool_call" as const,+ toolCallId: "provider-tool-1",+ title: "Read",+ kind: "read" as const,+ status: "pending",+ tag: "tool_call",+ text: "Reading",+ },+ ];+ if (fixtureOptions.runtimeEventFailure) {+ await fixtureOptions.runtimeEventFailure;+ }+ },+ },+ result: result.promise,+ cancel: vi.fn(async () => undefined),+ closeStream: vi.fn(async () => undefined),+ };+ const host = fakeHost(turn, () =>+ result.resolve({ status: "cancelled", stopReason: "session_closed" }),+ );+ let hostOptions: OpenAcpxRuntimeHostOptions | null = null;+ const openHost = vi.fn(+ fixtureOptions.openHost ??+ (async (options: OpenAcpxRuntimeHostOptions) => {+ hostOptions = options;+ return host;+ }),+ );+ const dependencies: CodexAcpxDriverDependencies = {+ openHost,+ closeSettlementTimeoutMs: fixtureOptions.closeSettlementTimeoutMs,+ maxBufferedEvents: fixtureOptions.maxBufferedEvents,+ terminalEventReserve: fixtureOptions.terminalEventReserve,+ };+ const driver = new CodexAcpxDriver(+ {+ runtimeDirectory: "/runtime",+ model: "gpt-5.6-sol",+ permissionMode: "approve-reads",+ dynamicTools: [+ {+ name: "documents.read",+ inputSchema: { type: "object" },+ },+ ],+ now: () => new Date("2026-08-27T12:00:00.000Z"),+ ...overrides,+ },+ dependencies,+ );+ return {+ driver,+ host,+ openHost,+ get hostOptions() {+ return hostOptions;+ },+ finishTurn: result.resolve,+ };+}++function fakeHost(turn: AcpxRuntimeTurn, onClose: () => void) {+ return {+ identity: () => ({+ schema: "paperclip.runner.acpx-identity.v1" as const,+ normalizedSessionId: "session-1",+ acpxRecordId: "record-1",+ backendSessionId: "backend-1",+ agentSessionId: "agent-1",+ profileDigest: `sha256:${"a".repeat(64)}`,+ workspaceDigest: `sha256:${"b".repeat(64)}`,+ requestedModel: "gpt-5.6-sol",+ effectiveModel: "gpt-5.6-sol",+ permissionMode: "approve-reads" as const,+ }),+ binding: () => ({+ normalizedSessionId: "session-1",+ workspacePath: "/workspace",+ workspaceDigest: `sha256:${"b".repeat(64)}`,+ runtimeRoot: "/runtime/acpx/session-1",+ profileDigest: `sha256:${"a".repeat(64)}`,+ requestedModel: "gpt-5.6-sol",+ effectiveModel: "gpt-5.6-sol",+ permissionMode: "approve-reads" as const,+ profileSessionKey: "paperclip-session",+ }),+ status: vi.fn(async () => ({+ agentSessionId: "agent-1",+ models: {+ currentModelId: "gpt-5.6-sol",+ availableModelIds: ["gpt-5.6-sol"],+ },+ })),+ startTurn: vi.fn(() => turn),+ interruptActiveTurn: vi.fn(async () => undefined),+ close: vi.fn(async () => {+ onClose();+ }),+ };+}++async function collectUntil(+ events: AsyncIterable<PrpEvent>,+ terminalType: PrpEvent["eventType"],+): Promise<PrpEvent[]> {+ const collected: PrpEvent[] = [];+ for await (const event of events) {+ collected.push(event);+ if (event.eventType === terminalType) return collected;+ }+ throw new Error(`Event stream closed before ${terminalType}`);+}++function completedResult() {+ return {+ schema: "paperclip.run_result.v1" as const,+ reportedWorkDisposition: "done" as const,+ summary: "The task is complete.",+ completionClaim: {+ contractRevision: "codex-acpx-test-v1",+ objectiveSatisfied: true,+ criteria: [],+ remainingWork: [],+ },+ evidence: [],+ verification: [+ { commandOrCheck: "Codex ACPX driver test", status: "passed" as const },+ ],+ attentionRequests: [],+ artifacts: [],+ };+}++function isCanonicalTurnTerminal(events: PrpEvent[], turnId: string) {+ return events.filter(+ (event) =>+ event.turnId === turnId &&+ (event.eventType === "turn.completed" ||+ event.eventType === "turn.failed" ||+ event.eventType === "turn.interrupted"),+ );+}++function deferred<T>() {+ let resolve!: (value: T) => void;+ let reject!: (error: unknown) => void;+ const promise = new Promise<T>((settle, fail) => {+ resolve = settle;+ reject = fail;+ });+ return { promise, reject, resolve };+}diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.tsnew file mode 100644index 0000000000..f28f669a94--- /dev/null+++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts@@ -0,0 +1,1403 @@+import { createHash, randomBytes } from "node:crypto";++import type { AcpRuntimeEvent } from "acpx/runtime";++import {+ PRP_BLOCK_TOOL_NAME,+ PRP_COMPLETION_TOOL_NAME,+} from "../../contracts/completion-result.js";+import {+ HarnessCapabilityUnavailableError,+ HarnessStaleTurnError,+ type HarnessDriver,+ type HarnessDriverConfigValidation,+ type HarnessDriverDescriptor,+ type HarnessSession,+ type HarnessTranscriptSnapshot,+ type OpenHarnessSessionInput,+ type PersistedHarnessSession,+} from "../../contracts/harness-driver.js";+import type { NativeAcpxPermissionMode } from "../../contracts/native-execution.js";+import type { NativeUserMessage } from "../../contracts/types.js";+import type {+ PrpEvent,+ PrpStructuredRunResult,+} from "../../protocol/replay-contract.js";+import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js";+import {+ canonicalProviderEventsFromAcpxRuntimeEvent,+ createAcpxToolEventNormalizer,+} from "../../provider-events.js";+import {+ canonicalRunnerToolName,+ type RunnerToolCall,+} from "../runner-tool-bridge.js";+import {+ DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS,+ openCodexAcpxRuntime,+} from "./codex-runtime-adapter.js";+import {+ acpxDriverDescriptor,+ validateAcpxDriverConfig,+} from "./driver-profile.js";+import {+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS,+ AcpxRuntimeHost,+ type AcpxRuntimeTurn,+ type OpenAcpxRuntimeHostOptions,+} from "./runtime-host.js";++const MAX_BUFFERED_EVENTS = 512;+const TERMINAL_EVENT_RESERVE = 3;+const TURN_START_EVENT_COUNT = 3;+const MAX_TRANSCRIPT_EVENTS = 1_024;+const MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024;+const CLOSE_TURN_SETTLEMENT_TIMEOUT_MS = 2_000;+const MAX_AUTONOMOUS_HOST_CLOSE_RETRIES = 3;+const MAX_QUARANTINED_HOST_CLOSE_RETRIES = 3;+const QUARANTINED_HOST_CLOSE_RETRY_MS = 60_000;+const MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS = 1_000;+// Host shutdown first bounds active-turn cancellation and then performs the+// adapter's bounded protocol/TERM/KILL cleanup. Bound each ownership phase that+// admission observes: an inherited owner may install a distinct replacement+// recovery, which receives its own finite grace rather than inheriting an+// almost-expired deadline. Cover every attempt and bounded inter-attempt delay,+// plus one second of scheduling margin.+const QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS =+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS ++ DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS;+const MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS = 1;+const QUARANTINED_HOST_ADMISSION_GRACE_MS =+ (MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS ++ MAX_QUARANTINED_HOST_CLOSE_RETRIES) *+ QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS ++ (MAX_QUARANTINED_HOST_CLOSE_RETRIES - 1) *+ MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS ++ 1_000;++export interface CodexAcpxDynamicToolCall {+ tool: string;+ callId: string;+ providerSessionId: string;+ turnId: string;+ arguments: unknown;+ signal: AbortSignal;+}++export interface CodexAcpxDriverOptions {+ runtimeDirectory: string;+ model: string;+ permissionMode?: NativeAcpxPermissionMode;+ systemInstructions?: string;+ environment?: NodeJS.ProcessEnv;+ managedCodexCredentialSourcePath?: string;+ dynamicTools?: readonly Readonly<Record<string, unknown>>[];+ dynamicToolHandler?: (call: CodexAcpxDynamicToolCall) => Promise<unknown>;+ now?: () => Date;+}++interface CodexAcpxHost {+ identity(): ReturnType<AcpxRuntimeHost["identity"]>;+ binding(): ReturnType<AcpxRuntimeHost["binding"]>;+ status(): ReturnType<AcpxRuntimeHost["status"]>;+ startTurn(+ input: Parameters<AcpxRuntimeHost["startTurn"]>[0],+ ): AcpxRuntimeTurn;+ interruptActiveTurn(reason: string): Promise<void>;+ close(input: { reason: string }): Promise<void>;+}++interface QuarantinedHostCleanup {+ host: CodexAcpxHost;+ reason: string;+ attempt: Promise<void> | null;+ recovery: Promise<void> | null;+ recoveryMaxAttempts: number | null;+ timer: ReturnType<typeof setTimeout> | null;+}++export interface CodexAcpxDriverDependencies {+ openHost?: (options: OpenAcpxRuntimeHostOptions) => Promise<CodexAcpxHost>;+ /** Internal test seam; production uses the fixed close-settlement bound. */+ closeSettlementTimeoutMs?: number;+ /** Internal test seam; production uses the fixed event-retention bound. */+ maxBufferedEvents?: number;+ /** Internal test seam; production reserves fixed terminal-event capacity. */+ terminalEventReserve?: number;+}++/** Codex-only HarnessDriver backed by the admitted ACPX runtime host. */+export class CodexAcpxDriver implements HarnessDriver {+ readonly #options: CodexAcpxDriverOptions;+ readonly #openHost: NonNullable<CodexAcpxDriverDependencies["openHost"]>;+ readonly #closeSettlementTimeoutMs: number;+ readonly #maxBufferedEvents: number;+ readonly #terminalEventReserve: number;+ readonly #cleanupOwners = new Set<Promise<void>>();+ readonly #quarantinedHostCleanups = new Set<QuarantinedHostCleanup>();++ constructor(+ options: CodexAcpxDriverOptions,+ dependencies: CodexAcpxDriverDependencies = {},+ ) {+ this.#options = {+ ...options,+ ...(options.environment+ ? { environment: { ...options.environment } }+ : {}),+ ...(options.dynamicTools+ ? { dynamicTools: structuredClone(options.dynamicTools) }+ : {}),+ };+ this.#openHost =+ dependencies.openHost ??+ ((hostOptions) =>+ AcpxRuntimeHost.open(hostOptions, {+ openRuntime: openCodexAcpxRuntime,+ reportRetainedCleanupFailure: reportRetainedAcpxCleanupFailure,+ } as Parameters<typeof AcpxRuntimeHost.open>[1] & {+ reportRetainedCleanupFailure: typeof reportRetainedAcpxCleanupFailure;+ }));+ this.#closeSettlementTimeoutMs =+ dependencies.closeSettlementTimeoutMs ?? CLOSE_TURN_SETTLEMENT_TIMEOUT_MS;+ this.#terminalEventReserve = Math.max(+ 0,+ Math.floor(dependencies.terminalEventReserve ?? TERMINAL_EVENT_RESERVE),+ );+ this.#maxBufferedEvents = Math.max(+ this.#terminalEventReserve + TURN_START_EVENT_COUNT,+ Math.floor(dependencies.maxBufferedEvents ?? MAX_BUFFERED_EVENTS),+ );+ }++ async descriptor(): Promise<HarnessDriverDescriptor> {+ const descriptor = acpxDriverDescriptor("codex");+ return {+ ...descriptor,+ displayName: "Codex via ACPX",+ runtimeContextCapabilities: {+ instructions: "native",+ skills: "unsupported",+ mcp: "native",+ },+ capabilities: {+ ...descriptor.capabilities,+ resume: false,+ runtimeRequestResolution: false,+ runtimeRequestHandoff: false,+ unsupported: [+ "resume",+ "steering",+ "runtimeRequestResolution",+ "runtimeRequestHandoff",+ "goals",+ "threadLineage",+ ],+ },+ };+ }++ async validateConfig(value: unknown): Promise<HarnessDriverConfigValidation> {+ const validation = validateAcpxDriverConfig(value);+ if (!validation.ok || validation.config.agent === "codex")+ return validation;+ return {+ ok: false,+ config: null,+ issues: [+ {+ path: "agent",+ code: "unsupported_agent",+ message: "The production ACPX driver currently supports Codex only.",+ },+ ],+ };+ }++ async openSession(input: OpenHarnessSessionInput): Promise<HarnessSession> {+ await runAbortableDriverAdmission(input.signal, () =>+ this.#retryQuarantinedHostCleanups(),+ );+ let session: CodexAcpxSession | null = null;+ const host = await this.#openHostForAdmission(+ {+ runtimeDirectory: this.#options.runtimeDirectory,+ normalizedSessionId: input.normalizedSessionId,+ workingDirectory: input.workingDirectory,+ agent: "codex",+ model: this.#options.model,+ permissionMode: this.#options.permissionMode ?? "approve-reads",+ systemInstructions: this.#options.systemInstructions,+ environment: this.#options.environment,+ managedCodexCredentialSourcePath:+ this.#options.managedCodexCredentialSourcePath,+ ...(input.signal === undefined ? {} : { signal: input.signal }),+ semanticTools: {+ tools: this.#options.dynamicTools ?? [],+ handler: (call) => {+ if (!session) {+ throw new Error("Codex ACPX session is not ready for tool calls");+ }+ return session.dispatchTool(call);+ },+ },+ },+ input.signal,+ );+ try {+ input.signal?.throwIfAborted();+ session = new CodexAcpxSession({+ host,+ input,+ dynamicToolHandler: this.#options.dynamicToolHandler,+ now: this.#options.now ?? (() => new Date()),+ closeSettlementTimeoutMs: this.#closeSettlementTimeoutMs,+ maxBufferedEvents: this.#maxBufferedEvents,+ terminalEventReserve: this.#terminalEventReserve,+ retainCleanup: (cleanup) => this.#retainCleanup(cleanup),+ quarantineCleanup: (hostToRetain, reason) =>+ this.#quarantineHostCleanup(hostToRetain, reason),+ });+ return session;+ } catch (error) {+ const reason = "Codex ACPX session initialization failed";+ const cleanup = Promise.resolve().then(() => host.close({ reason }));+ this.#retainCleanup(cleanup);+ void cleanup.catch(() => this.#quarantineHostCleanup(host, reason));+ await settleWithin(cleanup, this.#closeSettlementTimeoutMs).catch(+ () => undefined,+ );+ throw error;+ }+ }++ async #openHostForAdmission(+ options: OpenAcpxRuntimeHostOptions,+ signal: AbortSignal | undefined,+ ): Promise<CodexAcpxHost> {+ if (signal === undefined) return await this.#openHost(options);+ signal.throwIfAborted();+ const opening = Promise.resolve().then(() => this.#openHost(options));+ try {+ return await raceDriverAdmissionWithAbort(opening, signal);+ } catch (error) {+ if (signal.aborted) {+ const lateHostCleanup = opening.then(+ (host) => this.#closeLateAdmissionHost(host),+ () => undefined,+ );+ this.#retainCleanup(lateHostCleanup);+ }+ throw error;+ }+ }++ async #closeLateAdmissionHost(host: CodexAcpxHost): Promise<void> {+ const reason = "Codex ACPX host resolved after admission was aborted";+ const cleanup = Promise.resolve().then(() => host.close({ reason }));+ this.#retainCleanup(cleanup);+ try {+ await cleanup;+ } catch {+ this.#quarantineHostCleanup(host, reason);+ }+ }++ #retainCleanup(cleanup: Promise<void>): void {+ this.#cleanupOwners.add(cleanup);+ void cleanup+ .finally(() => this.#cleanupOwners.delete(cleanup))+ .catch(() => undefined);+ }++ #quarantineHostCleanup(host: CodexAcpxHost, reason: string): void {+ if (+ [...this.#quarantinedHostCleanups].some((entry) => entry.host === host)+ ) {+ return;+ }+ const cleanup: QuarantinedHostCleanup = {+ host,+ reason,+ attempt: null,+ recovery: null,+ recoveryMaxAttempts: null,+ timer: null,+ };+ this.#quarantinedHostCleanups.add(cleanup);+ this.#startQuarantinedHostCleanupRecovery(+ cleanup,+ MAX_QUARANTINED_HOST_CLOSE_RETRIES,+ "quarantined cleanup recovery",+ );+ }++ #startQuarantinedHostCleanupRecovery(+ cleanup: QuarantinedHostCleanup,+ maxAttempts: number,+ reason: string,+ ): Promise<void> {+ if (cleanup.recovery) return cleanup.recovery;+ const recovery = (async () => {+ for (+ let attemptCount = 0;+ attemptCount < maxAttempts &&+ this.#quarantinedHostCleanups.has(cleanup);+ attemptCount += 1+ ) {+ // Start the first retry immediately so admission's finite grace applies+ // to provider cleanup rather than to this rate-limit delay. Only later+ // attempts wait, preserving sequential bounded retry behavior.+ if (attemptCount > 0) {+ await waitForCleanupRetry(+ Math.max(+ 1,+ Math.min(+ MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS,+ this.#closeSettlementTimeoutMs,+ ),+ ),+ );+ }+ const attempt = Promise.resolve().then(() =>+ cleanup.host.close({+ reason: `${cleanup.reason} (${reason})`,+ }),+ );+ cleanup.attempt = attempt;+ try {+ await attempt;+ this.#quarantinedHostCleanups.delete(cleanup);+ } catch {+ // Retain the quarantine after this finite, sequential retry batch.+ } finally {+ if (cleanup.attempt === attempt) cleanup.attempt = null;+ }+ }+ })();+ cleanup.recovery = recovery;+ cleanup.recoveryMaxAttempts = maxAttempts;+ this.#retainCleanup(recovery);+ void recovery+ .finally(() => {+ if (cleanup.recovery === recovery) {+ cleanup.recovery = null;+ cleanup.recoveryMaxAttempts = null;+ }+ this.#scheduleQuarantinedHostCleanup(cleanup);+ })+ .catch(() => undefined);+ return recovery;+ }++ #scheduleQuarantinedHostCleanup(cleanup: QuarantinedHostCleanup): void {+ if (+ !this.#quarantinedHostCleanups.has(cleanup) ||+ cleanup.recovery ||+ cleanup.attempt ||+ cleanup.timer+ ) {+ return;+ }+ // A quarantined host remains actively owned, but recovery is rate-limited:+ // each entry holds at most one unref'd timer and one sequential close.+ // Admission may pull that timer forward; it never creates a parallel try.+ cleanup.timer = setTimeout(() => {+ cleanup.timer = null;+ if (!this.#quarantinedHostCleanups.has(cleanup)) return;+ this.#startQuarantinedHostCleanupRecovery(+ cleanup,+ 1,+ "scheduled quarantined cleanup recovery",+ );+ }, QUARANTINED_HOST_CLOSE_RETRY_MS);+ cleanup.timer.unref?.();+ }++ async #retryQuarantinedHostCleanups(): Promise<void> {+ const observedOwners = new Set<Promise<void>>();+ const acceleratedCleanups = new Set<QuarantinedHostCleanup>();+ try {+ while (true) {+ // A close that is still pending has not entered quarantine yet, but it+ // owns the same provider resources. Observe every retained owner and any+ // replacement quarantine recovery it installs before admitting a host.+ const cleanupOwners = new Set<Promise<void>>(this.#cleanupOwners);+ for (const cleanup of this.#quarantinedHostCleanups) {+ if (cleanup.timer) {+ clearTimeout(cleanup.timer);+ cleanup.timer = null;+ }+ if (cleanup.recovery) {+ // A scheduled autonomous owner has only one attempt. Admission+ // observes it first, then starts one complete bounded batch if that+ // attempt fails. A pre-existing full batch already consumed that+ // allowance and must never be duplicated.+ if (+ (cleanup.recoveryMaxAttempts ?? 0) >=+ MAX_QUARANTINED_HOST_CLOSE_RETRIES+ ) {+ acceleratedCleanups.add(cleanup);+ }+ cleanupOwners.add(cleanup.recovery);+ } else if (!acceleratedCleanups.has(cleanup)) {+ acceleratedCleanups.add(cleanup);+ cleanupOwners.add(+ this.#startQuarantinedHostCleanupRecovery(+ cleanup,+ MAX_QUARANTINED_HOST_CLOSE_RETRIES,+ "quarantined cleanup admission recovery",+ ),+ );+ }+ }+ const replacementOwners = [...cleanupOwners].filter(+ (owner) => !observedOwners.has(owner),+ );+ if (replacementOwners.length === 0) break;+ replacementOwners.forEach((owner) => observedOwners.add(owner));+ if (+ !(await settlesWithin(+ Promise.all(+ replacementOwners.map((owner) => owner.catch(() => undefined)),+ ),+ QUARANTINED_HOST_ADMISSION_GRACE_MS,+ ))+ ) {+ throw new Error(+ "Codex ACPX cannot open a new session because quarantined host cleanup exceeded the admission grace",+ );+ }+ }+ } finally {+ // Admission accelerates autonomous timers, but it must never strand a+ // different quarantined host if observing any owner exhausts the grace.+ for (const cleanup of this.#quarantinedHostCleanups) {+ this.#scheduleQuarantinedHostCleanup(cleanup);+ }+ }+ if (+ this.#cleanupOwners.size > 0 ||+ this.#quarantinedHostCleanups.size > 0+ ) {+ throw new Error(+ "Codex ACPX cannot open a new session while quarantined host cleanup remains incomplete",+ );+ }+ }+}++class CodexAcpxSession implements HarnessSession {+ readonly #host: CodexAcpxHost;+ readonly #input: OpenHarnessSessionInput;+ readonly #dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"];+ readonly #now: () => Date;+ readonly #closeSettlementTimeoutMs: number;+ readonly #maxBufferedEvents: number;+ readonly #terminalEventReserve: number;+ readonly #events: AsyncQueue<PrpEvent>;+ readonly #retainCleanup: (cleanup: Promise<void>) => void;+ readonly #quarantineCleanup: (host: CodexAcpxHost, reason: string) => void;+ readonly #transcript: Array<{ event: PrpEvent; bytes: number }> = [];+ readonly #terminalTurns = new Map<string, string>();+ readonly #sourceInstanceId: string;+ #sourceSequence = 0;+ #activeTurnId: string | null = null;+ #semanticResult: PrpStructuredRunResult | null = null;+ #semanticFingerprint: string | null = null;+ #semanticCallId: string | null = null;+ #semanticTurnId: string | null = null;+ #usage: Record<string, unknown> | null = null;+ #assistantText = "";+ #closed = false;+ #closingStarted = false;+ #eventStreamClosed = false;+ #activePump: Promise<void> | null = null;+ #closePromise: Promise<void> | null = null;+ #hostClosePromise: Promise<void> | null = null;+ #hostCloseRecoveryPromise: Promise<void> | null = null;+ #hostClosed = false;+ #transcriptBytes = 0;+ #transcriptEventCount = 0;+ #transcriptOmitted = false;+ #eventStreamOmitted = false;+ #pendingTerminal: {+ turnId: string;+ fingerprint: string;+ eventType: "turn.completed" | "turn.failed" | "turn.interrupted";+ payload: Record<string, unknown>;+ } | null = null;++ constructor(input: {+ host: CodexAcpxHost;+ input: OpenHarnessSessionInput;+ dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"];+ now: () => Date;+ closeSettlementTimeoutMs: number;+ maxBufferedEvents: number;+ terminalEventReserve: number;+ retainCleanup: (cleanup: Promise<void>) => void;+ quarantineCleanup: (host: CodexAcpxHost, reason: string) => void;+ }) {+ const identity = input.host.identity();+ if (identity.normalizedSessionId !== input.input.normalizedSessionId) {+ throw new Error("Codex ACPX host returned a different session identity");+ }+ this.#host = input.host;+ this.#input = structuredClone(input.input);+ this.#dynamicToolHandler = input.dynamicToolHandler;+ this.#now = input.now;+ this.#closeSettlementTimeoutMs = input.closeSettlementTimeoutMs;+ this.#maxBufferedEvents = input.maxBufferedEvents;+ this.#terminalEventReserve = input.terminalEventReserve;+ this.#retainCleanup = input.retainCleanup;+ this.#quarantineCleanup = input.quarantineCleanup;+ this.#events = new AsyncQueue<PrpEvent>(input.maxBufferedEvents);+ this.#sourceInstanceId = stableId(+ "paperclip-acpx",+ input.input.normalizedSessionId,+ );+ }++ ids() {+ const identity = this.#host.identity();+ return {+ driverSessionId: identity.acpxRecordId,+ providerSessionId: identity.agentSessionId,+ displayId: identity.agentSessionId,+ };+ }++ events(): AsyncIterable<PrpEvent> {+ return this.#events;+ }++ async startTurn(input: {+ message: NativeUserMessage;+ }): Promise<{ turnId: string }> {+ this.#assertOpen();+ if (this.#activeTurnId) {+ throw new Error("Codex ACPX session already has an active turn");+ }+ if (+ !this.#events.hasCapacity(+ this.#terminalEventReserve + TURN_START_EVENT_COUNT,+ )+ ) {+ throw new HarnessCapabilityUnavailableError(+ "turn.start",+ "the event consumer must drain the previous turn before another turn can start",+ );+ }+ if (this.#terminalTurns.size >= this.#maxBufferedEvents) {+ throw new HarnessCapabilityUnavailableError(+ "turn.start",+ "the bounded session turn limit was reached; open a new session",+ );+ }+ const turnId = `turn-${randomBytes(12).toString("hex")}`;+ this.#activeTurnId = turnId;+ this.#assistantText = "";+ this.#emit("turn.submitted", { text: input.message.text }, { turnId });+ this.#emit("turn.accepted", { turnId }, { turnId });+ this.#emit("turn.started", { status: "inProgress" }, { turnId });+ let turn: AcpxRuntimeTurn;+ try {+ turn = this.#host.startTurn({+ text: input.message.text,+ requestId: `${safeId(this.#input.runId, "run")}:${turnId}`,+ });+ } catch (error) {+ this.#publishTerminal(+ turnId,+ canonicalJson({ status: "failed" }),+ "turn.failed",+ { status: "failed", error: { message: safeMessage(error) } },+ );+ throw error;+ }+ const pump = this.#pumpTurn(turnId, turn);+ this.#activePump = pump;+ void pump+ .finally(() => {+ if (this.#activePump === pump) this.#activePump = null;+ })+ .catch(() => undefined);+ return { turnId };+ }++ async interrupt(input: { turnId?: string; reason?: string }): Promise<void> {+ this.#assertOpen();+ if (input.turnId && input.turnId !== this.#activeTurnId) {+ throw new HarnessStaleTurnError(input.turnId);+ }+ if (this.#pendingTerminal) {+ throw new HarnessCapabilityUnavailableError(+ "interruption",+ "the active turn is awaiting canonical terminal-event retention",+ );+ }+ if (!this.#activeTurnId) {+ throw new HarnessCapabilityUnavailableError(+ "interruption",+ "there is no active Codex ACPX turn",+ );+ }+ await this.#host.interruptActiveTurn(input.reason ?? "interrupted");+ }++ async dispatchTool(call: RunnerToolCall): Promise<unknown> {+ this.#assertOpen();+ if (this.#pendingTerminal) {+ throw new HarnessCapabilityUnavailableError(+ "tool.dispatch",+ "the active turn is awaiting canonical terminal-event retention",+ );+ }+ const turnId = this.#activeTurnId;+ if (!turnId) {+ throw new Error("Codex ACPX tool call is not bound to an active turn");+ }+ const tool = canonicalRunnerToolName(call.tool);+ if (tool === PRP_COMPLETION_TOOL_NAME || tool === PRP_BLOCK_TOOL_NAME) {+ const validation = validatePrpStructuredRunResult(call.arguments);+ if (!validation.ok) throw new Error("Invalid semantic run result");+ const blocked = validation.result.reportedWorkDisposition === "blocked";+ if (+ (tool === PRP_BLOCK_TOOL_NAME && !blocked) ||+ (tool === PRP_COMPLETION_TOOL_NAME && blocked)+ ) {+ throw new Error(+ "Semantic result disposition does not match the terminal tool",+ );+ }+ const fingerprint = canonicalJson(validation.result);+ if (+ this.#semanticFingerprint !== null &&+ this.#semanticFingerprint !== fingerprint+ ) {+ throw new Error("A different semantic result is already committed");+ }+ if (this.#semanticFingerprint === null) {+ if (+ !this.#emit("run.result.proposed", validation.result, {+ turnId,+ itemId: call.callId,+ })+ ) {+ throw new HarnessCapabilityUnavailableError(+ "run.result.proposed",+ "the event consumer must drain provider events before a semantic result can be accepted",+ );+ }+ this.#semanticResult = structuredClone(validation.result);+ this.#semanticFingerprint = fingerprint;+ this.#semanticCallId = call.callId;+ this.#semanticTurnId = turnId;+ }+ return { accepted: true };+ }+ if (!this.#dynamicToolHandler) {+ throw new Error(`Unsupported Paperclip operation ${tool}`);+ }+ return await this.#dynamicToolHandler({+ tool,+ callId: call.callId,+ providerSessionId: this.#host.identity().agentSessionId,+ turnId,+ arguments: structuredClone(call.arguments),+ signal: call.signal,+ });+ }++ async read(): Promise<Record<string, unknown>> {+ return {+ identity: this.#host.identity(),+ binding: this.#host.binding(),+ status: await this.#host.status(),+ };+ }++ async reconcile(): Promise<Record<string, unknown>> {+ const identity = this.#host.identity();+ const status = await this.#host.status();+ if (+ status.agentSessionId &&+ status.agentSessionId !== identity.agentSessionId+ ) {+ throw new Error("ACPX reconciliation changed the provider session");+ }+ return { identity, status };+ }++ async usage(): Promise<Record<string, unknown> | null> {+ return this.#usage === null ? null : structuredClone(this.#usage);+ }++ async transcript(): Promise<HarnessTranscriptSnapshot> {+ return {+ schema: "paperclip-runner/harness-transcript/v1",+ complete: !this.#transcriptOmitted,+ eventCount: this.#transcriptEventCount,+ events: structuredClone(this.#transcript.map(({ event }) => event)),+ omissionReason: this.#transcriptOmitted ? "retention_limit" : null,+ };+ }++ async snapshot(): Promise<PersistedHarnessSession> {+ const identity = this.#host.identity();+ return {+ driverKind: "acpx_runtime",+ driverSessionId: identity.acpxRecordId,+ providerSessionId: identity.agentSessionId,+ runId: this.#input.runId,+ normalizedSessionId: this.#input.normalizedSessionId,+ activeTurnId: this.#activeTurnId,+ lastSourceSequence: this.#sourceSequence,+ providerIdentity: {+ kind: "acpx",+ normalizedSessionId: identity.normalizedSessionId,+ acpxRecordId: identity.acpxRecordId,+ backendSessionId: identity.backendSessionId,+ agentSessionId: identity.agentSessionId,+ profileDigest: identity.profileDigest,+ workspaceDigest: identity.workspaceDigest,+ requestedModel: identity.requestedModel,+ effectiveModel: identity.effectiveModel,+ permissionMode: identity.permissionMode,+ },+ semanticResult:+ this.#semanticResult &&+ this.#semanticFingerprint &&+ this.#semanticTurnId+ ? {+ result: structuredClone(this.#semanticResult),+ fingerprint: this.#semanticFingerprint,+ callId: this.#semanticCallId,+ turnId: this.#semanticTurnId,+ }+ : null,+ terminalTurns: [...this.#terminalTurns].map(+ ([terminalTurnId, fingerprint]) => ({+ turnId: terminalTurnId,+ fingerprint,+ }),+ ),+ };+ }++ async close(input: { reason: string }): Promise<void> {+ if (this.#closePromise) return await this.#closePromise;+ if (this.#closed) return;+ this.#closingStarted = true;+ const closePromise = this.#finishClose(input.reason);+ this.#closePromise = closePromise;+ try {+ await closePromise;+ this.#closed = true;+ } catch (error) {+ this.#scheduleHostCloseRecovery(input.reason);+ throw error;+ } finally {+ if (this.#closePromise === closePromise) this.#closePromise = null;+ }+ }++ async #finishClose(reason: string): Promise<void> {+ const closingTurnId = this.#activeTurnId;+ const pump = this.#activePump;+ const hostClose =+ this.#hostClosePromise ?? this.#startHostClose({ reason });+ let hostCloseError: unknown = null;+ try {+ await settleWithin(hostClose, this.#closeSettlementTimeoutMs);+ } catch (error) {+ hostCloseError = error;+ }+ if (pump) {+ await settleWithin(+ pump.catch(() => undefined),+ this.#closeSettlementTimeoutMs,+ ).catch(() => undefined);+ }+ if (closingTurnId && !this.#terminalTurns.has(closingTurnId)) {+ const pendingTerminal = this.#pendingTerminal;+ if (pendingTerminal?.turnId === closingTurnId) {+ this.#publishTerminal(+ pendingTerminal.turnId,+ pendingTerminal.fingerprint,+ pendingTerminal.eventType,+ pendingTerminal.payload,+ );+ } else {+ this.#publishTerminal(+ closingTurnId,+ canonicalJson({ status: "interrupted" }),+ "turn.interrupted",+ { status: "interrupted", stopReason: "session_closed" },+ );+ }+ }+ if (hostCloseError) {+ this.#emit(+ "harness.diagnostic",+ {+ code: "acpx_host_cleanup_deferred",+ message:+ "ACPX host cleanup exceeded its caller wait bound; the driver retains the exact cleanup until it settles.",+ },+ closingTurnId ? { turnId: closingTurnId } : {},+ 0,+ );+ }+ this.#eventStreamClosed = true;+ this.#events.close();+ if (hostCloseError) {+ throw hostCloseError;+ }+ }++ #startHostClose(input: { reason: string }): Promise<void> {+ const closePromise = this.#host.close(input).then(() => {+ this.#hostClosed = true;+ });+ this.#hostClosePromise = closePromise;+ this.#retainCleanup(closePromise);+ return closePromise;+ }++ #scheduleHostCloseRecovery(reason: string): void {+ if (this.#hostClosed || this.#hostCloseRecoveryPromise) return;+ const failedOrPendingClose = this.#hostClosePromise;+ if (!failedOrPendingClose) return;+ // Never overlap the exact cleanup that exceeded the caller's wait bound.+ // Once an attempt rejects, keep one delay-bounded recovery owner alive for+ // a finite retry budget. A permanently pending attempt remains the sole+ // owner while the runtime adapter independently terminates its children;+ // repeated terminal failures settle instead of creating an immortal loop.+ // Host cleanup does not logically close the harness session: #finishClose+ // still owns terminal publication and event-stream closure after recovery.+ const recovery = (async () => {+ let attempt = failedOrPendingClose;+ let retryCount = 0;+ while (!this.#hostClosed) {+ try {+ await attempt;+ return;+ } catch (error) {+ if (retryCount >= MAX_AUTONOMOUS_HOST_CLOSE_RETRIES) {+ this.#quarantineCleanup(this.#host, reason);+ throw error;+ }+ retryCount += 1;+ }+ await waitForCleanupRetry(+ Math.max(1, Math.min(1_000, this.#closeSettlementTimeoutMs)),+ );+ if (this.#hostClosed) {+ return;+ }+ if (this.#hostClosePromise === attempt) {+ this.#hostClosePromise = null;+ }+ attempt = this.#startHostClose({+ reason: `${reason} (automatic cleanup recovery ${retryCount})`,+ });+ }+ })();+ this.#hostCloseRecoveryPromise = recovery;+ this.#retainCleanup(recovery);+ void recovery+ .finally(() => {+ if (this.#hostCloseRecoveryPromise === recovery) {+ this.#hostCloseRecoveryPromise = null;+ }+ })+ .catch(() => undefined);+ }++ async #pumpTurn(turnId: string, turn: AcpxRuntimeTurn): Promise<void> {+ try {+ let index = 0;+ const normalizeToolEvent =+ createAcpxToolEventNormalizer<AcpRuntimeEvent>();+ for await (const event of turn.events) {+ this.#mapRuntimeEvent(normalizeToolEvent(event), turnId, ++index);+ }+ const result = await turn.result;+ if (this.#terminalTurns.has(turnId)) return;+ if (result.status === "completed") {+ const finalText = this.#assistantText.trim();+ if (finalText) {+ this.#emit(+ "item.completed",+ { kind: "agentMessage", channel: "final", text: finalText },+ { turnId, itemId: `${turnId}:final-answer` },+ );+ }+ this.#publishTerminal(+ turnId,+ canonicalJson({+ status: "completed",+ semanticResult: this.#semanticFingerprint,+ }),+ "turn.completed",+ { status: "completed", stopReason: result.stopReason ?? null },+ );+ } else if (result.status === "cancelled") {+ this.#publishTerminal(+ turnId,+ canonicalJson({ status: "interrupted" }),+ "turn.interrupted",+ {+ status: "interrupted",+ stopReason: result.stopReason ?? "cancelled",+ },+ );+ } else {+ this.#emit(+ "provider.notice.recorded",+ {+ schema: "paperclip.provider.notice.v1",+ noticeId: `${turnId}:failure`,+ severity: "error",+ category: "acpx_turn_failed",+ scope: "turn",+ recoverable: result.error.retryable ?? false,+ userActionable: true,+ summary: safeMessage(result.error.message),+ },+ { turnId, itemId: `${turnId}:failure` },+ );+ this.#publishTerminal(+ turnId,+ canonicalJson({ status: "failed" }),+ "turn.failed",+ {+ status: "failed",+ error: {+ code: result.error.code ?? null,+ message: safeMessage(result.error.message),+ },+ },+ );+ }+ } catch (error) {+ if (this.#terminalTurns.has(turnId)) return;+ if (error instanceof TerminalEventCapacityError) throw error;+ if (this.#closed || this.#closingStarted) {+ this.#publishTerminal(+ turnId,+ canonicalJson({ status: "interrupted" }),+ "turn.interrupted",+ { status: "interrupted", stopReason: "session_closed" },+ );+ } else {+ this.#publishTerminal(+ turnId,+ canonicalJson({ status: "failed" }),+ "turn.failed",+ { status: "failed", error: { message: safeMessage(error) } },+ );+ }+ }+ }++ #publishTerminal(+ turnId: string,+ fingerprint: string,+ eventType: "turn.completed" | "turn.failed" | "turn.interrupted",+ payload: Record<string, unknown>,+ ): void {+ if (this.#terminalTurns.has(turnId)) return;+ const pendingTerminal = this.#pendingTerminal;+ if (+ pendingTerminal !== null &&+ (pendingTerminal.turnId !== turnId ||+ pendingTerminal.fingerprint !== fingerprint ||+ pendingTerminal.eventType !== eventType ||+ canonicalJson(pendingTerminal.payload) !== canonicalJson(payload))+ ) {+ throw new Error(+ `Codex ACPX terminal disposition changed while ${pendingTerminal.turnId} awaited stream capacity`,+ );+ }+ const terminal = pendingTerminal ?? {+ turnId,+ fingerprint,+ eventType,+ payload: structuredClone(payload),+ };+ this.#pendingTerminal = terminal;+ if (!this.#emit(terminal.eventType, terminal.payload, { turnId })) {+ // The persisted terminal index is the driver's settlement authority.+ // Never advance it unless the matching canonical event is already in+ // the bounded stream. A caller may drain and retry close, but it cannot+ // observe a settled turn whose terminal fact was omitted.+ throw new TerminalEventCapacityError(turnId, eventType);+ }+ this.#terminalTurns.set(turnId, fingerprint);+ this.#pendingTerminal = null;+ if (this.#activeTurnId === turnId) this.#activeTurnId = null;+ }++ #mapRuntimeEvent(+ event: AcpRuntimeEvent,+ turnId: string,+ index: number,+ ): void {+ const fallbackItemId = `${turnId}:acp:${index}`;+ if (event.type === "text_delta") {+ const output = boundedText(event.text, 64 * 1024);+ if (event.stream !== "thought" && event.tag !== "agent_thought_chunk") {+ this.#assistantText = boundedText(+ `${this.#assistantText}${output}`,+ 256 * 1024,+ );+ }+ this.#emit(+ "item.delta",+ {+ kind:+ event.stream === "thought" || event.tag === "agent_thought_chunk"+ ? "thinking"+ : "agent_message",+ text: output,+ },+ { turnId, itemId: fallbackItemId },+ );+ }+ if (event.type === "status" && event.tag === "usage_update") {+ this.#usage = boundedRecord({+ cumulative: event.breakdown,+ cost: event.cost,+ });+ }+ for (const canonical of canonicalProviderEventsFromAcpxRuntimeEvent(+ event,+ fallbackItemId,+ turnId,+ )) {+ this.#emit(canonical.eventType, canonical.payload, {+ turnId,+ itemId: canonical.itemId,+ });+ }+ }++ #emit(+ eventType: PrpEvent["eventType"],+ payload: Record<string, unknown>,+ refs: { turnId?: string; itemId?: string } = {},+ reservedAfter = isTerminalEvent(eventType)+ ? 0+ : eventType === "run.result.proposed"+ ? Math.max(0, this.#terminalEventReserve - 1)+ : this.#terminalEventReserve,+ ): boolean {+ if (this.#eventStreamClosed) return false;+ if (this.#eventStreamOmitted && isTerminalEvent(eventType)) {+ this.#eventStreamOmitted = false;+ const recordedOmission = this.#emit(+ "harness.diagnostic",+ {+ code: "event_stream_retention_limit",+ message:+ "Earlier provider events were omitted because the consumer exceeded the bounded event buffer.",+ },+ refs,+ 1,+ );+ if (!recordedOmission) this.#eventStreamOmitted = true;+ }+ const sourceSeq = this.#sourceSequence + 1;+ const event: PrpEvent = {+ schema: "paperclip.prp.event.v1",+ sourceEventId: `${this.#sourceInstanceId}:${sourceSeq}`,+ sourceSeq,+ sourceInstanceId: this.#sourceInstanceId,+ sourceKind: "runner",+ runId: this.#input.runId,+ normalizedSessionId: this.#input.normalizedSessionId,+ ...(refs.turnId ? { turnId: refs.turnId } : {}),+ ...(refs.itemId ? { itemId: refs.itemId } : {}),+ eventType,+ schemaVersion: 1,+ priority: eventType === "run.result.proposed" ? 0 : 1,+ emittedAt: this.#now().toISOString(),+ payload: structuredClone(payload),+ };+ if (!this.#events.push(event, reservedAfter)) {+ this.#eventStreamOmitted = true;+ this.#transcriptEventCount += 1;+ this.#transcriptOmitted = true;+ return false;+ }+ this.#sourceSequence = sourceSeq;+ this.#retainTranscriptEvent(event);+ return true;+ }++ #retainTranscriptEvent(event: PrpEvent): void {+ this.#transcriptEventCount += 1;+ const retained = structuredClone(event);+ let bytes: number;+ try {+ bytes = Buffer.byteLength(JSON.stringify(retained));+ } catch {+ this.#transcriptOmitted = true;+ return;+ }+ if (bytes > MAX_TRANSCRIPT_BYTES) {+ this.#transcriptOmitted = true;+ return;+ }+ this.#transcript.push({ event: retained, bytes });+ this.#transcriptBytes += bytes;+ while (+ this.#transcript.length > MAX_TRANSCRIPT_EVENTS ||+ this.#transcriptBytes > MAX_TRANSCRIPT_BYTES+ ) {+ const omitted = this.#transcript.shift();+ if (omitted) this.#transcriptBytes -= omitted.bytes;+ this.#transcriptOmitted = true;+ }+ }++ #assertOpen(): void {+ if (this.#closed || this.#closingStarted) {+ throw new Error("Codex ACPX session is closing or closed");+ }+ }+}++function waitForCleanupRetry(delayMs: number): Promise<void> {+ return new Promise((resolve) => {+ const timer = setTimeout(resolve, delayMs);+ timer.unref();+ });+}++function canonicalJson(value: unknown): string {+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;+ if (typeof value === "object" && value !== null) {+ const record = value as Record<string, unknown>;+ return `{${Object.keys(record)+ .sort()+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)+ .join(",")}}`;+ }+ return JSON.stringify(value) ?? "undefined";+}++function boundedRecord(value: unknown): Record<string, unknown> {+ const serialized = JSON.stringify(value);+ if (!serialized || Buffer.byteLength(serialized) > 64 * 1024) {+ return { omitted: true, reason: "payload_limit" };+ }+ const parsed: unknown = JSON.parse(serialized);+ return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)+ ? (parsed as Record<string, unknown>)+ : {};+}++function boundedText(value: string, maxBytes: number): string {+ const bytes = Buffer.from(value);+ return bytes.length <= maxBytes+ ? value+ : bytes.subarray(bytes.length - maxBytes).toString("utf8");+}++function safeId(value: string, fallback: string): string {+ const candidate = value.replace(/[^A-Za-z0-9._:-]/g, "-").slice(0, 160);+ return /^[A-Za-z0-9]/.test(candidate) ? candidate : fallback;+}++function stableId(prefix: string, value: string): string {+ const readable = safeId(value, "session").slice(0, 80);+ const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16);+ return `${prefix}-${readable}-${suffix}`;+}++function safeMessage(error: unknown): string {+ const message = error instanceof Error ? error.message : String(error);+ return message+ .replace(+ /(\bauthorization\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\r\n,}\]]*)/gi,+ "$1[REDACTED]",+ )+ .replace(+ /(\b(?:key|token|secret|password)\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\s,}\]]+)/gi,+ "$1[REDACTED]",+ )+ .slice(0, 4_000);+}++function reportRetainedAcpxCleanupFailure(input: {+ resource: "credential" | "command" | "runtime";+ attempt: number;+ error: unknown;+}): void {+ const errorName = input.error instanceof Error ? input.error.name : "Error";+ process.emitWarning(+ JSON.stringify({+ schema: "paperclip.runner.retained_cleanup_failure.v1",+ resource: input.resource,+ attempt: input.attempt,+ errorName,+ }),+ {+ code: "PAPERCLIP_ACPX_RETAINED_CLEANUP_FAILURE",+ type: "PaperclipRunnerCleanupWarning",+ },+ );+}++function isTerminalEvent(eventType: PrpEvent["eventType"]): boolean {+ return (+ eventType === "turn.completed" ||+ eventType === "turn.failed" ||+ eventType === "turn.interrupted"+ );+}++class TerminalEventCapacityError extends Error {+ constructor(turnId: string, eventType: PrpEvent["eventType"]) {+ super(+ `Codex ACPX cannot settle ${turnId} before its ${eventType} event is retained`,+ );+ this.name = "TerminalEventCapacityError";+ }+}++class AsyncQueue<T> implements AsyncIterable<T> {+ readonly #items: T[] = [];+ readonly #waiters: Array<(result: IteratorResult<T>) => void> = [];+ readonly #maxItems: number;+ #closed = false;++ constructor(maxItems: number) {+ this.#maxItems = maxItems;+ }++ hasCapacity(requiredItems: number): boolean {+ return (+ requiredItems <=+ this.#waiters.length + this.#maxItems - this.#items.length+ );+ }++ push(item: T, reservedAfter = 0): boolean {+ if (this.#closed) return false;+ // Admission and insertion are one synchronous operation so every regular+ // event leaves the terminal lane intact. In particular, a lagging+ // consumer cannot occupy terminal capacity between a check and a push.+ if (!this.hasCapacity(1 + reservedAfter)) return false;+ const waiter = this.#waiters.shift();+ if (waiter) {+ waiter({ done: false, value: item });+ return true;+ }+ if (this.#items.length >= this.#maxItems) return false;+ this.#items.push(item);+ return true;+ }++ close(): void {+ if (this.#closed) return;+ this.#closed = true;+ for (const waiter of this.#waiters.splice(0)) {+ waiter({ done: true, value: undefined });+ }+ }++ [Symbol.asyncIterator](): AsyncIterator<T> {+ return {+ next: () => {+ const item = this.#items.shift();+ if (item !== undefined) {+ return Promise.resolve({ done: false, value: item });+ }+ if (this.#closed) {+ return Promise.resolve({ done: true, value: undefined });+ }+ return new Promise((resolve) => this.#waiters.push(resolve));+ },+ };+ }+}++async function settleWithin(+ promise: Promise<void>,+ timeoutMs: number,+): Promise<void> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ promise,+ new Promise<void>((_resolve, reject) => {+ timer = setTimeout(() => {+ reject(new Error("ACPX host cleanup exceeded its shutdown timeout"));+ }, timeoutMs);+ timer.unref();+ }),+ ]);+ } finally {+ if (timer) clearTimeout(timer);+ }+}++async function runAbortableDriverAdmission<T>(+ signal: AbortSignal | undefined,+ operation: () => Promise<T>,+): Promise<T> {+ if (signal === undefined) return await operation();+ signal.throwIfAborted();+ return await raceDriverAdmissionWithAbort(+ Promise.resolve().then(operation),+ signal,+ );+}++function raceDriverAdmissionWithAbort<T>(+ pending: Promise<T>,+ signal: AbortSignal,+): Promise<T> {+ return new Promise<T>((resolve, reject) => {+ let settled = false;+ const settle = (operation: () => void): void => {+ if (settled) return;+ settled = true;+ signal.removeEventListener("abort", onAbort);+ operation();+ };+ const onAbort = (): void => settle(() => reject(signal.reason));+ signal.addEventListener("abort", onAbort, { once: true });+ if (signal.aborted) {+ onAbort();+ return;+ }+ void pending.then(+ (value) => settle(() => resolve(value)),+ (error: unknown) => settle(() => reject(error)),+ );+ });+}++async function settlesWithin(+ promise: Promise<unknown>,+ timeoutMs: number,+): Promise<boolean> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ return await Promise.race([+ promise.then(() => true),+ new Promise<false>((resolve) => {+ timer = setTimeout(() => resolve(false), timeoutMs);+ timer.unref?.();+ }),+ ]);+ } finally {+ if (timer) clearTimeout(timer);+ }+}diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.tsindex 4bea8aad78..0e89953bfe 100644--- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts+++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts@@ -32,14 +32,23 @@ describe("Codex ACPX runtime adapter", () => {controller.abort(cancellation);const command = fakeCommand();const createRuntime = vi.fn();+ const retainFailedAdmissionCleanup = vi.fn();await expect(openCodexAcpxRuntime(- { ...openOptions(command), signal: controller.signal },+ {+ ...openOptions(command),+ signal: controller.signal,+ retainFailedAdmissionCleanup,+ },{ createRuntime },),).rejects.toBe(cancellation);+ expect(retainFailedAdmissionCleanup).toHaveBeenCalledOnce();+ await expect(+ retainFailedAdmissionCleanup.mock.calls[0]?.[0],+ ).resolves.toBeUndefined();expect(createRuntime).not.toHaveBeenCalled();expect(command.spawn).not.toHaveBeenCalled();});@@ -248,6 +257,807 @@ describe("Codex ACPX runtime adapter", () => {});});+ it("never overlaps a retained protocol close that has not settled", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ const runtimeClose = new Promise<void>(() => {});+ vi.mocked(runtime.close)+ .mockReturnValueOnce(runtimeClose)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ });+ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });++ const firstClose = expect(+ port.close({ reason: "runtime close stalled" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ expect(runtime.close).toHaveBeenCalledOnce();+ await vi.advanceTimersByTimeAsync(2_000);+ expect(child.kill).toHaveBeenCalledWith("SIGTERM");+ await firstClose;++ // Repeated callers inherit the same bounded observation. A permanently+ // pending exact close remains the sole protocol attempt for this handle.+ expect(runtime.close).toHaveBeenCalledOnce();+ const secondClose = expect(+ port.close({ reason: "idempotent terminal close" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await vi.advanceTimersByTimeAsync(2_000);+ await secondClose;+ expect(runtime.close).toHaveBeenCalledOnce();+ } finally {+ vi.useRealTimers();+ }+ });++ it("allows a fresh close after a retained attempt rejects late", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectRuntimeClose!: (error: unknown) => void;+ const runtimeClose = new Promise<void>((_resolve, reject) => {+ rejectRuntimeClose = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(runtimeClose)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ });+ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });++ const firstClose = expect(+ port.close({ reason: "runtime close stalled" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(2_000);+ await firstClose;+ expect(child.kill).toHaveBeenCalledWith("SIGTERM");++ const protocolFailure = new Error("late protocol close failure");+ rejectRuntimeClose(protocolFailure);+ // The late-settlement observer schedules reconciliation asynchronously.+ // Wait for that fresh attempt instead of racing another caller against+ // the already-settled retained failure.+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));+ await expect(+ port.close({ reason: "retry after retained failure" }),+ ).resolves.toBeUndefined();+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("reconciles a retained close only after its late failure settles", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectFirstClose!: (error: unknown) => void;+ const firstRuntimeClose = new Promise<void>((_resolve, reject) => {+ rejectFirstClose = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(firstRuntimeClose)+ .mockResolvedValueOnce(undefined)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ });+ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });++ const firstClose = expect(+ port.close({ reason: "first protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(2_000);+ await firstClose;++ const inheritedClose = expect(+ port.close({ reason: "observe pending protocol close" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await vi.advanceTimersByTimeAsync(2_000);+ await inheritedClose;+ expect(runtime.close).toHaveBeenCalledOnce();++ rejectFirstClose(new Error("older protocol close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ await expect(+ port.close({ reason: "observe reconciled cleanup" }),+ ).resolves.toBeUndefined();+ expect(runtime.close).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("shares a pending close across concurrent callers before reconciliation", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectFirstClose!: (error: unknown) => void;+ const firstRuntimeClose = new Promise<void>((_resolve, reject) => {+ rejectFirstClose = reject;+ });+ let resolveFreshClose!: () => void;+ const freshRuntimeClose = new Promise<void>((resolve) => {+ resolveFreshClose = resolve;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(firstRuntimeClose)+ .mockReturnValueOnce(freshRuntimeClose)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ });+ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });++ const firstClose = expect(+ port.close({ reason: "first protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(2_000);+ await firstClose;++ const inheritedClose = expect(+ port.close({ reason: "observe pending protocol close" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ expect(runtime.close).toHaveBeenCalledOnce();+ rejectFirstClose(new Error("retained protocol close failed"));+ await inheritedClose;+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));++ resolveFreshClose();+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(2);+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ await expect(+ port.close({ reason: "observe reconciled cleanup" }),+ ).resolves.toBeUndefined();+ expect(runtime.close).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("does not let late settlements bypass the reconciliation retry bound", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"));+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ });++ const firstClose = expect(+ port.close({ reason: "first protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(2_000);+ await firstClose;+ rejectInitialClose(new Error("initial protocol close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(4);+ } finally {+ vi.useRealTimers();+ }+ });++ it("keeps timed-out reconciliation failures in their originating budget", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ const reconciliationRejectors: Array<(error: unknown) => void> = [];+ const reconciliationAttempts = Array.from(+ { length: 3 },+ () =>+ new Promise<void>((_resolve, reject) => {+ reconciliationRejectors.push(reject);+ }),+ );+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockReturnValueOnce(reconciliationAttempts[0]!)+ .mockReturnValueOnce(reconciliationAttempts[1]!)+ .mockReturnValueOnce(reconciliationAttempts[2]!)+ .mockResolvedValueOnce(undefined);+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const firstClose = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await firstClose;+ rejectInitialClose(new Error("external close failed late"));++ for (let index = 0; index < reconciliationRejectors.length; index += 1) {+ await vi.waitFor(() =>+ expect(runtime.close).toHaveBeenCalledTimes(index + 2),+ );+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: `ACPX late protocol cleanup reconciliation ${index + 1}`,+ discardPersistentState: false,+ });+ const overlappingExternalClose =+ index === 0+ ? expect(+ port.close({+ reason: "external observer of reconciliation",+ }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed")+ : null;+ // The external observer coalesces onto reconciliation attempt one. It+ // must not relabel that immutable attempt as an external generation.+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ if (overlappingExternalClose) await overlappingExternalClose;+ reconciliationRejectors[index]!(+ new Error(`reconciliation ${index + 1} failed late`),+ );+ }++ await vi.advanceTimersByTimeAsync(0);+ await Promise.resolve();+ expect(runtime.close).toHaveBeenCalledTimes(4);+ } finally {+ vi.useRealTimers();+ }+ });++ it("renews one budget when an external close joins the final reconciliation", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let rejectFinalReconciliation!: (error: unknown) => void;+ const finalReconciliation = new Promise<void>((_resolve, reject) => {+ rejectFinalReconciliation = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockReturnValueOnce(finalReconciliation)+ .mockResolvedValueOnce(undefined);+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const initialObserver = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await initialObserver;+ rejectInitialClose(new Error("external close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));++ const finalObserver = expect(+ port.close({ reason: "external observer of final reconciliation" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await finalObserver;+ expect(runtime.close).toHaveBeenCalledTimes(4);++ rejectFinalReconciliation(+ new Error("final reconciliation failed late"),+ );+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(5);+ } finally {+ vi.useRealTimers();+ }+ });++ it("batches coalesced external closes when the final reconciliation fails", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let rejectFinalReconciliation!: (error: unknown) => void;+ const finalReconciliation = new Promise<void>((_resolve, reject) => {+ rejectFinalReconciliation = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockReturnValueOnce(finalReconciliation)+ .mockRejectedValueOnce(new Error("renewed reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("renewed reconciliation 2 failed"))+ .mockRejectedValueOnce(new Error("renewed reconciliation 3 failed"));+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const initialObserver = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await initialObserver;+ rejectInitialClose(new Error("external close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));++ const coalescedObservers = ["first", "second", "third"].map((label) =>+ expect(+ port.close({ reason: `${label} external observer` }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed"),+ );+ rejectFinalReconciliation(+ new Error("final reconciliation failed with joined observers"),+ );+ expect(runtime.close).toHaveBeenCalledTimes(4);+ await Promise.all(coalescedObservers);+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(7));+ expect(+ vi+ .mocked(runtime.close)+ .mock.calls.slice(4)+ .map(([input]) => input.reason),+ ).toEqual([+ "ACPX late protocol cleanup reconciliation 1",+ "ACPX late protocol cleanup reconciliation 2",+ "ACPX late protocol cleanup reconciliation 3",+ ]);+ await vi.advanceTimersByTimeAsync(0);+ await Promise.resolve();+ expect(runtime.close).toHaveBeenCalledTimes(7);+ } finally {+ vi.useRealTimers();+ }+ });++ it("consumes an external intent when a timed-out final reconciliation succeeds late", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let resolveFinalReconciliation!: () => void;+ const finalReconciliation = new Promise<void>((resolve) => {+ resolveFinalReconciliation = resolve;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockReturnValueOnce(finalReconciliation);+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const initialObserver = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await initialObserver;+ rejectInitialClose(new Error("external close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));++ const finalObserver = expect(+ port.close({+ reason: "external observer of successful reconciliation",+ }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await finalObserver;+ expect(runtime.close).toHaveBeenCalledTimes(4);++ resolveFinalReconciliation();+ await vi.advanceTimersByTimeAsync(0);+ await Promise.resolve();+ expect(runtime.close).toHaveBeenCalledTimes(4);+ } finally {+ vi.useRealTimers();+ }+ });++ it("renews an external intent when bounded protocol success cannot terminate the provider", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let resolveFinalReconciliation!: () => void;+ const finalReconciliation = new Promise<void>((resolve) => {+ resolveFinalReconciliation = resolve;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockReturnValueOnce(finalReconciliation)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ child.kill = vi.fn(() => true);+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ runtimeCloseTimeoutMs: 1,+ });++ const initialObserver = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await initialObserver;+ rejectInitialClose(new Error("external close failed late"));+ for (+ let turn = 0;+ turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4;+ turn += 1+ ) {+ await Promise.resolve();+ }+ expect(runtime.close).toHaveBeenCalledTimes(4);++ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });+ const finalObserver = expect(+ port.close({ reason: "external observer of failed process cleanup" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ resolveFinalReconciliation();+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(4_000);+ await finalObserver;++ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ child.signalCode = "SIGKILL";+ child.emit("exit", null, "SIGKILL");+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(5);+ } finally {+ vi.useRealTimers();+ }+ });++ it("renews an external intent when late protocol success follows process cleanup failure", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let resolveFinalReconciliation!: () => void;+ const finalReconciliation = new Promise<void>((resolve) => {+ resolveFinalReconciliation = resolve;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockReturnValueOnce(finalReconciliation)+ .mockResolvedValueOnce(undefined);+ const child = fakeChild();+ child.kill = vi.fn(() => true);+ const command = fakeCommand();+ vi.mocked(command.spawn).mockReturnValue(child);+ let runtimeOptions: AcpRuntimeOptions | undefined;+ const port = await openCodexAcpxRuntime(openOptions(command), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: (options) => {+ runtimeOptions = options;+ return runtime;+ },+ runtimeCloseTimeoutMs: 1,+ });++ const initialObserver = expect(+ port.close({ reason: "external protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await initialObserver;+ rejectInitialClose(new Error("external close failed late"));+ for (+ let turn = 0;+ turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4;+ turn += 1+ ) {+ await Promise.resolve();+ }+ expect(runtime.close).toHaveBeenCalledTimes(4);++ runtimeOptions?.spawnAgent?.({+ command: "ignored",+ args: ["--stdio"],+ options: {},+ });+ const finalObserver = expect(+ port.close({ reason: "external observer of failed process cleanup" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await vi.advanceTimersByTimeAsync(4_000);+ await finalObserver;+ expect(runtime.close).toHaveBeenCalledTimes(4);++ child.signalCode = "SIGKILL";+ child.emit("exit", null, "SIGKILL");+ resolveFinalReconciliation();+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(5);+ } finally {+ vi.useRealTimers();+ }+ });++ it("shares one reconciliation budget across repeated bounded observers", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectRetainedClose!: (error: unknown) => void;+ const retainedClose = new Promise<void>((_resolve, reject) => {+ rejectRetainedClose = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(retainedClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"));+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ for (const reason of [+ "first observer",+ "second observer",+ "third observer",+ ]) {+ const close = expect(port.close({ reason })).rejects.toThrow(+ "ACPX runtime and provider cleanup failed",+ );+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await close;+ }+ expect(runtime.close).toHaveBeenCalledOnce();+ rejectRetainedClose(new Error("retained close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));+ await vi.advanceTimersByTimeAsync(0);+ expect(runtime.close).toHaveBeenCalledTimes(4);+ } finally {+ vi.useRealTimers();+ }+ });++ it("gives each late failure generation a bounded reconciliation budget", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let rejectInitialClose!: (error: unknown) => void;+ const initialClose = new Promise<void>((_resolve, reject) => {+ rejectInitialClose = reject;+ });+ let rejectNewerClose!: (error: unknown) => void;+ const newerClose = new Promise<void>((_resolve, reject) => {+ rejectNewerClose = reject;+ });+ vi.mocked(runtime.close)+ .mockReturnValueOnce(initialClose)+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"))+ .mockReturnValueOnce(newerClose)+ .mockResolvedValueOnce(undefined);+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const firstClose = expect(+ port.close({ reason: "first protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await firstClose;+ rejectInitialClose(new Error("initial close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));+ await vi.advanceTimersByTimeAsync(0);++ const secondClose = expect(+ port.close({ reason: "newer protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await secondClose;+ expect(runtime.close).toHaveBeenCalledTimes(5);++ rejectNewerClose(new Error("newer close failed late"));+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(6));+ expect(runtime.close).toHaveBeenLastCalledWith({+ handle: HANDLE,+ reason: "ACPX late protocol cleanup reconciliation 1",+ discardPersistentState: false,+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("marks a retained protocol close terminal when it succeeds late", async () => {+ vi.useFakeTimers();+ try {+ const runtime = fakeRuntime();+ let resolveRetainedClose!: () => void;+ const retainedClose = new Promise<void>((resolve) => {+ resolveRetainedClose = resolve;+ });+ vi.mocked(runtime.close).mockReturnValueOnce(retainedClose);+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {+ createRegistry: () => registry(),+ createStore: () => store(),+ createRuntime: () => runtime,+ runtimeCloseTimeoutMs: 1,+ });++ const close = expect(+ port.close({ reason: "protocol close stalls" }),+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");+ await Promise.resolve();+ await vi.advanceTimersByTimeAsync(1);+ await close;+ expect(runtime.close).toHaveBeenCalledOnce();++ resolveRetainedClose();+ await vi.advanceTimersByTimeAsync(0);+ await expect(+ port.close({ reason: "observe late success" }),+ ).resolves.toBeUndefined();+ expect(runtime.close).toHaveBeenCalledOnce();+ } finally {+ vi.useRealTimers();+ }+ });+it("maps prompt turns to the admitted ACPX handle", async () => {const runtime = fakeRuntime();const turn = {@@ -566,11 +1376,14 @@ describe("Codex ACPX runtime adapter", () => {.mockResolvedValueOnce(undefined);const controller = new AbortController();const cancellation = new Error("runtime admission cancelled");+ const retainedAdmissionCleanups: Promise<void>[] = [];const opening = openCodexAcpxRuntime({...openOptions(fakeCommand()),signal: controller.signal,+ retainFailedAdmissionCleanup: (cleanup) =>+ retainedAdmissionCleanups.push(cleanup),},{createRegistry: () => registry(),@@ -584,9 +1397,11 @@ describe("Codex ACPX runtime adapter", () => {controller.abort(cancellation);await expect(opening).rejects.toBe(cancellation);+ expect(retainedAdmissionCleanups).toHaveLength(1);resolveHandshake?.(HANDLE);await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));+ await expect(retainedAdmissionCleanups[0]).resolves.toBeUndefined();expect(vi.mocked(runtime.close).mock.calls[0]?.[0]).toEqual({handle: HANDLE,reason: "ACPX runtime admission aborted",@@ -631,7 +1446,9 @@ describe("Codex ACPX runtime adapter", () => {controller.abort(cancellation);await expect(opening).rejects.toBe(cancellation);- expect(retainedCleanups).toHaveLength(1);+ // The exact late-handshake owner and the host-facing aggregate proof are+ // distinct retained promises over the same cleanup obligation.+ expect(retainedCleanups).toHaveLength(2);resolveHandshake?.(HANDLE);await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce());@@ -649,7 +1466,7 @@ describe("Codex ACPX runtime adapter", () => {expect(runtime.close).toHaveBeenCalledOnce();resolveClose?.();- await retainedCleanups[0];+ await expect(Promise.all(retainedCleanups)).resolves.toBeDefined();expect(cleanupSettled).toBe(true);expect(runtime.close).toHaveBeenCalledOnce();});@@ -892,13 +1709,22 @@ describe("Codex ACPX runtime adapter", () => {} as never);await expect(opening).rejects.toBeInstanceOf(Error);- expect(retainedCleanups).toHaveLength(2);+ // Retain the pending handshake, the discovered runtime handle cleanup,+ // and their host-facing aggregate proof until the same obligation settles.+ expect(retainedCleanups).toHaveLength(3);+ const settledCleanups = new Set<Promise<void>>();+ for (const cleanup of retainedCleanups) {+ void cleanup+ .finally(() => settledCleanups.add(cleanup))+ .catch(() => undefined);+ }await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));- await expect(retainedCleanups[0]).resolves.toBeUndefined();+ await vi.waitFor(() => expect(settledCleanups.size).toBe(1));expect(runtime.close).toHaveBeenCalledTimes(5);rejectHandshake?.(new Error("test handshake stopped"));- await expect(retainedCleanups[1]).rejects.toThrow("test handshake stopped");+ await vi.waitFor(() => expect(settledCleanups.size).toBe(3));+ await expect(Promise.all(retainedCleanups)).resolves.toBeDefined();});it("aggregates asynchronous provider signal errors after a failed handshake", async () => {@@ -1267,6 +2093,7 @@ function openOptions(},systemInstructions: "Use Paperclip tools.",mcpServers: [],+ retainFailedAdmissionCleanup: vi.fn(),};}diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.tsindex 01a15540a8..3c4508be88 100644--- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts+++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts@@ -24,6 +24,22 @@ const VERIFIED_COMMAND_SENTINEL = "paperclip-verified-acpx-command";const DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS = 2_000;const RETAINED_ADMISSION_CLEANUP_RETRY_MIN_MS = 10;const RETAINED_ADMISSION_CLEANUP_RETRY_MAX_MS = 30_000;+const PROVIDER_TERM_EXIT_TIMEOUT_MS = 2_000;+const PROVIDER_KILL_EXIT_TIMEOUT_MS = 2_000;+const MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS = 3;+// Production shutdown waits for the protocol close bound before beginning the+// sequential TERM/KILL verification windows. Keep this exported package-local+// bound aligned with the implementation so admission can include the complete+// provider cleanup path instead of accounting for only part of it.+export const DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS =+ DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS ++ PROVIDER_TERM_EXIT_TIMEOUT_MS ++ PROVIDER_KILL_EXIT_TIMEOUT_MS;+// A close may outlive its caller-facing wait bound. Keep every exact attempt+// owned until it settles. A handle never starts a second protocol close while+// the first remains unresolved; late failure can start bounded reconciliation+// only after the exact attempt reaches a terminal outcome.+const activeRuntimeCleanupOwners = new Set<Promise<unknown>>();const activeCodexRuntimeCleanupOwners = new Set<Promise<unknown>>();class AcpxRuntimeCloseTimeoutError extends Error {@@ -53,12 +69,19 @@ export async function openCodexAcpxRuntime(options: AcpxRuntimePortOpenOptions,dependencies: CodexAcpxRuntimeDependencies = {},): Promise<AcpxRuntimePort> {+ if (options.signal?.aborted) {+ // The host may have already transferred its staged credential to this+ // pending admission before this microtask begins. No adapter resources+ // exist yet, so publish an already-complete cleanup proof before preserving+ // the caller's exact abort reason.+ options.retainFailedAdmissionCleanup(Promise.resolve());+ throw options.signal.reason;+ }if (options.profile.agent !== "codex") {throw new Error("The production ACPX runtime currently supports Codex only",);}- options.signal?.throwIfAborted();// The verified-command boundary already refuses to mint a Windows command// lease, because Node cannot pin its executable there. Repeat the platform// gate at this lower boundary so alternate host wiring cannot launch a@@ -81,6 +104,7 @@ export async function openCodexAcpxRuntime(const baseStore = createStore({ stateDir: options.stateDirectory });let failedHandshakeHandle: AcpRuntimeHandle | null = null;let admissionCleanup: RuntimeAdmissionCleanup | null = null;+ let abortedHandshakeCleanup: Promise<void> | null = null;const retainedCleanupOwners = new WeakSet<Promise<void>>();const retainCleanup = (cleanup: Promise<void>): void => {if (retainedCleanupOwners.has(cleanup)) {@@ -236,14 +260,19 @@ export async function openCodexAcpxRuntime(handle = await raceRuntimeHandshakeWithAbort(handshake, options.signal);} catch (error) {if (options.signal.aborted) {- retainCleanup(- handshake.then((lateHandle) =>+ abortedHandshakeCleanup = handshake.then(+ (lateHandle) =>admissionCleanup!.runRetained(lateHandle,"ACPX runtime admission aborted",),- ),+ () =>+ admissionCleanup!.runRetained(+ failedHandshakeHandle,+ "ACPX runtime admission aborted",+ ),);+ retainCleanup(abortedHandshakeCleanup);}throw error;}@@ -252,12 +281,30 @@ export async function openCodexAcpxRuntime(options.signal.throwIfAborted();}} catch (error) {+ const cleanupHandle = handle ?? failedHandshakeHandle;const cleanupErrors = await admissionCleanup.run(- handle ?? failedHandshakeHandle,+ cleanupHandle,options.signal?.aborted? "ACPX runtime admission aborted": "ACPX session handshake failed",);+ const retainedCleanup =+ cleanupErrors.length === 0+ ? Promise.resolve()+ : admissionCleanup.runRetained(+ cleanupHandle,+ options.signal?.aborted+ ? "ACPX runtime admission aborted"+ : "ACPX session handshake failed",+ );+ const cleanupProof =+ abortedHandshakeCleanup === null+ ? retainedCleanup+ : Promise.all([retainedCleanup, abortedHandshakeCleanup]).then(+ () => undefined,+ );+ options.retainFailedAdmissionCleanup(cleanupProof);+ retainCleanup(cleanupProof);if (cleanupErrors.length > 0) {throw new AggregateError([error, ...cleanupErrors],@@ -277,13 +324,23 @@ export async function openCodexAcpxRuntime(runtime,handle,requireIdentity(handle),- admissionCleanup,+ children,+ runtimeCloseTimeoutMs,);} catch (error) {const cleanupErrors = await admissionCleanup.run(handle,"ACPX runtime identity validation failed",);+ const cleanupProof =+ cleanupErrors.length === 0+ ? Promise.resolve()+ : admissionCleanup.runRetained(+ handle,+ "ACPX runtime identity validation failed",+ );+ options.retainFailedAdmissionCleanup(cleanupProof);+ retainCleanup(cleanupProof);if (cleanupErrors.length > 0) {throw new AggregateError([error, ...cleanupErrors],@@ -371,12 +428,12 @@ class RuntimeAdmissionCleanup {const targetKey = this.#resolveTargetKey(rawTargetKey, handle);const existing = this.#registeredTargets.get(targetKey);if (existing !== undefined) {- if (handle !== null) {- existing.handle =- existing.handle === null- ? handle+ existing.handle =+ existing.handle === null+ ? handle+ : handle === null+ ? existing.handle: preferRuntimeAdmissionCleanupHandle(existing.handle, handle);- }this.#targetAliases.set(rawTargetKey, targetKey);return existing.cleanup!;}@@ -407,8 +464,7 @@ class RuntimeAdmissionCleanup {const fallbackHandle =this.#registeredTargets.get(fallbackTargetKey)?.handle;if (- fallbackHandle !== undefined &&- fallbackHandle !== null &&+ fallbackHandle != null &&nonEmptyRuntimeIdentity(fallbackHandle.acpxRecordId) === undefined &&sameRuntimeAdmissionCleanupOwner(fallbackHandle, handle)) {@@ -590,9 +646,232 @@ function runtimePort(runtime: AcpRuntime,handle: AcpRuntimeHandle,identity: AcpxRuntimePortIdentity,- admissionCleanup: RuntimeAdmissionCleanup,+ children: SpawnedChildSet,+ runtimeCloseTimeoutMs: number,): AcpxRuntimePort {- return {+ type RuntimeCloseAttempt = {+ readonly outcome: Promise<unknown | null>;+ readonly reconciliationGeneration: number;+ readonly origin:+ | { readonly kind: "external" }+ | {+ readonly kind: "reconciliation";+ readonly generation: number;+ readonly attemptNumber: number;+ };+ pendingExternalIntent: boolean;+ };+ let runtimeClosed = false;+ let runtimeCloseAttempt: RuntimeCloseAttempt | undefined;+ let lateReconciliationOwner: Promise<void> | undefined;+ // Each independently observed late failure receives a bounded reconciliation+ // budget. Exhausting retries for an older generation must not prevent a newer+ // late failure from acquiring its own recovery owner.+ let lateReconciliationAttemptGeneration = 0;+ let lateReconciliationAttempts = 0;+ let lateFailureGeneration = 0;+ let reconciledLateFailureGeneration = 0;+ const watchedReleasedAttempts = new Set<RuntimeCloseAttempt>();++ const hasUnreconciledLateFailure = (): boolean =>+ reconciledLateFailureGeneration < lateFailureGeneration;++ const consumePendingExternalIntent = (+ attempt: RuntimeCloseAttempt,+ failed: boolean,+ ): boolean => {+ const pending = attempt.pendingExternalIntent;+ attempt.pendingExternalIntent = false;+ return (+ pending &&+ failed &&+ attempt.origin.kind === "reconciliation" &&+ attempt.origin.attemptNumber >=+ MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS+ );+ };++ const scheduleLateFailureReconciliation = (): void => {+ if (+ runtimeCloseAttempt ||+ lateReconciliationOwner ||+ !hasUnreconciledLateFailure()+ ) {+ return;+ }+ if (lateReconciliationAttemptGeneration !== lateFailureGeneration) {+ lateReconciliationAttemptGeneration = lateFailureGeneration;+ lateReconciliationAttempts = 0;+ }+ if (+ lateReconciliationAttempts >=+ MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS+ ) {+ return;+ }+ const attemptGeneration = lateFailureGeneration;+ const attemptNumber = lateReconciliationAttempts + 1;+ lateReconciliationAttempts = attemptNumber;+ let retry = false;+ const reconciliation = closeRuntime({+ reason: `ACPX late protocol cleanup reconciliation ${attemptNumber}`,+ reconciliation: {+ generation: attemptGeneration,+ attemptNumber,+ },+ }).then(+ () => {+ if (hasUnreconciledLateFailure()) {+ retry =+ lateFailureGeneration === attemptGeneration &&+ attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS;+ }+ },+ () => {+ retry =+ lateFailureGeneration === attemptGeneration &&+ attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS;+ },+ );+ const owner = reconciliation.finally(() => {+ if (lateReconciliationOwner === owner)+ lateReconciliationOwner = undefined;+ if (retry || hasUnreconciledLateFailure()) {+ queueMicrotask(scheduleLateFailureReconciliation);+ }+ });+ lateReconciliationOwner = owner;+ retainRuntimeCleanupOwner(owner);+ };++ const watchPendingAttempt = (+ attempt: RuntimeCloseAttempt,+ processCleanupSucceeded: boolean,+ ): void => {+ if (watchedReleasedAttempts.has(attempt)) return;+ watchedReleasedAttempts.add(attempt);+ void attempt.outcome.then((error) => {+ watchedReleasedAttempts.delete(attempt);+ if (runtimeCloseAttempt === attempt) runtimeCloseAttempt = undefined;+ const renewForExternalIntent = consumePendingExternalIntent(+ attempt,+ error !== null || !processCleanupSucceeded,+ );+ if (renewForExternalIntent) lateFailureGeneration += 1;+ if (error === null) {+ if (processCleanupSucceeded) {+ reconciledLateFailureGeneration = Math.max(+ reconciledLateFailureGeneration,+ attempt.reconciliationGeneration,+ );+ runtimeClosed = !hasUnreconciledLateFailure();+ }+ scheduleLateFailureReconciliation();+ return;+ }+ // A newer successful close cannot erase an older outcome that had not+ // settled yet. Re-open cleanup state and autonomously create a bounded+ // reconciliation generation so the late failure is not suppression-only.+ // An autonomous failure remains charged to the budget of the generation+ // that created it. If external callers coalesced onto the exhausted final+ // attempt, their single batched intent creates exactly one new generation;+ // joins on earlier attempts are satisfied by the remaining same-generation+ // retries.+ if (attempt.origin.kind === "external") {+ lateFailureGeneration += 1;+ }+ runtimeClosed = false;+ scheduleLateFailureReconciliation();+ });+ };++ async function closeRuntime(input: {+ reason: string;+ reconciliation?: {+ generation: number;+ attemptNumber: number;+ };+ }): Promise<void> {+ if (runtimeClosed) return;+ if (+ runtimeCloseAttempt?.origin.kind === "reconciliation" &&+ input.reconciliation === undefined+ ) {+ // Preserve the attempt's autonomous origin while remembering that one or+ // more external callers requested a fresh cleanup observation. The exact+ // outcome consumes this bit, so coalesced callers cannot mint generations+ // independently.+ runtimeCloseAttempt.pendingExternalIntent = true;+ }+ if (!runtimeCloseAttempt) {+ // A close can reconcile only failures already known when its protocol+ // attempt begins. A released older attempt may reject while this one is+ // in flight; that later generation must trigger a subsequent close.+ runtimeCloseAttempt = {+ outcome: ownedRuntimeCloseOutcome(runtime, handle, input.reason),+ reconciliationGeneration:+ input.reconciliation?.generation ?? lateFailureGeneration,+ origin:+ input.reconciliation === undefined+ ? { kind: "external" }+ : {+ kind: "reconciliation",+ generation: input.reconciliation.generation,+ attemptNumber: input.reconciliation.attemptNumber,+ },+ pendingExternalIntent: false,+ };+ }+ const observedAttempt = runtimeCloseAttempt;+ const processCleanup = terminateChildrenAfterCloseBound(+ observedAttempt.outcome,+ children,+ runtimeCloseTimeoutMs,+ );+ // The caller may stop waiting, but the exact ACPX protocol cleanup stays+ // owned and remains this handle's sole close attempt until it settles.+ // Provider termination still proceeds at the deadline.+ const [closeError, processErrors] = await Promise.all([+ boundedCloseOutcome(observedAttempt.outcome, runtimeCloseTimeoutMs),+ processCleanup,+ ]);+ if (closeError instanceof AcpxRuntimeCloseTimeoutError) {+ watchPendingAttempt(observedAttempt, processErrors.length === 0);+ } else {+ if (+ consumePendingExternalIntent(+ observedAttempt,+ closeError !== null || processErrors.length > 0,+ )+ ) {+ lateFailureGeneration += 1;+ }+ if (runtimeCloseAttempt === observedAttempt) {+ runtimeCloseAttempt = undefined;+ }+ }+ if (processErrors.length === 0 && closeError === null) {+ reconciledLateFailureGeneration = Math.max(+ reconciledLateFailureGeneration,+ observedAttempt.reconciliationGeneration,+ );+ runtimeClosed = !hasUnreconciledLateFailure();+ } else {+ runtimeClosed = false;+ }+ scheduleLateFailureReconciliation();+ if (closeError !== null || processErrors.length > 0) {+ const errors = [closeError, ...processErrors].filter(+ (error): error is unknown => error !== null,+ );+ throw new AggregateError(+ errors,+ "ACPX runtime and provider cleanup failed",+ );+ }+ }++ const port: AcpxRuntimePort = {async identity() {return structuredClone(identity);},@@ -622,16 +901,9 @@ function runtimePort(...(input.signal ? { signal: input.signal } : {}),});},- async close(input) {- const errors = await admissionCleanup.run(handle, input.reason);- if (errors.length > 0) {- throw new AggregateError(- errors,- "ACPX runtime and provider cleanup failed",- );- }- },+ close: closeRuntime,};+ return port;}function runtimeCloseOutcome(@@ -663,6 +935,70 @@ async function closeOutcomeWithin(return outcome;}+function ownedRuntimeCloseOutcome(+ runtime: AcpRuntime,+ handle: AcpRuntimeHandle,+ reason: string,+): Promise<unknown | null> {+ const cleanup = Promise.resolve()+ .then(() =>+ runtime.close({ handle, reason, discardPersistentState: false }),+ )+ .then(+ () => null,+ (error: unknown) => error,+ );+ return retainRuntimeCleanupOwner(cleanup);+}++function retainRuntimeCleanupOwner<T>(cleanup: Promise<T>): Promise<T> {+ activeRuntimeCleanupOwners.add(cleanup);+ void cleanup+ .finally(() => activeRuntimeCleanupOwners.delete(cleanup))+ .catch(() => undefined);+ return cleanup;+}++async function terminateChildrenAfterCloseBound(+ closeOutcome: Promise<unknown | null>,+ children: SpawnedChildSet,+ timeoutMs: number,+): Promise<unknown[]> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ closeOutcome.then(() => undefined),+ new Promise<void>((resolve) => {+ timer = setTimeout(resolve, Math.max(1, Math.floor(timeoutMs)));+ timer.unref();+ }),+ ]);+ } finally {+ if (timer) clearTimeout(timer);+ }+ return await children.terminate();+}++async function boundedCloseOutcome(+ closeOutcome: Promise<unknown | null>,+ timeoutMs: number,+): Promise<unknown | null> {+ const boundedTimeoutMs = Math.max(1, timeoutMs);+ let timer: ReturnType<typeof setTimeout> | undefined;+ const outcome = await Promise.race([+ closeOutcome.then((error) => ({ error })),+ new Promise<{ error: unknown }>((resolve) => {+ timer = setTimeout(+ () => resolve({ error: new AcpxRuntimeCloseTimeoutError() }),+ boundedTimeoutMs,+ );+ timer.unref();+ }),+ ]);+ if (timer) clearTimeout(timer);+ return outcome.error;+}+function delay(timeoutMs: number): Promise<void> {return new Promise((resolve) => {const timer = setTimeout(resolve, timeoutMs);@@ -703,7 +1039,7 @@ class SpawnedChildSet {const terminateOutcome = await signalAndWaitForExit(tracked,"SIGTERM",- 2_000,+ PROVIDER_TERM_EXIT_TIMEOUT_MS,);if (terminateOutcome.error !== undefined) {pushUnique(errors, terminateOutcome.error);@@ -712,7 +1048,7 @@ class SpawnedChildSet {const killOutcome = await signalAndWaitForExit(tracked,"SIGKILL",- 2_000,+ PROVIDER_KILL_EXIT_TIMEOUT_MS,);if (killOutcome.error !== undefined) {pushUnique(errors, killOutcome.error);diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.tsindex 5a35c44fa5..22cd50e916 100644--- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts+++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts@@ -4,6 +4,7 @@ import { join } from "node:path";import { afterEach, describe, expect, it, vi } from "vitest";+import { openCodexAcpxRuntime } from "./codex-runtime-adapter.js";import { stageManagedCodexCredential } from "./codex-credentials.js";import type {VerifiedAcpxCommandLease,@@ -14,6 +15,7 @@ import {AcpxRuntimeHost,type AcpxRuntimeHostDependencies,type AcpxRuntimePort,+ type AcpxRuntimePortOpenOptions,type AcpxRuntimeTurn,} from "./runtime-host.js";@@ -59,6 +61,65 @@ describe("ACPX runtime host", () => {expect(fixture.commandClose).not.toHaveBeenCalled();});+ it("scrubs credentials when abort wins before the adapter body starts", async () => {+ const fixture = await hostFixture();+ const controller = new AbortController();+ const cancellation = new Error("runtime admission cancelled before entry");+ const createRuntime = vi.fn();+ let credentialHome = "";+ const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => {+ credentialHome = options.launchEnvironment.CODEX_HOME!;+ // The host has scheduled its openRuntime callback and transferred the+ // staged credential to that pending admission. Abort before entering the+ // adapter so its pre-entry path must publish a completed cleanup proof.+ controller.abort(cancellation);+ return openCodexAcpxRuntime(options, { createRuntime });+ });++ await expect(+ AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "deny-all",+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },+ signal: controller.signal,+ },+ fixture.dependencies({ openRuntime }),+ ),+ ).rejects.toBe(cancellation);++ expect(openRuntime).toHaveBeenCalledOnce();+ expect(createRuntime).not.toHaveBeenCalled();+ expect(fixture.commandClose).toHaveBeenCalledOnce();+ const authPath = join(credentialHome, "auth.json");+ const contender = await vi.waitFor(() =>+ stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },+ }),+ );+ await contender.close();+ await expect(readFile(authPath)).rejects.toMatchObject({ code: "ENOENT" });++ const retryRuntime = runtimePort();+ const retryHost = await AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "deny-all",+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },+ },+ fixture.dependencies({ openRuntime: async () => retryRuntime }),+ );+ await retryHost.close({ reason: "retry admission complete" });+ expect(retryRuntime.close).toHaveBeenCalledOnce();+ });+it("composes admission, isolation, model verification, and cleanup", async () => {const fixture = await hostFixture();let capturedEnvironment: Readonly<NodeJS.ProcessEnv> = {};@@ -301,6 +362,77 @@ describe("ACPX runtime host", () => {expect(fixture.commandClose).toHaveBeenCalledOnce();});+ it("retains failed admission cleanup until provider shutdown permits credential scrub", async () => {+ const fixture = await hostFixture();+ let authPath = "";+ let credentialHome = "";+ let resolveRetryClose!: () => void;+ const retryClose = new Promise<void>((resolve) => {+ resolveRetryClose = resolve;+ });+ const runtime = runtimePort({+ getStatus: async () => ({+ models: {+ currentModelId: "wrong-model",+ availableModelIds: ["wrong-model"],+ },+ }),+ onClose: vi+ .fn()+ .mockRejectedValueOnce(new Error("first admission cleanup failed"))+ .mockImplementationOnce(() => retryClose),+ });++ await expect(+ AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "approve-all",+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET:+ '{"owner":"failed-admission"}',+ },+ },+ fixture.dependencies({+ openRuntime: async (options) => {+ credentialHome = options.launchEnvironment.CODEX_HOME!;+ authPath = join(credentialHome, "auth.json");+ return runtime;+ },+ }),+ ),+ ).rejects.toThrow(/initialization and cleanup failed/);++ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));+ await expect(readFile(authPath, "utf8")).resolves.toContain(+ "failed-admission",+ );+ await expect(+ stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },+ }),+ ).rejects.toThrow("already has an active lease");++ resolveRetryClose();+ await vi.waitFor(async () => {+ await expect(readFile(authPath)).rejects.toMatchObject({+ code: "ENOENT",+ });+ });+ const contender = await stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },+ });+ await contender.close();+ });+it("retains credential ownership when runtime shutdown fails until retry succeeds", async () => {const fixture = await hostFixture();let failClose = true;@@ -402,6 +534,70 @@ describe("ACPX runtime host", () => {await contender.close();});+ it("retains the exact pending cleanup while independent resources close", async () => {+ const fixture = await hostFixture();+ const firstClose = new Promise<void>(() => undefined);+ const runtime = runtimePort({+ onClose: vi+ .fn()+ .mockImplementationOnce(() => firstClose)+ .mockResolvedValueOnce(undefined),+ });+ const host = await AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "approve-all",+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },+ },+ fixture.dependencies({ openRuntime: async () => runtime }),+ );++ const first = host.close({ reason: "first close stalls" });+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce());+ const second = host.close({ reason: "same pending owner" });+ let settled = false;+ void Promise.all([first, second]).finally(() => {+ settled = true;+ });+ await Promise.resolve();+ expect(settled).toBe(false);+ expect(runtime.close).toHaveBeenCalledOnce();+ expect(fixture.commandClose).toHaveBeenCalledOnce();+ });++ it("retries only after the exact close outcome settles with failure", async () => {+ const fixture = await hostFixture();+ const runtime = runtimePort({+ onClose: vi+ .fn()+ .mockRejectedValueOnce(new Error("runtime close failed"))+ .mockResolvedValueOnce(undefined),+ });+ const host = await AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "approve-all",+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },+ },+ fixture.dependencies({ openRuntime: async () => runtime }),+ );++ await expect(host.close({ reason: "first close" })).rejects.toThrow(+ /cleanup failed/,+ );+ await expect(+ host.close({ reason: "fresh attempt" }),+ ).resolves.toBeUndefined();+ await expect(+ host.close({ reason: "ownership released" }),+ ).resolves.toBeUndefined();+ expect(runtime.close).toHaveBeenCalledTimes(2);+ });+it("admits one bounded turn and cancels it before shutdown", async () => {const fixture = await hostFixture();const turn = runtimeTurn();@@ -429,6 +625,9 @@ describe("ACPX runtime host", () => {host.startTurn({ text: "Concurrent", requestId: "turn-2" }),).toThrow("already has an active turn");+ await host.interruptActiveTurn("user interrupt");+ expect(turn.cancel).toHaveBeenCalledWith({ reason: "user interrupt" });+await host.close({ reason: "shutdown" });expect(turn.cancel).toHaveBeenCalledWith({ reason: "shutdown" });expect(runtime.close).toHaveBeenCalledOnce();@@ -693,13 +892,23 @@ describe("ACPX runtime host", () => {expect(fixture.commandClose).not.toHaveBeenCalled();});- it("forwards cancellation and closes a runtime that resolves after abort", async () => {+ it("retains managed credentials until an aborted late runtime is closed", async () => {const fixture = await hostFixture();const runtimeAdmission = deferred<AcpxRuntimePort>();- const lateRuntime = runtimePort();+ const retryClose = deferred<void>();+ const lateRuntime = runtimePort({+ onClose: vi+ .fn()+ .mockRejectedValueOnce(new Error("late runtime close failed"))+ .mockImplementationOnce(() => retryClose.promise),+ });let receivedSignal: AbortSignal | undefined;+ let credentialHome = "";+ let bridgeUrl = "";const openRuntime = vi.fn((options) => {receivedSignal = options.signal;+ credentialHome = options.launchEnvironment.CODEX_HOME!;+ bridgeUrl = options.mcpServers[0]!.url;return runtimeAdmission.promise;});const controller = new AbortController();@@ -714,6 +923,10 @@ describe("ACPX runtime host", () => {PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}",},signal: controller.signal,+ semanticTools: {+ tools: [],+ handler: async () => ({ ok: true }),+ },},fixture.dependencies({ openRuntime }),);@@ -723,13 +936,93 @@ describe("ACPX runtime host", () => {controller.abort(cancellation);await expect(opening).rejects.toBe(cancellation);expect(fixture.commandClose).toHaveBeenCalledOnce();+ await expect(fetch(bridgeUrl)).rejects.toThrow();+ const authPath = join(credentialHome, "auth.json");+ await expect(readFile(authPath, "utf8")).resolves.toBe("{}");+ await expect(+ stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },+ }),+ ).rejects.toThrow("already has an active lease");+runtimeAdmission.resolve(lateRuntime);+ await vi.waitFor(() => expect(lateRuntime.close).toHaveBeenCalledTimes(2));+ expect(lateRuntime.close).toHaveBeenNthCalledWith(1, {+ reason: "ACPX runtime admission aborted",+ });+ await expect(readFile(authPath, "utf8")).resolves.toBe("{}");+ await expect(+ stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },+ }),+ ).rejects.toThrow("already has an active lease");- await vi.waitFor(() =>- expect(lateRuntime.close).toHaveBeenCalledWith({- reason: "ACPX runtime admission aborted",+ retryClose.resolve(undefined);+ await vi.waitFor(async () => {+ await expect(readFile(authPath)).rejects.toMatchObject({+ code: "ENOENT",+ });+ });+ // File removal precedes kernel lease release. Wait for the lease itself so+ // this assertion cannot race between those two ordered cleanup steps.+ const contender = await vi.waitFor(() =>+ stageManagedCodexCredential({+ agentHomeDirectory: credentialHome,+ environment: {+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',+ },}),);+ await contender.close();+ });++ it("scrubs credentials after rejected runtime cleanup is proven", async () => {+ const fixture = await hostFixture();+ const runtimeAdmission = deferred<AcpxRuntimePort>();+ const providerCleanup = deferred<void>();+ const credentialClose = vi.fn(async () => undefined);+ const controller = new AbortController();+ const cancellation = new Error("runtime admission cancelled");+ const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => {+ options.retainFailedAdmissionCleanup(providerCleanup.promise);+ return runtimeAdmission.promise;+ });+ const opening = AcpxRuntimeHost.open(+ {+ ...fixture.options,+ agent: "codex",+ model: "gpt-5.6-sol",+ permissionMode: "deny-all",+ signal: controller.signal,+ },+ {+ ...fixture.dependencies({ openRuntime }),+ stageCredential: async () => ({+ path: join(fixture.root, "auth.json"),+ mode: "inline_json",+ close: credentialClose,+ }),+ },+ );+ await vi.waitFor(() => expect(openRuntime).toHaveBeenCalledOnce());++ controller.abort(cancellation);+ await expect(opening).rejects.toBe(cancellation);+ const cleanupFailure = new Error("provider survived forced cleanup");+ runtimeAdmission.reject(new AggregateError([cancellation, cleanupFailure]));+ await new Promise((resolve) => setImmediate(resolve));++ expect(credentialClose).not.toHaveBeenCalled();+ expect(fixture.commandClose).toHaveBeenCalledOnce();++ providerCleanup.resolve(undefined);+ await vi.waitFor(() => expect(credentialClose).toHaveBeenCalledOnce());});});@@ -824,10 +1117,13 @@ async function hostFixture() {function deferred<T>(): {promise: Promise<T>;resolve(value: T): void;+ reject(reason: unknown): void;} {let resolve!: (value: T) => void;- const promise = new Promise<T>((settle) => {+ let reject!: (reason: unknown) => void;+ const promise = new Promise<T>((settle, fail) => {resolve = settle;+ reject = fail;});- return { promise, resolve };+ return { promise, resolve, reject };}diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.tsindex 86c200744c..81e02b42c4 100644--- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts+++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts@@ -38,7 +38,11 @@ import {} from "./runtime-sandbox.js";import type { AcpxExpectedSessionIdentity } from "./sidecar-protocol.js";-const TURN_CANCELLATION_TIMEOUT_MS = 2_000;+export const ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS = 2_000;++const ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS = 8;+const ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS = 10;+const ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS = 1_000;export interface AcpxRuntimePortIdentity {acpxRecordId: string;@@ -83,6 +87,11 @@ export interface AcpxRuntimePortOpenOptions {/** Abort provider admission and clean any runtime that resolves too late. */signal?: AbortSignal;mcpServers: readonly AcpxMcpServerBinding[];+ /**+ * Transfer the provider cleanup proof before a failed open settles. The host+ * keeps credentials fenced until this exact cleanup succeeds.+ */+ retainFailedAdmissionCleanup(cleanup: Promise<void>): void;}export interface AcpxRetainedCleanupFailure {@@ -134,6 +143,35 @@ const activeRuntimeHostCleanupOwners = new Set<Promise<unknown>>();const RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS = 10;const RETAINED_CLEANUP_RETRY_MAX_DELAY_MS = 1_000;+interface RetainedRejectedRuntimeAdmission {+ readonly credential: ManagedCodexCredentialLease;+ cleanup: Promise<void>;+}++// A rejected provider open is not itself proof that provider processes are+// gone. Retain the credential with the adapter's exact cleanup proof and scrub+// it only after that proof succeeds. Terminal cleanup failure stays inert and+// fenced instead of exposing the credential to a replacement admission.+const retainedRejectedRuntimeAdmissions =+ new Set<RetainedRejectedRuntimeAdmission>();++interface RetainedAcpxAdmissionCleanup {+ readonly runtime: AcpxRuntimePort | null;+ readonly toolBridge: RunnerToolBridge | null;+ readonly credential: ManagedCodexCredentialLease | null;+ readonly command: VerifiedAcpxCommandLease | null;+ readonly reason: string;+ recovery: Promise<void> | null;+ timer: ReturnType<typeof setTimeout> | null;+}++// An admission failure has no AcpxRuntimeHost instance for a caller to close.+// Retain those resources here until provider shutdown succeeds and the staged+// credential can consequently be scrubbed. Each recovery batch is finite and+// sequential; an unref'd timer rate-limits later autonomous attempts without+// allowing the failed admission's ownership to become unreachable.+const retainedAcpxAdmissionCleanups = new Set<RetainedAcpxAdmissionCleanup>();+export class AcpxRuntimeHost {readonly #runtime: AcpxRuntimePort;readonly #binding: AcpxRecoveryBinding;@@ -205,6 +243,37 @@ export class AcpxRuntimeHost {let credential: ManagedCodexCredentialLease | null = null;let toolBridge: RunnerToolBridge | null = null;let runtime: AcpxRuntimePort | null = null;+ let pendingRuntimeOwnsCredential = false;+ let failedAdmissionCleanupTransferred = false;+ let resolveFailedAdmissionCleanupTransfer!: () => void;+ const failedAdmissionCleanupTransfer = new Promise<void>((resolve) => {+ resolveFailedAdmissionCleanupTransfer = resolve;+ });+ const retainFailedAdmissionCleanup = (cleanup: Promise<void>): void => {+ if (failedAdmissionCleanupTransferred) return;+ failedAdmissionCleanupTransferred = true;+ pendingRuntimeOwnsCredential = credential !== null;+ resolveFailedAdmissionCleanupTransfer();+ if (credential === null) {+ retainRuntimeHostCleanup(cleanup);+ return;+ }+ const retained: RetainedRejectedRuntimeAdmission = {+ credential,+ cleanup: Promise.resolve(),+ };+ const ownedCleanup = cleanup.then(async () => {+ await cleanupAbortedRuntimeAdmission(+ null,+ retained.credential,+ "ACPX rejected runtime admission cleanup confirmed",+ );+ retainedRejectedRuntimeAdmissions.delete(retained);+ });+ retained.cleanup = ownedCleanup;+ retainedRejectedRuntimeAdmissions.add(retained);+ retainRuntimeHostCleanup(ownedCleanup);+ };try {const sandbox = await runAbortableAdmissionStage(options.signal, () =>prepareAcpxRuntimeSandbox({@@ -272,6 +341,7 @@ export class AcpxRuntimeHost {},]: [],+ retainFailedAdmissionCleanup,}),resource: "runtime",releaseLate: (lateRuntime) =>@@ -280,6 +350,19 @@ export class AcpxRuntimeHost {}),reportFailure: (failure) =>dependencies.reportRetainedCleanupFailure(failure),+ onAbortedPending: (pendingRuntime) => {+ // A provider may already be running even though openRuntime has not+ // returned its port. Keep its credential lease with that exact+ // admission while the ordinary catch path revokes tools and releases+ // the consumed command snapshot without delaying cancellation.+ pendingRuntimeOwnsCredential = credential !== null;+ retainAbortedRuntimeAdmissionCleanup({+ pendingRuntime,+ credential,+ reason: "ACPX runtime admission aborted",+ failedAdmissionCleanupTransfer,+ });+ },});await runAbortableAdmissionStage(options.signal, () =>requireVerifiedAcpxModel(runtime!, profile),@@ -316,11 +399,18 @@ export class AcpxRuntimeHost {const cleanupError = await cleanupRuntimeResources(runtime,toolBridge,- credential,+ pendingRuntimeOwnsCredential ? null : credential,command,"ACPX runtime initialization failed",);if (cleanupError) {+ retainFailedAcpxAdmissionCleanup({+ runtime,+ toolBridge,+ credential: pendingRuntimeOwnsCredential ? null : credential,+ command,+ reason: "ACPX runtime initialization failed",+ });throw new AggregateError([error, ...cleanupError.errors],"ACPX runtime initialization and cleanup failed",@@ -346,6 +436,10 @@ export class AcpxRuntimeHost {return Object.freeze({ ...this.#sandbox.persistedEnvironment });}+ async status(): Promise<AcpxModelStatus> {+ return structuredClone(await this.#runtime.getStatus());+ }+startTurn(input: AcpxRuntimeTurnInput): AcpxRuntimeTurn {if (this.#closed || this.#closingStarted) {throw new Error("ACPX runtime host is closing");@@ -374,9 +468,20 @@ export class AcpxRuntimeHost {return turn;}+ async interruptActiveTurn(reason: string): Promise<void> {+ const turn = this.#activeTurn;+ if (!turn) throw new Error("ACPX runtime host has no active turn");+ const cancellationError = await boundedCancellation(+ turn.cancel({ reason: boundedReason(reason) }),+ );+ if (cancellationError) throw cancellationError;+ }+async close(input: { reason: string }): Promise<void> {if (this.#closed) return;- if (this.#closePromise) return await this.#closePromise;+ if (this.#closePromise) {+ return await this.#closePromise;+ }this.#closingStarted = true;const closePromise = this.#close(boundedReason(input.reason));this.#closePromise = closePromise;@@ -435,6 +540,7 @@ async function acquireAbortableAdmissionResource<T>(input: {resource: AcpxRetainedCleanupFailure["resource"];releaseLate: (resource: T) => Promise<void>;reportFailure: (failure: AcpxRetainedCleanupFailure) => void;+ onAbortedPending?: (pending: Promise<T>) => void;}): Promise<T> {if (input.signal === undefined) return await input.acquire();input.signal.throwIfAborted();@@ -443,16 +549,20 @@ async function acquireAbortableAdmissionResource<T>(input: {return await raceAdmissionWithAbort(pending, input.signal);} catch (error) {if (input.signal.aborted) {- retainRuntimeHostCleanup(- pending.then((resource) =>- releaseRetainedAdmissionResource({- resource,- resourceKind: input.resource,- release: input.releaseLate,- reportFailure: input.reportFailure,- }),- ),- );+ if (input.onAbortedPending) {+ input.onAbortedPending(pending);+ } else {+ retainRuntimeHostCleanup(+ pending.then((resource) =>+ releaseRetainedAdmissionResource({+ resource,+ resourceKind: input.resource,+ release: input.releaseLate,+ reportFailure: input.reportFailure,+ }),+ ),+ );+ }}throw error;}@@ -530,6 +640,123 @@ function retainRuntimeHostCleanup(cleanup: Promise<unknown>): void {.catch(() => undefined);}+function retainAbortedRuntimeAdmissionCleanup(input: {+ pendingRuntime: Promise<AcpxRuntimePort>;+ credential: ManagedCodexCredentialLease | null;+ reason: string;+ failedAdmissionCleanupTransfer: Promise<void>;+}): void {+ const cleanup = input.pendingRuntime.then(+ (runtime) =>+ cleanupAbortedRuntimeAdmission(runtime, input.credential, input.reason),+ () => input.failedAdmissionCleanupTransfer,+ );+ retainRuntimeHostCleanup(cleanup);+}++async function cleanupAbortedRuntimeAdmission(+ runtime: AcpxRuntimePort | null,+ credential: ManagedCodexCredentialLease | null,+ reason: string,+): Promise<void> {+ const cleanupError = await cleanupRuntimeResources(+ runtime,+ null,+ credential,+ null,+ reason,+ );+ if (!cleanupError) return;+ retainFailedAcpxAdmissionCleanup({+ runtime,+ toolBridge: null,+ credential,+ command: null,+ reason,+ });+}++function retainFailedAcpxAdmissionCleanup(input: {+ runtime: AcpxRuntimePort | null;+ toolBridge: RunnerToolBridge | null;+ credential: ManagedCodexCredentialLease | null;+ command: VerifiedAcpxCommandLease | null;+ reason: string;+}): void {+ const cleanup: RetainedAcpxAdmissionCleanup = {+ ...input,+ recovery: null,+ timer: null,+ };+ retainedAcpxAdmissionCleanups.add(cleanup);+ startRetainedAcpxAdmissionCleanup(cleanup);+}++function startRetainedAcpxAdmissionCleanup(+ cleanup: RetainedAcpxAdmissionCleanup,+): Promise<void> {+ if (cleanup.recovery) return cleanup.recovery;+ const recovery = (async () => {+ let retryDelayMs = ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS;+ for (+ let attempt = 1;+ attempt <= ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS;+ attempt += 1+ ) {+ const cleanupError = await cleanupRuntimeResources(+ cleanup.runtime,+ cleanup.toolBridge,+ cleanup.credential,+ cleanup.command,+ `${cleanup.reason} (automatic cleanup recovery ${attempt})`,+ );+ if (!cleanupError) {+ retainedAcpxAdmissionCleanups.delete(cleanup);+ if (cleanup.timer) clearTimeout(cleanup.timer);+ cleanup.timer = null;+ return;+ }+ if (attempt < ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS) {+ await waitForAdmissionCleanupRetry(retryDelayMs);+ retryDelayMs = Math.min(retryDelayMs * 2, 1_000);+ }+ }+ })();+ cleanup.recovery = recovery;+ void recovery+ .finally(() => {+ if (cleanup.recovery === recovery) cleanup.recovery = null;+ scheduleRetainedAcpxAdmissionCleanup(cleanup);+ })+ .catch(() => undefined);+ return recovery;+}++function scheduleRetainedAcpxAdmissionCleanup(+ cleanup: RetainedAcpxAdmissionCleanup,+): void {+ if (+ !retainedAcpxAdmissionCleanups.has(cleanup) ||+ cleanup.recovery ||+ cleanup.timer+ ) {+ return;+ }+ cleanup.timer = setTimeout(() => {+ cleanup.timer = null;+ if (!retainedAcpxAdmissionCleanups.has(cleanup)) return;+ startRetainedAcpxAdmissionCleanup(cleanup);+ }, ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS);+ cleanup.timer.unref?.();+}++async function waitForAdmissionCleanupRetry(delayMs: number): Promise<void> {+ await new Promise<void>((resolve) => {+ const timer = setTimeout(resolve, delayMs);+ timer.unref?.();+ });+}+async function boundedCancellation(cancellation: Promise<void>,): Promise<unknown | null> {@@ -547,7 +774,7 @@ async function boundedCancellation("ACPX turn cancellation exceeded its shutdown timeout",),}),- TURN_CANCELLATION_TIMEOUT_MS,+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS,);}),]);diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.tsindex 60c04b33ab..3cb1e0521e 100644--- a/packages/paperclip-runner/src/native-session-runtime.test.ts+++ b/packages/paperclip-runner/src/native-session-runtime.test.ts@@ -40,7 +40,9 @@ const result: PrpStructuredRunResult = {completionClaim: {contractRevision: "1",objectiveSatisfied: true,- criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }],+ criteria: [+ { criterionId: "objective", status: "satisfied", evidenceRefs: [] },+ ],remainingWork: [],},evidence: [],@@ -63,8 +65,16 @@ const yieldedResult: PrpStructuredRunResult = {completionClaim: {contractRevision: "1",objectiveSatisfied: false,- criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }],- remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }],+ criteria: [+ {+ criterionId: "objective",+ status: "unknown",+ evidenceRefs: ["interaction:pending"],+ },+ ],+ remainingWork: [+ { description: "Resume after the response.", blocksCompletion: true },+ ],},evidence: [{ ref: "interaction:pending" }],verification: [],@@ -93,8 +103,17 @@ const input: NativeExecutionInputV1 = {prompt: "# PAP-RECOVERY: Recover native work",workMode: "standard",},- workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },- session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 },+ workspace: {+ cwd: "/workspace",+ repoUrl: null,+ repoRef: null,+ branchName: null,+ },+ session: {+ normalizedSessionId: identity.sessionId,+ driverKind: "codex_app_server",+ protocolVersion: 1,+ },provider: { kind: "codex", model: null },completionContract: {id: "contract-recovery",@@ -154,24 +173,958 @@ function runnerEvent(};}-function highestContiguous(events: PrpEvent[]): number {- const sequences = new Set(events.map((event) => event.sourceSeq));- let cursor = 0;- while (sequences.has(cursor + 1)) cursor += 1;- return cursor;-}+function highestContiguous(events: PrpEvent[]): number {+ const sequences = new Set(events.map((event) => event.sourceSeq));+ let cursor = 0;+ while (sequences.has(cursor + 1)) cursor += 1;+ return cursor;+}++describe("executeNativeSession recovery", () => {+ it("keeps governed-wait discovery synchronous", () => {+ type GovernedWaitResolver = NonNullable<+ ExecuteNativeSessionOptions["resolveGovernedWait"]+ >;+ const resolver: GovernedWaitResolver = () => null;+ // An async resolver could retain control-plane mutation authority after+ // execution settles, so the public boundary rejects it at compile time.+ // @ts-expect-error governed-wait discovery must not return a promise+ const asynchronousResolver: GovernedWaitResolver = async () => null;++ expect(resolver).toBeTypeOf("function");+ void asynchronousResolver;+ });++ it("preserves durable success while quarantined cleanup stays bounded", async () => {+ vi.useFakeTimers();+ try {+ let executionCloseCount = 0;+ let quarantineAttempt = 0;+ const close = vi.fn(({ reason }: { reason: string }) => {+ if (reason === "native session quarantined cleanup recovery") {+ quarantineAttempt += 1;+ const attempt = quarantineAttempt;+ return new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (attempt < 3)+ reject(new Error("transient quarantine failure"));+ else resolve();+ }, 6_500);+ });+ }+ if (reason === "native session execution complete") {+ executionCloseCount += 1;+ if (executionCloseCount > 1) return Promise.resolve();+ }+ return Promise.reject(new Error("persistent close failure"));+ });+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ await expect(execute()).resolves.toMatchObject({ result });+ expect(close).toHaveBeenCalledOnce();+ await vi.advanceTimersByTimeAsync(3_000);+ expect(close).toHaveBeenCalledTimes(5);++ // Admission inherits the already-running three-attempt recovery and+ // waits through its bounded attempts instead of timing out after one.+ const recoveredExecution = execute();+ let admissionSettled = false;+ void recoveredExecution.then(+ () => {+ admissionSettled = true;+ },+ () => {+ admissionSettled = true;+ },+ );+ await vi.advanceTimersByTimeAsync(20_000);+ expect(admissionSettled).toBe(false);+ expect(openSession).toHaveBeenCalledTimes(1);+ expect(close).toHaveBeenCalledTimes(7);+ await vi.advanceTimersByTimeAsync(2_000);+ await expect(recoveredExecution).resolves.toMatchObject({ result });+ expect(quarantineAttempt).toBe(3);+ expect(close).toHaveBeenCalledTimes(8);+ expect(openSession).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("awaits the quarantine owner that replaces an exhausted close recovery", async () => {+ vi.useFakeTimers();+ try {+ let executionCloseCount = 0;+ const close = vi.fn(({ reason }: { reason: string }) => {+ if (reason === "native session execution complete") {+ executionCloseCount += 1;+ return executionCloseCount === 1+ ? Promise.reject(new Error("initial close failed"))+ : Promise.resolve();+ }+ if (+ reason.startsWith(+ "native session cleanup recovery after close failure",+ )+ ) {+ return Promise.reject(new Error("bounded close recovery failed"));+ }+ if (reason === "native session quarantined cleanup recovery") {+ return new Promise<void>((resolve) => setTimeout(resolve, 50));+ }+ return Promise.resolve();+ });+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ await expect(execute()).resolves.toMatchObject({ result });+ const admitted = execute();+ await vi.advanceTimersByTimeAsync(3_100);+ await expect(admitted).resolves.toMatchObject({ result });+ expect(openSession).toHaveBeenCalledTimes(2);+ expect(close).toHaveBeenCalledWith({+ reason: "native session quarantined cleanup recovery",+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("covers maximum-duration retained and replacement recovery phases", async () => {+ vi.useFakeTimers();+ try {+ let executionCloseCount = 0;+ let retainedRecoveryAttempt = 0;+ let quarantineRecoveryAttempt = 0;+ const close = vi.fn(({ reason }: { reason: string }) => {+ if (reason === "native session execution complete") {+ executionCloseCount += 1;+ if (executionCloseCount > 1) return Promise.resolve();+ return new Promise<void>((_resolve, reject) => {+ setTimeout(() => reject(new Error("initial close failed")), 6_900);+ });+ }+ if (+ reason.startsWith(+ "native session cleanup recovery after close failure",+ )+ ) {+ retainedRecoveryAttempt += 1;+ return new Promise<void>((_resolve, reject) => {+ setTimeout(+ () => reject(new Error("bounded retained recovery failed")),+ 6_900,+ );+ });+ }+ if (reason === "native session quarantined cleanup recovery") {+ quarantineRecoveryAttempt += 1;+ const attempt = quarantineRecoveryAttempt;+ return new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (attempt < 3)+ reject(new Error("transient quarantine recovery failure"));+ else resolve();+ }, 6_500);+ });+ }+ return Promise.resolve();+ });+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ const firstExecution = execute();+ await vi.advanceTimersByTimeAsync(100);+ await expect(firstExecution).resolves.toMatchObject({ result });+ const admitted = execute();+ let admissionSettled = false;+ void admitted.then(+ () => {+ admissionSettled = true;+ },+ () => {+ admissionSettled = true;+ },+ );++ // The initial close plus three near-bound retries exceed the old 23s+ // admission grace but remain within the configured 31s owner phase.+ await vi.advanceTimersByTimeAsync(23_100);+ expect(retainedRecoveryAttempt).toBe(2);+ expect(quarantineRecoveryAttempt).toBe(0);+ expect(admissionSettled).toBe(false);+ expect(openSession).toHaveBeenCalledOnce();++ await vi.advanceTimersByTimeAsync(7_600);+ expect(retainedRecoveryAttempt).toBe(3);+ expect(quarantineRecoveryAttempt).toBe(1);+ expect(admissionSettled).toBe(false);+ expect(openSession).toHaveBeenCalledOnce();++ // The replacement then receives its complete three-attempt bound rather+ // than inheriting only the remainder of the retained owner's deadline.+ await vi.advanceTimersByTimeAsync(21_600);+ await expect(admitted).resolves.toMatchObject({ result });+ expect(quarantineRecoveryAttempt).toBe(3);+ expect(openSession).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("runs a full admission batch after an inherited scheduled attempt fails", async () => {+ vi.useFakeTimers();+ try {+ let executionCloseCount = 0;+ let scheduledRecoveryAttempt = 0;+ let admissionRecoveryAttempt = 0;+ const close = vi.fn(({ reason }: { reason: string }) => {+ if (reason === "native session execution complete") {+ executionCloseCount += 1;+ return executionCloseCount === 1+ ? Promise.reject(new Error("initial close failed"))+ : Promise.resolve();+ }+ if (+ reason.startsWith(+ "native session cleanup recovery after close failure",+ )+ ) {+ return Promise.reject(new Error("bounded retained recovery failed"));+ }+ if (reason === "native session quarantined cleanup recovery") {+ return Promise.reject(+ new Error("bounded quarantine recovery failed"),+ );+ }+ if (+ reason === "native session scheduled quarantined cleanup recovery"+ ) {+ scheduledRecoveryAttempt += 1;+ return new Promise<void>((_resolve, reject) => {+ setTimeout(+ () => reject(new Error("scheduled cleanup failed")),+ 6_500,+ );+ });+ }+ if (reason === "native session quarantined admission recovery") {+ admissionRecoveryAttempt += 1;+ const attempt = admissionRecoveryAttempt;+ return new Promise<void>((resolve, reject) => {+ setTimeout(() => {+ if (attempt === 1)+ reject(new Error("transient admission cleanup failure"));+ else resolve();+ }, 6_500);+ });+ }+ return Promise.resolve();+ });+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ await expect(execute()).resolves.toMatchObject({ result });+ // Exhaust retained and initial quarantine batches, then enter the slow+ // autonomous one-attempt recovery scheduled sixty seconds later.+ await vi.advanceTimersByTimeAsync(65_001);+ expect(scheduledRecoveryAttempt).toBe(1);++ const admitted = execute();+ let admissionSettled = false;+ void admitted.then(+ () => {+ admissionSettled = true;+ },+ () => {+ admissionSettled = true;+ },+ );+ await vi.advanceTimersByTimeAsync(6_600);+ expect(admissionRecoveryAttempt).toBe(1);+ expect(admissionSettled).toBe(false);+ expect(openSession).toHaveBeenCalledOnce();++ await vi.advanceTimersByTimeAsync(14_100);+ await expect(admitted).resolves.toMatchObject({ result });+ expect(admissionRecoveryAttempt).toBe(2);+ expect(openSession).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("stops scheduled cleanup after the quarantine lifetime budget", async () => {+ vi.useFakeTimers();+ try {+ let executionCloseCount = 0;+ let admissionAttempt = 0;+ let scheduledAttempt = 0;+ const lifetimeIdentity = {+ ...identity,+ companyId: "company-cleanup-lifetime",+ };+ const close = vi.fn(({ reason }: { reason: string }) => {+ if (reason === "native session execution complete") {+ executionCloseCount += 1;+ return executionCloseCount === 1+ ? Promise.reject(new Error("initial close failed"))+ : Promise.resolve();+ }+ if (+ reason.startsWith(+ "native session cleanup recovery after close failure",+ )+ ) {+ return Promise.reject(new Error("bounded close recovery failed"));+ }+ if (reason === "native session quarantined cleanup recovery") {+ return Promise.reject(new Error("quarantined close recovery failed"));+ }+ if (reason === "native session quarantined admission recovery") {+ admissionAttempt += 1;+ return Promise.reject(new Error("admission cleanup failed"));+ }+ if (+ reason === "native session scheduled quarantined cleanup recovery"+ ) {+ scheduledAttempt += 1;+ return Promise.reject(new Error("scheduled cleanup failed"));+ }+ return Promise.resolve();+ });+ const session: NativeSession = {+ identity: () => lifetimeIdentity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity: lifetimeIdentity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input: {+ ...input,+ binding: {+ ...input.binding,+ companyId: lifetimeIdentity.companyId,+ },+ },+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ await expect(execute()).resolves.toMatchObject({ result });+ await vi.advanceTimersByTimeAsync(6_000);++ const recoveredExecution = expect(execute()).rejects.toThrow(+ "prior session cleanup remains incomplete",+ );+ await vi.advanceTimersByTimeAsync(2_100);+ await recoveredExecution;++ expect(admissionAttempt).toBe(3);+ expect(scheduledAttempt).toBe(0);+ await vi.advanceTimersByTimeAsync(180_000);+ expect(scheduledAttempt).toBe(3);+ await vi.advanceTimersByTimeAsync(300_000);+ expect(scheduledAttempt).toBe(3);+ expect(openSession).toHaveBeenCalledOnce();+ } finally {+ vi.useRealTimers();+ }+ });++ it("fails admission within a bound when quarantined close never settles", async () => {+ vi.useFakeTimers();+ try {+ let releaseBlockedClose = () => {};+ let blockClose = false;+ const close = vi.fn(() => {+ if (blockClose) {+ return new Promise<void>((resolve) => {+ releaseBlockedClose = resolve;+ });+ }+ return Promise.reject(new Error("persistent close failure"));+ });+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ await expect(execute()).resolves.toMatchObject({ result });+ await vi.advanceTimersByTimeAsync(6_000);+ blockClose = true;+ const blockedAdmission = execute();+ const blockedResult = expect(blockedAdmission).rejects.toThrow(+ "prior session cleanup exceeded the admission grace",+ );+ await vi.advanceTimersByTimeAsync(31_100);+ await blockedResult;+ expect(openSession).toHaveBeenCalledOnce();++ const isolatedSession: NativeSession = {+ ...session,+ identity: () => ({ ...identity, companyId: "company-isolated" }),+ close: vi.fn(async () => undefined),+ };+ const isolatedOpenSession = vi.fn(async () => isolatedSession);+ await expect(+ executeNativeSession({+ input: {+ ...input,+ binding: {+ ...input.binding,+ companyId: "company-isolated",+ },+ },+ backend: { ...backend, openSession: isolatedOpenSession },+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({ result });+ expect(isolatedOpenSession).toHaveBeenCalledOnce();++ blockClose = false;+ releaseBlockedClose();+ close.mockResolvedValue(undefined);+ await vi.runAllTimersAsync();+ await expect(execute()).resolves.toMatchObject({ result });+ } finally {+ vi.useRealTimers();+ }+ });++ it("quarantines a pending first close before another provider session can open", async () => {+ vi.useFakeTimers();+ try {+ let releaseClose = () => {};+ const pendingClose = new Promise<void>((resolve) => {+ releaseClose = resolve;+ });+ const close = vi.fn(() => pendingClose);+ const session: NativeSession = {+ identity: () => identity,+ async capabilities() {+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return {+ backendKind: "mock",+ sessionId: "driver-recovery",+ identity,+ providerSessionId: "provider-recovery",+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ };+ },+ close,+ };+ const openSession = vi.fn(async () => session);+ const backend: NativeSessionBackend = {+ async descriptor() {+ return {+ kind: "mock",+ name: "recovery-backend",+ version: "1",+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },+ };+ },+ openSession,+ };+ const port: ControlPlanePort = {+ async openRun() {},+ async checkpointSession() {},+ async appendEvent() {+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },+ async completeRun() {},+ };+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ });++ const firstExecution = execute();+ await vi.advanceTimersByTimeAsync(100);+ await expect(firstExecution).resolves.toMatchObject({ result });+ expect(close).toHaveBeenCalledOnce();-describe("executeNativeSession recovery", () => {- it("keeps governed-wait discovery synchronous", () => {- type GovernedWaitResolver = NonNullable<ExecuteNativeSessionOptions["resolveGovernedWait"]>;- const resolver: GovernedWaitResolver = () => null;- // An async resolver could retain control-plane mutation authority after- // execution settles, so the public boundary rejects it at compile time.- // @ts-expect-error governed-wait discovery must not return a promise- const asynchronousResolver: GovernedWaitResolver = async () => null;+ const blockedAdmission = execute();+ const blockedResult = expect(blockedAdmission).rejects.toThrow(+ "prior session cleanup exceeded the admission grace",+ );+ await vi.advanceTimersByTimeAsync(31_100);+ await blockedResult;+ expect(openSession).toHaveBeenCalledOnce();+ expect(close).toHaveBeenCalledOnce();- expect(resolver).toBeTypeOf("function");- void asynchronousResolver;+ releaseClose();+ await vi.runAllTimersAsync();+ } finally {+ vi.useRealTimers();+ }});it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => {@@ -217,27 +1170,33 @@ describe("executeNativeSession recovery", () => {const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input: {- ...input,- schema: "paperclip.native-execution-input.v3",- executionMode: "default",- planningContext: null,- runtimeContext: {- ...context,- aggregateDigest: canonicalNativeRuntimeContextDigest(context),+ await expect(+ executeNativeSession({+ input: {+ ...input,+ schema: "paperclip.native-execution-input.v3",+ executionMode: "default",+ planningContext: null,+ runtimeContext: {+ ...context,+ aggregateDigest: canonicalNativeRuntimeContextDigest(context),+ },},- },- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).rejects.toThrow("does not natively realize instructions, skills, mcp");+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).rejects.toThrow("does not natively realize instructions, skills, mcp");expect(openSession).not.toHaveBeenCalled();});@@ -266,31 +1225,38 @@ describe("executeNativeSession recovery", () => {};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },+ async appendEvent() {+ throw new Error("unexpected event");+ },async replayEvents() {return { events: [], highestContiguousSourceSeq: 0 };},async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).rejects.toBe(providerFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).rejects.toBe(providerFailure);- expect(openSession).toHaveBeenCalledWith(expect.objectContaining({- identity,- workingDirectory: input.workspace.cwd,- signal: expect.any(AbortSignal),- }));+ expect(openSession).toHaveBeenCalledWith(+ expect.objectContaining({+ identity,+ workingDirectory: input.workspace.cwd,+ signal: expect.any(AbortSignal),+ }),+ );expect(openRun).not.toHaveBeenCalled();});- it("bounds fresh session bootstrap and closes a session returned after timeout", async () => {+ it("retains late bootstrap cleanup through the next admission", async () => {vi.useFakeTimers();+ let releaseClose = () => {};try {let resolveBootstrap = (_value: NativeSession) => {};const stalledBootstrap = new Promise<NativeSession>((resolve) => {@@ -304,33 +1270,57 @@ describe("executeNativeSession recovery", () => {const closeStarted = new Promise<void>((resolve) => {markCloseStarted = resolve;});- const close = vi.fn(async () => { markCloseStarted(); });+ const closeReleased = new Promise<void>((resolve) => {+ releaseClose = resolve;+ });+ const close = vi.fn(async () => {+ markCloseStarted();+ await closeReleased;+ });const lateSession: NativeSession = {identity: () => identity,async capabilities() {- return { resume: false, typedEvents: true, steering: false, interruption: true };+ return {+ resume: false,+ typedEvents: true,+ steering: false,+ interruption: true,+ };},async *events() {},- async startTurn() { throw new Error("late fresh session must not start"); },- async result() { return null; },- async snapshot() { throw new Error("late fresh session must not snapshot"); },+ async startTurn() {+ throw new Error("late fresh session must not start");+ },+ async result() {+ return null;+ },+ async snapshot() {+ throw new Error("late fresh session must not snapshot");+ },close,};let bootstrapSignal: AbortSignal | undefined;- const openSession = vi.fn((bootstrapInput: {- identity: NativeRunIdentity;- workingDirectory?: string;- signal?: AbortSignal;- }) => {- bootstrapSignal = bootstrapInput.signal;- bootstrapInput.signal?.addEventListener(- "abort",- () => resolveBootstrap(lateSession),- { once: true },- );- markBootstrapStarted();- return stalledBootstrap;- });+ let bootstrapCount = 0;+ const openSession = vi.fn(+ (bootstrapInput: {+ identity: NativeRunIdentity;+ workingDirectory?: string;+ signal?: AbortSignal;+ }) => {+ bootstrapCount += 1;+ if (bootstrapCount > 1) {+ throw new Error("replacement bootstrap launched");+ }+ bootstrapSignal = bootstrapInput.signal;+ bootstrapInput.signal?.addEventListener(+ "abort",+ () => resolveBootstrap(lateSession),+ { once: true },+ );+ markBootstrapStarted();+ return stalledBootstrap;+ },+ );const openRun = vi.fn(async () => undefined);const backend: NativeSessionBackend = {async descriptor() {@@ -345,19 +1335,24 @@ describe("executeNativeSession recovery", () => {};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};-- const execution = executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- timeoutMs: 5,- });+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ timeoutMs: 5,+ });+ const execution = execute();const rejection = expect(execution).rejects.toThrow("native session bootstrap timed out after 5ms",);@@ -373,7 +1368,25 @@ describe("executeNativeSession recovery", () => {expect(close).toHaveBeenCalledWith({reason: "native session bootstrap timed out",});++ // Even after the detached disposer exhausts its short settlement grace,+ // the exact close remains admission-visible until it releases provider+ // resources. A replacement bootstrap cannot start concurrently.+ await vi.advanceTimersByTimeAsync(101);+ const blockedAdmission = execute();+ const blockedAdmissionRejection = expect(blockedAdmission).rejects.toThrow(+ "replacement bootstrap launched",+ );+ await vi.advanceTimersByTimeAsync(1_000);+ expect(openSession).toHaveBeenCalledOnce();++ releaseClose();+ await vi.advanceTimersByTimeAsync(0);+ await blockedAdmissionRejection;+ expect(openSession).toHaveBeenCalledTimes(2);+ expect(openRun).not.toHaveBeenCalled();} finally {+ releaseClose();vi.useRealTimers();}});@@ -384,12 +1397,23 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: false, typedEvents: true, steering: false, interruption: true };+ return {+ resume: false,+ typedEvents: true,+ steering: false,+ interruption: true,+ };},async *events() {},- async startTurn() { throw new Error("unexpected turn"); },- async result() { return null; },- async snapshot() { throw snapshotFailure; },+ async startTurn() {+ throw new Error("unexpected turn");+ },+ async result() {+ return null;+ },+ async snapshot() {+ throw snapshotFailure;+ },close,};const backend: NativeSessionBackend = {@@ -401,29 +1425,37 @@ describe("executeNativeSession recovery", () => {capabilities: await session.capabilities(),};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};const retainedSessions: Array<NativeSession | null> = [];- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- keepSessionOpen: true,- onSession(current) {- retainedSessions.push(current);- if (current === null) throw new Error("owner notification failed");- },- })).rejects.toBe(snapshotFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ keepSessionOpen: true,+ onSession(current) {+ retainedSessions.push(current);+ if (current === null) throw new Error("owner notification failed");+ },+ }),+ ).rejects.toBe(snapshotFailure);expect(retainedSessions).toEqual([session, null]);expect(close).toHaveBeenCalledOnce();@@ -445,14 +1477,13 @@ describe("executeNativeSession recovery", () => {});let checkpointSignal: AbortSignal | undefined;let controlPlaneCheckpointCount = 0;- const checkpointSession: NonNullable<ControlPlanePort["checkpointSession"]> = async (- _snapshot,- checkpointOptions,- ) => {+ const checkpointSession: NonNullable<+ ControlPlanePort["checkpointSession"]+ > = async (_snapshot, checkpointOptions) => {controlPlaneCheckpointCount += 1;if (- stalledBoundary === "control-plane checkpoint"- && controlPlaneCheckpointCount === 2+ stalledBoundary === "control-plane checkpoint" &&+ controlPlaneCheckpointCount === 2) {checkpointSignal = checkpointOptions?.signal;markCheckpointStalled();@@ -460,21 +1491,22 @@ describe("executeNativeSession recovery", () => {}};let ownerCheckpointCount = 0;- const onCheckpoint: NonNullable<ExecuteNativeSessionOptions["onCheckpoint"]> = async (- _snapshot,- checkpointOptions,- ) => {+ const onCheckpoint: NonNullable<+ ExecuteNativeSessionOptions["onCheckpoint"]+ > = async (_snapshot, checkpointOptions) => {ownerCheckpointCount += 1;if (- stalledBoundary === "owner checkpoint"- && ownerCheckpointCount === 2+ stalledBoundary === "owner checkpoint" &&+ ownerCheckpointCount === 2) {checkpointSignal = checkpointOptions?.signal;markCheckpointStalled();await never;}};- const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled" }));+ const startTurn = vi.fn(async () => ({+ turnId: "turn-checkpoint-stalled",+ }));const close = vi.fn(async () => {releaseStream();});@@ -493,7 +1525,9 @@ describe("executeNativeSession recovery", () => {await streamReleased;},startTurn,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -517,13 +1551,17 @@ describe("executeNativeSession recovery", () => {capabilities: await session.capabilities(),};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const openRun = vi.fn(async () => undefined);const port: ControlPlanePort = {openRun,checkpointSession,- async appendEvent() { throw new Error("unexpected event"); },+ async appendEvent() {+ throw new Error("unexpected event");+ },async replayEvents() {return { events: [], highestContiguousSourceSeq: 0 };},@@ -596,8 +1634,12 @@ describe("executeNativeSession recovery", () => {structuredResult: true,};},- async *events() { yield runnerEvent(1, "turn.completed"); },- async startTurn() { return { turnId: "turn-recovery" }; },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },async result() {resultCalls += 1;if (stalledBoundary === "provider result") {@@ -630,7 +1672,9 @@ describe("executeNativeSession recovery", () => {capabilities: await session.capabilities(),};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -644,8 +1688,8 @@ describe("executeNativeSession recovery", () => {},async appendEvent(event, operationOptions) {if (- stalledBoundary === "final event append"- && (event as PrpEvent).sourceKind === "control_plane"+ stalledBoundary === "final event append" &&+ (event as PrpEvent).sourceKind === "control_plane") {stalledSignal = operationOptions?.signal;markFinalizationStalled();@@ -711,10 +1755,7 @@ describe("executeNativeSession recovery", () => {},);- it.each([- "final event append",- "run completion",- ] as const)(+ it.each(["final event append", "run completion"] as const)("confirms durable completion when %s commits before its acknowledgement stalls",async (stalledBoundary) => {vi.useFakeTimers();@@ -739,9 +1780,15 @@ describe("executeNativeSession recovery", () => {structuredResult: true,};},- async *events() { yield runnerEvent(1, "turn.completed"); },- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return { result, terminal, turnId: "turn-recovery" }; },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },async snapshot() {return {backendKind: "mock",@@ -765,22 +1812,25 @@ describe("executeNativeSession recovery", () => {capabilities: await session.capabilities(),};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},async appendEvent(event, operationOptions) {const appended = structuredClone(event as PrpEvent);- const existing = events.find((candidate) =>- candidate.sourceInstanceId === appended.sourceInstanceId- && candidate.sourceSeq === appended.sourceSeq+ const existing = events.find(+ (candidate) =>+ candidate.sourceInstanceId === appended.sourceInstanceId &&+ candidate.sourceSeq === appended.sourceSeq,);if (existing === undefined) events.push(appended);if (- stalledBoundary === "final event append"- && appended.sourceKind === "control_plane"- && !stalledOnce+ stalledBoundary === "final event append" &&+ appended.sourceKind === "control_plane" &&+ !stalledOnce) {stalledOnce = true;stalledSignal = operationOptions?.signal;@@ -788,7 +1838,8 @@ describe("executeNativeSession recovery", () => {return await never;}const sourceEvents = events.filter(- (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId,+ (candidate) =>+ candidate.sourceInstanceId === appended.sourceInstanceId,);return {cursor: events.length,@@ -801,9 +1852,11 @@ describe("executeNativeSession recovery", () => {(event) => event.sourceInstanceId === replay.sourceInstanceId,);return {- events: structuredClone(sourceEvents.filter(- (event) => event.sourceSeq > replay.afterSourceSeq,- )),+ events: structuredClone(+ sourceEvents.filter(+ (event) => event.sourceSeq > replay.afterSourceSeq,+ ),+ ),highestContiguousSourceSeq: highestContiguous(sourceEvents),};},@@ -842,8 +1895,9 @@ describe("executeNativeSession recovery", () => {});expect(stalledSignal?.aborted).toBe(true);expect(durableCompletion).toMatchObject({ result, terminal });- expect(events.filter((event) => event.sourceKind === "control_plane"))- .toHaveLength(2);+ expect(+ events.filter((event) => event.sourceKind === "control_plane"),+ ).toHaveLength(2);} finally {vi.useRealTimers();}@@ -878,9 +1932,15 @@ describe("executeNativeSession recovery", () => {structuredResult: true,};},- async *events() { yield runnerEvent(1, "turn.completed"); },- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return { result, terminal, turnId: "turn-recovery" }; },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },async snapshot(snapshotOptions) {if (stalledBoundary === "provider snapshot" && runCompleted) {stalledSignal = snapshotOptions?.signal;@@ -916,7 +1976,9 @@ describe("executeNativeSession recovery", () => {capabilities: await session.capabilities(),};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const completeRun = vi.fn(async () => {runCompleted = true;@@ -924,7 +1986,10 @@ describe("executeNativeSession recovery", () => {const port: ControlPlanePort = {async openRun() {},async checkpointSession(_snapshot, operationOptions) {- if (stalledBoundary === "post-completion checkpoint" && runCompleted) {+ if (+ stalledBoundary === "post-completion checkpoint" &&+ runCompleted+ ) {stalledSignal = operationOptions?.signal;markEnrichmentStalled();await never;@@ -955,7 +2020,8 @@ describe("executeNativeSession recovery", () => {onSession: (current) => retainedSessions.push(current),});await enrichmentStalled;- if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false);+ if (stalledSignal !== undefined)+ expect(stalledSignal.aborted).toBe(false);await vi.advanceTimersByTimeAsync(10);@@ -966,7 +2032,8 @@ describe("executeNativeSession recovery", () => {driverVersion: "1",usage: null,});- if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true);+ if (stalledSignal !== undefined)+ expect(stalledSignal.aborted).toBe(true);expect(completeRun).toHaveBeenCalledOnce();expect(close).toHaveBeenCalledOnce();expect(retainedSessions).toEqual([session, null]);@@ -978,22 +2045,36 @@ describe("executeNativeSession recovery", () => {it("contains a consumer rejection when starting the turn fails first", async () => {let markAppendStarted = () => {};- const appendStarted = new Promise<void>((resolve) => { markAppendStarted = resolve; });+ const appendStarted = new Promise<void>((resolve) => {+ markAppendStarted = resolve;+ });let releaseAppend = () => {};- const appendReleased = new Promise<void>((resolve) => { releaseAppend = resolve; });+ const appendReleased = new Promise<void>((resolve) => {+ releaseAppend = resolve;+ });let appendCommitted = false;const close = vi.fn(async () => undefined);const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.started");},- async *events() { yield runnerEvent(1, "turn.started"); },async startTurn() {await appendStarted;throw new Error("start turn failed");},- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1014,10 +2095,18 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -1027,9 +2116,13 @@ describe("executeNativeSession recovery", () => {await Promise.race([appendReleased,new Promise<never>((_resolve, reject) => {- const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted"));+ const rejectAbort = () =>+ reject(options?.signal.reason ?? new Error("append aborted"));if (options?.signal.aborted) rejectAbort();- else options?.signal.addEventListener("abort", rejectAbort, { once: true });+ else+ options?.signal.addEventListener("abort", rejectAbort, {+ once: true,+ });}),]);appendCommitted = true;@@ -1039,7 +2132,9 @@ describe("executeNativeSession recovery", () => {disposition: "committed" as const,};},- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -1061,42 +2156,59 @@ describe("executeNativeSession recovery", () => {it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => {let markAppendStarted = () => {};- const appendStarted = new Promise<void>((resolve) => { markAppendStarted = resolve; });+ const appendStarted = new Promise<void>((resolve) => {+ markAppendStarted = resolve;+ });let releaseAppend = () => {};- const appendReleased = new Promise<void>((resolve) => { releaseAppend = resolve; });+ const appendReleased = new Promise<void>((resolve) => {+ releaseAppend = resolve;+ });let releaseTeardown = () => {};- const teardownReleased = new Promise<void>((resolve) => { releaseTeardown = resolve; });- const iteratorTeardown = vi.fn();- let appendCommitted = false;- const appendEvent = vi.fn(async (- _event: PrpEvent,- options?: { signal: AbortSignal },- ) => {- markAppendStarted();- await Promise.race([- appendReleased,- new Promise<never>((_resolve, reject) => {- const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted"));- if (options?.signal.aborted) rejectAbort();- else options?.signal.addEventListener("abort", rejectAbort, { once: true });- }),- ]);- appendCommitted = true;- return {- cursor: 1,- highestContiguousSourceSeq: 1,- disposition: "committed" as const,- };+ const teardownReleased = new Promise<void>((resolve) => {+ releaseTeardown = resolve;});+ const iteratorTeardown = vi.fn();+ let appendCommitted = false;+ const appendEvent = vi.fn(+ async (_event: PrpEvent, options?: { signal: AbortSignal }) => {+ markAppendStarted();+ await Promise.race([+ appendReleased,+ new Promise<never>((_resolve, reject) => {+ const rejectAbort = () =>+ reject(options?.signal.reason ?? new Error("append aborted"));+ if (options?.signal.aborted) rejectAbort();+ else+ options?.signal.addEventListener("abort", rejectAbort, {+ once: true,+ });+ }),+ ]);+ appendCommitted = true;+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed" as const,+ };+ },+ );const cancel = vi.fn(() => {releaseTeardown();return { cleanup: Promise.resolve() };});- const close = vi.fn(async () => { releaseTeardown(); });+ const close = vi.fn(async () => {+ releaseTeardown();+ });const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {try {@@ -1106,9 +2218,13 @@ describe("executeNativeSession recovery", () => {await teardownReleased;}},- async startTurn() { return { turnId: "turn-recovery" }; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },cancel,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1129,16 +2245,26 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},appendEvent,- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -1151,7 +2277,9 @@ describe("executeNativeSession recovery", () => {timeoutMs: 1,keepSessionOpen: true,});- const rejection = expect(execution).rejects.toThrow("native session timed out");+ const rejection = expect(execution).rejects.toThrow(+ "native session timed out",+ );await appendStarted;await vi.waitFor(() => expect(iteratorTeardown).toHaveBeenCalledOnce());await rejection;@@ -1166,19 +2294,33 @@ describe("executeNativeSession recovery", () => {it("closes a failed session while retaining an uncancellable event read", async () => {let releaseStream = () => {};- const streamReleased = new Promise<void>((resolve) => { releaseStream = resolve; });- const close = vi.fn(async () => { releaseStream(); });+ const streamReleased = new Promise<void>((resolve) => {+ releaseStream = resolve;+ });+ const close = vi.fn(async () => {+ releaseStream();+ });const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ };},async *events() {await streamReleased;yield runnerEvent(1, "turn.completed");},- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return null; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1199,56 +2341,86 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- timeoutMs: 1,- keepSessionOpen: true,- })).rejects.toThrow("native session timed out");+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ timeoutMs: 1,+ keepSessionOpen: true,+ }),+ ).rejects.toThrow("native session timed out");expect(close).toHaveBeenCalledOnce();});it("commits cancellation before bounding failed provider cleanup", async () => {let releaseStream = () => {};- const streamReleased = new Promise<void>((resolve) => { releaseStream = resolve; });+ const streamReleased = new Promise<void>((resolve) => {+ releaseStream = resolve;+ });let releaseCancellation = () => {};- const cancellationReleased = new Promise<void>((resolve) => { releaseCancellation = resolve; });+ const cancellationReleased = new Promise<void>((resolve) => {+ releaseCancellation = resolve;+ });const interrupt = vi.fn(() => cancellationReleased);let cancellationCommitted = false;const cancel = vi.fn(() => {cancellationCommitted = true;return { cleanup: cancellationReleased };});- const close = vi.fn(async () => { releaseStream(); });+ const close = vi.fn(async () => {+ releaseStream();+ });const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {await streamReleased;yield runnerEvent(1, "turn.completed");},- async startTurn() { return { turnId: "turn-recovery" }; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },interrupt,cancel,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1269,16 +2441,28 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -1301,18 +2485,32 @@ describe("executeNativeSession recovery", () => {it("bounds failure when iterator teardown and provider close never settle", async () => {const never = new Promise<void>(() => undefined);- const close = vi.fn(() => never);+ let releaseClose = () => {};+ const pendingClose = new Promise<void>((resolve) => {+ releaseClose = resolve;+ });+ const close = vi.fn(() => pendingClose);const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ };},async *events() {await never;yield runnerEvent(1, "turn.completed");},- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return null; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1333,43 +2531,74 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- timeoutMs: 1,- keepSessionOpen: true,- })).rejects.toThrow("native session timed out");+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ timeoutMs: 1,+ keepSessionOpen: true,+ }),+ ).rejects.toThrow("native session timed out");expect(close).toHaveBeenCalledOnce();+ releaseClose();+ await pendingClose;});it("preserves durable success when provider close never settles", async () => {- const never = new Promise<void>(() => undefined);- const close = vi.fn(() => never);+ let releaseClose = () => {};+ const pendingClose = new Promise<void>((resolve) => {+ releaseClose = resolve;+ });+ const close = vi.fn(() => pendingClose);const completeRun = vi.fn(async () => undefined);const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };},- async *events() { yield runnerEvent(1, "turn.completed"); },- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return { result, terminal, turnId: "turn-recovery" }; },async snapshot() {return {backendKind: "mock",@@ -1390,10 +2619,18 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: false,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const events: PrpEvent[] = [];const port: ControlPlanePort = {@@ -1407,35 +2644,53 @@ describe("executeNativeSession recovery", () => {disposition: "committed",};},- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },completeRun,};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).resolves.toMatchObject({ result, terminal });+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({ result, terminal });expect(completeRun).toHaveBeenCalledOnce();expect(close).toHaveBeenCalledOnce();+ releaseClose();+ await pendingClose;});it("closes after a synchronous governed-wait probe returns no result", async () => {const resolveGovernedWait = vi.fn(() => null);const lifecycle: string[] = [];- const close = vi.fn(async () => { lifecycle.push("closed"); });+ const close = vi.fn(async () => {+ lifecycle.push("closed");+ });const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {yield runnerEvent(1, "item.completed");},- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return null; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1456,18 +2711,32 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},async checkpointSession() {},async appendEvent() {- return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" };+ return {+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };},- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },async completeRun() {},};@@ -1494,20 +2763,32 @@ describe("executeNativeSession recovery", () => {lifecycle.push("cancelled");return { cleanup: new Promise<void>(() => undefined) };});- const close = vi.fn(async () => { lifecycle.push("closed"); });+ const close = vi.fn(async () => {+ lifecycle.push("closed");+ });const events: PrpEvent[] = [];const retainedSessions: Array<NativeSession | null> = [];const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {yield runnerEvent(1, "item.completed");},- async startTurn() { return { turnId: "turn-recovery" }; },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },cancel,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -1528,10 +2809,18 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -1544,7 +2833,9 @@ describe("executeNativeSession recovery", () => {disposition: "committed",};},- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -1582,7 +2873,13 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},openSession,@@ -1597,18 +2894,24 @@ describe("executeNativeSession recovery", () => {};},checkpointSession,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).rejects.toThrow("native_session_checkpoint_binding_mismatch");+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).rejects.toThrow("native_session_checkpoint_binding_mismatch");expect(openRun).not.toHaveBeenCalled();expect(checkpointSession).not.toHaveBeenCalled();expect(openSession).not.toHaveBeenCalled();@@ -1620,13 +2923,25 @@ describe("executeNativeSession recovery", () => {const existingSession: NativeSession = {identity: () => ({ ...identity, companyId: "other-company" }),async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},attachRun,async *events() {},- async startTurn() { return { turnId: "unexpected" }; },- async result() { return null; },- async snapshot() { throw new Error("unexpected snapshot"); },+ async startTurn() {+ return { turnId: "unexpected" };+ },+ async result() {+ return null;+ },+ async snapshot() {+ throw new Error("unexpected snapshot");+ },async close() {},};const backend: NativeSessionBackend = {@@ -1635,26 +2950,40 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "existing-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected open"); },+ async openSession() {+ throw new Error("unexpected open");+ },};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- existingSession,- })).rejects.toThrow("native_session_attach_binding_mismatch");+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ existingSession,+ }),+ ).rejects.toThrow("native_session_attach_binding_mismatch");expect(openRun).not.toHaveBeenCalled();expect(attachRun).not.toHaveBeenCalled();});@@ -1673,42 +3002,66 @@ describe("executeNativeSession recovery", () => {const existingSession: NativeSession = {identity: () => structuredClone(retainedIdentity),async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},attachRun,async *events() {},startTurn,- async result() { return null; },- async snapshot() { throw new Error("unexpected snapshot"); },+ async result() {+ return null;+ },+ async snapshot() {+ throw new Error("unexpected snapshot");+ },close,};const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",- name: "existing-backend",+ name: "attachment-failure-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected open"); },+ async openSession() {+ throw new Error("unexpected open");+ },};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- existingSession,- onSession,- })).rejects.toBe(attachmentFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ existingSession,+ onSession,+ }),+ ).rejects.toBe(attachmentFailure);expect(attachRun).toHaveBeenCalledWith({ identity });expect(retainedIdentity).toEqual(identity);expect(onSession).toHaveBeenCalledOnce();@@ -1732,42 +3085,66 @@ describe("executeNativeSession recovery", () => {const existingSession: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},attachRun,async *events() {},startTurn,- async result() { return null; },- async snapshot() { throw new Error("unexpected snapshot"); },+ async result() {+ return null;+ },+ async snapshot() {+ throw new Error("unexpected snapshot");+ },close,};const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",- name: "existing-backend",+ name: "control-plane-admission-failure-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected open"); },+ async openSession() {+ throw new Error("unexpected open");+ },};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- existingSession,- onSession,- })).rejects.toBe(admissionFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ existingSession,+ onSession,+ }),+ ).rejects.toBe(admissionFailure);expect(attachRun).toHaveBeenCalledWith({ identity });expect(openRun).toHaveBeenCalledOnce();expect(onSession).toHaveBeenCalledOnce();@@ -1787,7 +3164,13 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},openSession,@@ -1801,21 +3184,27 @@ describe("executeNativeSession recovery", () => {identity: { ...identity, sessionId: "other-session" },};},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};- await expect(executeNativeSession({- input: {- ...input,- session: { ...input.session, normalizedSessionId: null },- },- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).rejects.toThrow("native_session_checkpoint_binding_mismatch");+ await expect(+ executeNativeSession({+ input: {+ ...input,+ session: { ...input.session, normalizedSessionId: null },+ },+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).rejects.toThrow("native_session_checkpoint_binding_mismatch");expect(openRun).not.toHaveBeenCalled();expect(openSession).not.toHaveBeenCalled();});@@ -1838,14 +3227,22 @@ describe("executeNativeSession recovery", () => {recovered: false as const,reason: "provider session no longer exists",}));- const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); });+ const openSession = vi.fn(async () => {+ throw new Error("replacement is forbidden");+ });const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},openSession,@@ -1853,20 +3250,28 @@ describe("executeNativeSession recovery", () => {};const port: ControlPlanePort = {openRun,- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },completeRun,};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).rejects.toThrow(+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).rejects.toThrow("native_session_recovery_failed: provider session no longer exists",);@@ -1921,15 +3326,25 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected replacement"); },+ async openSession() {+ throw new Error("unexpected replacement");+ },recoverSession,};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },+ async appendEvent() {+ throw new Error("unexpected event");+ },replayEvents,async completeRun() {},};@@ -1983,10 +3398,16 @@ describe("executeNativeSession recovery", () => {pendingRuntimeRequests: [],lineage: [],};- let resolveRecovery = (_value: { recovered: true; session: NativeSession }) => {};- const stalledRecovery = new Promise<{ recovered: true; session: NativeSession }>(- (resolve) => { resolveRecovery = resolve; },- );+ let resolveRecovery = (_value: {+ recovered: true;+ session: NativeSession;+ }) => {};+ const stalledRecovery = new Promise<{+ recovered: true;+ session: NativeSession;+ }>((resolve) => {+ resolveRecovery = resolve;+ });let markRecoveryStarted = () => {};const recoveryStarted = new Promise<void>((resolve) => {markRecoveryStarted = resolve;@@ -1995,21 +3416,37 @@ describe("executeNativeSession recovery", () => {const closeStarted = new Promise<void>((resolve) => {markCloseStarted = resolve;});- const close = vi.fn(async () => { markCloseStarted(); });+ const close = vi.fn(async () => {+ markCloseStarted();+ });const lateSession: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ };},async *events() {},- async startTurn() { throw new Error("late recovery session must not start"); },- async result() { return null; },- async snapshot() { return structuredClone(checkpoint); },+ async startTurn() {+ throw new Error("late recovery session must not start");+ },+ async result() {+ return null;+ },+ async snapshot() {+ return structuredClone(checkpoint);+ },close,};let recoverySignal: AbortSignal | undefined;const recoverSession = vi.fn(- (_checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }) => {+ (+ _checkpoint: PersistedNativeSession,+ recoveryOptions: { signal: AbortSignal },+ ) => {recoverySignal = recoveryOptions.signal;recoveryOptions.signal.addEventListener("abort",@@ -2028,16 +3465,27 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ },};},- async openSession() { throw new Error("unexpected replacement"); },+ async openSession() {+ throw new Error("unexpected replacement");+ },recoverSession,};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -2099,33 +3547,48 @@ describe("executeNativeSession recovery", () => {const closeStarted = new Promise<void>((resolve) => {markCloseStarted = resolve;});- const close = vi.fn(async () => { markCloseStarted(); });+ const close = vi.fn(async () => {+ markCloseStarted();+ });const lateSession: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ };},async *events() {},- async startTurn() { throw new Error("late replacement session must not start"); },- async result() { return null; },- async snapshot() { return structuredClone(checkpoint); },+ async startTurn() {+ throw new Error("late replacement session must not start");+ },+ async result() {+ return null;+ },+ async snapshot() {+ return structuredClone(checkpoint);+ },close,};let replacementSignal: AbortSignal | undefined;- const openReplacementSession = vi.fn((replacementInput: {- identity: NativeRunIdentity;- workingDirectory?: string;- signal?: AbortSignal;- }) => {- replacementSignal = replacementInput.signal;- replacementInput.signal?.addEventListener(- "abort",- () => resolveReplacement(lateSession),- { once: true },- );- markReplacementStarted();- return stalledReplacement;- });+ const openReplacementSession = vi.fn(+ (replacementInput: {+ identity: NativeRunIdentity;+ workingDirectory?: string;+ signal?: AbortSignal;+ }) => {+ replacementSignal = replacementInput.signal;+ replacementInput.signal?.addEventListener(+ "abort",+ () => resolveReplacement(lateSession),+ { once: true },+ );+ markReplacementStarted();+ return stalledReplacement;+ },+ );const openRun = vi.fn(async () => undefined);const onSession = vi.fn();const backend: NativeSessionBackend = {@@ -2134,10 +3597,17 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "replacement-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ },};},- async openSession() { throw new Error("replacement seam must be used"); },+ async openSession() {+ throw new Error("replacement seam must be used");+ },async recoverSession() {return { recovered: false, reason: "provider session is missing" };},@@ -2145,8 +3615,12 @@ describe("executeNativeSession recovery", () => {};const port: ControlPlanePort = {openRun,- async appendEvent() { throw new Error("unexpected event"); },- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },+ async appendEvent() {+ throw new Error("unexpected event");+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };+ },async completeRun() {},};@@ -2198,27 +3672,41 @@ describe("executeNativeSession recovery", () => {const openRun = vi.fn(async () => undefined);const checkpointSession = vi.fn(async () => undefined);const onCheckpoint = vi.fn(async () => undefined);- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {- expect(recoveryCheckpoint.cursor).toBe("1");- throw recoveryFailure;- });+ const recoverSession = vi.fn(+ async (recoveryCheckpoint: PersistedNativeSession) => {+ expect(recoveryCheckpoint.cursor).toBe("1");+ throw recoveryFailure;+ },+ );const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected replacement"); },+ async openSession() {+ throw new Error("unexpected replacement");+ },recoverSession,};const port: ControlPlanePort = {openRun,- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },checkpointSession,- async appendEvent() { throw new Error("unexpected event"); },+ async appendEvent() {+ throw new Error("unexpected event");+ },async replayEvents(replay) {return {events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [],@@ -2228,14 +3716,16 @@ describe("executeNativeSession recovery", () => {async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- onCheckpoint,- })).rejects.toBe(recoveryFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ onCheckpoint,+ }),+ ).rejects.toBe(recoveryFailure);expect(recoverSession).toHaveBeenCalledOnce();expect(openRun).not.toHaveBeenCalled();@@ -2256,7 +3746,9 @@ describe("executeNativeSession recovery", () => {lineage: [],};const admissionFailure = new Error("control-plane admission rejected");- const openRun = vi.fn(async () => { throw admissionFailure; });+ const openRun = vi.fn(async () => {+ throw admissionFailure;+ });const checkpointSession = vi.fn(async () => undefined);const onCheckpoint = vi.fn(async () => undefined);const onSession = vi.fn();@@ -2264,35 +3756,61 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {},- async startTurn() { throw new Error("unexpected turn"); },- async result() { return null; },- async snapshot() { throw new Error("unexpected snapshot"); },+ async startTurn() {+ throw new Error("unexpected turn");+ },+ async result() {+ return null;+ },+ async snapshot() {+ throw new Error("unexpected snapshot");+ },close,};- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {- expect(recoveryCheckpoint.cursor).toBe("1");- return { recovered: true as const, session };- });+ const recoverSession = vi.fn(+ async (recoveryCheckpoint: PersistedNativeSession) => {+ expect(recoveryCheckpoint.cursor).toBe("1");+ return { recovered: true as const, session };+ },+ );const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("unexpected replacement"); },+ async openSession() {+ throw new Error("unexpected replacement");+ },recoverSession,};const port: ControlPlanePort = {openRun,- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },checkpointSession,- async appendEvent() { throw new Error("unexpected event"); },+ async appendEvent() {+ throw new Error("unexpected event");+ },async replayEvents(replay) {return {events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [],@@ -2302,15 +3820,17 @@ describe("executeNativeSession recovery", () => {async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- onCheckpoint,- onSession,- })).rejects.toBe(admissionFailure);+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ onCheckpoint,+ onSession,+ }),+ ).rejects.toBe(admissionFailure);expect(recoverSession).toHaveBeenCalledOnce();expect(openRun).toHaveBeenCalledOnce();@@ -2356,16 +3876,30 @@ describe("executeNativeSession recovery", () => {};const bySource = new Map<string, PrpEvent[]>();const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" }));- const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); });+ const openSession = vi.fn(async () => {+ throw new Error("must recover the provider session");+ });const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield terminalEvent;},- async *events() { yield terminalEvent; },startTurn,- async result() { return { result, terminal, turnId: "turn-recovery" }; },- async snapshot() { return structuredClone(providerSnapshot); },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return structuredClone(providerSnapshot);+ },async close() {},};const backend: NativeSessionBackend = {@@ -2374,15 +3908,25 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},openSession,- async recoverSession() { return { recovered: true, session }; },+ async recoverSession() {+ return { recovered: true, session };+ },};const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},async appendEvent(event) {const list = bySource.get(event.sourceInstanceId) ?? [];@@ -2397,20 +3941,27 @@ describe("executeNativeSession recovery", () => {async replayEvents(replay) {const list = bySource.get(replay.sourceInstanceId) ?? [];return {- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),+ events: structuredClone(+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),+ ),highestContiguousSourceSeq: highestContiguous(list),};},async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery" });+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({+ turnId: "turn-recovery",+ providerSessionId: "provider-recovery",+ });expect(openSession).not.toHaveBeenCalled();expect(startTurn).not.toHaveBeenCalled();});@@ -2441,39 +3992,74 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield terminalEvent;+ },+ async startTurn() {+ return { turnId: "turn-recovery" };+ },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };},- async *events() { yield terminalEvent; },- async startTurn() { return { turnId: "turn-recovery" }; },- async result() { return { result, terminal, turnId: "turn-recovery" }; },async snapshot() {- return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null };+ return {+ ...checkpoint,+ cursor: String(terminalSequence),+ activeTurnId: null,+ };},async close() {},};- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {- expect(recoveryCheckpoint.cursor).toBe(expectedCursor);- return { recovered: true, session };- });+ const recoverSession = vi.fn(+ async (recoveryCheckpoint: PersistedNativeSession) => {+ expect(recoveryCheckpoint.cursor).toBe(expectedCursor);+ return { recovered: true, session };+ },+ );const backend: NativeSessionBackend = {async descriptor() {return {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("must recover the provider session"); },+ async openSession() {+ throw new Error("must recover the provider session");+ },recoverSession,};const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },- async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },+ async checkpointSession(snapshot) {+ checkpoints.push(structuredClone(snapshot));+ },async appendEvent(event) {- const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents;- if (target.some((existing) => existing.sourceSeq === event.sourceSeq)) {+ const target =+ event.sourceInstanceId === "runner-recovery"+ ? runnerEvents+ : controlEvents;+ if (+ target.some((existing) => existing.sourceSeq === event.sourceSeq)+ ) {throw new Error(`native_event_replay_conflict:${event.sourceSeq}`);}target.push(structuredClone(event));@@ -2484,26 +4070,36 @@ describe("executeNativeSession recovery", () => {};},async replayEvents(replay) {- const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents;+ const source =+ replay.sourceInstanceId === "runner-recovery"+ ? runnerEvents+ : controlEvents;const events = source.filter((event) => event.sourceSeq > replay.afterSourceSeq).sort((left, right) => left.sourceSeq - right.sourceSeq).slice(0, replay.limit);- return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source) };+ return {+ events: structuredClone(events),+ highestContiguousSourceSeq: highestContiguous(source),+ };},async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).resolves.toMatchObject({ turnId: "turn-recovery" });+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({ turnId: "turn-recovery" });expect(recoverSession).toHaveBeenCalledOnce();- expect(runnerEvents.some((event) => event.sourceSeq === terminalSequence)).toBe(true);+ expect(+ runnerEvents.some((event) => event.sourceSeq === terminalSequence),+ ).toBe(true);if (checkpointCursor === "12") {expect(checkpoints[0]).toMatchObject({ cursor: "41" });}@@ -2531,12 +4127,26 @@ describe("executeNativeSession recovery", () => {const replacementSession: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield runnerEvent(1, "turn.completed");+ },+ async startTurn() {+ return { turnId: "turn-replacement" };+ },+ async result() {+ return { result, terminal, turnId: "turn-replacement" };+ },+ async snapshot() {+ return structuredClone(replacementSnapshot);},- async *events() { yield runnerEvent(1, "turn.completed"); },- async startTurn() { return { turnId: "turn-replacement" }; },- async result() { return { result, terminal, turnId: "turn-replacement" }; },- async snapshot() { return structuredClone(replacementSnapshot); },async close() {},};const recoverSession = vi.fn(async () => ({@@ -2551,34 +4161,52 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "replacement-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("replacement seam must be used"); },+ async openSession() {+ throw new Error("replacement seam must be used");+ },recoverSession,openReplacementSession,};const events: PrpEvent[] = [];const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},async appendEvent(event) {events.push(structuredClone(event));- return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" };+ return {+ cursor: events.length,+ highestContiguousSourceSeq: highestContiguous(events),+ disposition: "committed",+ };+ },+ async replayEvents() {+ return { events: [], highestContiguousSourceSeq: 0 };},- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-replacement",- controlPlaneInstanceId: "control-replacement",- onContinuityBreak,- })).resolves.toMatchObject({ providerSessionId: "provider-new" });+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-replacement",+ controlPlaneInstanceId: "control-replacement",+ onContinuityBreak,+ }),+ ).resolves.toMatchObject({ providerSessionId: "provider-new" });expect(recoverSession).toHaveBeenCalledOnce();expect(openReplacementSession).toHaveBeenCalledOnce();@@ -2599,7 +4227,9 @@ describe("executeNativeSession recovery", () => {providerSessionId: "provider-recovery",cursor: "1",activeTurnId: null,- terminalTurns: [{ turnId: "turn-work", fingerprint: "terminal-fingerprint" }],+ terminalTurns: [+ { turnId: "turn-work", fingerprint: "terminal-fingerprint" },+ ],dispositionOnlyRecoveryConsumed: true,dispositionOnlyRecoveryTurnId: "turn-missing-disposition",pendingRuntimeRequests: [],@@ -2629,12 +4259,24 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield terminalEvent;},- async *events() { yield terminalEvent; },startTurn,- async result() { return { result, terminal, turnId: "turn-continuation" }; },- async snapshot() { return structuredClone(recoveredSnapshot); },+ async result() {+ return { result, terminal, turnId: "turn-continuation" };+ },+ async snapshot() {+ return structuredClone(recoveredSnapshot);+ },async close() {},};const backend: NativeSessionBackend = {@@ -2643,26 +4285,42 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("must recover the provider session"); },- async recoverSession() { return { recovered: true, session }; },+ async openSession() {+ throw new Error("must recover the provider session");+ },+ async recoverSession() {+ return { recovered: true, session };+ },};const bySource = new Map<string, PrpEvent[]>();const replayedPages: PrpEvent[][] = [];- const replayEvents = vi.fn(async (replay: Parameters<ControlPlanePort["replayEvents"]>[0]) => {- const list = bySource.get(replay.sourceInstanceId) ?? [];- const events = structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq));- replayedPages.push(events);- return {- events,- highestContiguousSourceSeq: highestContiguous(list),- };- });+ const replayEvents = vi.fn(+ async (replay: Parameters<ControlPlanePort["replayEvents"]>[0]) => {+ const list = bySource.get(replay.sourceInstanceId) ?? [];+ const events = structuredClone(+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),+ );+ replayedPages.push(events);+ return {+ events,+ highestContiguousSourceSeq: highestContiguous(list),+ };+ },+ );const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},async appendEvent(event) {const list = bySource.get(event.sourceInstanceId) ?? [];@@ -2678,13 +4336,18 @@ describe("executeNativeSession recovery", () => {async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" });+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({+ turnId: "turn-continuation",+ providerSessionId: "provider-recovery",+ });expect(startTurn).toHaveBeenCalledOnce();expect(replayEvents).toHaveBeenCalledWith({runId: identity.runId,@@ -2708,19 +4371,23 @@ describe("executeNativeSession recovery", () => {recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined;startTurn.mockClear();bySource.clear();- bySource.set("runner-recovery", [{- ...terminalEvent,- sourceEventId: "runner-recovery:stale-terminal",- turnId: "turn-stale-unbound",- }]);+ bySource.set("runner-recovery", [+ {+ ...terminalEvent,+ sourceEventId: "runner-recovery:stale-terminal",+ turnId: "turn-stale-unbound",+ },+ ]);- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- })).resolves.toMatchObject({+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ }),+ ).resolves.toMatchObject({turnId: "turn-continuation",providerSessionId: "provider-recovery",});@@ -2770,12 +4437,24 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield terminalEvent;},- async *events() { yield terminalEvent; },startTurn,- async result() { return null; },- async snapshot() { return structuredClone(recoveredSnapshot); },+ async result() {+ return null;+ },+ async snapshot() {+ return structuredClone(recoveredSnapshot);+ },async close() {},};const events: PrpEvent[] = [];@@ -2785,24 +4464,42 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("must recover the provider session"); },- async recoverSession() { return { recovered: true, session }; },+ async openSession() {+ throw new Error("must recover the provider session");+ },+ async recoverSession() {+ return { recovered: true, session };+ },};const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession(snapshot) {if (- snapshot.terminalTurns?.some((turn) => turn.turnId === "turn-disposition")- && !dispositionTerminalCommitted- ) prematureDispositionCheckpoint = true;+ snapshot.terminalTurns?.some(+ (turn) => turn.turnId === "turn-disposition",+ ) &&+ !dispositionTerminalCommitted+ )+ prematureDispositionCheckpoint = true;},async appendEvent(event) {events.push(structuredClone(event));- if (event.eventType === "turn.completed" && event.turnId === "turn-disposition") {+ if (+ event.eventType === "turn.completed" &&+ event.turnId === "turn-disposition"+ ) {dispositionTerminalCommitted = true;}return {@@ -2813,24 +4510,29 @@ describe("executeNativeSession recovery", () => {},async replayEvents(replay) {return {- events: structuredClone(events.filter((event) =>- event.sourceInstanceId === replay.sourceInstanceId- && event.sourceSeq > replay.afterSourceSeq- )),+ events: structuredClone(+ events.filter(+ (event) =>+ event.sourceInstanceId === replay.sourceInstanceId &&+ event.sourceSeq > replay.afterSourceSeq,+ ),+ ),highestContiguousSourceSeq: highestContiguous(events),};},async completeRun() {},};- await expect(executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- resolveMissingResult: async () => result,- })).resolves.toMatchObject({+ await expect(+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ resolveMissingResult: async () => result,+ }),+ ).resolves.toMatchObject({result,turnId: "turn-disposition",});@@ -2896,11 +4598,21 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield structuredClone(terminalEvent);},- async *events() { yield structuredClone(terminalEvent); },startTurn,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {...structuredClone(checkpoint),@@ -2924,21 +4636,36 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("must recover the provider session"); },- async recoverSession() { return { recovered: true, session }; },+ async openSession() {+ throw new Error("must recover the provider session");+ },+ async recoverSession() {+ return { recovered: true, session };+ },};const bySource = new Map<string, PrpEvent[]>([- ["runner-recovery", [- structuredClone(originalTaskProposal),- structuredClone(originalTaskTerminal),- ]],+ [+ "runner-recovery",+ [+ structuredClone(originalTaskProposal),+ structuredClone(originalTaskTerminal),+ ],+ ],]);const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},async appendEvent(event) {const list = bySource.get(event.sourceInstanceId) ?? [];@@ -2953,24 +4680,27 @@ describe("executeNativeSession recovery", () => {async replayEvents(replay) {const list = bySource.get(replay.sourceInstanceId) ?? [];return {- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),+ events: structuredClone(+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),+ ),highestContiguousSourceSeq: highestContiguous(list),};},async completeRun() {},};- const execute = () => executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- resolveMissingResult: async ({ terminalEvent: replayed }) => {- expect(replayed).toEqual(terminalEvent);- return result;- },- });+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ resolveMissingResult: async ({ terminalEvent: replayed }) => {+ expect(replayed).toEqual(terminalEvent);+ return result;+ },+ });await expect(execute()).resolves.toMatchObject({result,turnId: "turn-disposition",@@ -2997,11 +4727,9 @@ describe("executeNativeSession recovery", () => {resultProposalEvent,terminalEvent,]);- expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([- "run.result.accepted",- "run.terminal",- ]);-+ expect(+ bySource.get("control-recovery")?.map((event) => event.eventType),+ ).toEqual(["run.result.accepted", "run.terminal"]);});it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => {@@ -3029,20 +4757,33 @@ describe("executeNativeSession recovery", () => {pendingRuntimeRequests: [],lineage: [],};- const recoveredCheckpoint: PersistedNativeSession = structuredClone(checkpoint);+ const recoveredCheckpoint: PersistedNativeSession =+ structuredClone(checkpoint);const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));- const events = vi.fn(() => (async function* () {- throw new Error("checkpoint fallback must not consume provider events");- })());+ const events = vi.fn(() =>+ (async function* () {+ throw new Error("checkpoint fallback must not consume provider events");+ })(),+ );const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},events,startTurn,- async result() { return null; },- async snapshot() { return structuredClone(recoveredCheckpoint); },+ async result() {+ return null;+ },+ async snapshot() {+ return structuredClone(recoveredCheckpoint);+ },async close() {},};const backend: NativeSessionBackend = {@@ -3051,11 +4792,21 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { throw new Error("must recover the provider session"); },- async recoverSession() { return { recovered: true, session }; },+ async openSession() {+ throw new Error("must recover the provider session");+ },+ async recoverSession() {+ return { recovered: true, session };+ },};const bySource = new Map<string, PrpEvent[]>([["runner-recovery", [workProposal, workTerminal]],@@ -3063,7 +4814,9 @@ describe("executeNativeSession recovery", () => {const completeRun = vi.fn(async () => undefined);const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },async checkpointSession() {},async appendEvent(event) {const list = bySource.get(event.sourceInstanceId) ?? [];@@ -3078,36 +4831,39 @@ describe("executeNativeSession recovery", () => {async replayEvents(replay) {const list = bySource.get(replay.sourceInstanceId) ?? [];return {- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),+ events: structuredClone(+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),+ ),highestContiguousSourceSeq: highestContiguous(list),};},completeRun,};- const execute = () => executeNativeSession({- input,- backend,- controlPlane: port,- runnerInstanceId: "runner-recovery",- controlPlaneInstanceId: "control-recovery",- resolveMissingResult: async ({ turnId, terminalEvent }) => {- expect(turnId).toBe("turn-disposition");- expect(terminalEvent).toMatchObject({- sourceInstanceId: "control-recovery",- sourceKind: "control_plane",- runId: identity.runId,- normalizedSessionId: identity.sessionId,- turnId: "turn-disposition",- eventType: "turn.completed",- payload: {- recovery: "checkpointed_resultless_disposition",- terminalFingerprint: "disposition-terminal",- },- });- return result;- },- });+ const execute = () =>+ executeNativeSession({+ input,+ backend,+ controlPlane: port,+ runnerInstanceId: "runner-recovery",+ controlPlaneInstanceId: "control-recovery",+ resolveMissingResult: async ({ turnId, terminalEvent }) => {+ expect(turnId).toBe("turn-disposition");+ expect(terminalEvent).toMatchObject({+ sourceInstanceId: "control-recovery",+ sourceKind: "control_plane",+ runId: identity.runId,+ normalizedSessionId: identity.sessionId,+ turnId: "turn-disposition",+ eventType: "turn.completed",+ payload: {+ recovery: "checkpointed_resultless_disposition",+ terminalFingerprint: "disposition-terminal",+ },+ });+ return result;+ },+ });await expect(execute()).resolves.toMatchObject({result,turnId: "turn-disposition",@@ -3119,10 +4875,9 @@ describe("executeNativeSession recovery", () => {workProposal,workTerminal,]);- expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([- "run.result.accepted",- "run.terminal",- ]);+ expect(+ bySource.get("control-recovery")?.map((event) => event.eventType),+ ).toEqual(["run.result.accepted", "run.terminal"]);recoveredCheckpoint.terminalTurns![1]!.fingerprint = "conflicting-terminal";await expect(execute()).rejects.toThrow(@@ -3143,7 +4898,9 @@ describe("executeNativeSession recovery", () => {semanticResult: result,terminal,activeTurnId: "turn-recovery",- terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }],+ terminalTurns: [+ { turnId: "turn-recovery", fingerprint: "terminal-fingerprint" },+ ],pendingRuntimeRequests: [],lineage: [],};@@ -3152,17 +4909,29 @@ describe("executeNativeSession recovery", () => {const completeRun = vi.fn(async () => undefined);const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));const openSession = vi.fn(async () => {- throw new Error("a recovered run must not open a second provider session");+ throw new Error(+ "a recovered run must not open a second provider session",+ );});const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };},async *events() {},startTurn,- async result() { return { result, terminal, turnId: "turn-recovery" }; },- async snapshot() { return structuredClone(checkpoint); },+ async result() {+ return { result, terminal, turnId: "turn-recovery" };+ },+ async snapshot() {+ return structuredClone(checkpoint);+ },async close() {},};const recoverSession = vi.fn(async () => ({ recovered: true, session }));@@ -3172,7 +4941,13 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "recovery-backend",version: "1",- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},openSession,@@ -3180,8 +4955,12 @@ describe("executeNativeSession recovery", () => {};const port: ControlPlanePort = {async openRun() {},- async loadSessionCheckpoint() { return structuredClone(checkpoint); },- async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },+ async loadSessionCheckpoint() {+ return structuredClone(checkpoint);+ },+ async checkpointSession(snapshot) {+ checkpoints.push(structuredClone(snapshot));+ },async appendEvent(event) {events.push(structuredClone(event as PrpEvent));return {@@ -3191,8 +4970,13 @@ describe("executeNativeSession recovery", () => {};},async replayEvents(replay) {- const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq);- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) };+ const replayed = events.filter(+ (event) => event.sourceSeq > replay.afterSourceSeq,+ );+ return {+ events: structuredClone(replayed),+ highestContiguousSourceSeq: highestContiguous(events),+ };},completeRun,};@@ -3208,11 +4992,20 @@ describe("executeNativeSession recovery", () => {expect(openSession).not.toHaveBeenCalled();expect(recoverSession).toHaveBeenCalledOnce();expect(startTurn).not.toHaveBeenCalled();- expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]);+ expect(events.map((event) => event.eventType)).toEqual([+ "run.result.accepted",+ "run.terminal",+ ]);expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]);expect(completeRun).toHaveBeenCalledOnce();- expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 });- expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal });+ expect(completed).toMatchObject({+ nativeEventCount: 1,+ highestContiguousSourceSeq: 2,+ });+ expect(checkpoints.at(-1)).toMatchObject({+ semanticResult: result,+ terminal,+ });});it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => {@@ -3238,8 +5031,16 @@ describe("executeNativeSession recovery", () => {completionClaim: {contractRevision: "1",objectiveSatisfied: false,- criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }],- remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }],+ criteria: [+ {+ criterionId: "objective",+ status: "unknown",+ evidenceRefs: ["interaction:pending"],+ },+ ],+ remainingWork: [+ { description: "Resume after the response.", blocksCompletion: true },+ ],},evidence: [{ ref: "interaction:pending" }],verification: [],@@ -3257,11 +5058,23 @@ describe("executeNativeSession recovery", () => {const session: NativeSession = {identity: () => identity,async capabilities() {- return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true };+ return {+ resume: false,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ },+ async *events() {+ yield terminalEvent;+ },+ async startTurn() {+ return { turnId: "turn-waiting" };+ },+ async result() {+ return null;},- async *events() { yield terminalEvent; },- async startTurn() { return { turnId: "turn-waiting" }; },- async result() { return null; },async snapshot() {return {backendKind: "mock",@@ -3282,10 +5095,18 @@ describe("executeNativeSession recovery", () => {kind: "mock",name: "governed-wait-backend",version: "1",- capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true },+ capabilities: {+ resume: false,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ },};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -3299,10 +5120,15 @@ describe("executeNativeSession recovery", () => {};},async replayEvents(replay) {- const replayed = events.filter((event) =>- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq+ const replayed = events.filter(+ (event) =>+ event.sourceInstanceId === replay.sourceInstanceId &&+ event.sourceSeq > replay.afterSourceSeq,);- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };+ return {+ events: structuredClone(replayed),+ highestContiguousSourceSeq: highestContiguous(replayed),+ };},completeRun,};@@ -3316,10 +5142,16 @@ describe("executeNativeSession recovery", () => {resolveMissingResult,});- expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent });+ expect(resolveMissingResult).toHaveBeenCalledWith({+ turnId: "turn-waiting",+ terminalEvent,+ });expect(completed).toMatchObject({result: yielded,- terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" },+ terminal: {+ runTerminalState: "succeeded",+ reportedWorkDisposition: "yielded",+ },turnId: "turn-waiting",});expect(completeRun).toHaveBeenCalledWith(@@ -3501,13 +5333,18 @@ describe("executeNativeSession recovery", () => {try {const questionSet = {schema: "paperclip.question_set.v1" as const,- questions: [{- id: "region",- prompt: "Which region?",- required: true,- answerMode: "single_select" as const,- options: [{ id: "us", label: "US" }, { id: "eu", label: "Europe" }],- }],+ questions: [+ {+ id: "region",+ prompt: "Which region?",+ required: true,+ answerMode: "single_select" as const,+ options: [+ { id: "us", label: "US" },+ { id: "eu", label: "Europe" },+ ],+ },+ ],};const request = {schema: "paperclip.runtime_request.v2",@@ -3521,24 +5358,39 @@ describe("executeNativeSession recovery", () => {turnId: "turn-waiting",itemId: "input-1",};- const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting" };- const expired = { ...runnerEvent(2, "runtime_request.expired", {- requestId: "input-1",- requestKind: "runtime",+ const created = {+ ...runnerEvent(1, "runtime_request.created", { request }),turnId: "turn-waiting",- itemId: "input-1",- reason: "durable_handoff",- replayAllowed: false,- requestType: "input",- request,- }), turnId: "turn-waiting" };- const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting" };+ };+ const expired = {+ ...runnerEvent(2, "runtime_request.expired", {+ requestId: "input-1",+ requestKind: "runtime",+ turnId: "turn-waiting",+ itemId: "input-1",+ reason: "durable_handoff",+ replayAllowed: false,+ requestType: "input",+ request,+ }),+ turnId: "turn-waiting",+ };+ const interrupted = {+ ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }),+ turnId: "turn-waiting",+ };let releaseHandoff!: () => void;- const handedOff = new Promise<void>((resolve) => { releaseHandoff = resolve; });+ const handedOff = new Promise<void>((resolve) => {+ releaseHandoff = resolve;+ });let releaseCancelled!: () => void;- const cancelled = new Promise<void>((resolve) => { releaseCancelled = resolve; });+ const cancelled = new Promise<void>((resolve) => {+ releaseCancelled = resolve;+ });let releaseCreated!: () => void;- const createdCommitted = new Promise<void>((resolve) => { releaseCreated = resolve; });+ const createdCommitted = new Promise<void>((resolve) => {+ releaseCreated = resolve;+ });const handoffRuntimeRequest = vi.fn(() => {releaseHandoff();return { result: "handed_off" as const, cleanup: Promise.resolve() };@@ -3567,10 +5419,14 @@ describe("executeNativeSession recovery", () => {await cancelled;yield interrupted;},- async startTurn() { return { turnId: "turn-waiting" }; },+ async startTurn() {+ return { turnId: "turn-waiting" };+ },handoffRuntimeRequest,cancel,- async result() { return null; },+ async result() {+ return null;+ },async snapshot() {return {backendKind: "mock",@@ -3601,7 +5457,9 @@ describe("executeNativeSession recovery", () => {},};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -3609,7 +5467,10 @@ describe("executeNativeSession recovery", () => {async appendEvent(event) {events.push(structuredClone(event as PrpEvent));if (event.eventType === "runtime_request.created") releaseCreated();- const sourceEvents = events.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId);+ const sourceEvents = events.filter(+ (candidate) =>+ candidate.sourceInstanceId === event.sourceInstanceId,+ );return {cursor: events.length,highestContiguousSourceSeq: highestContiguous(sourceEvents),@@ -3617,10 +5478,15 @@ describe("executeNativeSession recovery", () => {};},async replayEvents(replay) {- const replayed = events.filter((event) =>- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq+ const replayed = events.filter(+ (event) =>+ event.sourceInstanceId === replay.sourceInstanceId &&+ event.sourceSeq > replay.afterSourceSeq,);- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };+ return {+ events: structuredClone(replayed),+ highestContiguousSourceSeq: highestContiguous(replayed),+ };},async completeRun() {},};@@ -3649,7 +5515,9 @@ describe("executeNativeSession recovery", () => {signal: expect.any(AbortSignal),});expect(cancel).toHaveBeenCalledOnce();- expect(events.map((event) => event.eventType)).toContain("runtime_request.expired");+ expect(events.map((event) => event.eventType)).toContain(+ "runtime_request.expired",+ );} finally {vi.useRealTimers();}@@ -3665,12 +5533,14 @@ describe("executeNativeSession recovery", () => {prompt: "Which region?",input: {schema: "paperclip.question_set.v1",- questions: [{- id: "region",- prompt: "Which region?",- required: true,- answerMode: "text",- }],+ questions: [+ {+ id: "region",+ prompt: "Which region?",+ required: true,+ answerMode: "text",+ },+ ],},origin: { adapter: "mock" },turnId: "turn-stalled",@@ -3715,7 +5585,9 @@ describe("executeNativeSession recovery", () => {markHandoffStarted();return {result: "handed_off",- cleanup: new Promise<void>((resolve) => { releaseHandoff = resolve; }),+ cleanup: new Promise<void>((resolve) => {+ releaseHandoff = resolve;+ }),};},cancel() {@@ -3807,7 +5679,10 @@ describe("executeNativeSession recovery", () => {},async *events() {try {- yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal" };+ yield {+ ...runnerEvent(1, "turn.completed"),+ turnId: "turn-terminal",+ };} finally {teardownStarted();await new Promise<void>((resolve) => {@@ -3891,12 +5766,14 @@ describe("executeNativeSession recovery", () => {prompt: "Which region?",input: {schema: "paperclip.question_set.v1",- questions: [{- id: "region",- prompt: "Which region?",- required: true,- answerMode: "text",- }],+ questions: [+ {+ id: "region",+ prompt: "Which region?",+ required: true,+ answerMode: "text",+ },+ ],},origin: { adapter: "mock" },turnId: "turn-terminal",@@ -4030,19 +5907,24 @@ describe("executeNativeSession recovery", () => {prompt: "Which region?",input: {schema: "paperclip.question_set.v1",- questions: [{- id: "region",- prompt: "Which region?",- required: true,- answerMode: "text",- }],+ questions: [+ {+ id: "region",+ prompt: "Which region?",+ required: true,+ answerMode: "text",+ },+ ],},origin: { adapter: "mock" },turnId: "turn-live",itemId: "input-live",};const providerEvents = [- { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live" },+ {+ ...runnerEvent(1, "runtime_request.created", { request }),+ turnId: "turn-live",+ },{...runnerEvent(2, "runtime_request.resolved", {requestId: "input-live",@@ -4081,10 +5963,16 @@ describe("executeNativeSession recovery", () => {runtimeRequestHandoff: true,};},- async *events() { yield* providerEvents; },- async startTurn() { return { turnId: "turn-live" }; },+ async *events() {+ yield* providerEvents;+ },+ async startTurn() {+ return { turnId: "turn-live" };+ },handoffRuntimeRequest,- async result() { return { result, terminal, turnId: "turn-live" }; },+ async result() {+ return { result, terminal, turnId: "turn-live" };+ },async snapshot() {return {backendKind: "mock",@@ -4115,7 +6003,9 @@ describe("executeNativeSession recovery", () => {},};},- async openSession() { return session; },+ async openSession() {+ return session;+ },};const port: ControlPlanePort = {async openRun() {},@@ -4126,7 +6016,9 @@ describe("executeNativeSession recovery", () => {await settlementAppendReleased;}appended.push(structuredClone(event as PrpEvent));- const sourceEvents = appended.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId);+ const sourceEvents = appended.filter(+ (candidate) => candidate.sourceInstanceId === event.sourceInstanceId,+ );return {cursor: appended.length,highestContiguousSourceSeq: highestContiguous(sourceEvents),@@ -4134,17 +6026,26 @@ describe("executeNativeSession recovery", () => {};},async replayEvents(replay) {- const replayed = appended.filter((event) =>- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq+ const replayed = appended.filter(+ (event) =>+ event.sourceInstanceId === replay.sourceInstanceId &&+ event.sourceSeq > replay.afterSourceSeq,);- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };+ return {+ events: structuredClone(replayed),+ highestContiguousSourceSeq: highestContiguous(replayed),+ };},async completeRun() {},};const originalSetTimeout = globalThis.setTimeout;let queuedHandoffCallback: (() => void) | null = null;- const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback, delay, ...args) => {+ const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((+ callback,+ delay,+ ...args+ ) => {if (delay === 123_456) {queuedHandoffCallback = () => callback(...args);const handle = originalSetTimeout(() => undefined, 60_000);@@ -4172,7 +6073,9 @@ describe("executeNativeSession recovery", () => {queuedHandoffCallback?.();await Promise.resolve();expect(handoffRuntimeRequest).not.toHaveBeenCalled();- expect(appended.some((event) => event.eventType === "runtime_request.expired")).toBe(false);+ expect(+ appended.some((event) => event.eventType === "runtime_request.expired"),+ ).toBe(false);} finally {releaseSettlementAppend();timeoutSpy.mockRestore();diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.tsindex 31d89f3ddf..cf608b941d 100644--- a/packages/paperclip-runner/src/native-session-runtime.ts+++ b/packages/paperclip-runner/src/native-session-runtime.ts@@ -4,17 +4,66 @@ import type {CheckpointControlPlaneSessionOptions,ControlPlanePort,} from "./contracts/control-plane-port.js";-import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contracts/native-execution.js";-import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js";-import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js";+import type {+ NativeExecutionInput,+ NativeSessionExecutionResult,+} from "./contracts/native-execution.js";+import {+ buildNativeModelEnvelope,+ parseNativeExecutionInput,+} from "./contracts/native-execution.js";+import type {+ NativeSession,+ NativeSessionBackend,+} from "./contracts/native-session-backend.js";import type { PersistedNativeSession } from "./contracts/native-session-backend.js";-import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js";+import type {+ PrpEvent,+ PrpStructuredRunResult,+ PrpTerminalState,+} from "./protocol/replay-contract.js";import { parsePaperclipQuestionSet } from "./contracts/question-set.js";export const DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS = 120_000;const OPTIONAL_SESSION_CANCELLATION_GRACE_MS = 100;const FAILED_OPERATION_SETTLEMENT_GRACE_MS = 100;const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000;+type NativeSessionCleanupDomain = string;++const failedSessionCleanupOwners = new Map<+ Promise<void>,+ NativeSessionCleanupDomain+>();+const FAILED_SESSION_CLOSE_RETRY_MS = 1_000;+const MAX_FAILED_SESSION_CLOSE_RETRIES = 3;+const MAX_QUARANTINED_SESSION_CLOSE_RETRIES = 3;+const MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS = 9;+const QUARANTINED_SESSION_CLOSE_RETRY_MS = 60_000;+// Admission can inherit the initial close plus all three retained retries.+// Keep the wait finite while covering every bounded close attempt and all+// three production retry delays instead of timing out midway through recovery.+const QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS = 7_000;+const QUARANTINED_SESSION_ADMISSION_GRACE_MS =+ (MAX_FAILED_SESSION_CLOSE_RETRIES + 1) *+ QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS ++ MAX_FAILED_SESSION_CLOSE_RETRIES * FAILED_SESSION_CLOSE_RETRY_MS;+// Admission can observe a retained close recovery and then the one quarantine+// recovery that retained owner installs when it exhausts its retries. Give+// each owner generation a complete finite grace, while rejecting any+// unexpected third generation instead of permitting an unbounded owner chain.+const MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES = 2;++interface QuarantinedSessionCleanup {+ session: NativeSession;+ domain: NativeSessionCleanupDomain;+ automaticAttempts: number;+ attempt: Promise<void> | null;+ recovery: Promise<void> | null;+ recoveryMaxAttempts: number | null;+ timer: ReturnType<typeof setTimeout> | null;+}++const quarantinedSessionCleanups = new Set<QuarantinedSessionCleanup>();export interface ExecuteNativeSessionOptions {input: NativeExecutionInput;@@ -23,6 +72,8 @@ export interface ExecuteNativeSessionOptions {runnerInstanceId: string;controlPlaneInstanceId: string;timeoutMs?: number;+ /** Abort admission while waiting for prior cleanup in the same domain. */+ signal?: AbortSignal;/** Internal test seam; production bounds checkpoint persistence to 30 seconds. */checkpointTimeoutMs?: number;/** Internal test seam; production uses the fixed 120-second platform policy. */@@ -68,18 +119,43 @@ export interface ExecuteNativeSessionOptions {}function isTurnTerminal(event: PrpEvent): boolean {- return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType);+ return [+ "turn.completed",+ "turn.failed",+ "turn.interrupted",+ "turn.cancelled",+ ].includes(event.eventType);}-function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["reportedWorkDisposition"]): PrpTerminalState {- const states = event.eventType === "turn.completed"- ? { turnTerminalState: "completed" as const, runTerminalState: "succeeded" as const }- : event.eventType === "turn.failed"- ? { turnTerminalState: "failed" as const, runTerminalState: "failed" as const }- : event.eventType === "turn.interrupted"- ? { turnTerminalState: "interrupted" as const, runTerminalState: "cancelled" as const }- : { turnTerminalState: "cancelled" as const, runTerminalState: "cancelled" as const };- return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition };+function terminalFromEvent(+ event: PrpEvent,+ disposition: PrpTerminalState["reportedWorkDisposition"],+): PrpTerminalState {+ const states =+ event.eventType === "turn.completed"+ ? {+ turnTerminalState: "completed" as const,+ runTerminalState: "succeeded" as const,+ }+ : event.eventType === "turn.failed"+ ? {+ turnTerminalState: "failed" as const,+ runTerminalState: "failed" as const,+ }+ : event.eventType === "turn.interrupted"+ ? {+ turnTerminalState: "interrupted" as const,+ runTerminalState: "cancelled" as const,+ }+ : {+ turnTerminalState: "cancelled" as const,+ runTerminalState: "cancelled" as const,+ };+ return {+ schema: "paperclip.prp.terminal.v1",+ ...states,+ reportedWorkDisposition: disposition,+ };}async function attemptOptionalSessionCancellation(@@ -104,21 +180,48 @@ async function attemptOptionalSessionCancellation(if (await settlesWithin(attempts, OPTIONAL_SESSION_CANCELLATION_GRACE_MS)) {return null;}- cancellationAbort.abort(new Error("native session cancellation grace expired"));+ cancellationAbort.abort(+ new Error("native session cancellation grace expired"),+ );return { settlement: attempts };}-async function settlesWithin(operation: Promise<unknown>, timeoutMs: number): Promise<boolean> {+async function settlesWithin(+ operation: Promise<unknown>,+ timeoutMs: number,+ signal?: AbortSignal,+): Promise<boolean> {+ signal?.throwIfAborted();let graceTimer: ReturnType<typeof setTimeout> | undefined;- const settled = await Promise.race([- operation.then(() => true),- new Promise<false>((resolve) => {- graceTimer = setTimeout(() => resolve(false), timeoutMs);- graceTimer.unref?.();- }),- ]);- if (graceTimer !== undefined) clearTimeout(graceTimer);- return settled;+ let removeAbort = () => {};+ try {+ const aborted = signal+ ? new Promise<never>((_resolve, reject) => {+ const onAbort = () =>+ reject(+ signal.reason ??+ new Error("native session cleanup admission aborted"),+ );+ if (signal.aborted) {+ onAbort();+ } else {+ signal.addEventListener("abort", onAbort, { once: true });+ removeAbort = () => signal.removeEventListener("abort", onAbort);+ }+ })+ : new Promise<never>(() => undefined);+ return await Promise.race([+ operation.then(() => true),+ new Promise<false>((resolve) => {+ graceTimer = setTimeout(() => resolve(false), timeoutMs);+ graceTimer.unref?.();+ }),+ aborted,+ ]);+ } finally {+ if (graceTimer !== undefined) clearTimeout(graceTimer);+ removeAbort();+ }}async function runAbortableOperationWithin<T>(input: {@@ -152,7 +255,8 @@ async function runAbortableOperationWithin<T>(input: {operation,new Promise<never>((_resolve, reject) => {timer = setTimeout(() => {- const error = input.timeoutError?.() ?? new Error(input.timeoutMessage);+ const error =+ input.timeoutError?.() ?? new Error(input.timeoutMessage);timedOut = true;timeoutError = error;reject(error);@@ -174,14 +278,54 @@ async function runAbortableOperationWithin<T>(input: {async function disposeUnadmittedSession(session: NativeSession,reason: string,+ cleanupDomain: NativeSessionCleanupDomain,): Promise<void> {// A provider may ignore abort and return a session after its caller has// already timed out. That session was never published through onSession, so// close it without clearing ownership that a later execution may establish.- const closeSettlement = Promise.allSettled([- Promise.resolve().then(() => session.close({ reason })),- ]);- await settlesWithin(closeSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS);+ // Retain slow or failed cleanup in the same admission gate as an admitted+ // session: the absence of an onSession publication does not mean provider+ // resources have already been released.+ const cleanup = retainUnadmittedSessionCleanup(+ session,+ reason,+ cleanupDomain,+ );+ await settlesWithin(+ Promise.allSettled([cleanup]),+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,+ );+}++function retainUnadmittedSessionCleanup(+ session: NativeSession,+ reason: string,+ cleanupDomain: NativeSessionCleanupDomain,+): Promise<void> {+ const cleanup = (async () => {+ let attempt = Promise.resolve().then(() => session.close({ reason }));+ let retryCount = 0;+ while (true) {+ try {+ await attempt;+ return;+ } catch (error) {+ if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {+ quarantineSessionCleanup(session, cleanupDomain);+ throw error;+ }+ retryCount += 1;+ await waitForSessionCloseRetry();+ attempt = Promise.resolve().then(() =>+ session.close({+ reason: `${reason} cleanup recovery (${retryCount})`,+ }),+ );+ }+ }+ })();+ retainFailedSessionCleanupOwner(cleanup, cleanupDomain);+ return cleanup;}class NativeSessionFinalizationTimeoutError extends Error {@@ -198,7 +342,8 @@ async function finalizeWithin<T>(input: {return runAbortableOperationWithin({...input,timeoutMessage: `native session finalization timed out after ${input.timeoutMs}ms`,- timeoutError: () => new NativeSessionFinalizationTimeoutError(input.timeoutMs),+ timeoutError: () =>+ new NativeSessionFinalizationTimeoutError(input.timeoutMs),});}@@ -223,13 +368,14 @@ async function quarantineRetainedSession(session: NativeSession,onSession: ExecuteNativeSessionOptions["onSession"],reason: string,+ cleanupDomain: NativeSessionCleanupDomain,): Promise<void> {// Eviction and provider cleanup are independent obligations. Keep both// observed so a throwing owner callback cannot prevent close from starting,// and a broken provider cannot keep the failed execution pending forever.const quarantineSettlement = Promise.allSettled([Promise.resolve().then(() => onSession?.(null)),- Promise.resolve().then(() => session.close({ reason })),+ retainUnadmittedSessionCleanup(session, reason, cleanupDomain),]);await settlesWithin(quarantineSettlement,@@ -251,8 +397,9 @@ async function persistCheckpointWithin(input: {const externalAbortFailure = externalSignal? new Promise<never>((_resolve, reject) => {const abort = () => {- const reason = externalSignal.reason- ?? new Error("native session checkpoint aborted");+ const reason =+ externalSignal.reason ??+ new Error("native session checkpoint aborted");checkpointAbort.abort(reason);reject(reason);};@@ -260,7 +407,8 @@ async function persistCheckpointWithin(input: {abort();} else {externalSignal.addEventListener("abort", abort, { once: true });- removeExternalAbort = () => externalSignal.removeEventListener("abort", abort);+ removeExternalAbort = () =>+ externalSignal.removeEventListener("abort", abort);}}): new Promise<never>(() => undefined);@@ -271,8 +419,10 @@ async function persistCheckpointWithin(input: {checkpointOptions,);if (checkpointAbort.signal.aborted) {- throw checkpointAbort.signal.reason- ?? new Error("native session checkpoint aborted");+ throw (+ checkpointAbort.signal.reason ??+ new Error("native session checkpoint aborted")+ );}await input.onCheckpoint?.(input.snapshot, checkpointOptions);})();@@ -300,6 +450,226 @@ async function persistCheckpointWithin(input: {}}+function waitForSessionCloseRetry(): Promise<void> {+ return new Promise((resolve) => {+ const timer = setTimeout(resolve, FAILED_SESSION_CLOSE_RETRY_MS);+ timer.unref?.();+ });+}+function retainFailedSessionCleanupOwner(+ cleanup: Promise<void>,+ cleanupDomain: NativeSessionCleanupDomain,+): void {+ failedSessionCleanupOwners.set(cleanup, cleanupDomain);+ void cleanup+ .finally(() => failedSessionCleanupOwners.delete(cleanup))+ .catch(() => undefined);+}++function quarantineSessionCleanup(+ session: NativeSession,+ cleanupDomain: NativeSessionCleanupDomain,+): void {+ if (+ [...quarantinedSessionCleanups].some((entry) => entry.session === session)+ ) {+ return;+ }+ const cleanup: QuarantinedSessionCleanup = {+ session,+ domain: cleanupDomain,+ automaticAttempts: 0,+ attempt: null,+ recovery: null,+ recoveryMaxAttempts: null,+ timer: null,+ };+ quarantinedSessionCleanups.add(cleanup);+ startQuarantinedSessionCleanupRecovery(+ cleanup,+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES,+ "native session quarantined cleanup recovery",+ );+}++function startQuarantinedSessionCleanupRecovery(+ cleanup: QuarantinedSessionCleanup,+ maxAttempts: number,+ reason: string,+): Promise<void> {+ if (cleanup.recovery) return cleanup.recovery;+ const remainingAttempts =+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS -+ cleanup.automaticAttempts;+ const boundedMaxAttempts = Math.min(maxAttempts, remainingAttempts);+ if (boundedMaxAttempts <= 0) return Promise.resolve();+ const recovery = (async () => {+ for (+ let attemptCount = 0;+ attemptCount < boundedMaxAttempts &&+ quarantinedSessionCleanups.has(cleanup);+ attemptCount += 1+ ) {+ // The first retry starts immediately so an admission-triggered recovery+ // receives the complete close grace. Later attempts retain the bounded+ // delay that prevents a hot retry loop.+ if (attemptCount > 0) await waitForSessionCloseRetry();+ cleanup.automaticAttempts += 1;+ const attempt = Promise.resolve().then(() =>+ cleanup.session.close({+ reason,+ }),+ );+ cleanup.attempt = attempt;+ try {+ await attempt;+ quarantinedSessionCleanups.delete(cleanup);+ } catch {+ // Retain the quarantine after this finite, sequential retry batch.+ } finally {+ if (cleanup.attempt === attempt) cleanup.attempt = null;+ }+ }+ })();+ cleanup.recovery = recovery;+ cleanup.recoveryMaxAttempts = boundedMaxAttempts;+ failedSessionCleanupOwners.set(recovery, cleanup.domain);+ void recovery+ .finally(() => {+ failedSessionCleanupOwners.delete(recovery);+ if (cleanup.recovery === recovery) {+ cleanup.recovery = null;+ cleanup.recoveryMaxAttempts = null;+ }+ scheduleQuarantinedSessionCleanup(cleanup);+ })+ .catch(() => undefined);+ return recovery;+}++function scheduleQuarantinedSessionCleanup(+ cleanup: QuarantinedSessionCleanup,+): void {+ if (+ !quarantinedSessionCleanups.has(cleanup) ||+ cleanup.recovery ||+ cleanup.attempt ||+ cleanup.timer ||+ cleanup.automaticAttempts >=+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS+ ) {+ return;+ }+ // Keep cleanup live without a hot retry loop: one unref'd timer and one+ // sequential close are the maximum background work owned by each entry.+ // A later admission may accelerate, but never overlap, the scheduled try.+ cleanup.timer = setTimeout(() => {+ cleanup.timer = null;+ if (!quarantinedSessionCleanups.has(cleanup)) return;+ startQuarantinedSessionCleanupRecovery(+ cleanup,+ 1,+ "native session scheduled quarantined cleanup recovery",+ );+ }, QUARANTINED_SESSION_CLOSE_RETRY_MS);+ cleanup.timer.unref?.();+}++async function retryQuarantinedSessionCleanups(+ cleanupDomain: NativeSessionCleanupDomain,+ signal?: AbortSignal,+): Promise<void> {+ // A close attempt becomes admission-visible as soon as the runtime retains+ // its exact cleanup owner. It may not have rejected yet, so it may not have+ // entered the retry quarantine below. Observe both states through the same+ // finite gate to prevent a later execution from opening a second provider+ // session while the first close still owns provider resources.+ const observedOwners = new Set<Promise<void>>();+ const acceleratedCleanups = new Set<QuarantinedSessionCleanup>();+ let observedOwnerPhases = 0;+ while (true) {+ signal?.throwIfAborted();+ const cleanupOwners = new Set<Promise<void>>(+ [...failedSessionCleanupOwners]+ .filter(([, domain]) => domain === cleanupDomain)+ .map(([owner]) => owner),+ );+ for (const cleanup of quarantinedSessionCleanups) {+ if (cleanup.domain !== cleanupDomain) continue;+ if (cleanup.timer) {+ clearTimeout(cleanup.timer);+ cleanup.timer = null;+ }+ if (cleanup.recovery) {+ // An autonomous scheduled owner receives one attempt. Admission must+ // observe it without mistaking it for the complete three-attempt+ // admission batch; if that attempt fails, the next bounded owner+ // phase accelerates one full batch. Existing full recoveries already+ // consumed that allowance and are never duplicated.+ if (+ (cleanup.recoveryMaxAttempts ?? 0) >=+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES+ ) {+ acceleratedCleanups.add(cleanup);+ }+ cleanupOwners.add(cleanup.recovery);+ } else if (!acceleratedCleanups.has(cleanup)) {+ acceleratedCleanups.add(cleanup);+ if (+ cleanup.automaticAttempts <+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS+ ) {+ cleanupOwners.add(+ startQuarantinedSessionCleanupRecovery(+ cleanup,+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES,+ "native session quarantined admission recovery",+ ),+ );+ }+ }+ }+ const replacementOwners = [...cleanupOwners].filter(+ (owner) => !observedOwners.has(owner),+ );+ if (replacementOwners.length === 0) break;+ replacementOwners.forEach((owner) => observedOwners.add(owner));+ observedOwnerPhases += 1;+ if (+ observedOwnerPhases > MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES ||+ !(await settlesWithin(+ Promise.all(+ replacementOwners.map((owner) => owner.catch(() => undefined)),+ ),+ QUARANTINED_SESSION_ADMISSION_GRACE_MS,+ signal,+ ))+ ) {+ throw new Error(+ "native_session_cleanup_quarantined: prior session cleanup exceeded the admission grace",+ );+ }+ }+ if (+ [...failedSessionCleanupOwners.values()].some(+ (domain) => domain === cleanupDomain,+ ) ||+ [...quarantinedSessionCleanups].some(+ (cleanup) => cleanup.domain === cleanupDomain,+ )+ ) {+ // Admission may have pulled a scheduled retry forward and observed a+ // complete recovery batch that still failed. Restore autonomous ownership+ // before rejecting so cleanup cannot remain dormant until another run.+ for (const cleanup of quarantinedSessionCleanups) {+ if (cleanup.domain !== cleanupDomain) continue;+ scheduleQuarantinedSessionCleanup(cleanup);+ }+ throw new Error(+ "native_session_cleanup_quarantined: prior session cleanup remains incomplete",+ );+ }+}async function consumeTurn(session: NativeSession,controlPlane: ControlPlanePort,@@ -328,7 +698,9 @@ async function consumeTurn(const externalAbortFailure = externalSignal? new Promise<never>((_resolve, reject) => {const abort = () => {- const reason = externalSignal.reason ?? new Error("native event consumption aborted");+ const reason =+ externalSignal.reason ??+ new Error("native event consumption aborted");stopConsumer = true;appendAbort.abort(reason);reject(reason);@@ -337,7 +709,8 @@ async function consumeTurn(abort();} else {externalSignal.addEventListener("abort", abort, { once: true });- removeExternalAbort = () => externalSignal.removeEventListener("abort", abort);+ removeExternalAbort = () =>+ externalSignal.removeEventListener("abort", abort);}}): new Promise<never>(() => undefined);@@ -347,128 +720,155 @@ async function consumeTurn(inputTimers.delete(requestId);};const consumer = (async () => {- let eventCount = 0;- let highestContiguousSourceSeq = 0;- let governedResult: PrpStructuredRunResult | null = null;- while (true) {- const next = await eventIterator.next();- if (stopConsumer) throw new Error("native event consumer stopped");- if (next.done) throw new Error("native event stream closed before a turn terminal fact");- const event = next.value;- const payload = event.payload as Record<string, unknown>;- const settlingRequestId =- ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]- .includes(event.eventType)- && typeof payload.requestId === "string"- ? payload.requestId- : null;- // Beginning settlement revokes the expiry timer's handoff authority.- // appendEvent may remain pending across the live-window deadline; if- // the timer stayed live until the receipt returned, both settlement- // and a durable handoff could commit for the same request.- if (settlingRequestId !== null) clearInputTimer(settlingRequestId);- const receipt = await controlPlane.appendEvent(event, {- signal: appendAbort.signal,- });- if (stopConsumer) throw new Error("native event consumer stopped");- eventCount += receipt.disposition === "committed" ? 1 : 0;- highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq);- const request = payload.request && typeof payload.request === "object" && !Array.isArray(payload.request)- ? payload.request as Record<string, unknown>- : null;- if (- receipt.disposition === "committed"- && event.eventType === "runtime_request.created"- && request?.schema === "paperclip.runtime_request.v2"- && request.type === "input"- && typeof request.requestId === "string"- && typeof request.turnId === "string"- ) {- try {- parsePaperclipQuestionSet(request.input);- const requestId = request.requestId;- const turnId = request.turnId;- clearInputTimer(requestId);- const inputTimer = setTimeout(() => {- // Clearing a timeout does not revoke a callback that is already- // queued. The map entry is the per-request authority token.- if (inputTimers.get(requestId) !== inputTimer) return;- inputTimers.delete(requestId);- // A timer callback can already be queued when teardown clears- // its handle. Re-check the live-turn authority inside the- // callback before starting or registering durable work.- if (stopConsumer || appendAbort.signal.aborted) return;- if (session.handoffRuntimeRequest === undefined) {- rejectHandoff?.(new Error("native_runtime_request_handoff_unavailable"));- return;- }- let handoffCleanup: Promise<unknown>;- try {- const handoff = session.handoffRuntimeRequest({- requestId,- turnId,- reason: "durable_handoff",- signal: appendAbort.signal,- });- // Durable handoff mutation is synchronous. The returned- // promise owns provider interruption only, so it can remain- // observed without acquiring authority to delay or reverse- // a provider terminal fact.- handoffCleanup = handoff.cleanup;- } catch (error) {- rejectHandoff?.(error);- return;- }- handoffCleanupOperations.add(handoffCleanup);- void handoffCleanup- .catch((error) => {- if (!stopConsumer && !appendAbort.signal.aborted) {- rejectHandoff?.(error);- }- })- .finally(() => handoffCleanupOperations.delete(handoffCleanup));- }, runtimeInputLiveWindowMs);- inputTimer.unref?.();- inputTimers.set(requestId, inputTimer);- } catch {- // Invalid structured inputs remain rejected by the driver and never become durable questions.- }- }- if (governedResult === null && resolveGovernedWait) {- if (appendAbort.signal.aborted) {- throw appendAbort.signal.reason ?? new Error("native event consumption aborted");+ let eventCount = 0;+ let highestContiguousSourceSeq = 0;+ let governedResult: PrpStructuredRunResult | null = null;+ while (true) {+ const next = await eventIterator.next();+ if (stopConsumer) throw new Error("native event consumer stopped");+ if (next.done)+ throw new Error(+ "native event stream closed before a turn terminal fact",+ );+ const event = next.value;+ const payload = event.payload as Record<string, unknown>;+ const settlingRequestId =+ [+ "runtime_request.resolved",+ "runtime_request.cancelled",+ "runtime_request.expired",+ ].includes(event.eventType) && typeof payload.requestId === "string"+ ? payload.requestId+ : null;+ // Beginning settlement revokes the expiry timer's handoff authority.+ // appendEvent may remain pending across the live-window deadline; if+ // the timer stayed live until the receipt returned, both settlement+ // and a durable handoff could commit for the same request.+ if (settlingRequestId !== null) clearInputTimer(settlingRequestId);+ const receipt = await controlPlane.appendEvent(event, {+ signal: appendAbort.signal,+ });+ if (stopConsumer) throw new Error("native event consumer stopped");+ eventCount += receipt.disposition === "committed" ? 1 : 0;+ highestContiguousSourceSeq = Math.max(+ highestContiguousSourceSeq,+ receipt.highestContiguousSourceSeq,+ );+ const request =+ payload.request &&+ typeof payload.request === "object" &&+ !Array.isArray(payload.request)+ ? (payload.request as Record<string, unknown>)+ : null;+ if (+ receipt.disposition === "committed" &&+ event.eventType === "runtime_request.created" &&+ request?.schema === "paperclip.runtime_request.v2" &&+ request.type === "input" &&+ typeof request.requestId === "string" &&+ typeof request.turnId === "string"+ ) {+ try {+ parsePaperclipQuestionSet(request.input);+ const requestId = request.requestId;+ const turnId = request.turnId;+ clearInputTimer(requestId);+ const inputTimer = setTimeout(() => {+ // Clearing a timeout does not revoke a callback that is already+ // queued. The map entry is the per-request authority token.+ if (inputTimers.get(requestId) !== inputTimer) return;+ inputTimers.delete(requestId);+ // A timer callback can already be queued when teardown clears+ // its handle. Re-check the live-turn authority inside the+ // callback before starting or registering durable work.+ if (stopConsumer || appendAbort.signal.aborted) return;+ if (session.handoffRuntimeRequest === undefined) {+ rejectHandoff?.(+ new Error("native_runtime_request_handoff_unavailable"),+ );+ return;}- governedResult = resolveGovernedWait({- turnId: event.turnId ?? null,- event,- });- if (governedResult !== null && !isTurnTerminal(event)) {- if (session.cancel === undefined) {- throw new Error("native_governed_wait_cancellation_unavailable");- }- // A governed result is already durable. Commit cancellation- // synchronously, then stop consuming provider output now rather- // than waiting for an abort-insensitive cleanup or terminal event.- // The returned promise owns cleanup only and remains observed in- // finally, where its wait is bounded and the session quarantined.- const cancellation = session.cancel({- reason: "Paperclip parked this turn on a durable governed interaction.",+ let handoffCleanup: Promise<unknown>;+ try {+ const handoff = session.handoffRuntimeRequest({+ requestId,+ turnId,+ reason: "durable_handoff",signal: appendAbort.signal,});- governedCancellationCommitted = true;- const cleanup = cancellation.cleanup;- governedCleanupOperations.add(cleanup);- void cleanup- .catch(() => quarantineSession())- .finally(() => governedCleanupOperations.delete(cleanup));- return { event, eventCount, highestContiguousSourceSeq, governedResult };+ // Durable handoff mutation is synchronous. The returned+ // promise owns provider interruption only, so it can remain+ // observed without acquiring authority to delay or reverse+ // a provider terminal fact.+ handoffCleanup = handoff.cleanup;+ } catch (error) {+ rejectHandoff?.(error);+ return;}+ handoffCleanupOperations.add(handoffCleanup);+ void handoffCleanup+ .catch((error) => {+ if (!stopConsumer && !appendAbort.signal.aborted) {+ rejectHandoff?.(error);+ }+ })+ .finally(() => handoffCleanupOperations.delete(handoffCleanup));+ }, runtimeInputLiveWindowMs);+ inputTimer.unref?.();+ inputTimers.set(requestId, inputTimer);+ } catch {+ // Invalid structured inputs remain rejected by the driver and never become durable questions.+ }+ }+ if (governedResult === null && resolveGovernedWait) {+ if (appendAbort.signal.aborted) {+ throw (+ appendAbort.signal.reason ??+ new Error("native event consumption aborted")+ );+ }+ governedResult = resolveGovernedWait({+ turnId: event.turnId ?? null,+ event,+ });+ if (governedResult !== null && !isTurnTerminal(event)) {+ if (session.cancel === undefined) {+ throw new Error("native_governed_wait_cancellation_unavailable");}- if (isTurnTerminal(event)) {- return { event, eventCount, highestContiguousSourceSeq, governedResult };- }+ // A governed result is already durable. Commit cancellation+ // synchronously, then stop consuming provider output now rather+ // than waiting for an abort-insensitive cleanup or terminal event.+ // The returned promise owns cleanup only and remains observed in+ // finally, where its wait is bounded and the session quarantined.+ const cancellation = session.cancel({+ reason:+ "Paperclip parked this turn on a durable governed interaction.",+ signal: appendAbort.signal,+ });+ governedCancellationCommitted = true;+ const cleanup = cancellation.cleanup;+ governedCleanupOperations.add(cleanup);+ void cleanup+ .catch(() => quarantineSession())+ .finally(() => governedCleanupOperations.delete(cleanup));+ return {+ event,+ eventCount,+ highestContiguousSourceSeq,+ governedResult,+ };}- })();+ }+ if (isTurnTerminal(event)) {+ return {+ event,+ eventCount,+ highestContiguousSourceSeq,+ governedResult,+ };+ }+ }+ })();// A timeout can win the race while an iterator is still waiting for data.// Observe any later consumer rejection so it cannot become process-fatal.void consumer.catch(() => undefined);@@ -494,8 +894,15 @@ async function consumeTurn(// cancellation has already committed synchronously. Bound only the// authority-free provider cleanup while keeping its outcome observed.if (governedCleanupOperations.size > 0) {- const governedCleanupSettlement = Promise.allSettled([...governedCleanupOperations]);- if (!(await settlesWithin(governedCleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS))) {+ const governedCleanupSettlement = Promise.allSettled([+ ...governedCleanupOperations,+ ]);+ if (+ !(await settlesWithin(+ governedCleanupSettlement,+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,+ ))+ ) {deferredGovernedCleanupSettlement = governedCleanupSettlement;}}@@ -504,20 +911,23 @@ async function consumeTurn(session,"Native session event consumption failed.",);- deferredSessionCancellationSettlement = deferredCancellation?.settlement ?? null;+ deferredSessionCancellationSettlement =+ deferredCancellation?.settlement ?? null;}throw error;} finally {stopConsumer = true;for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer);inputTimers.clear();- const activeHandoffCleanupSettlement = handoffCleanupOperations.size > 0- ? Promise.allSettled([...handoffCleanupOperations])- : null;- const activeGovernedCleanupSettlement = deferredGovernedCleanupSettlement === null- && governedCleanupOperations.size > 0- ? Promise.allSettled([...governedCleanupOperations])- : null;+ const activeHandoffCleanupSettlement =+ handoffCleanupOperations.size > 0+ ? Promise.allSettled([...handoffCleanupOperations])+ : null;+ const activeGovernedCleanupSettlement =+ deferredGovernedCleanupSettlement === null &&+ governedCleanupOperations.size > 0+ ? Promise.allSettled([...governedCleanupOperations])+ : null;if (!consumptionFailed && !appendAbort.signal.aborted) {// A provider terminal or synchronous governed cancellation revokes// live-turn authority. Handoff state was already committed; abort only@@ -537,9 +947,15 @@ async function consumeTurn(const passiveTeardownSettlement = Promise.allSettled([iteratorTeardown,consumer,- ...(activeHandoffCleanupSettlement ? [activeHandoffCleanupSettlement] : []),- ...(activeGovernedCleanupSettlement ? [activeGovernedCleanupSettlement] : []),- ...(deferredGovernedCleanupSettlement ? [deferredGovernedCleanupSettlement] : []),+ ...(activeHandoffCleanupSettlement+ ? [activeHandoffCleanupSettlement]+ : []),+ ...(activeGovernedCleanupSettlement+ ? [activeGovernedCleanupSettlement]+ : []),+ ...(deferredGovernedCleanupSettlement+ ? [deferredGovernedCleanupSettlement]+ : []),...(deferredSessionCancellationSettlement? [deferredSessionCancellationSettlement]: []),@@ -556,7 +972,10 @@ async function consumeTurn(passiveTeardownSettlement,Promise.resolve().then(closeFailedSession),]);- await settlesWithin(cleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS);+ await settlesWithin(+ cleanupSettlement,+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,+ );} else {// Iterator and provider cleanup own no control-plane mutation authority.// A slow subscription or cleanup remains observed and is released by the@@ -611,7 +1030,10 @@ async function reconcileRecoveryCursor(input: {persistedHighWater = Math.max(persistedHighWater, pageHighWater);afterSourceSeq = pageHighWater;}- if (persistedHighWater === checkpointHighWater && input.checkpoint.cursor === String(checkpointHighWater)) {+ if (+ persistedHighWater === checkpointHighWater &&+ input.checkpoint.cursor === String(checkpointHighWater)+ ) {return input.checkpoint;}return { ...input.checkpoint, cursor: String(persistedHighWater) };@@ -639,9 +1061,12 @@ async function replayCheckpointedTurnTerminal(input: {// Disposition recovery owns the newest durable provider terminal. An// older task turn may also have a valid proposal, but selecting it would// finalize stale work and strand the actual recovery terminal.- const terminal = [...terminals]- .sort((left, right) => right.sourceSeq - left.sourceSeq)[0] ?? null;- const proposalSequence = latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0;+ const terminal =+ [...terminals].sort(+ (left, right) => right.sourceSeq - left.sourceSeq,+ )[0] ?? null;+ const proposalSequence =+ latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0;return terminal === null? null: {@@ -663,11 +1088,9 @@ async function replayCheckpointedTurnTerminal(input: {if (event.turnId &&isTurnTerminal(event) &&- (- input.expectedTurnId !== undefined && input.expectedTurnId !== null- ? event.turnId === input.expectedTurnId- : !priorTerminalTurnIds.has(event.turnId)- )+ (input.expectedTurnId !== undefined && input.expectedTurnId !== null+ ? event.turnId === input.expectedTurnId+ : !priorTerminalTurnIds.has(event.turnId))) {terminals.push(structuredClone(event));}@@ -690,30 +1113,37 @@ function checkpointedResultlessDispositionFallback(input: {}): PrpEvent | null {const turnId = input.persisted.dispositionOnlyRecoveryTurnId;if (- !input.persisted.dispositionOnlyRecoveryConsumed- || input.persisted.semanticResult- || input.persisted.activeTurnId- || typeof turnId !== "string"- || turnId.length === 0- ) return null;- const persistedTerminal = input.persisted.terminalTurns?.filter(- (terminal) => terminal.turnId === turnId,- ) ?? [];- if (persistedTerminal.length !== 1 || persistedTerminal[0]!.fingerprint.length === 0) {+ !input.persisted.dispositionOnlyRecoveryConsumed ||+ input.persisted.semanticResult ||+ input.persisted.activeTurnId ||+ typeof turnId !== "string" ||+ turnId.length === 0+ )+ return null;+ const persistedTerminal =+ input.persisted.terminalTurns?.filter(+ (terminal) => terminal.turnId === turnId,+ ) ?? [];+ if (+ persistedTerminal.length !== 1 ||+ persistedTerminal[0]!.fingerprint.length === 0+ ) {return null;}const recoveredTurnId = input.recovered.dispositionOnlyRecoveryTurnId;- const recoveredTerminal = input.recovered.terminalTurns?.filter(- (terminal) => terminal.turnId === recoveredTurnId,- ) ?? [];+ const recoveredTerminal =+ input.recovered.terminalTurns?.filter(+ (terminal) => terminal.turnId === recoveredTurnId,+ ) ?? [];if (- !input.recovered.dispositionOnlyRecoveryConsumed- || input.recovered.semanticResult- || input.recovered.activeTurnId- || recoveredTurnId !== turnId- || recoveredTerminal.length !== 1- || recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint- || canonicalJson(input.recovered.identity) !== canonicalJson(input.persisted.identity)+ !input.recovered.dispositionOnlyRecoveryConsumed ||+ input.recovered.semanticResult ||+ input.recovered.activeTurnId ||+ recoveredTurnId !== turnId ||+ recoveredTerminal.length !== 1 ||+ recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint ||+ canonicalJson(input.recovered.identity) !==+ canonicalJson(input.persisted.identity)) {throw new Error("native_disposition_recovery_checkpoint_conflict");}@@ -741,43 +1171,57 @@ function checkpointedResultlessDispositionFallback(input: {* Package-owned normalized session loop. Paperclip supplies persistence and* authority through ControlPlanePort; provider/session behavior stays here.*/-export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise<NativeSessionExecutionResult> {+export async function executeNativeSession(+ options: ExecuteNativeSessionOptions,+): Promise<NativeSessionExecutionResult> {const input = parseNativeExecutionInput(options.input);const descriptor = await options.backend.descriptor();+ const cleanupDomain = JSON.stringify([+ input.binding.companyId,+ descriptor.kind,+ descriptor.name,+ ]);+ await retryQuarantinedSessionCleanups(cleanupDomain, options.signal);if ("runtimeContext" in input) {const capabilities = descriptor.runtimeContextCapabilities;- const unsupported = (["instructions", "skills", "mcp"] as const).filter((key) => capabilities?.[key] !== "native");- if (unsupported.length) throw new Error(`native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`);+ const unsupported = (["instructions", "skills", "mcp"] as const).filter(+ (key) => capabilities?.[key] !== "native",+ );+ if (unsupported.length)+ throw new Error(+ `native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`,+ );}let persistedSession = options.existingSession? null- : options.persistedSession ?? await options.controlPlane.loadSessionCheckpoint?.() ?? null;+ : (options.persistedSession ??+ (await options.controlPlane.loadSessionCheckpoint?.()) ??+ null);if (- persistedSession- && (- persistedSession.identity.runId !== input.binding.runId- || persistedSession.identity.companyId !== input.binding.companyId- || persistedSession.identity.issueId !== input.binding.issueId- || persistedSession.identity.agentId !== input.binding.agentId- || input.session.normalizedSessionId === null- || persistedSession.identity.sessionId !== input.session.normalizedSessionId- )- ) throw new Error("native_session_checkpoint_binding_mismatch");+ persistedSession &&+ (persistedSession.identity.runId !== input.binding.runId ||+ persistedSession.identity.companyId !== input.binding.companyId ||+ persistedSession.identity.issueId !== input.binding.issueId ||+ persistedSession.identity.agentId !== input.binding.agentId ||+ input.session.normalizedSessionId === null ||+ persistedSession.identity.sessionId !== input.session.normalizedSessionId)+ )+ throw new Error("native_session_checkpoint_binding_mismatch");const existingIdentity = options.existingSession?.identity() ?? null;if (- existingIdentity- && (- existingIdentity.companyId !== input.binding.companyId- || existingIdentity.issueId !== input.binding.issueId- || existingIdentity.agentId !== input.binding.agentId- || input.session.normalizedSessionId === null- || existingIdentity.sessionId !== input.session.normalizedSessionId- )- ) throw new Error("native_session_attach_binding_mismatch");- const normalizedSessionId = persistedSession?.identity.sessionId- ?? existingIdentity?.sessionId- ?? input.session.normalizedSessionId- ?? randomUUID();+ existingIdentity &&+ (existingIdentity.companyId !== input.binding.companyId ||+ existingIdentity.issueId !== input.binding.issueId ||+ existingIdentity.agentId !== input.binding.agentId ||+ input.session.normalizedSessionId === null ||+ existingIdentity.sessionId !== input.session.normalizedSessionId)+ )+ throw new Error("native_session_attach_binding_mismatch");+ const normalizedSessionId =+ persistedSession?.identity.sessionId ??+ existingIdentity?.sessionId ??+ input.session.normalizedSessionId ??+ randomUUID();const identity = {runId: input.binding.runId,sessionId: normalizedSessionId,@@ -811,6 +1255,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)options.existingSession,options.onSession,"native session attachment failed",+ cleanupDomain,);throw error;}@@ -826,13 +1271,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)persistedSession = await runAbortableOperationWithin({timeoutMs: recoveryTimeoutMs,timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`,- operation: (signal) => reconcileRecoveryCursor({- controlPlane: options.controlPlane,- checkpoint: recoveryCheckpoint,- runId: input.binding.runId,- sourceInstanceId: options.runnerInstanceId,- signal,- }),+ operation: (signal) =>+ reconcileRecoveryCursor({+ controlPlane: options.controlPlane,+ checkpoint: recoveryCheckpoint,+ runId: input.binding.runId,+ sourceInstanceId: options.runnerInstanceId,+ signal,+ }),});reconciledRecoveryCheckpoint = persistedSession;const providerRecoveryCheckpoint = persistedSession;@@ -844,28 +1290,35 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)? await runAbortableOperationWithin({timeoutMs: recoveryTimeoutMs,timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`,- operation: (signal) => options.backend.recoverSession!(- providerRecoveryCheckpoint,- { signal },- ),+ operation: (signal) =>+ options.backend.recoverSession!(providerRecoveryCheckpoint, {+ signal,+ }),onLateResolution: async (lateRecovery) => {if (lateRecovery.session) {await disposeUnadmittedSession(lateRecovery.session,"native session provider recovery timed out",+ cleanupDomain,);}},})- : { recovered: false as const, reason: "driver does not support recovery" };+ : {+ recovered: false as const,+ reason: "driver does not support recovery",+ };if (!recovery.recovered || !recovery.session) {if (!replacementAllowed) {- throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`);+ throw new Error(+ `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`,+ );}continuityBreak = {reason: recovery.reason ?? "provider session is no longer recoverable",previousDriverSessionId: providerRecoveryCheckpoint.sessionId,- previousProviderSessionId: providerRecoveryCheckpoint.providerSessionId ?? null,+ previousProviderSessionId:+ providerRecoveryCheckpoint.providerSessionId ?? null,};const replacementInput = {identity,@@ -883,10 +1336,12 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)): options.backend.openSession(abortableReplacementInput);},- onLateResolution: (lateSession) => disposeUnadmittedSession(- lateSession,- "native session replacement bootstrap timed out",- ),+ onLateResolution: (lateSession) =>+ disposeUnadmittedSession(+ lateSession,+ "native session replacement bootstrap timed out",+ cleanupDomain,+ ),});} else {session = recovery.session;@@ -905,14 +1360,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)session = await runAbortableOperationWithin({timeoutMs: bootstrapTimeoutMs,timeoutMessage: `native session bootstrap timed out after ${bootstrapTimeoutMs}ms`,- operation: (signal) => options.backend.openSession({- ...bootstrapInput,- signal,- }),- onLateResolution: (lateSession) => disposeUnadmittedSession(- lateSession,- "native session bootstrap timed out",- ),+ operation: (signal) =>+ options.backend.openSession({+ ...bootstrapInput,+ signal,+ }),+ onLateResolution: (lateSession) =>+ disposeUnadmittedSession(+ lateSession,+ "native session bootstrap timed out",+ cleanupDomain,+ ),});}try {@@ -930,6 +1388,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)session,options.onSession,"native control-plane run admission failed",+ cleanupDomain,);}throw error;@@ -945,14 +1404,66 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)// Owner notification cannot prevent provider cleanup.}};+ let failedCleanupDeferred = false;+ let sessionCloseRecoveryPromise: Promise<void> | null = null;+ const retainFailedCleanup = (cleanup: Promise<void>) => {+ failedCleanupDeferred = true;+ quarantineSession();+ retainFailedSessionCleanupOwner(cleanup, cleanupDomain);+ };+ const startSessionClose = (reason: string) => {+ const attempt = session.close({ reason });+ sessionClosePromise = attempt;+ return attempt;+ };const closeSession = () => {if (sessionClosePromise === null) {quarantineSession();- sessionClosePromise = session.close({- reason: "native session execution complete",- });+ const firstAttempt = startSessionClose(+ "native session execution complete",+ );+ // Preserve the first close outcome for its caller. If an exact attempt+ // fails, retain one ordered, delay-bounded production recovery within a+ // finite retry budget. A still-pending attempt is never overlapped or+ // replaced, and repeated terminal failure cannot create an immortal loop.+ const recovery = (async () => {+ let attempt = firstAttempt;+ let retryCount = 0;+ while (true) {+ try {+ await attempt;+ return;+ } catch (error) {+ if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {+ quarantineSessionCleanup(session, cleanupDomain);+ throw error;+ }+ retryCount += 1;+ await waitForSessionCloseRetry();+ if (sessionClosePromise === attempt) {+ sessionClosePromise = null;+ }+ attempt = startSessionClose(+ `native session cleanup recovery after close failure (${retryCount})`,+ );+ }+ }+ })();+ sessionCloseRecoveryPromise = recovery;+ retainFailedCleanup(recovery);+ void recovery+ .finally(() => {+ if (sessionCloseRecoveryPromise === recovery) {+ sessionCloseRecoveryPromise = null;+ }+ })+ .catch(() => undefined);+ }+ const activeClose = sessionClosePromise;+ if (activeClose === null) {+ throw new Error("native session cleanup lost its active close attempt");}- return sessionClosePromise;+ return activeClose;};let executionSucceeded = false;try {@@ -960,8 +1471,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)// callback fails, the finally block below still quarantines and closes the// provider session.options.onSession?.(session);- const checkpointTimeoutMs = options.checkpointTimeoutMs- ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS;+ const checkpointTimeoutMs =+ options.checkpointTimeoutMs ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS;const persistCheckpoint = (snapshot: PersistedNativeSession,externalSignal?: AbortSignal,@@ -987,14 +1498,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)};const recoveredSnapshot = await session.snapshot();const recoveredActiveTurnId = recovered- ? recoveredSnapshot.activeTurnId ?? null- : persistedSession?.activeTurnId ?? null;+ ? (recoveredSnapshot.activeTurnId ?? null)+ : (persistedSession?.activeTurnId ?? null);const adoptedDispositionTerminal = Boolean(- recovered- && recoveredSnapshot.dispositionOnlyRecoveryConsumed- && !recoveredActiveTurnId- && (recoveredSnapshot.terminalTurns?.length ?? 0)- > (persistedSession?.terminalTurns?.length ?? 0)+ recovered &&+ recoveredSnapshot.dispositionOnlyRecoveryConsumed &&+ !recoveredActiveTurnId &&+ (recoveredSnapshot.terminalTurns?.length ?? 0) >+ (persistedSession?.terminalTurns?.length ?? 0),);if (continuityBreak) {await options.onContinuityBreak?.({@@ -1020,16 +1531,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)highestContiguousSourceSeq: 0,governedResult: null as PrpStructuredRunResult | null,};- const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal- ? recoveredSnapshot- : persistedSession;- let completed = completionSnapshot?.semanticResult && completionSnapshot.terminal- ? {- result: completionSnapshot.semanticResult,- terminal: completionSnapshot.terminal,- turnId: completionSnapshot.activeTurnId ?? null,- }- : null;+ const completionSnapshot =+ recoveredSnapshot.semanticResult && recoveredSnapshot.terminal+ ? recoveredSnapshot+ : persistedSession;+ let completed =+ completionSnapshot?.semanticResult && completionSnapshot.terminal+ ? {+ result: completionSnapshot.semanticResult,+ terminal: completionSnapshot.terminal,+ turnId: completionSnapshot.activeTurnId ?? null,+ }+ : null;if (!completed) {// A recovered driver is authoritative about whether a provider turn is// still active. In particular, drivers normalize the checkpoint race@@ -1038,45 +1551,48 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)// resurrects that terminal turn and waits forever for an event that was// already consumed.const dispositionRecoveryWasSubmitted = Boolean(- recovered- && persistedSession?.dispositionOnlyRecoveryConsumed- && !recoveredSnapshot.semanticResult- && !recoveredActiveTurnId+ recovered &&+ persistedSession?.dispositionOnlyRecoveryConsumed &&+ !recoveredSnapshot.semanticResult &&+ !recoveredActiveTurnId,);const dispositionRecoveryTurnId =- persistedSession?.dispositionOnlyRecoveryTurnId- ?? recoveredSnapshot.dispositionOnlyRecoveryTurnId- ?? null;+ persistedSession?.dispositionOnlyRecoveryTurnId ??+ recoveredSnapshot.dispositionOnlyRecoveryTurnId ??+ null;const recoveredDispositionTurnObserved = Boolean(- dispositionRecoveryTurnId- && recoveredSnapshot.terminalTurns?.some(+ dispositionRecoveryTurnId &&+ recoveredSnapshot.terminalTurns?.some((terminal) => terminal.turnId === dispositionRecoveryTurnId,- )+ ),);const dispositionRecoveryStillOwned = Boolean(- dispositionRecoveryWasSubmitted- && recoveredSnapshot.dispositionOnlyRecoveryConsumed- && dispositionRecoveryTurnId !== null- && recoveredDispositionTurnObserved+ dispositionRecoveryWasSubmitted &&+ recoveredSnapshot.dispositionOnlyRecoveryConsumed &&+ dispositionRecoveryTurnId !== null &&+ recoveredDispositionTurnObserved,);const replayedDisposition =dispositionRecoveryWasSubmitted && dispositionRecoveryTurnId !== null- ? await replayCheckpointedTurnTerminal({- controlPlane: options.controlPlane,- runId: input.binding.runId,- sourceInstanceId: options.runnerInstanceId,- priorTerminalTurnIds: (persistedSession?.terminalTurns ?? [])- .map((terminal) => terminal.turnId),- expectedTurnId: dispositionRecoveryTurnId,- })- : null;+ ? await replayCheckpointedTurnTerminal({+ controlPlane: options.controlPlane,+ runId: input.binding.runId,+ sourceInstanceId: options.runnerInstanceId,+ priorTerminalTurnIds: (persistedSession?.terminalTurns ?? []).map(+ (terminal) => terminal.turnId,+ ),+ expectedTurnId: dispositionRecoveryTurnId,+ })+ : null;// Durable replay remains authoritative. If it has no terminal, an exact// consumed marker bound to the same terminal fingerprint in both// checkpoints proves provider completion without reconstructing provider// output. Give only that non-provider fact to control-plane policy; a// mismatch fails closed and a null policy result fails finalization.const dispositionFallback =- dispositionRecoveryWasSubmitted && replayedDisposition === null && persistedSession+ dispositionRecoveryWasSubmitted &&+ replayedDisposition === null &&+ persistedSession? checkpointedResultlessDispositionFallback({persisted: persistedSession,recovered: recoveredSnapshot,@@ -1085,26 +1601,29 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions): null;const checkpointedDispositionTerminal =replayedDisposition !== null || dispositionFallback !== null;- const recoveryTerminal = replayedDisposition?.terminal ?? dispositionFallback;+ const recoveryTerminal =+ replayedDisposition?.terminal ?? dispositionFallback;const consumptionAbort = new AbortController();- const consuming = recoveryTerminal === null- ? consumeTurn(- session,- options.controlPlane,- options.timeoutMs ?? 900_000,- options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS,- closeSession,- quarantineSession,- options.resolveGovernedWait,- consumptionAbort.signal,- )- : Promise.resolve({- event: recoveryTerminal,- eventCount: 0,- highestContiguousSourceSeq:- replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq,- governedResult: null,- });+ const consuming =+ recoveryTerminal === null+ ? consumeTurn(+ session,+ options.controlPlane,+ options.timeoutMs ?? 900_000,+ options.runtimeInputLiveWindowMs ??+ DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS,+ closeSession,+ quarantineSession,+ options.resolveGovernedWait,+ consumptionAbort.signal,+ )+ : Promise.resolve({+ event: recoveryTerminal,+ eventCount: 0,+ highestContiguousSourceSeq:+ replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq,+ governedResult: null,+ });// Event consumption must begin before startTurn so an eager provider cannot// outrun us. Observe its rejection immediately, though: if startTurn or// checkpointing fails first, the outer finally closes the session and the@@ -1113,20 +1632,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)void consuming.catch(() => undefined);try {if (- !recovered- || (- !recoveredActiveTurnId- && !adoptedDispositionTerminal- && !checkpointedDispositionTerminal- && !dispositionRecoveryStillOwned- )+ !recovered ||+ (!recoveredActiveTurnId &&+ !adoptedDispositionTerminal &&+ !checkpointedDispositionTerminal &&+ !dispositionRecoveryStillOwned)) {const modelEnvelope = buildNativeModelEnvelope(input);const dispositionOnlyRecovery = Boolean(recovered &&!recoveredSnapshot.semanticResult &&(persistedSession?.terminalTurns?.length ?? 0) > 0 &&- !recoveredActiveTurnId+ !recoveredActiveTurnId,);if (dispositionOnlyRecovery) {modelEnvelope.task.prompt = [@@ -1138,7 +1655,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)}await session.startTurn({message: { role: "user", text: JSON.stringify(modelEnvelope) },- requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default",+ requestedCollaborationMode:+ "executionMode" in input ? input.executionMode : "default",});await checkpoint();}@@ -1162,21 +1680,24 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)if (settledCompletion === null) {const terminalEvent = consumed.event;if (terminalEvent === null) {- throw new Error("native_finalization_missing: session returned no terminal event");+ throw new Error(+ "native_finalization_missing: session returned no terminal event",+ );}- settledCompletion = consumed.governedResult === null- ? await session.result()- : {- result: consumed.governedResult,- terminal: {- schema: "paperclip.prp.terminal.v1",- turnTerminalState: "completed",- runTerminalState: "succeeded",- reportedWorkDisposition:- consumed.governedResult.reportedWorkDisposition,- },- turnId: terminalEvent.turnId ?? null,- };+ settledCompletion =+ consumed.governedResult === null+ ? await session.result()+ : {+ result: consumed.governedResult,+ terminal: {+ schema: "paperclip.prp.terminal.v1",+ turnTerminalState: "completed",+ runTerminalState: "succeeded",+ reportedWorkDisposition:+ consumed.governedResult.reportedWorkDisposition,+ },+ turnId: terminalEvent.turnId ?? null,+ };signal.throwIfAborted();if (settledCompletion === null && options.resolveMissingResult) {const recoveredResult = await options.resolveMissingResult({@@ -1203,12 +1724,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)completed = settledCompletion;}if (settledCompletion === null) {- throw new Error("native_finalization_missing: session returned no semantic result");+ throw new Error(+ "native_finalization_missing: session returned no semantic result",+ );}let terminal: PrpTerminalState;if (consumed.governedResult !== null) {terminal = settledCompletion.terminal;- } else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) {+ } else if (+ completionSnapshot?.semanticResult &&+ completionSnapshot.terminal+ ) {terminal = completionSnapshot.terminal;} else {terminal = terminalFromEvent(@@ -1216,10 +1742,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)settledCompletion.result.reportedWorkDisposition,);}- const eventTurnId = settledCompletion.turnId- ?? persistedSession?.activeTurnId- ?? persistedSession?.terminalTurns?.at(-1)?.turnId- ?? consumed.event?.turnId;+ const eventTurnId =+ settledCompletion.turnId ??+ persistedSession?.activeTurnId ??+ persistedSession?.terminalTurns?.at(-1)?.turnId ??+ consumed.event?.turnId;const controlEvent = (sourceSeq: number,eventType: PrpEvent["eventType"],@@ -1261,22 +1788,30 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({timeoutMs: finalizationTimeoutMs,operation: async (signal) => {- const controlReplay = await options.controlPlane.replayEvents({- runId: input.binding.runId,- sourceInstanceId: options.controlPlaneInstanceId,- afterSourceSeq: 0,- limit: 10,- }, { signal });+ const controlReplay = await options.controlPlane.replayEvents(+ {+ runId: input.binding.runId,+ sourceInstanceId: options.controlPlaneInstanceId,+ afterSourceSeq: 0,+ limit: 10,+ },+ { signal },+ );signal.throwIfAborted();- const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event]));+ const replayBySequence = new Map(+ controlReplay.events.map((event) => [event.sourceSeq, event]),+ );for (const existing of controlReplay.events) {- const expected = preparedFinalization.expectedControlEvents[existing.sourceSeq - 1];+ const expected =+ preparedFinalization.expectedControlEvents[existing.sourceSeq - 1];if (- expected === undefined- || existing.eventType !== expected.eventType- || canonicalJson(existing.payload) !== canonicalJson(expected.payload)+ expected === undefined ||+ existing.eventType !== expected.eventType ||+ canonicalJson(existing.payload) !== canonicalJson(expected.payload)) {- throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`);+ throw new Error(+ `native_control_event_replay_conflict:${existing.sourceSeq}`,+ );}}if (!baselineControlReplayCaptured) {@@ -1287,8 +1822,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)} else {for (const existing of controlReplay.events) {if (- !baselineControlEventSequences.has(existing.sourceSeq)- && !accountedControlEventSequences.has(existing.sourceSeq)+ !baselineControlEventSequences.has(existing.sourceSeq) &&+ !accountedControlEventSequences.has(existing.sourceSeq)) {accountedControlEventSequences.add(existing.sourceSeq);consumed.eventCount += 1;@@ -1301,11 +1836,13 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions));for (const event of preparedFinalization.expectedControlEvents) {if (replayBySequence.has(event.sourceSeq)) continue;- const receipt = await options.controlPlane.appendEvent(event, { signal });+ const receipt = await options.controlPlane.appendEvent(event, {+ signal,+ });signal.throwIfAborted();if (- receipt.disposition === "committed"- && !accountedControlEventSequences.has(event.sourceSeq)+ receipt.disposition === "committed" &&+ !accountedControlEventSequences.has(event.sourceSeq)) {accountedControlEventSequences.add(event.sourceSeq);consumed.eventCount += 1;@@ -1315,13 +1852,16 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)receipt.highestContiguousSourceSeq,);}- await options.controlPlane.completeRun({- result: preparedFinalization.completed.result,- terminal: preparedFinalization.terminal,- turnId: preparedFinalization.completed.turnId,- callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`,- callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,- }, { signal });+ await options.controlPlane.completeRun(+ {+ result: preparedFinalization.completed.result,+ terminal: preparedFinalization.terminal,+ turnId: preparedFinalization.completed.turnId,+ callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`,+ callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,+ },+ { signal },+ );return {result: preparedFinalization.completed.result,terminal: preparedFinalization.terminal,@@ -1351,13 +1891,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)};await persistCheckpoint(completedSnapshot, signal);signal.throwIfAborted();- const usage = await session.usage?.() ?? null;+ const usage = (await session.usage?.()) ?? null;signal.throwIfAborted();return {providerSessionId: snapshot.providerSessionId ?? null,- driverVersion: typeof usage?.driverVersion === "string"- ? usage.driverVersion- : descriptor.version,+ driverVersion:+ typeof usage?.driverVersion === "string"+ ? usage.driverVersion+ : descriptor.version,usage,};},@@ -1376,7 +1917,10 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)executionSucceeded = true;return { ...durableExecutionResult, ...enrichment };} finally {- if (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) {+ if (+ (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) &&+ !failedCleanupDeferred+ ) {// A provider that ignores close must not keep execution pending forever.// closeSession removes it from the caller before invoking the backend;// retain observation of the promise, but bound the final join. Provider