Skip to content

The open-source app everyone uses to manage agents at work

feat(runner): add Codex ACPX harness driver (#12405)

Dotta74aabb7ea6c5
diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts
new file mode 100644
index 0000000000..1806909fcf
--- /dev/null
+++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts
@@ -0,0 +1,1483 @@
+import { describe, expect, it, vi } from "vitest";
+
+import type { AcpRuntimeEvent } from "acpx/runtime";
+
+import {
+ PRP_BLOCK_TOOL_NAME,
+ PRP_COMPLETION_TOOL_NAME,
+} from "../../contracts/completion-result.js";
+import type { PrpEvent } from "../../protocol/replay-contract.js";
+import { validatePrpEvent } from "../../protocol/replay-contract.js";
+import {
+ CodexAcpxDriver,
+ type CodexAcpxDriverDependencies,
+ type CodexAcpxDriverOptions,
+} from "./codex-acpx-driver.js";
+import type {
+ AcpxRuntimeTurn,
+ OpenAcpxRuntimeHostOptions,
+} from "./runtime-host.js";
+
+describe("Codex ACPX harness driver", () => {
+ it("rejects a pre-aborted open before starting host admission", async () => {
+ const fixture = driverFixture();
+ const controller = new AbortController();
+ const cancellation = new Error("session open cancelled");
+ controller.abort(cancellation);
+
+ await expect(
+ fixture.driver.openSession({
+ runId: "run-pre-abort",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ signal: controller.signal,
+ }),
+ ).rejects.toBe(cancellation);
+
+ expect(fixture.openHost).not.toHaveBeenCalled();
+ });
+
+ it("forwards the exact open signal to host admission", async () => {
+ const fixture = driverFixture();
+ const controller = new AbortController();
+ const session = await fixture.driver.openSession({
+ runId: "run-open-signal",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ signal: controller.signal,
+ });
+
+ expect(fixture.hostOptions?.signal).toBe(controller.signal);
+ await session.close({ reason: "signal forwarding verified" });
+ });
+
+ it("closes and quarantines a host that resolves after open is aborted", async () => {
+ const hostAdmission = deferred<ReturnType<typeof fakeHost>>();
+ const fixture = driverFixture(
+ {},
+ {
+ openHost: () => hostAdmission.promise,
+ closeSettlementTimeoutMs: 5,
+ },
+ );
+ fixture.host.close.mockRejectedValueOnce(
+ new Error("late host close failed once"),
+ );
+ const controller = new AbortController();
+ const cancellation = new Error("host admission cancelled");
+ const opening = fixture.driver.openSession({
+ runId: "run-late-host",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ signal: controller.signal,
+ });
+ await vi.waitFor(() => expect(fixture.openHost).toHaveBeenCalledOnce());
+
+ controller.abort(cancellation);
+ await expect(opening).rejects.toBe(cancellation);
+ hostAdmission.resolve(fixture.host);
+
+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));
+ expect(fixture.host.close).toHaveBeenNthCalledWith(1, {
+ reason: "Codex ACPX host resolved after admission was aborted",
+ });
+ expect(fixture.host.close).toHaveBeenNthCalledWith(2, {
+ reason: expect.stringContaining("quarantined cleanup recovery"),
+ });
+ });
+
+ it("quarantines failed cleanup after session construction fails", async () => {
+ const quarantineRecovery = deferred<void>();
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const identity = fixture.host.identity();
+ fixture.host.identity = vi
+ .fn()
+ .mockReturnValueOnce({
+ ...identity,
+ normalizedSessionId: "different-session",
+ })
+ .mockReturnValue(identity);
+ fixture.host.close
+ .mockRejectedValueOnce(new Error("initial cleanup failed"))
+ .mockImplementationOnce(() => quarantineRecovery.promise)
+ .mockResolvedValue(undefined);
+
+ await expect(
+ fixture.driver.openSession({
+ runId: "run-construction-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ }),
+ ).rejects.toThrow("Codex ACPX host returned a different session identity");
+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));
+
+ const nextAdmission = fixture.driver.openSession({
+ runId: "run-after-construction-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ await Promise.resolve();
+ expect(fixture.openHost).toHaveBeenCalledOnce();
+
+ quarantineRecovery.resolve();
+ const session = await nextAdmission;
+ expect(fixture.openHost).toHaveBeenCalledTimes(2);
+ expect(fixture.host.close).toHaveBeenNthCalledWith(2, {
+ reason: expect.stringContaining("quarantined cleanup recovery"),
+ });
+ await session.close({ reason: "construction cleanup recovered" });
+ });
+
+ it("advertises only the implemented Codex production surface", async () => {
+ const fixture = driverFixture();
+ const descriptor = await fixture.driver.descriptor();
+
+ expect(descriptor).toMatchObject({
+ kind: "acpx_runtime",
+ displayName: "Codex via ACPX",
+ capabilities: {
+ resume: false,
+ interruption: true,
+ dynamicTools: true,
+ runtimeRequestResolution: false,
+ },
+ runtimeContextCapabilities: {
+ instructions: "native",
+ skills: "unsupported",
+ mcp: "native",
+ },
+ });
+ await expect(
+ fixture.driver.validateConfig({
+ agent: "claude",
+ model: "claude-sonnet-5",
+ permissionMode: "approve-reads",
+ }),
+ ).resolves.toMatchObject({
+ ok: false,
+ issues: [{ code: "unsupported_agent" }],
+ });
+ });
+
+ it("maps one turn, dispatches tools, and commits one semantic result", async () => {
+ const dynamicToolHandler = vi.fn(async () => ({ title: "Document" }));
+ const fixture = driverFixture({ dynamicToolHandler });
+ const session = await fixture.driver.openSession({
+ runId: "run-1",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const terminalEvents = collectUntil(session.events(), "turn.completed");
+
+ const { turnId } = await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;
+ await expect(
+ bridgeHandler({
+ tool: "documents.read",
+ callId: "tool-1",
+ arguments: { id: "doc-1" },
+ signal: new AbortController().signal,
+ }),
+ ).resolves.toEqual({ title: "Document" });
+ expect(dynamicToolHandler).toHaveBeenCalledWith({
+ tool: "documents.read",
+ callId: "tool-1",
+ providerSessionId: "agent-1",
+ turnId,
+ arguments: { id: "doc-1" },
+ signal: expect.any(AbortSignal),
+ });
+
+ await expect(
+ bridgeHandler({
+ tool: PRP_COMPLETION_TOOL_NAME,
+ callId: "finish-1",
+ arguments: completedResult(),
+ signal: new AbortController().signal,
+ }),
+ ).resolves.toEqual({ accepted: true });
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+
+ const events = await terminalEvents;
+ expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true);
+ expect(events.map((event) => event.eventType)).toEqual(
+ expect.arrayContaining([
+ "turn.submitted",
+ "turn.accepted",
+ "turn.started",
+ "item.delta",
+ "tool.execution.started",
+ "run.result.proposed",
+ "item.completed",
+ "turn.completed",
+ ]),
+ );
+ expect(
+ events.findIndex((event) => event.eventType === "run.result.proposed"),
+ ).toBeLessThan(
+ events.findIndex((event) => event.eventType === "turn.completed"),
+ );
+ await expect(session.snapshot()).resolves.toMatchObject({
+ driverKind: "acpx_runtime",
+ activeTurnId: null,
+ providerIdentity: {
+ kind: "acpx",
+ agentSessionId: "agent-1",
+ },
+ semanticResult: {
+ callId: "finish-1",
+ turnId,
+ result: { reportedWorkDisposition: "done" },
+ },
+ });
+ await session.close({ reason: "complete" });
+ await session.close({ reason: "idempotent close" });
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ });
+
+ it("rejects terminal disposition drift and bounds interruption", async () => {
+ const fixture = driverFixture();
+ const session = await fixture.driver.openSession({
+ runId: "run-2",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const { turnId } = await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;
+
+ await expect(
+ bridgeHandler({
+ tool: PRP_BLOCK_TOOL_NAME,
+ callId: "block-1",
+ arguments: completedResult(),
+ signal: new AbortController().signal,
+ }),
+ ).rejects.toThrow("does not match");
+ await session.interrupt({ turnId, reason: "user cancelled" });
+ expect(fixture.host.interruptActiveTurn).toHaveBeenCalledWith(
+ "user cancelled",
+ );
+ await expect(session.interrupt({ turnId: "stale-turn" })).rejects.toThrow(
+ "is not the active turn",
+ );
+ await session.close({ reason: "cancelled" });
+ });
+
+ it("emits an interrupted terminal before closing an active stream", async () => {
+ const fixture = driverFixture();
+ const session = await fixture.driver.openSession({
+ runId: "run-close",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");
+ await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+
+ await Promise.all([
+ session.close({ reason: "operator shutdown" }),
+ session.close({ reason: "duplicate shutdown" }),
+ ]);
+
+ await expect(terminalEvents).resolves.toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ eventType: "turn.interrupted" }),
+ ]),
+ );
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: null,
+ terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })],
+ });
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ });
+
+ it("classifies a pump rejection during close as interrupted", async () => {
+ const runtimeEventFailure = deferred<never>();
+ const fixture = driverFixture(
+ {},
+ { runtimeEventFailure: runtimeEventFailure.promise },
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-close-pump-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");
+ await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+
+ const closing = session.close({ reason: "operator shutdown" });
+ runtimeEventFailure.reject(
+ new Error("provider stream closed during shutdown"),
+ );
+ await expect(closing).resolves.toBeUndefined();
+
+ const emitted = await terminalEvents;
+ expect(
+ emitted.filter((event) => event.eventType === "turn.interrupted"),
+ ).toHaveLength(1);
+ expect(emitted.some((event) => event.eventType === "turn.failed")).toBe(
+ false,
+ );
+ });
+
+ it("reports a host close timeout while retaining the exact cleanup", async () => {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const hostClose = deferred<void>();
+ fixture.host.close.mockImplementation(() => hostClose.promise);
+ const session = await fixture.driver.openSession({
+ runId: "run-close-timeout",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const terminalEvents = collectUntil(session.events(), "turn.interrupted");
+ await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+
+ await expect(
+ session.close({ reason: "runtime close stalled" }),
+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ await expect(terminalEvents).resolves.toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ eventType: "turn.interrupted" }),
+ ]),
+ );
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: null,
+ terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })],
+ });
+
+ fixture.finishTurn({ status: "cancelled", stopReason: "session_closed" });
+ hostClose.resolve();
+ await new Promise<void>((resolve) => setTimeout(resolve, 10));
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ });
+
+ it("reconciles the retained close when it settles after the wait bound", async () => {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const retainedClose = deferred<void>();
+ fixture.host.close.mockImplementation(() => retainedClose.promise);
+ const session = await fixture.driver.openSession({
+ runId: "run-close-recovery",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ await expect(
+ session.close({ reason: "runtime close stalled" }),
+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");
+ retainedClose.resolve();
+ await expect(
+ session.close({ reason: "observe retained completion" }),
+ ).resolves.toBeUndefined();
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ });
+
+ it("does not spin while the exact host cleanup remains pending", async () => {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const stalledClose = deferred<void>();
+ fixture.host.close.mockImplementation(() => stalledClose.promise);
+ const session = await fixture.driver.openSession({
+ runId: "run-close-stalled-recovery",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ await expect(
+ session.close({ reason: "runtime close never settles" }),
+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ await new Promise<void>((resolve) => setTimeout(resolve, 10));
+ expect(fixture.host.close).toHaveBeenCalledOnce();
+ });
+
+ it("blocks admission while the first retained host close is still pending", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const stalledClose = deferred<void>();
+ fixture.host.close.mockImplementation(() => stalledClose.promise);
+ const session = await fixture.driver.openSession({
+ runId: "run-close-pending-admission",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "runtime close remains pending" }),
+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ const admission = fixture.driver.openSession({
+ runId: "run-before-pending-close-settles",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const blocked = expect(admission).rejects.toThrow(
+ "exceeded the admission grace",
+ );
+ // Drive the implementation's complete admission grace instead of
+ // duplicating its derived shutdown bound in this test. Later transport
+ // layers may add another bounded cleanup phase without weakening the
+ // invariant exercised here: no replacement host opens while the exact
+ // retained close is still pending.
+ await vi.runAllTimersAsync();
+ await blocked;
+ expect(fixture.openHost).toHaveBeenCalledOnce();
+ stalledClose.resolve();
+ await vi.runAllTimersAsync();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("retains autonomous host cleanup recovery through repeated failure", async () => {
+ const fixture = driverFixture(
+ {},
+ {
+ closeSettlementTimeoutMs: 1,
+ },
+ );
+ fixture.host.close
+ .mockRejectedValueOnce(new Error("transient cleanup failure"))
+ .mockRejectedValueOnce(new Error("second cleanup failure"))
+ .mockResolvedValueOnce(undefined);
+ const session = await fixture.driver.openSession({
+ runId: "run-close-permanent-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const diagnosticEvents = collectUntil(
+ session.events(),
+ "harness.diagnostic",
+ );
+
+ await expect(
+ session.close({ reason: "runtime close initially failed" }),
+ ).rejects.toThrow("transient cleanup failure");
+ await expect(diagnosticEvents).resolves.toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ eventType: "harness.diagnostic",
+ payload: expect.objectContaining({
+ code: "acpx_host_cleanup_deferred",
+ }),
+ }),
+ ]),
+ );
+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(3));
+ expect(fixture.host.close).toHaveBeenLastCalledWith({
+ reason: "runtime close initially failed (automatic cleanup recovery 2)",
+ });
+ await expect(
+ session.close({ reason: "observe recovered cleanup" }),
+ ).resolves.toBeUndefined();
+ expect(fixture.host.close).toHaveBeenCalledTimes(3);
+ });
+
+ it("keeps quarantined cleanup scheduled and lets admission await its exact owner", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockRejectedValue(
+ new Error("persistent cleanup failure"),
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-close-retry-bound",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "runtime close persistently failed" }),
+ ).rejects.toThrow("persistent cleanup failure");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ await vi.advanceTimersByTimeAsync(10);
+ expect(fixture.host.close).toHaveBeenCalledTimes(7);
+ await vi.advanceTimersByTimeAsync(59_000);
+ expect(fixture.host.close).toHaveBeenCalledTimes(7);
+
+ fixture.host.close.mockImplementation(({ reason }) =>
+ reason.includes("scheduled quarantined cleanup recovery")
+ ? // Exercise the complete production host bound: two seconds for
+ // active-turn cancellation plus six seconds for protocol/TERM/KILL.
+ new Promise<void>((resolve) => setTimeout(resolve, 8_500))
+ : Promise.resolve(),
+ );
+ await vi.advanceTimersToNextTimerAsync();
+ let admissionSettled = false;
+ const admission = fixture.driver.openSession({
+ runId: "run-after-recovery",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ void admission.then(
+ () => {
+ admissionSettled = true;
+ },
+ () => {
+ admissionSettled = true;
+ },
+ );
+ expect(fixture.host.close).toHaveBeenCalledTimes(8);
+ expect(fixture.host.close).toHaveBeenLastCalledWith({
+ reason:
+ "runtime close persistently failed (scheduled quarantined cleanup recovery)",
+ });
+ await vi.advanceTimersByTimeAsync(8_499);
+ expect(admissionSettled).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ await expect(admission).resolves.toBeDefined();
+ expect(fixture.host.close).toHaveBeenCalledTimes(8);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("runs a full admission batch after an inherited scheduled attempt fails", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockRejectedValue(
+ new Error("persistent cleanup failure"),
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-scheduled-owner-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "exhaust cleanup before scheduled recovery" }),
+ ).rejects.toThrow("persistent cleanup failure");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ await vi.advanceTimersByTimeAsync(10);
+
+ let scheduledAttempt = 0;
+ let admissionAttempt = 0;
+ fixture.host.close.mockImplementation(({ reason }) => {
+ if (reason.includes("scheduled quarantined cleanup recovery")) {
+ scheduledAttempt += 1;
+ return new Promise<void>((_resolve, reject) => {
+ setTimeout(
+ () => reject(new Error("scheduled cleanup failed")),
+ 8_000,
+ );
+ });
+ }
+ if (reason.includes("quarantined cleanup admission recovery")) {
+ admissionAttempt += 1;
+ const attempt = admissionAttempt;
+ return new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (attempt < 3) reject(new Error("admission cleanup failed"));
+ else resolve();
+ }, 8_000);
+ });
+ }
+ return Promise.reject(new Error("unexpected cleanup reason"));
+ });
+
+ await vi.advanceTimersToNextTimerAsync();
+ const admission = fixture.driver.openSession({
+ runId: "run-after-scheduled-owner-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ let admissionSettled = false;
+ void admission
+ .finally(() => {
+ admissionSettled = true;
+ })
+ .catch(() => undefined);
+ // The inherited attempt plus the replacement three-attempt batch takes
+ // just over 32 seconds with this fixture. The production grace must cover
+ // that complete bounded owner chain rather than expiring at 27 seconds.
+ await vi.advanceTimersByTimeAsync(32_001);
+ expect(admissionSettled).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ const admitted = await admission;
+
+ expect(scheduledAttempt).toBe(1);
+ expect(admissionAttempt).toBe(3);
+ expect(fixture.openHost).toHaveBeenCalledTimes(2);
+ fixture.host.close.mockResolvedValue(undefined);
+ await admitted.close({
+ reason: "complete after scheduled owner recovery",
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("lets admission observe a complete bounded multi-attempt recovery", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ let quarantineAttempt = 0;
+ fixture.host.close.mockImplementation(({ reason }) => {
+ if (!reason.includes("quarantined cleanup recovery")) {
+ return Promise.reject(new Error("persistent cleanup failure"));
+ }
+ quarantineAttempt += 1;
+ const attempt = quarantineAttempt;
+ return new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (attempt < 3) reject(new Error("transient quarantine failure"));
+ else resolve();
+ }, 8_000);
+ });
+ });
+ const session = await fixture.driver.openSession({
+ runId: "run-multi-attempt-recovery",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "runtime close persistently failed" }),
+ ).rejects.toThrow("persistent cleanup failure");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ await vi.advanceTimersByTimeAsync(4);
+ expect(fixture.host.close).toHaveBeenCalledTimes(5);
+
+ let admissionSettled = false;
+ const admission = fixture.driver.openSession({
+ runId: "run-after-multi-attempt-recovery",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ void admission.then(
+ () => {
+ admissionSettled = true;
+ },
+ () => {
+ admissionSettled = true;
+ },
+ );
+ await vi.advanceTimersByTimeAsync(23_000);
+ expect(admissionSettled).toBe(false);
+ expect(fixture.host.close).toHaveBeenCalledTimes(7);
+ await vi.advanceTimersByTimeAsync(2_000);
+ await expect(admission).resolves.toBeDefined();
+ expect(quarantineAttempt).toBe(3);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("awaits quarantine recovery installed by an exhausted retained owner", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockImplementation(({ reason }) => {
+ if (reason.includes("quarantined cleanup recovery")) {
+ return new Promise<void>((resolve) => setTimeout(resolve, 2));
+ }
+ return Promise.reject(new Error("bounded host recovery failed"));
+ });
+ const session = await fixture.driver.openSession({
+ runId: "run-replacement-cleanup-owner",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const closing = expect(
+ session.close({ reason: "host cleanup must be replaced" }),
+ ).rejects.toThrow("bounded host recovery failed");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+
+ const admission = fixture.driver.openSession({
+ runId: "run-after-replacement-cleanup",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ await vi.advanceTimersByTimeAsync(10);
+ const admitted = await admission;
+ expect(fixture.openHost).toHaveBeenCalledTimes(2);
+ fixture.host.close.mockResolvedValue(undefined);
+ await admitted.close({ reason: "complete after replacement cleanup" });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("gives replacement quarantine recovery a fresh admission grace", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockImplementation(
+ ({ reason }) =>
+ new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (reason.includes("quarantined cleanup recovery")) resolve();
+ else reject(new Error("bounded autonomous cleanup failed"));
+ }, 8_000);
+ }),
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-owner-phase-grace",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "exhaust cleanup near admission deadline" }),
+ ).rejects.toThrow("host cleanup exceeded its shutdown timeout");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+
+ let admissionSettled = false;
+ const admission = fixture.driver.openSession({
+ runId: "run-after-owner-phase-grace",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ void admission
+ .finally(() => {
+ admissionSettled = true;
+ })
+ .catch(() => undefined);
+
+ // The inherited autonomous owner consumes about 32 seconds before it
+ // installs a distinct bounded quarantine recovery. That replacement
+ // owner must not inherit the nearly exhausted admission deadline.
+ await vi.advanceTimersByTimeAsync(35_001);
+ expect(admissionSettled).toBe(false);
+ expect(fixture.openHost).toHaveBeenCalledOnce();
+
+ await vi.advanceTimersByTimeAsync(5_002);
+ const admitted = await admission;
+ expect(fixture.openHost).toHaveBeenCalledTimes(2);
+ fixture.host.close.mockResolvedValue(undefined);
+ await admitted.close({ reason: "complete after replacement owner" });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("reschedules host cleanup after an admission batch is exhausted", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockRejectedValue(
+ new Error("persistent cleanup failure"),
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-transient-admission-cleanup",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "exhaust cleanup before admission" }),
+ ).rejects.toThrow("persistent cleanup failure");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ await vi.advanceTimersByTimeAsync(10);
+
+ let admissionAttempt = 0;
+ let scheduledAttempt = 0;
+ fixture.host.close.mockImplementation(({ reason }) => {
+ if (reason.includes("quarantined cleanup admission recovery")) {
+ admissionAttempt += 1;
+ return Promise.reject(new Error("admission cleanup failed"));
+ }
+ if (reason.includes("scheduled quarantined cleanup recovery")) {
+ scheduledAttempt += 1;
+ return Promise.resolve();
+ }
+ return Promise.reject(new Error("unexpected cleanup reason"));
+ });
+
+ const admission = expect(
+ fixture.driver.openSession({
+ runId: "run-after-transient-admission-cleanup",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ }),
+ ).rejects.toThrow("quarantined host cleanup remains incomplete");
+ await vi.advanceTimersByTimeAsync(3);
+ await admission;
+
+ expect(admissionAttempt).toBe(3);
+ expect(scheduledAttempt).toBe(0);
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(scheduledAttempt).toBe(1);
+
+ const admitted = await fixture.driver.openSession({
+ runId: "run-after-autonomous-cleanup",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ expect(fixture.openHost).toHaveBeenCalledTimes(2);
+ fixture.host.close.mockResolvedValue(undefined);
+ await admitted.close({ reason: "complete after admission retry" });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("restores another host's retry timer when admission recovery times out", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ const hostA = {
+ ...fixture.host,
+ close: vi.fn(async () => {
+ throw new Error("host A cleanup failed");
+ }),
+ };
+ const hostB = {
+ ...fixture.host,
+ close: vi.fn(async () => {
+ throw new Error("host B cleanup failed");
+ }),
+ };
+ fixture.openHost
+ .mockReset()
+ .mockResolvedValueOnce(hostB)
+ .mockResolvedValueOnce(hostA)
+ .mockResolvedValue(fixture.host);
+
+ const sessionB = await fixture.driver.openSession({
+ runId: "run-quarantined-host-b",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const sessionA = await fixture.driver.openSession({
+ runId: "run-quarantined-host-a",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closingB = expect(
+ sessionB.close({ reason: "quarantine host B" }),
+ ).rejects.toThrow("host B cleanup failed");
+ await vi.advanceTimersByTimeAsync(1);
+ await closingB;
+ await vi.advanceTimersByTimeAsync(10);
+ expect(hostB.close).toHaveBeenCalledTimes(7);
+
+ // Offset the autonomous retry timers so host B can own an inherited
+ // scheduled attempt while host A still has a dormant retry timer.
+ await vi.advanceTimersByTimeAsync(100);
+ const closingA = expect(
+ sessionA.close({ reason: "quarantine host A" }),
+ ).rejects.toThrow("host A cleanup failed");
+ await vi.advanceTimersByTimeAsync(1);
+ await closingA;
+ await vi.advanceTimersByTimeAsync(10);
+ expect(hostA.close).toHaveBeenCalledTimes(7);
+
+ let hostAAdmissionAttempts = 0;
+ let hostAScheduledAttempts = 0;
+ hostA.close.mockImplementation(({ reason }) => {
+ if (reason.includes("quarantined cleanup admission recovery")) {
+ hostAAdmissionAttempts += 1;
+ return Promise.reject(new Error("host A admission cleanup failed"));
+ }
+ if (reason.includes("scheduled quarantined cleanup recovery")) {
+ hostAScheduledAttempts += 1;
+ return Promise.resolve();
+ }
+ return Promise.reject(new Error("unexpected host A cleanup reason"));
+ });
+ let hostBAdmissionAttempts = 0;
+ hostB.close.mockImplementation(({ reason }) => {
+ if (reason.includes("scheduled quarantined cleanup recovery")) {
+ return new Promise<void>((_resolve, reject) => {
+ setTimeout(
+ () => reject(new Error("host B scheduled cleanup failed")),
+ 1,
+ );
+ });
+ }
+ if (reason.includes("quarantined cleanup admission recovery")) {
+ hostBAdmissionAttempts += 1;
+ return new Promise<void>(() => undefined);
+ }
+ return Promise.reject(new Error("unexpected host B cleanup reason"));
+ });
+
+ await vi.advanceTimersToNextTimerAsync();
+ expect(hostB.close).toHaveBeenLastCalledWith({
+ reason: expect.stringContaining(
+ "scheduled quarantined cleanup recovery",
+ ),
+ });
+ expect(hostAScheduledAttempts).toBe(0);
+
+ const admission = expect(
+ fixture.driver.openSession({
+ runId: "run-after-multi-host-quarantine",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ }),
+ ).rejects.toThrow("exceeded the admission grace");
+ await vi.advanceTimersByTimeAsync(40_000);
+ await admission;
+
+ expect(hostAAdmissionAttempts).toBe(3);
+ expect(hostBAdmissionAttempts).toBe(1);
+ expect(hostAScheduledAttempts).toBe(0);
+
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(hostAScheduledAttempts).toBe(1);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("fails admission within a finite grace when quarantined cleanup never settles", async () => {
+ vi.useFakeTimers();
+ try {
+ const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 });
+ fixture.host.close.mockRejectedValue(
+ new Error("persistent cleanup failure"),
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-close-never-settles",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+
+ const closing = expect(
+ session.close({ reason: "runtime close persistently failed" }),
+ ).rejects.toThrow("persistent cleanup failure");
+ await vi.advanceTimersByTimeAsync(1);
+ await closing;
+ await vi.advanceTimersByTimeAsync(10);
+ expect(fixture.host.close).toHaveBeenCalledTimes(7);
+
+ fixture.host.close.mockImplementation(() => new Promise<void>(() => {}));
+ let admissionSettled = false;
+ const admission = fixture.driver.openSession({
+ runId: "run-after-stalled-quarantine",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ void admission
+ .finally(() => {
+ admissionSettled = true;
+ })
+ .catch(() => undefined);
+ await vi.advanceTimersByTimeAsync(34_999);
+ expect(admissionSettled).toBe(false);
+ await vi.advanceTimersByTimeAsync(2);
+ await expect(admission).rejects.toThrow("exceeded the admission grace");
+ expect(fixture.host.close).toHaveBeenCalledTimes(8);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("bounds lagging streams without introducing source sequence gaps", async () => {
+ const fixture = driverFixture(
+ {},
+ {
+ runtimeEvents: Array.from({ length: 1_100 }, (_, index) => ({
+ type: "text_delta" as const,
+ text: `chunk-${index}`,
+ stream: "output" as const,
+ })),
+ },
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-bounds",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ await session.startTurn({
+ message: { role: "user", text: "Produce many events." },
+ });
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+
+ await vi.waitFor(async () => {
+ const snapshot = await session.snapshot();
+ expect(snapshot.terminalTurns).toHaveLength(1);
+ });
+ await vi.waitFor(async () => {
+ const snapshot = await session.snapshot();
+ expect(snapshot.terminalTurns).toHaveLength(1);
+ const transcript = await session.transcript!();
+ expect(transcript.eventCount).toBeGreaterThan(1_024);
+ expect(transcript.complete).toBe(false);
+ expect(transcript.events.length).toBeLessThanOrEqual(1_024);
+ expect(transcript.omissionReason).toBe("retention_limit");
+ });
+ await expect(
+ session.startTurn({
+ message: {
+ role: "user",
+ text: "Do not overtake the lagging consumer.",
+ },
+ }),
+ ).rejects.toThrow("event consumer must drain");
+
+ await session.close({ reason: "bounds verified" });
+ const iterator = session.events()[Symbol.asyncIterator]();
+ const retained: PrpEvent[] = [];
+ for (;;) {
+ const next = await iterator.next();
+ if (next.done) break;
+ retained.push(next.value);
+ }
+ expect(retained.length).toBeLessThanOrEqual(512);
+ expect(retained).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ eventType: "harness.diagnostic",
+ payload: expect.objectContaining({
+ code: "event_stream_retention_limit",
+ }),
+ }),
+ ]),
+ );
+ expect(
+ retained.filter((event) => event.eventType === "turn.completed"),
+ ).toHaveLength(1);
+ expect(retained.map((event) => event.sourceSeq)).toEqual(
+ Array.from({ length: retained.length }, (_, index) => index + 1),
+ );
+ });
+
+ it("retains a committed semantic proposal under terminal queue pressure", async () => {
+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });
+ const session = await fixture.driver.openSession({
+ runId: "run-semantic-bounds",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const { turnId } = await session.startTurn({
+ message: { role: "user", text: "Complete the task." },
+ });
+ await expect(
+ fixture.hostOptions!.semanticTools!.handler({
+ tool: PRP_COMPLETION_TOOL_NAME,
+ callId: "finish-bounded",
+ arguments: completedResult(),
+ signal: new AbortController().signal,
+ }),
+ ).resolves.toEqual({ accepted: true });
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+
+ await vi.waitFor(async () => {
+ await expect(session.snapshot()).resolves.toMatchObject({
+ terminalTurns: [expect.objectContaining({ turnId })],
+ });
+ });
+ await session.close({ reason: "bounded result verified" });
+ const retained: PrpEvent[] = [];
+ for await (const event of session.events()) retained.push(event);
+
+ const proposalIndex = retained.findIndex(
+ (event) => event.eventType === "run.result.proposed",
+ );
+ const terminalIndex = retained.findIndex(
+ (event) => event.eventType === "turn.completed",
+ );
+ expect(proposalIndex).toBeGreaterThanOrEqual(0);
+ expect(terminalIndex).toBeGreaterThan(proposalIndex);
+ expect(retained).toHaveLength(6);
+ });
+
+ it("keeps a full-queue terminal pending until it can be streamed", async () => {
+ const dynamicToolHandler = vi.fn(async () => ({ title: "late result" }));
+ const fixture = driverFixture(
+ { dynamicToolHandler },
+ {
+ closeSettlementTimeoutMs: 1,
+ maxBufferedEvents: 6,
+ terminalEventReserve: 0,
+ runtimeEvents: Array.from({ length: 8 }, (_, index) => ({
+ type: "text_delta" as const,
+ text: `pressure-${index}`,
+ stream: "output" as const,
+ })),
+ },
+ );
+ const session = await fixture.driver.openSession({
+ runId: "run-terminal-capacity-failure",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const { turnId } = await session.startTurn({
+ message: { role: "user", text: "Fill the complete event queue." },
+ });
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+
+ await vi.waitFor(async () => {
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: turnId,
+ terminalTurns: [],
+ });
+ const transcript = await session.transcript!();
+ expect(transcript.complete).toBe(false);
+ expect(transcript.eventCount).toBe(14);
+ });
+
+ const bridgeHandler = fixture.hostOptions!.semanticTools!.handler;
+ await expect(
+ bridgeHandler({
+ tool: PRP_COMPLETION_TOOL_NAME,
+ callId: "late-completion",
+ arguments: completedResult(),
+ signal: new AbortController().signal,
+ }),
+ ).rejects.toThrow("awaiting canonical terminal-event retention");
+ await expect(
+ bridgeHandler({
+ tool: "documents.read",
+ callId: "late-dynamic-tool",
+ arguments: { id: "doc-after-terminal" },
+ signal: new AbortController().signal,
+ }),
+ ).rejects.toThrow("awaiting canonical terminal-event retention");
+ expect(dynamicToolHandler).not.toHaveBeenCalled();
+ await expect(
+ session.interrupt({ turnId, reason: "too late" }),
+ ).rejects.toThrow("awaiting canonical terminal-event retention");
+ expect(fixture.host.interruptActiveTurn).not.toHaveBeenCalled();
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: turnId,
+ semanticResult: null,
+ terminalTurns: [],
+ });
+ fixture.host.close
+ .mockRejectedValueOnce(new Error("initial host cleanup failed"))
+ .mockResolvedValueOnce(undefined);
+ await expect(
+ session.close({ reason: "queue is still full" }),
+ ).rejects.toThrow("before its turn.completed event is retained");
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: turnId,
+ semanticResult: null,
+ terminalTurns: [],
+ });
+ await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2));
+ expect(fixture.host.close).toHaveBeenLastCalledWith({
+ reason: "queue is still full (automatic cleanup recovery 1)",
+ });
+
+ const iterator = session.events()[Symbol.asyncIterator]();
+ const pressureEvents: PrpEvent[] = [];
+ for (let index = 0; index < 6; index += 1) {
+ const next = await iterator.next();
+ expect(next.done).toBe(false);
+ if (!next.done) pressureEvents.push(next.value);
+ }
+ expect(pressureEvents).toHaveLength(6);
+ expect(isCanonicalTurnTerminal(pressureEvents, turnId)).toHaveLength(0);
+
+ await session.close({ reason: "capacity became available" });
+ const settlementEvents: PrpEvent[] = [];
+ for (;;) {
+ const next = await iterator.next();
+ if (next.done) break;
+ settlementEvents.push(next.value);
+ }
+ expect(isCanonicalTurnTerminal(settlementEvents, turnId)).toEqual([
+ expect.objectContaining({
+ eventType: "turn.completed",
+ turnId,
+ }),
+ ]);
+ await expect(session.snapshot()).resolves.toMatchObject({
+ activeTurnId: null,
+ terminalTurns: [expect.objectContaining({ turnId })],
+ });
+ });
+
+ it("rejects turn admission when only terminal reserve capacity remains", async () => {
+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });
+ const session = await fixture.driver.openSession({
+ runId: "run-admission-bounds",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ await session.startTurn({
+ message: { role: "user", text: "Fill the regular event capacity." },
+ });
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+ await vi.waitFor(async () => {
+ await expect(session.snapshot()).resolves.toMatchObject({
+ terminalTurns: [
+ expect.objectContaining({ turnId: expect.any(String) }),
+ ],
+ });
+ });
+ const iterator = session.events()[Symbol.asyncIterator]();
+ await iterator.next();
+ await iterator.next();
+
+ await expect(
+ session.startTurn({
+ message: { role: "user", text: "Must wait for the remaining events." },
+ }),
+ ).rejects.toThrow("event consumer must drain");
+ expect(fixture.host.startTurn).toHaveBeenCalledOnce();
+ await session.close({ reason: "admission bound verified" });
+ });
+
+ it("redacts a complete Authorization credential from events and transcripts", async () => {
+ const fixture = driverFixture();
+ const session = await fixture.driver.openSession({
+ runId: "run-redaction",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ fixture.host.startTurn.mockImplementationOnce(() => {
+ throw new Error(
+ "Authorization: Bearer first-secret second-secret, code=denied",
+ );
+ });
+ const terminalEvents = collectUntil(session.events(), "turn.failed");
+
+ await expect(
+ session.startTurn({
+ message: { role: "user", text: "Do not expose provider credentials." },
+ }),
+ ).rejects.toThrow("Authorization");
+
+ const serializedEvents = JSON.stringify(await terminalEvents);
+ const serializedTranscript = JSON.stringify(await session.transcript!());
+ for (const serialized of [serializedEvents, serializedTranscript]) {
+ expect(serialized).toContain("[REDACTED]");
+ expect(serialized).not.toContain("Bearer");
+ expect(serialized).not.toContain("first-secret");
+ expect(serialized).not.toContain("second-secret");
+ }
+ await session.close({ reason: "redaction verified" });
+ });
+
+ it("preserves every terminal when the bounded consumer keeps draining", async () => {
+ const fixture = driverFixture({}, { maxBufferedEvents: 6 });
+ const session = await fixture.driver.openSession({
+ runId: "run-critical-bounds",
+ normalizedSessionId: "session-1",
+ workingDirectory: "/workspace",
+ });
+ const terminalTurnIds: string[] = [];
+ for (let index = 0; index < 6; index += 1) {
+ const terminalEvents = collectUntil(session.events(), "turn.completed");
+ const { turnId } = await session.startTurn({
+ message: { role: "user", text: `Complete turn ${index}.` },
+ });
+ terminalTurnIds.push(turnId);
+ fixture.finishTurn({ status: "completed", stopReason: "end_turn" });
+ const emitted = await terminalEvents;
+ expect(emitted.at(-1)).toMatchObject({
+ eventType: "turn.completed",
+ turnId,
+ });
+ }
+ await expect(
+ session.startTurn({
+ message: { role: "user", text: "Exceed the bounded turn limit." },
+ }),
+ ).rejects.toThrow("bounded session turn limit");
+
+ await session.close({ reason: "critical bounds verified" });
+ await expect(session.snapshot()).resolves.toMatchObject({
+ terminalTurns: terminalTurnIds.map((turnId) =>
+ expect.objectContaining({ turnId }),
+ ),
+ });
+ });
+});
+
+function driverFixture(
+ overrides: Partial<CodexAcpxDriverOptions> = {},
+ fixtureOptions: {
+ runtimeEvents?: readonly AcpRuntimeEvent[];
+ runtimeEventFailure?: Promise<never>;
+ closeSettlementTimeoutMs?: number;
+ maxBufferedEvents?: number;
+ terminalEventReserve?: number;
+ openHost?: NonNullable<CodexAcpxDriverDependencies["openHost"]>;
+ } = {},
+): {
+ driver: CodexAcpxDriver;
+ host: ReturnType<typeof fakeHost>;
+ openHost: ReturnType<typeof vi.fn>;
+ hostOptions: OpenAcpxRuntimeHostOptions | null;
+ finishTurn(result: Awaited<AcpxRuntimeTurn["result"]>): void;
+} {
+ const result = deferred<Awaited<AcpxRuntimeTurn["result"]>>();
+ const turn: AcpxRuntimeTurn = {
+ requestId: "provider-turn-1",
+ promptStarted: Promise.resolve(),
+ events: {
+ async *[Symbol.asyncIterator]() {
+ yield* fixtureOptions.runtimeEvents ?? [
+ {
+ type: "text_delta" as const,
+ text: "Task complete.",
+ stream: "output" as const,
+ },
+ {
+ type: "tool_call" as const,
+ toolCallId: "provider-tool-1",
+ title: "Read",
+ kind: "read" as const,
+ status: "pending",
+ tag: "tool_call",
+ text: "Reading",
+ },
+ ];
+ if (fixtureOptions.runtimeEventFailure) {
+ await fixtureOptions.runtimeEventFailure;
+ }
+ },
+ },
+ result: result.promise,
+ cancel: vi.fn(async () => undefined),
+ closeStream: vi.fn(async () => undefined),
+ };
+ const host = fakeHost(turn, () =>
+ result.resolve({ status: "cancelled", stopReason: "session_closed" }),
+ );
+ let hostOptions: OpenAcpxRuntimeHostOptions | null = null;
+ const openHost = vi.fn(
+ fixtureOptions.openHost ??
+ (async (options: OpenAcpxRuntimeHostOptions) => {
+ hostOptions = options;
+ return host;
+ }),
+ );
+ const dependencies: CodexAcpxDriverDependencies = {
+ openHost,
+ closeSettlementTimeoutMs: fixtureOptions.closeSettlementTimeoutMs,
+ maxBufferedEvents: fixtureOptions.maxBufferedEvents,
+ terminalEventReserve: fixtureOptions.terminalEventReserve,
+ };
+ const driver = new CodexAcpxDriver(
+ {
+ runtimeDirectory: "/runtime",
+ model: "gpt-5.6-sol",
+ permissionMode: "approve-reads",
+ dynamicTools: [
+ {
+ name: "documents.read",
+ inputSchema: { type: "object" },
+ },
+ ],
+ now: () => new Date("2026-08-27T12:00:00.000Z"),
+ ...overrides,
+ },
+ dependencies,
+ );
+ return {
+ driver,
+ host,
+ openHost,
+ get hostOptions() {
+ return hostOptions;
+ },
+ finishTurn: result.resolve,
+ };
+}
+
+function fakeHost(turn: AcpxRuntimeTurn, onClose: () => void) {
+ return {
+ identity: () => ({
+ schema: "paperclip.runner.acpx-identity.v1" as const,
+ normalizedSessionId: "session-1",
+ acpxRecordId: "record-1",
+ backendSessionId: "backend-1",
+ agentSessionId: "agent-1",
+ profileDigest: `sha256:${"a".repeat(64)}`,
+ workspaceDigest: `sha256:${"b".repeat(64)}`,
+ requestedModel: "gpt-5.6-sol",
+ effectiveModel: "gpt-5.6-sol",
+ permissionMode: "approve-reads" as const,
+ }),
+ binding: () => ({
+ normalizedSessionId: "session-1",
+ workspacePath: "/workspace",
+ workspaceDigest: `sha256:${"b".repeat(64)}`,
+ runtimeRoot: "/runtime/acpx/session-1",
+ profileDigest: `sha256:${"a".repeat(64)}`,
+ requestedModel: "gpt-5.6-sol",
+ effectiveModel: "gpt-5.6-sol",
+ permissionMode: "approve-reads" as const,
+ profileSessionKey: "paperclip-session",
+ }),
+ status: vi.fn(async () => ({
+ agentSessionId: "agent-1",
+ models: {
+ currentModelId: "gpt-5.6-sol",
+ availableModelIds: ["gpt-5.6-sol"],
+ },
+ })),
+ startTurn: vi.fn(() => turn),
+ interruptActiveTurn: vi.fn(async () => undefined),
+ close: vi.fn(async () => {
+ onClose();
+ }),
+ };
+}
+
+async function collectUntil(
+ events: AsyncIterable<PrpEvent>,
+ terminalType: PrpEvent["eventType"],
+): Promise<PrpEvent[]> {
+ const collected: PrpEvent[] = [];
+ for await (const event of events) {
+ collected.push(event);
+ if (event.eventType === terminalType) return collected;
+ }
+ throw new Error(`Event stream closed before ${terminalType}`);
+}
+
+function completedResult() {
+ return {
+ schema: "paperclip.run_result.v1" as const,
+ reportedWorkDisposition: "done" as const,
+ summary: "The task is complete.",
+ completionClaim: {
+ contractRevision: "codex-acpx-test-v1",
+ objectiveSatisfied: true,
+ criteria: [],
+ remainingWork: [],
+ },
+ evidence: [],
+ verification: [
+ { commandOrCheck: "Codex ACPX driver test", status: "passed" as const },
+ ],
+ attentionRequests: [],
+ artifacts: [],
+ };
+}
+
+function isCanonicalTurnTerminal(events: PrpEvent[], turnId: string) {
+ return events.filter(
+ (event) =>
+ event.turnId === turnId &&
+ (event.eventType === "turn.completed" ||
+ event.eventType === "turn.failed" ||
+ event.eventType === "turn.interrupted"),
+ );
+}
+
+function deferred<T>() {
+ let resolve!: (value: T) => void;
+ let reject!: (error: unknown) => void;
+ const promise = new Promise<T>((settle, fail) => {
+ resolve = settle;
+ reject = fail;
+ });
+ return { promise, reject, resolve };
+}
diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts
new file mode 100644
index 0000000000..f28f669a94
--- /dev/null
+++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts
@@ -0,0 +1,1403 @@
+import { createHash, randomBytes } from "node:crypto";
+
+import type { AcpRuntimeEvent } from "acpx/runtime";
+
+import {
+ PRP_BLOCK_TOOL_NAME,
+ PRP_COMPLETION_TOOL_NAME,
+} from "../../contracts/completion-result.js";
+import {
+ HarnessCapabilityUnavailableError,
+ HarnessStaleTurnError,
+ type HarnessDriver,
+ type HarnessDriverConfigValidation,
+ type HarnessDriverDescriptor,
+ type HarnessSession,
+ type HarnessTranscriptSnapshot,
+ type OpenHarnessSessionInput,
+ type PersistedHarnessSession,
+} from "../../contracts/harness-driver.js";
+import type { NativeAcpxPermissionMode } from "../../contracts/native-execution.js";
+import type { NativeUserMessage } from "../../contracts/types.js";
+import type {
+ PrpEvent,
+ PrpStructuredRunResult,
+} from "../../protocol/replay-contract.js";
+import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js";
+import {
+ canonicalProviderEventsFromAcpxRuntimeEvent,
+ createAcpxToolEventNormalizer,
+} from "../../provider-events.js";
+import {
+ canonicalRunnerToolName,
+ type RunnerToolCall,
+} from "../runner-tool-bridge.js";
+import {
+ DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS,
+ openCodexAcpxRuntime,
+} from "./codex-runtime-adapter.js";
+import {
+ acpxDriverDescriptor,
+ validateAcpxDriverConfig,
+} from "./driver-profile.js";
+import {
+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS,
+ AcpxRuntimeHost,
+ type AcpxRuntimeTurn,
+ type OpenAcpxRuntimeHostOptions,
+} from "./runtime-host.js";
+
+const MAX_BUFFERED_EVENTS = 512;
+const TERMINAL_EVENT_RESERVE = 3;
+const TURN_START_EVENT_COUNT = 3;
+const MAX_TRANSCRIPT_EVENTS = 1_024;
+const MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024;
+const CLOSE_TURN_SETTLEMENT_TIMEOUT_MS = 2_000;
+const MAX_AUTONOMOUS_HOST_CLOSE_RETRIES = 3;
+const MAX_QUARANTINED_HOST_CLOSE_RETRIES = 3;
+const QUARANTINED_HOST_CLOSE_RETRY_MS = 60_000;
+const MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS = 1_000;
+// Host shutdown first bounds active-turn cancellation and then performs the
+// adapter's bounded protocol/TERM/KILL cleanup. Bound each ownership phase that
+// admission observes: an inherited owner may install a distinct replacement
+// recovery, which receives its own finite grace rather than inheriting an
+// almost-expired deadline. Cover every attempt and bounded inter-attempt delay,
+// plus one second of scheduling margin.
+const QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS =
+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS +
+ DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS;
+const MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS = 1;
+const QUARANTINED_HOST_ADMISSION_GRACE_MS =
+ (MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS +
+ MAX_QUARANTINED_HOST_CLOSE_RETRIES) *
+ QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS +
+ (MAX_QUARANTINED_HOST_CLOSE_RETRIES - 1) *
+ MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS +
+ 1_000;
+
+export interface CodexAcpxDynamicToolCall {
+ tool: string;
+ callId: string;
+ providerSessionId: string;
+ turnId: string;
+ arguments: unknown;
+ signal: AbortSignal;
+}
+
+export interface CodexAcpxDriverOptions {
+ runtimeDirectory: string;
+ model: string;
+ permissionMode?: NativeAcpxPermissionMode;
+ systemInstructions?: string;
+ environment?: NodeJS.ProcessEnv;
+ managedCodexCredentialSourcePath?: string;
+ dynamicTools?: readonly Readonly<Record<string, unknown>>[];
+ dynamicToolHandler?: (call: CodexAcpxDynamicToolCall) => Promise<unknown>;
+ now?: () => Date;
+}
+
+interface CodexAcpxHost {
+ identity(): ReturnType<AcpxRuntimeHost["identity"]>;
+ binding(): ReturnType<AcpxRuntimeHost["binding"]>;
+ status(): ReturnType<AcpxRuntimeHost["status"]>;
+ startTurn(
+ input: Parameters<AcpxRuntimeHost["startTurn"]>[0],
+ ): AcpxRuntimeTurn;
+ interruptActiveTurn(reason: string): Promise<void>;
+ close(input: { reason: string }): Promise<void>;
+}
+
+interface QuarantinedHostCleanup {
+ host: CodexAcpxHost;
+ reason: string;
+ attempt: Promise<void> | null;
+ recovery: Promise<void> | null;
+ recoveryMaxAttempts: number | null;
+ timer: ReturnType<typeof setTimeout> | null;
+}
+
+export interface CodexAcpxDriverDependencies {
+ openHost?: (options: OpenAcpxRuntimeHostOptions) => Promise<CodexAcpxHost>;
+ /** Internal test seam; production uses the fixed close-settlement bound. */
+ closeSettlementTimeoutMs?: number;
+ /** Internal test seam; production uses the fixed event-retention bound. */
+ maxBufferedEvents?: number;
+ /** Internal test seam; production reserves fixed terminal-event capacity. */
+ terminalEventReserve?: number;
+}
+
+/** Codex-only HarnessDriver backed by the admitted ACPX runtime host. */
+export class CodexAcpxDriver implements HarnessDriver {
+ readonly #options: CodexAcpxDriverOptions;
+ readonly #openHost: NonNullable<CodexAcpxDriverDependencies["openHost"]>;
+ readonly #closeSettlementTimeoutMs: number;
+ readonly #maxBufferedEvents: number;
+ readonly #terminalEventReserve: number;
+ readonly #cleanupOwners = new Set<Promise<void>>();
+ readonly #quarantinedHostCleanups = new Set<QuarantinedHostCleanup>();
+
+ constructor(
+ options: CodexAcpxDriverOptions,
+ dependencies: CodexAcpxDriverDependencies = {},
+ ) {
+ this.#options = {
+ ...options,
+ ...(options.environment
+ ? { environment: { ...options.environment } }
+ : {}),
+ ...(options.dynamicTools
+ ? { dynamicTools: structuredClone(options.dynamicTools) }
+ : {}),
+ };
+ this.#openHost =
+ dependencies.openHost ??
+ ((hostOptions) =>
+ AcpxRuntimeHost.open(hostOptions, {
+ openRuntime: openCodexAcpxRuntime,
+ reportRetainedCleanupFailure: reportRetainedAcpxCleanupFailure,
+ } as Parameters<typeof AcpxRuntimeHost.open>[1] & {
+ reportRetainedCleanupFailure: typeof reportRetainedAcpxCleanupFailure;
+ }));
+ this.#closeSettlementTimeoutMs =
+ dependencies.closeSettlementTimeoutMs ?? CLOSE_TURN_SETTLEMENT_TIMEOUT_MS;
+ this.#terminalEventReserve = Math.max(
+ 0,
+ Math.floor(dependencies.terminalEventReserve ?? TERMINAL_EVENT_RESERVE),
+ );
+ this.#maxBufferedEvents = Math.max(
+ this.#terminalEventReserve + TURN_START_EVENT_COUNT,
+ Math.floor(dependencies.maxBufferedEvents ?? MAX_BUFFERED_EVENTS),
+ );
+ }
+
+ async descriptor(): Promise<HarnessDriverDescriptor> {
+ const descriptor = acpxDriverDescriptor("codex");
+ return {
+ ...descriptor,
+ displayName: "Codex via ACPX",
+ runtimeContextCapabilities: {
+ instructions: "native",
+ skills: "unsupported",
+ mcp: "native",
+ },
+ capabilities: {
+ ...descriptor.capabilities,
+ resume: false,
+ runtimeRequestResolution: false,
+ runtimeRequestHandoff: false,
+ unsupported: [
+ "resume",
+ "steering",
+ "runtimeRequestResolution",
+ "runtimeRequestHandoff",
+ "goals",
+ "threadLineage",
+ ],
+ },
+ };
+ }
+
+ async validateConfig(value: unknown): Promise<HarnessDriverConfigValidation> {
+ const validation = validateAcpxDriverConfig(value);
+ if (!validation.ok || validation.config.agent === "codex")
+ return validation;
+ return {
+ ok: false,
+ config: null,
+ issues: [
+ {
+ path: "agent",
+ code: "unsupported_agent",
+ message: "The production ACPX driver currently supports Codex only.",
+ },
+ ],
+ };
+ }
+
+ async openSession(input: OpenHarnessSessionInput): Promise<HarnessSession> {
+ await runAbortableDriverAdmission(input.signal, () =>
+ this.#retryQuarantinedHostCleanups(),
+ );
+ let session: CodexAcpxSession | null = null;
+ const host = await this.#openHostForAdmission(
+ {
+ runtimeDirectory: this.#options.runtimeDirectory,
+ normalizedSessionId: input.normalizedSessionId,
+ workingDirectory: input.workingDirectory,
+ agent: "codex",
+ model: this.#options.model,
+ permissionMode: this.#options.permissionMode ?? "approve-reads",
+ systemInstructions: this.#options.systemInstructions,
+ environment: this.#options.environment,
+ managedCodexCredentialSourcePath:
+ this.#options.managedCodexCredentialSourcePath,
+ ...(input.signal === undefined ? {} : { signal: input.signal }),
+ semanticTools: {
+ tools: this.#options.dynamicTools ?? [],
+ handler: (call) => {
+ if (!session) {
+ throw new Error("Codex ACPX session is not ready for tool calls");
+ }
+ return session.dispatchTool(call);
+ },
+ },
+ },
+ input.signal,
+ );
+ try {
+ input.signal?.throwIfAborted();
+ session = new CodexAcpxSession({
+ host,
+ input,
+ dynamicToolHandler: this.#options.dynamicToolHandler,
+ now: this.#options.now ?? (() => new Date()),
+ closeSettlementTimeoutMs: this.#closeSettlementTimeoutMs,
+ maxBufferedEvents: this.#maxBufferedEvents,
+ terminalEventReserve: this.#terminalEventReserve,
+ retainCleanup: (cleanup) => this.#retainCleanup(cleanup),
+ quarantineCleanup: (hostToRetain, reason) =>
+ this.#quarantineHostCleanup(hostToRetain, reason),
+ });
+ return session;
+ } catch (error) {
+ const reason = "Codex ACPX session initialization failed";
+ const cleanup = Promise.resolve().then(() => host.close({ reason }));
+ this.#retainCleanup(cleanup);
+ void cleanup.catch(() => this.#quarantineHostCleanup(host, reason));
+ await settleWithin(cleanup, this.#closeSettlementTimeoutMs).catch(
+ () => undefined,
+ );
+ throw error;
+ }
+ }
+
+ async #openHostForAdmission(
+ options: OpenAcpxRuntimeHostOptions,
+ signal: AbortSignal | undefined,
+ ): Promise<CodexAcpxHost> {
+ if (signal === undefined) return await this.#openHost(options);
+ signal.throwIfAborted();
+ const opening = Promise.resolve().then(() => this.#openHost(options));
+ try {
+ return await raceDriverAdmissionWithAbort(opening, signal);
+ } catch (error) {
+ if (signal.aborted) {
+ const lateHostCleanup = opening.then(
+ (host) => this.#closeLateAdmissionHost(host),
+ () => undefined,
+ );
+ this.#retainCleanup(lateHostCleanup);
+ }
+ throw error;
+ }
+ }
+
+ async #closeLateAdmissionHost(host: CodexAcpxHost): Promise<void> {
+ const reason = "Codex ACPX host resolved after admission was aborted";
+ const cleanup = Promise.resolve().then(() => host.close({ reason }));
+ this.#retainCleanup(cleanup);
+ try {
+ await cleanup;
+ } catch {
+ this.#quarantineHostCleanup(host, reason);
+ }
+ }
+
+ #retainCleanup(cleanup: Promise<void>): void {
+ this.#cleanupOwners.add(cleanup);
+ void cleanup
+ .finally(() => this.#cleanupOwners.delete(cleanup))
+ .catch(() => undefined);
+ }
+
+ #quarantineHostCleanup(host: CodexAcpxHost, reason: string): void {
+ if (
+ [...this.#quarantinedHostCleanups].some((entry) => entry.host === host)
+ ) {
+ return;
+ }
+ const cleanup: QuarantinedHostCleanup = {
+ host,
+ reason,
+ attempt: null,
+ recovery: null,
+ recoveryMaxAttempts: null,
+ timer: null,
+ };
+ this.#quarantinedHostCleanups.add(cleanup);
+ this.#startQuarantinedHostCleanupRecovery(
+ cleanup,
+ MAX_QUARANTINED_HOST_CLOSE_RETRIES,
+ "quarantined cleanup recovery",
+ );
+ }
+
+ #startQuarantinedHostCleanupRecovery(
+ cleanup: QuarantinedHostCleanup,
+ maxAttempts: number,
+ reason: string,
+ ): Promise<void> {
+ if (cleanup.recovery) return cleanup.recovery;
+ const recovery = (async () => {
+ for (
+ let attemptCount = 0;
+ attemptCount < maxAttempts &&
+ this.#quarantinedHostCleanups.has(cleanup);
+ attemptCount += 1
+ ) {
+ // Start the first retry immediately so admission's finite grace applies
+ // to provider cleanup rather than to this rate-limit delay. Only later
+ // attempts wait, preserving sequential bounded retry behavior.
+ if (attemptCount > 0) {
+ await waitForCleanupRetry(
+ Math.max(
+ 1,
+ Math.min(
+ MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS,
+ this.#closeSettlementTimeoutMs,
+ ),
+ ),
+ );
+ }
+ const attempt = Promise.resolve().then(() =>
+ cleanup.host.close({
+ reason: `${cleanup.reason} (${reason})`,
+ }),
+ );
+ cleanup.attempt = attempt;
+ try {
+ await attempt;
+ this.#quarantinedHostCleanups.delete(cleanup);
+ } catch {
+ // Retain the quarantine after this finite, sequential retry batch.
+ } finally {
+ if (cleanup.attempt === attempt) cleanup.attempt = null;
+ }
+ }
+ })();
+ cleanup.recovery = recovery;
+ cleanup.recoveryMaxAttempts = maxAttempts;
+ this.#retainCleanup(recovery);
+ void recovery
+ .finally(() => {
+ if (cleanup.recovery === recovery) {
+ cleanup.recovery = null;
+ cleanup.recoveryMaxAttempts = null;
+ }
+ this.#scheduleQuarantinedHostCleanup(cleanup);
+ })
+ .catch(() => undefined);
+ return recovery;
+ }
+
+ #scheduleQuarantinedHostCleanup(cleanup: QuarantinedHostCleanup): void {
+ if (
+ !this.#quarantinedHostCleanups.has(cleanup) ||
+ cleanup.recovery ||
+ cleanup.attempt ||
+ cleanup.timer
+ ) {
+ return;
+ }
+ // A quarantined host remains actively owned, but recovery is rate-limited:
+ // each entry holds at most one unref'd timer and one sequential close.
+ // Admission may pull that timer forward; it never creates a parallel try.
+ cleanup.timer = setTimeout(() => {
+ cleanup.timer = null;
+ if (!this.#quarantinedHostCleanups.has(cleanup)) return;
+ this.#startQuarantinedHostCleanupRecovery(
+ cleanup,
+ 1,
+ "scheduled quarantined cleanup recovery",
+ );
+ }, QUARANTINED_HOST_CLOSE_RETRY_MS);
+ cleanup.timer.unref?.();
+ }
+
+ async #retryQuarantinedHostCleanups(): Promise<void> {
+ const observedOwners = new Set<Promise<void>>();
+ const acceleratedCleanups = new Set<QuarantinedHostCleanup>();
+ try {
+ while (true) {
+ // A close that is still pending has not entered quarantine yet, but it
+ // owns the same provider resources. Observe every retained owner and any
+ // replacement quarantine recovery it installs before admitting a host.
+ const cleanupOwners = new Set<Promise<void>>(this.#cleanupOwners);
+ for (const cleanup of this.#quarantinedHostCleanups) {
+ if (cleanup.timer) {
+ clearTimeout(cleanup.timer);
+ cleanup.timer = null;
+ }
+ if (cleanup.recovery) {
+ // A scheduled autonomous owner has only one attempt. Admission
+ // observes it first, then starts one complete bounded batch if that
+ // attempt fails. A pre-existing full batch already consumed that
+ // allowance and must never be duplicated.
+ if (
+ (cleanup.recoveryMaxAttempts ?? 0) >=
+ MAX_QUARANTINED_HOST_CLOSE_RETRIES
+ ) {
+ acceleratedCleanups.add(cleanup);
+ }
+ cleanupOwners.add(cleanup.recovery);
+ } else if (!acceleratedCleanups.has(cleanup)) {
+ acceleratedCleanups.add(cleanup);
+ cleanupOwners.add(
+ this.#startQuarantinedHostCleanupRecovery(
+ cleanup,
+ MAX_QUARANTINED_HOST_CLOSE_RETRIES,
+ "quarantined cleanup admission recovery",
+ ),
+ );
+ }
+ }
+ const replacementOwners = [...cleanupOwners].filter(
+ (owner) => !observedOwners.has(owner),
+ );
+ if (replacementOwners.length === 0) break;
+ replacementOwners.forEach((owner) => observedOwners.add(owner));
+ if (
+ !(await settlesWithin(
+ Promise.all(
+ replacementOwners.map((owner) => owner.catch(() => undefined)),
+ ),
+ QUARANTINED_HOST_ADMISSION_GRACE_MS,
+ ))
+ ) {
+ throw new Error(
+ "Codex ACPX cannot open a new session because quarantined host cleanup exceeded the admission grace",
+ );
+ }
+ }
+ } finally {
+ // Admission accelerates autonomous timers, but it must never strand a
+ // different quarantined host if observing any owner exhausts the grace.
+ for (const cleanup of this.#quarantinedHostCleanups) {
+ this.#scheduleQuarantinedHostCleanup(cleanup);
+ }
+ }
+ if (
+ this.#cleanupOwners.size > 0 ||
+ this.#quarantinedHostCleanups.size > 0
+ ) {
+ throw new Error(
+ "Codex ACPX cannot open a new session while quarantined host cleanup remains incomplete",
+ );
+ }
+ }
+}
+
+class CodexAcpxSession implements HarnessSession {
+ readonly #host: CodexAcpxHost;
+ readonly #input: OpenHarnessSessionInput;
+ readonly #dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"];
+ readonly #now: () => Date;
+ readonly #closeSettlementTimeoutMs: number;
+ readonly #maxBufferedEvents: number;
+ readonly #terminalEventReserve: number;
+ readonly #events: AsyncQueue<PrpEvent>;
+ readonly #retainCleanup: (cleanup: Promise<void>) => void;
+ readonly #quarantineCleanup: (host: CodexAcpxHost, reason: string) => void;
+ readonly #transcript: Array<{ event: PrpEvent; bytes: number }> = [];
+ readonly #terminalTurns = new Map<string, string>();
+ readonly #sourceInstanceId: string;
+ #sourceSequence = 0;
+ #activeTurnId: string | null = null;
+ #semanticResult: PrpStructuredRunResult | null = null;
+ #semanticFingerprint: string | null = null;
+ #semanticCallId: string | null = null;
+ #semanticTurnId: string | null = null;
+ #usage: Record<string, unknown> | null = null;
+ #assistantText = "";
+ #closed = false;
+ #closingStarted = false;
+ #eventStreamClosed = false;
+ #activePump: Promise<void> | null = null;
+ #closePromise: Promise<void> | null = null;
+ #hostClosePromise: Promise<void> | null = null;
+ #hostCloseRecoveryPromise: Promise<void> | null = null;
+ #hostClosed = false;
+ #transcriptBytes = 0;
+ #transcriptEventCount = 0;
+ #transcriptOmitted = false;
+ #eventStreamOmitted = false;
+ #pendingTerminal: {
+ turnId: string;
+ fingerprint: string;
+ eventType: "turn.completed" | "turn.failed" | "turn.interrupted";
+ payload: Record<string, unknown>;
+ } | null = null;
+
+ constructor(input: {
+ host: CodexAcpxHost;
+ input: OpenHarnessSessionInput;
+ dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"];
+ now: () => Date;
+ closeSettlementTimeoutMs: number;
+ maxBufferedEvents: number;
+ terminalEventReserve: number;
+ retainCleanup: (cleanup: Promise<void>) => void;
+ quarantineCleanup: (host: CodexAcpxHost, reason: string) => void;
+ }) {
+ const identity = input.host.identity();
+ if (identity.normalizedSessionId !== input.input.normalizedSessionId) {
+ throw new Error("Codex ACPX host returned a different session identity");
+ }
+ this.#host = input.host;
+ this.#input = structuredClone(input.input);
+ this.#dynamicToolHandler = input.dynamicToolHandler;
+ this.#now = input.now;
+ this.#closeSettlementTimeoutMs = input.closeSettlementTimeoutMs;
+ this.#maxBufferedEvents = input.maxBufferedEvents;
+ this.#terminalEventReserve = input.terminalEventReserve;
+ this.#retainCleanup = input.retainCleanup;
+ this.#quarantineCleanup = input.quarantineCleanup;
+ this.#events = new AsyncQueue<PrpEvent>(input.maxBufferedEvents);
+ this.#sourceInstanceId = stableId(
+ "paperclip-acpx",
+ input.input.normalizedSessionId,
+ );
+ }
+
+ ids() {
+ const identity = this.#host.identity();
+ return {
+ driverSessionId: identity.acpxRecordId,
+ providerSessionId: identity.agentSessionId,
+ displayId: identity.agentSessionId,
+ };
+ }
+
+ events(): AsyncIterable<PrpEvent> {
+ return this.#events;
+ }
+
+ async startTurn(input: {
+ message: NativeUserMessage;
+ }): Promise<{ turnId: string }> {
+ this.#assertOpen();
+ if (this.#activeTurnId) {
+ throw new Error("Codex ACPX session already has an active turn");
+ }
+ if (
+ !this.#events.hasCapacity(
+ this.#terminalEventReserve + TURN_START_EVENT_COUNT,
+ )
+ ) {
+ throw new HarnessCapabilityUnavailableError(
+ "turn.start",
+ "the event consumer must drain the previous turn before another turn can start",
+ );
+ }
+ if (this.#terminalTurns.size >= this.#maxBufferedEvents) {
+ throw new HarnessCapabilityUnavailableError(
+ "turn.start",
+ "the bounded session turn limit was reached; open a new session",
+ );
+ }
+ const turnId = `turn-${randomBytes(12).toString("hex")}`;
+ this.#activeTurnId = turnId;
+ this.#assistantText = "";
+ this.#emit("turn.submitted", { text: input.message.text }, { turnId });
+ this.#emit("turn.accepted", { turnId }, { turnId });
+ this.#emit("turn.started", { status: "inProgress" }, { turnId });
+ let turn: AcpxRuntimeTurn;
+ try {
+ turn = this.#host.startTurn({
+ text: input.message.text,
+ requestId: `${safeId(this.#input.runId, "run")}:${turnId}`,
+ });
+ } catch (error) {
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({ status: "failed" }),
+ "turn.failed",
+ { status: "failed", error: { message: safeMessage(error) } },
+ );
+ throw error;
+ }
+ const pump = this.#pumpTurn(turnId, turn);
+ this.#activePump = pump;
+ void pump
+ .finally(() => {
+ if (this.#activePump === pump) this.#activePump = null;
+ })
+ .catch(() => undefined);
+ return { turnId };
+ }
+
+ async interrupt(input: { turnId?: string; reason?: string }): Promise<void> {
+ this.#assertOpen();
+ if (input.turnId && input.turnId !== this.#activeTurnId) {
+ throw new HarnessStaleTurnError(input.turnId);
+ }
+ if (this.#pendingTerminal) {
+ throw new HarnessCapabilityUnavailableError(
+ "interruption",
+ "the active turn is awaiting canonical terminal-event retention",
+ );
+ }
+ if (!this.#activeTurnId) {
+ throw new HarnessCapabilityUnavailableError(
+ "interruption",
+ "there is no active Codex ACPX turn",
+ );
+ }
+ await this.#host.interruptActiveTurn(input.reason ?? "interrupted");
+ }
+
+ async dispatchTool(call: RunnerToolCall): Promise<unknown> {
+ this.#assertOpen();
+ if (this.#pendingTerminal) {
+ throw new HarnessCapabilityUnavailableError(
+ "tool.dispatch",
+ "the active turn is awaiting canonical terminal-event retention",
+ );
+ }
+ const turnId = this.#activeTurnId;
+ if (!turnId) {
+ throw new Error("Codex ACPX tool call is not bound to an active turn");
+ }
+ const tool = canonicalRunnerToolName(call.tool);
+ if (tool === PRP_COMPLETION_TOOL_NAME || tool === PRP_BLOCK_TOOL_NAME) {
+ const validation = validatePrpStructuredRunResult(call.arguments);
+ if (!validation.ok) throw new Error("Invalid semantic run result");
+ const blocked = validation.result.reportedWorkDisposition === "blocked";
+ if (
+ (tool === PRP_BLOCK_TOOL_NAME && !blocked) ||
+ (tool === PRP_COMPLETION_TOOL_NAME && blocked)
+ ) {
+ throw new Error(
+ "Semantic result disposition does not match the terminal tool",
+ );
+ }
+ const fingerprint = canonicalJson(validation.result);
+ if (
+ this.#semanticFingerprint !== null &&
+ this.#semanticFingerprint !== fingerprint
+ ) {
+ throw new Error("A different semantic result is already committed");
+ }
+ if (this.#semanticFingerprint === null) {
+ if (
+ !this.#emit("run.result.proposed", validation.result, {
+ turnId,
+ itemId: call.callId,
+ })
+ ) {
+ throw new HarnessCapabilityUnavailableError(
+ "run.result.proposed",
+ "the event consumer must drain provider events before a semantic result can be accepted",
+ );
+ }
+ this.#semanticResult = structuredClone(validation.result);
+ this.#semanticFingerprint = fingerprint;
+ this.#semanticCallId = call.callId;
+ this.#semanticTurnId = turnId;
+ }
+ return { accepted: true };
+ }
+ if (!this.#dynamicToolHandler) {
+ throw new Error(`Unsupported Paperclip operation ${tool}`);
+ }
+ return await this.#dynamicToolHandler({
+ tool,
+ callId: call.callId,
+ providerSessionId: this.#host.identity().agentSessionId,
+ turnId,
+ arguments: structuredClone(call.arguments),
+ signal: call.signal,
+ });
+ }
+
+ async read(): Promise<Record<string, unknown>> {
+ return {
+ identity: this.#host.identity(),
+ binding: this.#host.binding(),
+ status: await this.#host.status(),
+ };
+ }
+
+ async reconcile(): Promise<Record<string, unknown>> {
+ const identity = this.#host.identity();
+ const status = await this.#host.status();
+ if (
+ status.agentSessionId &&
+ status.agentSessionId !== identity.agentSessionId
+ ) {
+ throw new Error("ACPX reconciliation changed the provider session");
+ }
+ return { identity, status };
+ }
+
+ async usage(): Promise<Record<string, unknown> | null> {
+ return this.#usage === null ? null : structuredClone(this.#usage);
+ }
+
+ async transcript(): Promise<HarnessTranscriptSnapshot> {
+ return {
+ schema: "paperclip-runner/harness-transcript/v1",
+ complete: !this.#transcriptOmitted,
+ eventCount: this.#transcriptEventCount,
+ events: structuredClone(this.#transcript.map(({ event }) => event)),
+ omissionReason: this.#transcriptOmitted ? "retention_limit" : null,
+ };
+ }
+
+ async snapshot(): Promise<PersistedHarnessSession> {
+ const identity = this.#host.identity();
+ return {
+ driverKind: "acpx_runtime",
+ driverSessionId: identity.acpxRecordId,
+ providerSessionId: identity.agentSessionId,
+ runId: this.#input.runId,
+ normalizedSessionId: this.#input.normalizedSessionId,
+ activeTurnId: this.#activeTurnId,
+ lastSourceSequence: this.#sourceSequence,
+ providerIdentity: {
+ kind: "acpx",
+ normalizedSessionId: identity.normalizedSessionId,
+ acpxRecordId: identity.acpxRecordId,
+ backendSessionId: identity.backendSessionId,
+ agentSessionId: identity.agentSessionId,
+ profileDigest: identity.profileDigest,
+ workspaceDigest: identity.workspaceDigest,
+ requestedModel: identity.requestedModel,
+ effectiveModel: identity.effectiveModel,
+ permissionMode: identity.permissionMode,
+ },
+ semanticResult:
+ this.#semanticResult &&
+ this.#semanticFingerprint &&
+ this.#semanticTurnId
+ ? {
+ result: structuredClone(this.#semanticResult),
+ fingerprint: this.#semanticFingerprint,
+ callId: this.#semanticCallId,
+ turnId: this.#semanticTurnId,
+ }
+ : null,
+ terminalTurns: [...this.#terminalTurns].map(
+ ([terminalTurnId, fingerprint]) => ({
+ turnId: terminalTurnId,
+ fingerprint,
+ }),
+ ),
+ };
+ }
+
+ async close(input: { reason: string }): Promise<void> {
+ if (this.#closePromise) return await this.#closePromise;
+ if (this.#closed) return;
+ this.#closingStarted = true;
+ const closePromise = this.#finishClose(input.reason);
+ this.#closePromise = closePromise;
+ try {
+ await closePromise;
+ this.#closed = true;
+ } catch (error) {
+ this.#scheduleHostCloseRecovery(input.reason);
+ throw error;
+ } finally {
+ if (this.#closePromise === closePromise) this.#closePromise = null;
+ }
+ }
+
+ async #finishClose(reason: string): Promise<void> {
+ const closingTurnId = this.#activeTurnId;
+ const pump = this.#activePump;
+ const hostClose =
+ this.#hostClosePromise ?? this.#startHostClose({ reason });
+ let hostCloseError: unknown = null;
+ try {
+ await settleWithin(hostClose, this.#closeSettlementTimeoutMs);
+ } catch (error) {
+ hostCloseError = error;
+ }
+ if (pump) {
+ await settleWithin(
+ pump.catch(() => undefined),
+ this.#closeSettlementTimeoutMs,
+ ).catch(() => undefined);
+ }
+ if (closingTurnId && !this.#terminalTurns.has(closingTurnId)) {
+ const pendingTerminal = this.#pendingTerminal;
+ if (pendingTerminal?.turnId === closingTurnId) {
+ this.#publishTerminal(
+ pendingTerminal.turnId,
+ pendingTerminal.fingerprint,
+ pendingTerminal.eventType,
+ pendingTerminal.payload,
+ );
+ } else {
+ this.#publishTerminal(
+ closingTurnId,
+ canonicalJson({ status: "interrupted" }),
+ "turn.interrupted",
+ { status: "interrupted", stopReason: "session_closed" },
+ );
+ }
+ }
+ if (hostCloseError) {
+ this.#emit(
+ "harness.diagnostic",
+ {
+ code: "acpx_host_cleanup_deferred",
+ message:
+ "ACPX host cleanup exceeded its caller wait bound; the driver retains the exact cleanup until it settles.",
+ },
+ closingTurnId ? { turnId: closingTurnId } : {},
+ 0,
+ );
+ }
+ this.#eventStreamClosed = true;
+ this.#events.close();
+ if (hostCloseError) {
+ throw hostCloseError;
+ }
+ }
+
+ #startHostClose(input: { reason: string }): Promise<void> {
+ const closePromise = this.#host.close(input).then(() => {
+ this.#hostClosed = true;
+ });
+ this.#hostClosePromise = closePromise;
+ this.#retainCleanup(closePromise);
+ return closePromise;
+ }
+
+ #scheduleHostCloseRecovery(reason: string): void {
+ if (this.#hostClosed || this.#hostCloseRecoveryPromise) return;
+ const failedOrPendingClose = this.#hostClosePromise;
+ if (!failedOrPendingClose) return;
+ // Never overlap the exact cleanup that exceeded the caller's wait bound.
+ // Once an attempt rejects, keep one delay-bounded recovery owner alive for
+ // a finite retry budget. A permanently pending attempt remains the sole
+ // owner while the runtime adapter independently terminates its children;
+ // repeated terminal failures settle instead of creating an immortal loop.
+ // Host cleanup does not logically close the harness session: #finishClose
+ // still owns terminal publication and event-stream closure after recovery.
+ const recovery = (async () => {
+ let attempt = failedOrPendingClose;
+ let retryCount = 0;
+ while (!this.#hostClosed) {
+ try {
+ await attempt;
+ return;
+ } catch (error) {
+ if (retryCount >= MAX_AUTONOMOUS_HOST_CLOSE_RETRIES) {
+ this.#quarantineCleanup(this.#host, reason);
+ throw error;
+ }
+ retryCount += 1;
+ }
+ await waitForCleanupRetry(
+ Math.max(1, Math.min(1_000, this.#closeSettlementTimeoutMs)),
+ );
+ if (this.#hostClosed) {
+ return;
+ }
+ if (this.#hostClosePromise === attempt) {
+ this.#hostClosePromise = null;
+ }
+ attempt = this.#startHostClose({
+ reason: `${reason} (automatic cleanup recovery ${retryCount})`,
+ });
+ }
+ })();
+ this.#hostCloseRecoveryPromise = recovery;
+ this.#retainCleanup(recovery);
+ void recovery
+ .finally(() => {
+ if (this.#hostCloseRecoveryPromise === recovery) {
+ this.#hostCloseRecoveryPromise = null;
+ }
+ })
+ .catch(() => undefined);
+ }
+
+ async #pumpTurn(turnId: string, turn: AcpxRuntimeTurn): Promise<void> {
+ try {
+ let index = 0;
+ const normalizeToolEvent =
+ createAcpxToolEventNormalizer<AcpRuntimeEvent>();
+ for await (const event of turn.events) {
+ this.#mapRuntimeEvent(normalizeToolEvent(event), turnId, ++index);
+ }
+ const result = await turn.result;
+ if (this.#terminalTurns.has(turnId)) return;
+ if (result.status === "completed") {
+ const finalText = this.#assistantText.trim();
+ if (finalText) {
+ this.#emit(
+ "item.completed",
+ { kind: "agentMessage", channel: "final", text: finalText },
+ { turnId, itemId: `${turnId}:final-answer` },
+ );
+ }
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({
+ status: "completed",
+ semanticResult: this.#semanticFingerprint,
+ }),
+ "turn.completed",
+ { status: "completed", stopReason: result.stopReason ?? null },
+ );
+ } else if (result.status === "cancelled") {
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({ status: "interrupted" }),
+ "turn.interrupted",
+ {
+ status: "interrupted",
+ stopReason: result.stopReason ?? "cancelled",
+ },
+ );
+ } else {
+ this.#emit(
+ "provider.notice.recorded",
+ {
+ schema: "paperclip.provider.notice.v1",
+ noticeId: `${turnId}:failure`,
+ severity: "error",
+ category: "acpx_turn_failed",
+ scope: "turn",
+ recoverable: result.error.retryable ?? false,
+ userActionable: true,
+ summary: safeMessage(result.error.message),
+ },
+ { turnId, itemId: `${turnId}:failure` },
+ );
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({ status: "failed" }),
+ "turn.failed",
+ {
+ status: "failed",
+ error: {
+ code: result.error.code ?? null,
+ message: safeMessage(result.error.message),
+ },
+ },
+ );
+ }
+ } catch (error) {
+ if (this.#terminalTurns.has(turnId)) return;
+ if (error instanceof TerminalEventCapacityError) throw error;
+ if (this.#closed || this.#closingStarted) {
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({ status: "interrupted" }),
+ "turn.interrupted",
+ { status: "interrupted", stopReason: "session_closed" },
+ );
+ } else {
+ this.#publishTerminal(
+ turnId,
+ canonicalJson({ status: "failed" }),
+ "turn.failed",
+ { status: "failed", error: { message: safeMessage(error) } },
+ );
+ }
+ }
+ }
+
+ #publishTerminal(
+ turnId: string,
+ fingerprint: string,
+ eventType: "turn.completed" | "turn.failed" | "turn.interrupted",
+ payload: Record<string, unknown>,
+ ): void {
+ if (this.#terminalTurns.has(turnId)) return;
+ const pendingTerminal = this.#pendingTerminal;
+ if (
+ pendingTerminal !== null &&
+ (pendingTerminal.turnId !== turnId ||
+ pendingTerminal.fingerprint !== fingerprint ||
+ pendingTerminal.eventType !== eventType ||
+ canonicalJson(pendingTerminal.payload) !== canonicalJson(payload))
+ ) {
+ throw new Error(
+ `Codex ACPX terminal disposition changed while ${pendingTerminal.turnId} awaited stream capacity`,
+ );
+ }
+ const terminal = pendingTerminal ?? {
+ turnId,
+ fingerprint,
+ eventType,
+ payload: structuredClone(payload),
+ };
+ this.#pendingTerminal = terminal;
+ if (!this.#emit(terminal.eventType, terminal.payload, { turnId })) {
+ // The persisted terminal index is the driver's settlement authority.
+ // Never advance it unless the matching canonical event is already in
+ // the bounded stream. A caller may drain and retry close, but it cannot
+ // observe a settled turn whose terminal fact was omitted.
+ throw new TerminalEventCapacityError(turnId, eventType);
+ }
+ this.#terminalTurns.set(turnId, fingerprint);
+ this.#pendingTerminal = null;
+ if (this.#activeTurnId === turnId) this.#activeTurnId = null;
+ }
+
+ #mapRuntimeEvent(
+ event: AcpRuntimeEvent,
+ turnId: string,
+ index: number,
+ ): void {
+ const fallbackItemId = `${turnId}:acp:${index}`;
+ if (event.type === "text_delta") {
+ const output = boundedText(event.text, 64 * 1024);
+ if (event.stream !== "thought" && event.tag !== "agent_thought_chunk") {
+ this.#assistantText = boundedText(
+ `${this.#assistantText}${output}`,
+ 256 * 1024,
+ );
+ }
+ this.#emit(
+ "item.delta",
+ {
+ kind:
+ event.stream === "thought" || event.tag === "agent_thought_chunk"
+ ? "thinking"
+ : "agent_message",
+ text: output,
+ },
+ { turnId, itemId: fallbackItemId },
+ );
+ }
+ if (event.type === "status" && event.tag === "usage_update") {
+ this.#usage = boundedRecord({
+ cumulative: event.breakdown,
+ cost: event.cost,
+ });
+ }
+ for (const canonical of canonicalProviderEventsFromAcpxRuntimeEvent(
+ event,
+ fallbackItemId,
+ turnId,
+ )) {
+ this.#emit(canonical.eventType, canonical.payload, {
+ turnId,
+ itemId: canonical.itemId,
+ });
+ }
+ }
+
+ #emit(
+ eventType: PrpEvent["eventType"],
+ payload: Record<string, unknown>,
+ refs: { turnId?: string; itemId?: string } = {},
+ reservedAfter = isTerminalEvent(eventType)
+ ? 0
+ : eventType === "run.result.proposed"
+ ? Math.max(0, this.#terminalEventReserve - 1)
+ : this.#terminalEventReserve,
+ ): boolean {
+ if (this.#eventStreamClosed) return false;
+ if (this.#eventStreamOmitted && isTerminalEvent(eventType)) {
+ this.#eventStreamOmitted = false;
+ const recordedOmission = this.#emit(
+ "harness.diagnostic",
+ {
+ code: "event_stream_retention_limit",
+ message:
+ "Earlier provider events were omitted because the consumer exceeded the bounded event buffer.",
+ },
+ refs,
+ 1,
+ );
+ if (!recordedOmission) this.#eventStreamOmitted = true;
+ }
+ const sourceSeq = this.#sourceSequence + 1;
+ const event: PrpEvent = {
+ schema: "paperclip.prp.event.v1",
+ sourceEventId: `${this.#sourceInstanceId}:${sourceSeq}`,
+ sourceSeq,
+ sourceInstanceId: this.#sourceInstanceId,
+ sourceKind: "runner",
+ runId: this.#input.runId,
+ normalizedSessionId: this.#input.normalizedSessionId,
+ ...(refs.turnId ? { turnId: refs.turnId } : {}),
+ ...(refs.itemId ? { itemId: refs.itemId } : {}),
+ eventType,
+ schemaVersion: 1,
+ priority: eventType === "run.result.proposed" ? 0 : 1,
+ emittedAt: this.#now().toISOString(),
+ payload: structuredClone(payload),
+ };
+ if (!this.#events.push(event, reservedAfter)) {
+ this.#eventStreamOmitted = true;
+ this.#transcriptEventCount += 1;
+ this.#transcriptOmitted = true;
+ return false;
+ }
+ this.#sourceSequence = sourceSeq;
+ this.#retainTranscriptEvent(event);
+ return true;
+ }
+
+ #retainTranscriptEvent(event: PrpEvent): void {
+ this.#transcriptEventCount += 1;
+ const retained = structuredClone(event);
+ let bytes: number;
+ try {
+ bytes = Buffer.byteLength(JSON.stringify(retained));
+ } catch {
+ this.#transcriptOmitted = true;
+ return;
+ }
+ if (bytes > MAX_TRANSCRIPT_BYTES) {
+ this.#transcriptOmitted = true;
+ return;
+ }
+ this.#transcript.push({ event: retained, bytes });
+ this.#transcriptBytes += bytes;
+ while (
+ this.#transcript.length > MAX_TRANSCRIPT_EVENTS ||
+ this.#transcriptBytes > MAX_TRANSCRIPT_BYTES
+ ) {
+ const omitted = this.#transcript.shift();
+ if (omitted) this.#transcriptBytes -= omitted.bytes;
+ this.#transcriptOmitted = true;
+ }
+ }
+
+ #assertOpen(): void {
+ if (this.#closed || this.#closingStarted) {
+ throw new Error("Codex ACPX session is closing or closed");
+ }
+ }
+}
+
+function waitForCleanupRetry(delayMs: number): Promise<void> {
+ return new Promise((resolve) => {
+ const timer = setTimeout(resolve, delayMs);
+ timer.unref();
+ });
+}
+
+function canonicalJson(value: unknown): string {
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+ if (typeof value === "object" && value !== null) {
+ const record = value as Record<string, unknown>;
+ return `{${Object.keys(record)
+ .sort()
+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
+ .join(",")}}`;
+ }
+ return JSON.stringify(value) ?? "undefined";
+}
+
+function boundedRecord(value: unknown): Record<string, unknown> {
+ const serialized = JSON.stringify(value);
+ if (!serialized || Buffer.byteLength(serialized) > 64 * 1024) {
+ return { omitted: true, reason: "payload_limit" };
+ }
+ const parsed: unknown = JSON.parse(serialized);
+ return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
+ ? (parsed as Record<string, unknown>)
+ : {};
+}
+
+function boundedText(value: string, maxBytes: number): string {
+ const bytes = Buffer.from(value);
+ return bytes.length <= maxBytes
+ ? value
+ : bytes.subarray(bytes.length - maxBytes).toString("utf8");
+}
+
+function safeId(value: string, fallback: string): string {
+ const candidate = value.replace(/[^A-Za-z0-9._:-]/g, "-").slice(0, 160);
+ return /^[A-Za-z0-9]/.test(candidate) ? candidate : fallback;
+}
+
+function stableId(prefix: string, value: string): string {
+ const readable = safeId(value, "session").slice(0, 80);
+ const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16);
+ return `${prefix}-${readable}-${suffix}`;
+}
+
+function safeMessage(error: unknown): string {
+ const message = error instanceof Error ? error.message : String(error);
+ return message
+ .replace(
+ /(\bauthorization\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\r\n,}\]]*)/gi,
+ "$1[REDACTED]",
+ )
+ .replace(
+ /(\b(?:key|token|secret|password)\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\s,}\]]+)/gi,
+ "$1[REDACTED]",
+ )
+ .slice(0, 4_000);
+}
+
+function reportRetainedAcpxCleanupFailure(input: {
+ resource: "credential" | "command" | "runtime";
+ attempt: number;
+ error: unknown;
+}): void {
+ const errorName = input.error instanceof Error ? input.error.name : "Error";
+ process.emitWarning(
+ JSON.stringify({
+ schema: "paperclip.runner.retained_cleanup_failure.v1",
+ resource: input.resource,
+ attempt: input.attempt,
+ errorName,
+ }),
+ {
+ code: "PAPERCLIP_ACPX_RETAINED_CLEANUP_FAILURE",
+ type: "PaperclipRunnerCleanupWarning",
+ },
+ );
+}
+
+function isTerminalEvent(eventType: PrpEvent["eventType"]): boolean {
+ return (
+ eventType === "turn.completed" ||
+ eventType === "turn.failed" ||
+ eventType === "turn.interrupted"
+ );
+}
+
+class TerminalEventCapacityError extends Error {
+ constructor(turnId: string, eventType: PrpEvent["eventType"]) {
+ super(
+ `Codex ACPX cannot settle ${turnId} before its ${eventType} event is retained`,
+ );
+ this.name = "TerminalEventCapacityError";
+ }
+}
+
+class AsyncQueue<T> implements AsyncIterable<T> {
+ readonly #items: T[] = [];
+ readonly #waiters: Array<(result: IteratorResult<T>) => void> = [];
+ readonly #maxItems: number;
+ #closed = false;
+
+ constructor(maxItems: number) {
+ this.#maxItems = maxItems;
+ }
+
+ hasCapacity(requiredItems: number): boolean {
+ return (
+ requiredItems <=
+ this.#waiters.length + this.#maxItems - this.#items.length
+ );
+ }
+
+ push(item: T, reservedAfter = 0): boolean {
+ if (this.#closed) return false;
+ // Admission and insertion are one synchronous operation so every regular
+ // event leaves the terminal lane intact. In particular, a lagging
+ // consumer cannot occupy terminal capacity between a check and a push.
+ if (!this.hasCapacity(1 + reservedAfter)) return false;
+ const waiter = this.#waiters.shift();
+ if (waiter) {
+ waiter({ done: false, value: item });
+ return true;
+ }
+ if (this.#items.length >= this.#maxItems) return false;
+ this.#items.push(item);
+ return true;
+ }
+
+ close(): void {
+ if (this.#closed) return;
+ this.#closed = true;
+ for (const waiter of this.#waiters.splice(0)) {
+ waiter({ done: true, value: undefined });
+ }
+ }
+
+ [Symbol.asyncIterator](): AsyncIterator<T> {
+ return {
+ next: () => {
+ const item = this.#items.shift();
+ if (item !== undefined) {
+ return Promise.resolve({ done: false, value: item });
+ }
+ if (this.#closed) {
+ return Promise.resolve({ done: true, value: undefined });
+ }
+ return new Promise((resolve) => this.#waiters.push(resolve));
+ },
+ };
+ }
+}
+
+async function settleWithin(
+ promise: Promise<void>,
+ timeoutMs: number,
+): Promise<void> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ promise,
+ new Promise<void>((_resolve, reject) => {
+ timer = setTimeout(() => {
+ reject(new Error("ACPX host cleanup exceeded its shutdown timeout"));
+ }, timeoutMs);
+ timer.unref();
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
+async function runAbortableDriverAdmission<T>(
+ signal: AbortSignal | undefined,
+ operation: () => Promise<T>,
+): Promise<T> {
+ if (signal === undefined) return await operation();
+ signal.throwIfAborted();
+ return await raceDriverAdmissionWithAbort(
+ Promise.resolve().then(operation),
+ signal,
+ );
+}
+
+function raceDriverAdmissionWithAbort<T>(
+ pending: Promise<T>,
+ signal: AbortSignal,
+): Promise<T> {
+ return new Promise<T>((resolve, reject) => {
+ let settled = false;
+ const settle = (operation: () => void): void => {
+ if (settled) return;
+ settled = true;
+ signal.removeEventListener("abort", onAbort);
+ operation();
+ };
+ const onAbort = (): void => settle(() => reject(signal.reason));
+ signal.addEventListener("abort", onAbort, { once: true });
+ if (signal.aborted) {
+ onAbort();
+ return;
+ }
+ void pending.then(
+ (value) => settle(() => resolve(value)),
+ (error: unknown) => settle(() => reject(error)),
+ );
+ });
+}
+
+async function settlesWithin(
+ promise: Promise<unknown>,
+ timeoutMs: number,
+): Promise<boolean> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ return await Promise.race([
+ promise.then(() => true),
+ new Promise<false>((resolve) => {
+ timer = setTimeout(() => resolve(false), timeoutMs);
+ timer.unref?.();
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts
index 4bea8aad78..0e89953bfe 100644
--- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts
@@ -32,14 +32,23 @@ describe("Codex ACPX runtime adapter", () => {
controller.abort(cancellation);
const command = fakeCommand();
const createRuntime = vi.fn();
+ const retainFailedAdmissionCleanup = vi.fn();
await expect(
openCodexAcpxRuntime(
- { ...openOptions(command), signal: controller.signal },
+ {
+ ...openOptions(command),
+ signal: controller.signal,
+ retainFailedAdmissionCleanup,
+ },
{ createRuntime },
),
).rejects.toBe(cancellation);
+ expect(retainFailedAdmissionCleanup).toHaveBeenCalledOnce();
+ await expect(
+ retainFailedAdmissionCleanup.mock.calls[0]?.[0],
+ ).resolves.toBeUndefined();
expect(createRuntime).not.toHaveBeenCalled();
expect(command.spawn).not.toHaveBeenCalled();
});
@@ -248,6 +257,807 @@ describe("Codex ACPX runtime adapter", () => {
});
});
+ it("never overlaps a retained protocol close that has not settled", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ const runtimeClose = new Promise<void>(() => {});
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(runtimeClose)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ });
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "runtime close stalled" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ expect(runtime.close).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(child.kill).toHaveBeenCalledWith("SIGTERM");
+ await firstClose;
+
+ // Repeated callers inherit the same bounded observation. A permanently
+ // pending exact close remains the sole protocol attempt for this handle.
+ expect(runtime.close).toHaveBeenCalledOnce();
+ const secondClose = expect(
+ port.close({ reason: "idempotent terminal close" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await vi.advanceTimersByTimeAsync(2_000);
+ await secondClose;
+ expect(runtime.close).toHaveBeenCalledOnce();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("allows a fresh close after a retained attempt rejects late", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectRuntimeClose!: (error: unknown) => void;
+ const runtimeClose = new Promise<void>((_resolve, reject) => {
+ rejectRuntimeClose = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(runtimeClose)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ });
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "runtime close stalled" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(2_000);
+ await firstClose;
+ expect(child.kill).toHaveBeenCalledWith("SIGTERM");
+
+ const protocolFailure = new Error("late protocol close failure");
+ rejectRuntimeClose(protocolFailure);
+ // The late-settlement observer schedules reconciliation asynchronously.
+ // Wait for that fresh attempt instead of racing another caller against
+ // the already-settled retained failure.
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));
+ await expect(
+ port.close({ reason: "retry after retained failure" }),
+ ).resolves.toBeUndefined();
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("reconciles a retained close only after its late failure settles", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectFirstClose!: (error: unknown) => void;
+ const firstRuntimeClose = new Promise<void>((_resolve, reject) => {
+ rejectFirstClose = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(firstRuntimeClose)
+ .mockResolvedValueOnce(undefined)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ });
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "first protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(2_000);
+ await firstClose;
+
+ const inheritedClose = expect(
+ port.close({ reason: "observe pending protocol close" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await vi.advanceTimersByTimeAsync(2_000);
+ await inheritedClose;
+ expect(runtime.close).toHaveBeenCalledOnce();
+
+ rejectFirstClose(new Error("older protocol close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ await expect(
+ port.close({ reason: "observe reconciled cleanup" }),
+ ).resolves.toBeUndefined();
+ expect(runtime.close).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("shares a pending close across concurrent callers before reconciliation", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectFirstClose!: (error: unknown) => void;
+ const firstRuntimeClose = new Promise<void>((_resolve, reject) => {
+ rejectFirstClose = reject;
+ });
+ let resolveFreshClose!: () => void;
+ const freshRuntimeClose = new Promise<void>((resolve) => {
+ resolveFreshClose = resolve;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(firstRuntimeClose)
+ .mockReturnValueOnce(freshRuntimeClose)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ });
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "first protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(2_000);
+ await firstClose;
+
+ const inheritedClose = expect(
+ port.close({ reason: "observe pending protocol close" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ expect(runtime.close).toHaveBeenCalledOnce();
+ rejectFirstClose(new Error("retained protocol close failed"));
+ await inheritedClose;
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));
+
+ resolveFreshClose();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(2);
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ await expect(
+ port.close({ reason: "observe reconciled cleanup" }),
+ ).resolves.toBeUndefined();
+ expect(runtime.close).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("does not let late settlements bypass the reconciliation retry bound", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"));
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "first protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(2_000);
+ await firstClose;
+ rejectInitialClose(new Error("initial protocol close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("keeps timed-out reconciliation failures in their originating budget", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ const reconciliationRejectors: Array<(error: unknown) => void> = [];
+ const reconciliationAttempts = Array.from(
+ { length: 3 },
+ () =>
+ new Promise<void>((_resolve, reject) => {
+ reconciliationRejectors.push(reject);
+ }),
+ );
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockReturnValueOnce(reconciliationAttempts[0]!)
+ .mockReturnValueOnce(reconciliationAttempts[1]!)
+ .mockReturnValueOnce(reconciliationAttempts[2]!)
+ .mockResolvedValueOnce(undefined);
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await firstClose;
+ rejectInitialClose(new Error("external close failed late"));
+
+ for (let index = 0; index < reconciliationRejectors.length; index += 1) {
+ await vi.waitFor(() =>
+ expect(runtime.close).toHaveBeenCalledTimes(index + 2),
+ );
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: `ACPX late protocol cleanup reconciliation ${index + 1}`,
+ discardPersistentState: false,
+ });
+ const overlappingExternalClose =
+ index === 0
+ ? expect(
+ port.close({
+ reason: "external observer of reconciliation",
+ }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed")
+ : null;
+ // The external observer coalesces onto reconciliation attempt one. It
+ // must not relabel that immutable attempt as an external generation.
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ if (overlappingExternalClose) await overlappingExternalClose;
+ reconciliationRejectors[index]!(
+ new Error(`reconciliation ${index + 1} failed late`),
+ );
+ }
+
+ await vi.advanceTimersByTimeAsync(0);
+ await Promise.resolve();
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("renews one budget when an external close joins the final reconciliation", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let rejectFinalReconciliation!: (error: unknown) => void;
+ const finalReconciliation = new Promise<void>((_resolve, reject) => {
+ rejectFinalReconciliation = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockReturnValueOnce(finalReconciliation)
+ .mockResolvedValueOnce(undefined);
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const initialObserver = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await initialObserver;
+ rejectInitialClose(new Error("external close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+
+ const finalObserver = expect(
+ port.close({ reason: "external observer of final reconciliation" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await finalObserver;
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+
+ rejectFinalReconciliation(
+ new Error("final reconciliation failed late"),
+ );
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(5);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("batches coalesced external closes when the final reconciliation fails", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let rejectFinalReconciliation!: (error: unknown) => void;
+ const finalReconciliation = new Promise<void>((_resolve, reject) => {
+ rejectFinalReconciliation = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockReturnValueOnce(finalReconciliation)
+ .mockRejectedValueOnce(new Error("renewed reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("renewed reconciliation 2 failed"))
+ .mockRejectedValueOnce(new Error("renewed reconciliation 3 failed"));
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const initialObserver = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await initialObserver;
+ rejectInitialClose(new Error("external close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+
+ const coalescedObservers = ["first", "second", "third"].map((label) =>
+ expect(
+ port.close({ reason: `${label} external observer` }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed"),
+ );
+ rejectFinalReconciliation(
+ new Error("final reconciliation failed with joined observers"),
+ );
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+ await Promise.all(coalescedObservers);
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(7));
+ expect(
+ vi
+ .mocked(runtime.close)
+ .mock.calls.slice(4)
+ .map(([input]) => input.reason),
+ ).toEqual([
+ "ACPX late protocol cleanup reconciliation 1",
+ "ACPX late protocol cleanup reconciliation 2",
+ "ACPX late protocol cleanup reconciliation 3",
+ ]);
+ await vi.advanceTimersByTimeAsync(0);
+ await Promise.resolve();
+ expect(runtime.close).toHaveBeenCalledTimes(7);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("consumes an external intent when a timed-out final reconciliation succeeds late", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let resolveFinalReconciliation!: () => void;
+ const finalReconciliation = new Promise<void>((resolve) => {
+ resolveFinalReconciliation = resolve;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockReturnValueOnce(finalReconciliation);
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const initialObserver = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await initialObserver;
+ rejectInitialClose(new Error("external close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+
+ const finalObserver = expect(
+ port.close({
+ reason: "external observer of successful reconciliation",
+ }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await finalObserver;
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+
+ resolveFinalReconciliation();
+ await vi.advanceTimersByTimeAsync(0);
+ await Promise.resolve();
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("renews an external intent when bounded protocol success cannot terminate the provider", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let resolveFinalReconciliation!: () => void;
+ const finalReconciliation = new Promise<void>((resolve) => {
+ resolveFinalReconciliation = resolve;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockReturnValueOnce(finalReconciliation)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ child.kill = vi.fn(() => true);
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const initialObserver = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await initialObserver;
+ rejectInitialClose(new Error("external close failed late"));
+ for (
+ let turn = 0;
+ turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4;
+ turn += 1
+ ) {
+ await Promise.resolve();
+ }
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+ const finalObserver = expect(
+ port.close({ reason: "external observer of failed process cleanup" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ resolveFinalReconciliation();
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(4_000);
+ await finalObserver;
+
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ child.signalCode = "SIGKILL";
+ child.emit("exit", null, "SIGKILL");
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(5);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("renews an external intent when late protocol success follows process cleanup failure", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let resolveFinalReconciliation!: () => void;
+ const finalReconciliation = new Promise<void>((resolve) => {
+ resolveFinalReconciliation = resolve;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockReturnValueOnce(finalReconciliation)
+ .mockResolvedValueOnce(undefined);
+ const child = fakeChild();
+ child.kill = vi.fn(() => true);
+ const command = fakeCommand();
+ vi.mocked(command.spawn).mockReturnValue(child);
+ let runtimeOptions: AcpRuntimeOptions | undefined;
+ const port = await openCodexAcpxRuntime(openOptions(command), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: (options) => {
+ runtimeOptions = options;
+ return runtime;
+ },
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const initialObserver = expect(
+ port.close({ reason: "external protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await initialObserver;
+ rejectInitialClose(new Error("external close failed late"));
+ for (
+ let turn = 0;
+ turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4;
+ turn += 1
+ ) {
+ await Promise.resolve();
+ }
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+
+ runtimeOptions?.spawnAgent?.({
+ command: "ignored",
+ args: ["--stdio"],
+ options: {},
+ });
+ const finalObserver = expect(
+ port.close({ reason: "external observer of failed process cleanup" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await vi.advanceTimersByTimeAsync(4_000);
+ await finalObserver;
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+
+ child.signalCode = "SIGKILL";
+ child.emit("exit", null, "SIGKILL");
+ resolveFinalReconciliation();
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(5);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("shares one reconciliation budget across repeated bounded observers", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectRetainedClose!: (error: unknown) => void;
+ const retainedClose = new Promise<void>((_resolve, reject) => {
+ rejectRetainedClose = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(retainedClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"));
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ for (const reason of [
+ "first observer",
+ "second observer",
+ "third observer",
+ ]) {
+ const close = expect(port.close({ reason })).rejects.toThrow(
+ "ACPX runtime and provider cleanup failed",
+ );
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await close;
+ }
+ expect(runtime.close).toHaveBeenCalledOnce();
+ rejectRetainedClose(new Error("retained close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtime.close).toHaveBeenCalledTimes(4);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("gives each late failure generation a bounded reconciliation budget", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let rejectInitialClose!: (error: unknown) => void;
+ const initialClose = new Promise<void>((_resolve, reject) => {
+ rejectInitialClose = reject;
+ });
+ let rejectNewerClose!: (error: unknown) => void;
+ const newerClose = new Promise<void>((_resolve, reject) => {
+ rejectNewerClose = reject;
+ });
+ vi.mocked(runtime.close)
+ .mockReturnValueOnce(initialClose)
+ .mockRejectedValueOnce(new Error("reconciliation 1 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 2 failed"))
+ .mockRejectedValueOnce(new Error("reconciliation 3 failed"))
+ .mockReturnValueOnce(newerClose)
+ .mockResolvedValueOnce(undefined);
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const firstClose = expect(
+ port.close({ reason: "first protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await firstClose;
+ rejectInitialClose(new Error("initial close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4));
+ await vi.advanceTimersByTimeAsync(0);
+
+ const secondClose = expect(
+ port.close({ reason: "newer protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await secondClose;
+ expect(runtime.close).toHaveBeenCalledTimes(5);
+
+ rejectNewerClose(new Error("newer close failed late"));
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(6));
+ expect(runtime.close).toHaveBeenLastCalledWith({
+ handle: HANDLE,
+ reason: "ACPX late protocol cleanup reconciliation 1",
+ discardPersistentState: false,
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("marks a retained protocol close terminal when it succeeds late", async () => {
+ vi.useFakeTimers();
+ try {
+ const runtime = fakeRuntime();
+ let resolveRetainedClose!: () => void;
+ const retainedClose = new Promise<void>((resolve) => {
+ resolveRetainedClose = resolve;
+ });
+ vi.mocked(runtime.close).mockReturnValueOnce(retainedClose);
+ const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), {
+ createRegistry: () => registry(),
+ createStore: () => store(),
+ createRuntime: () => runtime,
+ runtimeCloseTimeoutMs: 1,
+ });
+
+ const close = expect(
+ port.close({ reason: "protocol close stalls" }),
+ ).rejects.toThrow("ACPX runtime and provider cleanup failed");
+ await Promise.resolve();
+ await vi.advanceTimersByTimeAsync(1);
+ await close;
+ expect(runtime.close).toHaveBeenCalledOnce();
+
+ resolveRetainedClose();
+ await vi.advanceTimersByTimeAsync(0);
+ await expect(
+ port.close({ reason: "observe late success" }),
+ ).resolves.toBeUndefined();
+ expect(runtime.close).toHaveBeenCalledOnce();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
it("maps prompt turns to the admitted ACPX handle", async () => {
const runtime = fakeRuntime();
const turn = {
@@ -566,11 +1376,14 @@ describe("Codex ACPX runtime adapter", () => {
.mockResolvedValueOnce(undefined);
const controller = new AbortController();
const cancellation = new Error("runtime admission cancelled");
+ const retainedAdmissionCleanups: Promise<void>[] = [];
const opening = openCodexAcpxRuntime(
{
...openOptions(fakeCommand()),
signal: controller.signal,
+ retainFailedAdmissionCleanup: (cleanup) =>
+ retainedAdmissionCleanups.push(cleanup),
},
{
createRegistry: () => registry(),
@@ -584,9 +1397,11 @@ describe("Codex ACPX runtime adapter", () => {
controller.abort(cancellation);
await expect(opening).rejects.toBe(cancellation);
+ expect(retainedAdmissionCleanups).toHaveLength(1);
resolveHandshake?.(HANDLE);
await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));
+ await expect(retainedAdmissionCleanups[0]).resolves.toBeUndefined();
expect(vi.mocked(runtime.close).mock.calls[0]?.[0]).toEqual({
handle: HANDLE,
reason: "ACPX runtime admission aborted",
@@ -631,7 +1446,9 @@ describe("Codex ACPX runtime adapter", () => {
controller.abort(cancellation);
await expect(opening).rejects.toBe(cancellation);
- expect(retainedCleanups).toHaveLength(1);
+ // The exact late-handshake owner and the host-facing aggregate proof are
+ // distinct retained promises over the same cleanup obligation.
+ expect(retainedCleanups).toHaveLength(2);
resolveHandshake?.(HANDLE);
await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce());
@@ -649,7 +1466,7 @@ describe("Codex ACPX runtime adapter", () => {
expect(runtime.close).toHaveBeenCalledOnce();
resolveClose?.();
- await retainedCleanups[0];
+ await expect(Promise.all(retainedCleanups)).resolves.toBeDefined();
expect(cleanupSettled).toBe(true);
expect(runtime.close).toHaveBeenCalledOnce();
});
@@ -892,13 +1709,22 @@ describe("Codex ACPX runtime adapter", () => {
} as never);
await expect(opening).rejects.toBeInstanceOf(Error);
- expect(retainedCleanups).toHaveLength(2);
+ // Retain the pending handshake, the discovered runtime handle cleanup,
+ // and their host-facing aggregate proof until the same obligation settles.
+ expect(retainedCleanups).toHaveLength(3);
+ const settledCleanups = new Set<Promise<void>>();
+ for (const cleanup of retainedCleanups) {
+ void cleanup
+ .finally(() => settledCleanups.add(cleanup))
+ .catch(() => undefined);
+ }
await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5));
- await expect(retainedCleanups[0]).resolves.toBeUndefined();
+ await vi.waitFor(() => expect(settledCleanups.size).toBe(1));
expect(runtime.close).toHaveBeenCalledTimes(5);
rejectHandshake?.(new Error("test handshake stopped"));
- await expect(retainedCleanups[1]).rejects.toThrow("test handshake stopped");
+ await vi.waitFor(() => expect(settledCleanups.size).toBe(3));
+ await expect(Promise.all(retainedCleanups)).resolves.toBeDefined();
});
it("aggregates asynchronous provider signal errors after a failed handshake", async () => {
@@ -1267,6 +2093,7 @@ function openOptions(
},
systemInstructions: "Use Paperclip tools.",
mcpServers: [],
+ retainFailedAdmissionCleanup: vi.fn(),
};
}
diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts
index 01a15540a8..3c4508be88 100644
--- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts
@@ -24,6 +24,22 @@ const VERIFIED_COMMAND_SENTINEL = "paperclip-verified-acpx-command";
const DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS = 2_000;
const RETAINED_ADMISSION_CLEANUP_RETRY_MIN_MS = 10;
const RETAINED_ADMISSION_CLEANUP_RETRY_MAX_MS = 30_000;
+const PROVIDER_TERM_EXIT_TIMEOUT_MS = 2_000;
+const PROVIDER_KILL_EXIT_TIMEOUT_MS = 2_000;
+const MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS = 3;
+// Production shutdown waits for the protocol close bound before beginning the
+// sequential TERM/KILL verification windows. Keep this exported package-local
+// bound aligned with the implementation so admission can include the complete
+// provider cleanup path instead of accounting for only part of it.
+export const DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS =
+ DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS +
+ PROVIDER_TERM_EXIT_TIMEOUT_MS +
+ PROVIDER_KILL_EXIT_TIMEOUT_MS;
+// A close may outlive its caller-facing wait bound. Keep every exact attempt
+// owned until it settles. A handle never starts a second protocol close while
+// the first remains unresolved; late failure can start bounded reconciliation
+// only after the exact attempt reaches a terminal outcome.
+const activeRuntimeCleanupOwners = new Set<Promise<unknown>>();
const activeCodexRuntimeCleanupOwners = new Set<Promise<unknown>>();
class AcpxRuntimeCloseTimeoutError extends Error {
@@ -53,12 +69,19 @@ export async function openCodexAcpxRuntime(
options: AcpxRuntimePortOpenOptions,
dependencies: CodexAcpxRuntimeDependencies = {},
): Promise<AcpxRuntimePort> {
+ if (options.signal?.aborted) {
+ // The host may have already transferred its staged credential to this
+ // pending admission before this microtask begins. No adapter resources
+ // exist yet, so publish an already-complete cleanup proof before preserving
+ // the caller's exact abort reason.
+ options.retainFailedAdmissionCleanup(Promise.resolve());
+ throw options.signal.reason;
+ }
if (options.profile.agent !== "codex") {
throw new Error(
"The production ACPX runtime currently supports Codex only",
);
}
- options.signal?.throwIfAborted();
// The verified-command boundary already refuses to mint a Windows command
// lease, because Node cannot pin its executable there. Repeat the platform
// gate at this lower boundary so alternate host wiring cannot launch a
@@ -81,6 +104,7 @@ export async function openCodexAcpxRuntime(
const baseStore = createStore({ stateDir: options.stateDirectory });
let failedHandshakeHandle: AcpRuntimeHandle | null = null;
let admissionCleanup: RuntimeAdmissionCleanup | null = null;
+ let abortedHandshakeCleanup: Promise<void> | null = null;
const retainedCleanupOwners = new WeakSet<Promise<void>>();
const retainCleanup = (cleanup: Promise<void>): void => {
if (retainedCleanupOwners.has(cleanup)) {
@@ -236,14 +260,19 @@ export async function openCodexAcpxRuntime(
handle = await raceRuntimeHandshakeWithAbort(handshake, options.signal);
} catch (error) {
if (options.signal.aborted) {
- retainCleanup(
- handshake.then((lateHandle) =>
+ abortedHandshakeCleanup = handshake.then(
+ (lateHandle) =>
admissionCleanup!.runRetained(
lateHandle,
"ACPX runtime admission aborted",
),
- ),
+ () =>
+ admissionCleanup!.runRetained(
+ failedHandshakeHandle,
+ "ACPX runtime admission aborted",
+ ),
);
+ retainCleanup(abortedHandshakeCleanup);
}
throw error;
}
@@ -252,12 +281,30 @@ export async function openCodexAcpxRuntime(
options.signal.throwIfAborted();
}
} catch (error) {
+ const cleanupHandle = handle ?? failedHandshakeHandle;
const cleanupErrors = await admissionCleanup.run(
- handle ?? failedHandshakeHandle,
+ cleanupHandle,
options.signal?.aborted
? "ACPX runtime admission aborted"
: "ACPX session handshake failed",
);
+ const retainedCleanup =
+ cleanupErrors.length === 0
+ ? Promise.resolve()
+ : admissionCleanup.runRetained(
+ cleanupHandle,
+ options.signal?.aborted
+ ? "ACPX runtime admission aborted"
+ : "ACPX session handshake failed",
+ );
+ const cleanupProof =
+ abortedHandshakeCleanup === null
+ ? retainedCleanup
+ : Promise.all([retainedCleanup, abortedHandshakeCleanup]).then(
+ () => undefined,
+ );
+ options.retainFailedAdmissionCleanup(cleanupProof);
+ retainCleanup(cleanupProof);
if (cleanupErrors.length > 0) {
throw new AggregateError(
[error, ...cleanupErrors],
@@ -277,13 +324,23 @@ export async function openCodexAcpxRuntime(
runtime,
handle,
requireIdentity(handle),
- admissionCleanup,
+ children,
+ runtimeCloseTimeoutMs,
);
} catch (error) {
const cleanupErrors = await admissionCleanup.run(
handle,
"ACPX runtime identity validation failed",
);
+ const cleanupProof =
+ cleanupErrors.length === 0
+ ? Promise.resolve()
+ : admissionCleanup.runRetained(
+ handle,
+ "ACPX runtime identity validation failed",
+ );
+ options.retainFailedAdmissionCleanup(cleanupProof);
+ retainCleanup(cleanupProof);
if (cleanupErrors.length > 0) {
throw new AggregateError(
[error, ...cleanupErrors],
@@ -371,12 +428,12 @@ class RuntimeAdmissionCleanup {
const targetKey = this.#resolveTargetKey(rawTargetKey, handle);
const existing = this.#registeredTargets.get(targetKey);
if (existing !== undefined) {
- if (handle !== null) {
- existing.handle =
- existing.handle === null
- ? handle
+ existing.handle =
+ existing.handle === null
+ ? handle
+ : handle === null
+ ? existing.handle
: preferRuntimeAdmissionCleanupHandle(existing.handle, handle);
- }
this.#targetAliases.set(rawTargetKey, targetKey);
return existing.cleanup!;
}
@@ -407,8 +464,7 @@ class RuntimeAdmissionCleanup {
const fallbackHandle =
this.#registeredTargets.get(fallbackTargetKey)?.handle;
if (
- fallbackHandle !== undefined &&
- fallbackHandle !== null &&
+ fallbackHandle != null &&
nonEmptyRuntimeIdentity(fallbackHandle.acpxRecordId) === undefined &&
sameRuntimeAdmissionCleanupOwner(fallbackHandle, handle)
) {
@@ -590,9 +646,232 @@ function runtimePort(
runtime: AcpRuntime,
handle: AcpRuntimeHandle,
identity: AcpxRuntimePortIdentity,
- admissionCleanup: RuntimeAdmissionCleanup,
+ children: SpawnedChildSet,
+ runtimeCloseTimeoutMs: number,
): AcpxRuntimePort {
- return {
+ type RuntimeCloseAttempt = {
+ readonly outcome: Promise<unknown | null>;
+ readonly reconciliationGeneration: number;
+ readonly origin:
+ | { readonly kind: "external" }
+ | {
+ readonly kind: "reconciliation";
+ readonly generation: number;
+ readonly attemptNumber: number;
+ };
+ pendingExternalIntent: boolean;
+ };
+ let runtimeClosed = false;
+ let runtimeCloseAttempt: RuntimeCloseAttempt | undefined;
+ let lateReconciliationOwner: Promise<void> | undefined;
+ // Each independently observed late failure receives a bounded reconciliation
+ // budget. Exhausting retries for an older generation must not prevent a newer
+ // late failure from acquiring its own recovery owner.
+ let lateReconciliationAttemptGeneration = 0;
+ let lateReconciliationAttempts = 0;
+ let lateFailureGeneration = 0;
+ let reconciledLateFailureGeneration = 0;
+ const watchedReleasedAttempts = new Set<RuntimeCloseAttempt>();
+
+ const hasUnreconciledLateFailure = (): boolean =>
+ reconciledLateFailureGeneration < lateFailureGeneration;
+
+ const consumePendingExternalIntent = (
+ attempt: RuntimeCloseAttempt,
+ failed: boolean,
+ ): boolean => {
+ const pending = attempt.pendingExternalIntent;
+ attempt.pendingExternalIntent = false;
+ return (
+ pending &&
+ failed &&
+ attempt.origin.kind === "reconciliation" &&
+ attempt.origin.attemptNumber >=
+ MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS
+ );
+ };
+
+ const scheduleLateFailureReconciliation = (): void => {
+ if (
+ runtimeCloseAttempt ||
+ lateReconciliationOwner ||
+ !hasUnreconciledLateFailure()
+ ) {
+ return;
+ }
+ if (lateReconciliationAttemptGeneration !== lateFailureGeneration) {
+ lateReconciliationAttemptGeneration = lateFailureGeneration;
+ lateReconciliationAttempts = 0;
+ }
+ if (
+ lateReconciliationAttempts >=
+ MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS
+ ) {
+ return;
+ }
+ const attemptGeneration = lateFailureGeneration;
+ const attemptNumber = lateReconciliationAttempts + 1;
+ lateReconciliationAttempts = attemptNumber;
+ let retry = false;
+ const reconciliation = closeRuntime({
+ reason: `ACPX late protocol cleanup reconciliation ${attemptNumber}`,
+ reconciliation: {
+ generation: attemptGeneration,
+ attemptNumber,
+ },
+ }).then(
+ () => {
+ if (hasUnreconciledLateFailure()) {
+ retry =
+ lateFailureGeneration === attemptGeneration &&
+ attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS;
+ }
+ },
+ () => {
+ retry =
+ lateFailureGeneration === attemptGeneration &&
+ attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS;
+ },
+ );
+ const owner = reconciliation.finally(() => {
+ if (lateReconciliationOwner === owner)
+ lateReconciliationOwner = undefined;
+ if (retry || hasUnreconciledLateFailure()) {
+ queueMicrotask(scheduleLateFailureReconciliation);
+ }
+ });
+ lateReconciliationOwner = owner;
+ retainRuntimeCleanupOwner(owner);
+ };
+
+ const watchPendingAttempt = (
+ attempt: RuntimeCloseAttempt,
+ processCleanupSucceeded: boolean,
+ ): void => {
+ if (watchedReleasedAttempts.has(attempt)) return;
+ watchedReleasedAttempts.add(attempt);
+ void attempt.outcome.then((error) => {
+ watchedReleasedAttempts.delete(attempt);
+ if (runtimeCloseAttempt === attempt) runtimeCloseAttempt = undefined;
+ const renewForExternalIntent = consumePendingExternalIntent(
+ attempt,
+ error !== null || !processCleanupSucceeded,
+ );
+ if (renewForExternalIntent) lateFailureGeneration += 1;
+ if (error === null) {
+ if (processCleanupSucceeded) {
+ reconciledLateFailureGeneration = Math.max(
+ reconciledLateFailureGeneration,
+ attempt.reconciliationGeneration,
+ );
+ runtimeClosed = !hasUnreconciledLateFailure();
+ }
+ scheduleLateFailureReconciliation();
+ return;
+ }
+ // A newer successful close cannot erase an older outcome that had not
+ // settled yet. Re-open cleanup state and autonomously create a bounded
+ // reconciliation generation so the late failure is not suppression-only.
+ // An autonomous failure remains charged to the budget of the generation
+ // that created it. If external callers coalesced onto the exhausted final
+ // attempt, their single batched intent creates exactly one new generation;
+ // joins on earlier attempts are satisfied by the remaining same-generation
+ // retries.
+ if (attempt.origin.kind === "external") {
+ lateFailureGeneration += 1;
+ }
+ runtimeClosed = false;
+ scheduleLateFailureReconciliation();
+ });
+ };
+
+ async function closeRuntime(input: {
+ reason: string;
+ reconciliation?: {
+ generation: number;
+ attemptNumber: number;
+ };
+ }): Promise<void> {
+ if (runtimeClosed) return;
+ if (
+ runtimeCloseAttempt?.origin.kind === "reconciliation" &&
+ input.reconciliation === undefined
+ ) {
+ // Preserve the attempt's autonomous origin while remembering that one or
+ // more external callers requested a fresh cleanup observation. The exact
+ // outcome consumes this bit, so coalesced callers cannot mint generations
+ // independently.
+ runtimeCloseAttempt.pendingExternalIntent = true;
+ }
+ if (!runtimeCloseAttempt) {
+ // A close can reconcile only failures already known when its protocol
+ // attempt begins. A released older attempt may reject while this one is
+ // in flight; that later generation must trigger a subsequent close.
+ runtimeCloseAttempt = {
+ outcome: ownedRuntimeCloseOutcome(runtime, handle, input.reason),
+ reconciliationGeneration:
+ input.reconciliation?.generation ?? lateFailureGeneration,
+ origin:
+ input.reconciliation === undefined
+ ? { kind: "external" }
+ : {
+ kind: "reconciliation",
+ generation: input.reconciliation.generation,
+ attemptNumber: input.reconciliation.attemptNumber,
+ },
+ pendingExternalIntent: false,
+ };
+ }
+ const observedAttempt = runtimeCloseAttempt;
+ const processCleanup = terminateChildrenAfterCloseBound(
+ observedAttempt.outcome,
+ children,
+ runtimeCloseTimeoutMs,
+ );
+ // The caller may stop waiting, but the exact ACPX protocol cleanup stays
+ // owned and remains this handle's sole close attempt until it settles.
+ // Provider termination still proceeds at the deadline.
+ const [closeError, processErrors] = await Promise.all([
+ boundedCloseOutcome(observedAttempt.outcome, runtimeCloseTimeoutMs),
+ processCleanup,
+ ]);
+ if (closeError instanceof AcpxRuntimeCloseTimeoutError) {
+ watchPendingAttempt(observedAttempt, processErrors.length === 0);
+ } else {
+ if (
+ consumePendingExternalIntent(
+ observedAttempt,
+ closeError !== null || processErrors.length > 0,
+ )
+ ) {
+ lateFailureGeneration += 1;
+ }
+ if (runtimeCloseAttempt === observedAttempt) {
+ runtimeCloseAttempt = undefined;
+ }
+ }
+ if (processErrors.length === 0 && closeError === null) {
+ reconciledLateFailureGeneration = Math.max(
+ reconciledLateFailureGeneration,
+ observedAttempt.reconciliationGeneration,
+ );
+ runtimeClosed = !hasUnreconciledLateFailure();
+ } else {
+ runtimeClosed = false;
+ }
+ scheduleLateFailureReconciliation();
+ if (closeError !== null || processErrors.length > 0) {
+ const errors = [closeError, ...processErrors].filter(
+ (error): error is unknown => error !== null,
+ );
+ throw new AggregateError(
+ errors,
+ "ACPX runtime and provider cleanup failed",
+ );
+ }
+ }
+
+ const port: AcpxRuntimePort = {
async identity() {
return structuredClone(identity);
},
@@ -622,16 +901,9 @@ function runtimePort(
...(input.signal ? { signal: input.signal } : {}),
});
},
- async close(input) {
- const errors = await admissionCleanup.run(handle, input.reason);
- if (errors.length > 0) {
- throw new AggregateError(
- errors,
- "ACPX runtime and provider cleanup failed",
- );
- }
- },
+ close: closeRuntime,
};
+ return port;
}
function runtimeCloseOutcome(
@@ -663,6 +935,70 @@ async function closeOutcomeWithin(
return outcome;
}
+function ownedRuntimeCloseOutcome(
+ runtime: AcpRuntime,
+ handle: AcpRuntimeHandle,
+ reason: string,
+): Promise<unknown | null> {
+ const cleanup = Promise.resolve()
+ .then(() =>
+ runtime.close({ handle, reason, discardPersistentState: false }),
+ )
+ .then(
+ () => null,
+ (error: unknown) => error,
+ );
+ return retainRuntimeCleanupOwner(cleanup);
+}
+
+function retainRuntimeCleanupOwner<T>(cleanup: Promise<T>): Promise<T> {
+ activeRuntimeCleanupOwners.add(cleanup);
+ void cleanup
+ .finally(() => activeRuntimeCleanupOwners.delete(cleanup))
+ .catch(() => undefined);
+ return cleanup;
+}
+
+async function terminateChildrenAfterCloseBound(
+ closeOutcome: Promise<unknown | null>,
+ children: SpawnedChildSet,
+ timeoutMs: number,
+): Promise<unknown[]> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ closeOutcome.then(() => undefined),
+ new Promise<void>((resolve) => {
+ timer = setTimeout(resolve, Math.max(1, Math.floor(timeoutMs)));
+ timer.unref();
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+ return await children.terminate();
+}
+
+async function boundedCloseOutcome(
+ closeOutcome: Promise<unknown | null>,
+ timeoutMs: number,
+): Promise<unknown | null> {
+ const boundedTimeoutMs = Math.max(1, timeoutMs);
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const outcome = await Promise.race([
+ closeOutcome.then((error) => ({ error })),
+ new Promise<{ error: unknown }>((resolve) => {
+ timer = setTimeout(
+ () => resolve({ error: new AcpxRuntimeCloseTimeoutError() }),
+ boundedTimeoutMs,
+ );
+ timer.unref();
+ }),
+ ]);
+ if (timer) clearTimeout(timer);
+ return outcome.error;
+}
+
function delay(timeoutMs: number): Promise<void> {
return new Promise((resolve) => {
const timer = setTimeout(resolve, timeoutMs);
@@ -703,7 +1039,7 @@ class SpawnedChildSet {
const terminateOutcome = await signalAndWaitForExit(
tracked,
"SIGTERM",
- 2_000,
+ PROVIDER_TERM_EXIT_TIMEOUT_MS,
);
if (terminateOutcome.error !== undefined) {
pushUnique(errors, terminateOutcome.error);
@@ -712,7 +1048,7 @@ class SpawnedChildSet {
const killOutcome = await signalAndWaitForExit(
tracked,
"SIGKILL",
- 2_000,
+ PROVIDER_KILL_EXIT_TIMEOUT_MS,
);
if (killOutcome.error !== undefined) {
pushUnique(errors, killOutcome.error);
diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts
index 5a35c44fa5..22cd50e916 100644
--- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts
@@ -4,6 +4,7 @@ import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { openCodexAcpxRuntime } from "./codex-runtime-adapter.js";
import { stageManagedCodexCredential } from "./codex-credentials.js";
import type {
VerifiedAcpxCommandLease,
@@ -14,6 +15,7 @@ import {
AcpxRuntimeHost,
type AcpxRuntimeHostDependencies,
type AcpxRuntimePort,
+ type AcpxRuntimePortOpenOptions,
type AcpxRuntimeTurn,
} from "./runtime-host.js";
@@ -59,6 +61,65 @@ describe("ACPX runtime host", () => {
expect(fixture.commandClose).not.toHaveBeenCalled();
});
+ it("scrubs credentials when abort wins before the adapter body starts", async () => {
+ const fixture = await hostFixture();
+ const controller = new AbortController();
+ const cancellation = new Error("runtime admission cancelled before entry");
+ const createRuntime = vi.fn();
+ let credentialHome = "";
+ const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => {
+ credentialHome = options.launchEnvironment.CODEX_HOME!;
+ // The host has scheduled its openRuntime callback and transferred the
+ // staged credential to that pending admission. Abort before entering the
+ // adapter so its pre-entry path must publish a completed cleanup proof.
+ controller.abort(cancellation);
+ return openCodexAcpxRuntime(options, { createRuntime });
+ });
+
+ await expect(
+ AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "deny-all",
+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },
+ signal: controller.signal,
+ },
+ fixture.dependencies({ openRuntime }),
+ ),
+ ).rejects.toBe(cancellation);
+
+ expect(openRuntime).toHaveBeenCalledOnce();
+ expect(createRuntime).not.toHaveBeenCalled();
+ expect(fixture.commandClose).toHaveBeenCalledOnce();
+ const authPath = join(credentialHome, "auth.json");
+ const contender = await vi.waitFor(() =>
+ stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
+ }),
+ );
+ await contender.close();
+ await expect(readFile(authPath)).rejects.toMatchObject({ code: "ENOENT" });
+
+ const retryRuntime = runtimePort();
+ const retryHost = await AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "deny-all",
+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },
+ },
+ fixture.dependencies({ openRuntime: async () => retryRuntime }),
+ );
+ await retryHost.close({ reason: "retry admission complete" });
+ expect(retryRuntime.close).toHaveBeenCalledOnce();
+ });
+
it("composes admission, isolation, model verification, and cleanup", async () => {
const fixture = await hostFixture();
let capturedEnvironment: Readonly<NodeJS.ProcessEnv> = {};
@@ -301,6 +362,77 @@ describe("ACPX runtime host", () => {
expect(fixture.commandClose).toHaveBeenCalledOnce();
});
+ it("retains failed admission cleanup until provider shutdown permits credential scrub", async () => {
+ const fixture = await hostFixture();
+ let authPath = "";
+ let credentialHome = "";
+ let resolveRetryClose!: () => void;
+ const retryClose = new Promise<void>((resolve) => {
+ resolveRetryClose = resolve;
+ });
+ const runtime = runtimePort({
+ getStatus: async () => ({
+ models: {
+ currentModelId: "wrong-model",
+ availableModelIds: ["wrong-model"],
+ },
+ }),
+ onClose: vi
+ .fn()
+ .mockRejectedValueOnce(new Error("first admission cleanup failed"))
+ .mockImplementationOnce(() => retryClose),
+ });
+
+ await expect(
+ AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "approve-all",
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET:
+ '{"owner":"failed-admission"}',
+ },
+ },
+ fixture.dependencies({
+ openRuntime: async (options) => {
+ credentialHome = options.launchEnvironment.CODEX_HOME!;
+ authPath = join(credentialHome, "auth.json");
+ return runtime;
+ },
+ }),
+ ),
+ ).rejects.toThrow(/initialization and cleanup failed/);
+
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2));
+ await expect(readFile(authPath, "utf8")).resolves.toContain(
+ "failed-admission",
+ );
+ await expect(
+ stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
+ }),
+ ).rejects.toThrow("already has an active lease");
+
+ resolveRetryClose();
+ await vi.waitFor(async () => {
+ await expect(readFile(authPath)).rejects.toMatchObject({
+ code: "ENOENT",
+ });
+ });
+ const contender = await stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
+ });
+ await contender.close();
+ });
+
it("retains credential ownership when runtime shutdown fails until retry succeeds", async () => {
const fixture = await hostFixture();
let failClose = true;
@@ -402,6 +534,70 @@ describe("ACPX runtime host", () => {
await contender.close();
});
+ it("retains the exact pending cleanup while independent resources close", async () => {
+ const fixture = await hostFixture();
+ const firstClose = new Promise<void>(() => undefined);
+ const runtime = runtimePort({
+ onClose: vi
+ .fn()
+ .mockImplementationOnce(() => firstClose)
+ .mockResolvedValueOnce(undefined),
+ });
+ const host = await AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "approve-all",
+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },
+ },
+ fixture.dependencies({ openRuntime: async () => runtime }),
+ );
+
+ const first = host.close({ reason: "first close stalls" });
+ await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce());
+ const second = host.close({ reason: "same pending owner" });
+ let settled = false;
+ void Promise.all([first, second]).finally(() => {
+ settled = true;
+ });
+ await Promise.resolve();
+ expect(settled).toBe(false);
+ expect(runtime.close).toHaveBeenCalledOnce();
+ expect(fixture.commandClose).toHaveBeenCalledOnce();
+ });
+
+ it("retries only after the exact close outcome settles with failure", async () => {
+ const fixture = await hostFixture();
+ const runtime = runtimePort({
+ onClose: vi
+ .fn()
+ .mockRejectedValueOnce(new Error("runtime close failed"))
+ .mockResolvedValueOnce(undefined),
+ });
+ const host = await AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "approve-all",
+ environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" },
+ },
+ fixture.dependencies({ openRuntime: async () => runtime }),
+ );
+
+ await expect(host.close({ reason: "first close" })).rejects.toThrow(
+ /cleanup failed/,
+ );
+ await expect(
+ host.close({ reason: "fresh attempt" }),
+ ).resolves.toBeUndefined();
+ await expect(
+ host.close({ reason: "ownership released" }),
+ ).resolves.toBeUndefined();
+ expect(runtime.close).toHaveBeenCalledTimes(2);
+ });
+
it("admits one bounded turn and cancels it before shutdown", async () => {
const fixture = await hostFixture();
const turn = runtimeTurn();
@@ -429,6 +625,9 @@ describe("ACPX runtime host", () => {
host.startTurn({ text: "Concurrent", requestId: "turn-2" }),
).toThrow("already has an active turn");
+ await host.interruptActiveTurn("user interrupt");
+ expect(turn.cancel).toHaveBeenCalledWith({ reason: "user interrupt" });
+
await host.close({ reason: "shutdown" });
expect(turn.cancel).toHaveBeenCalledWith({ reason: "shutdown" });
expect(runtime.close).toHaveBeenCalledOnce();
@@ -693,13 +892,23 @@ describe("ACPX runtime host", () => {
expect(fixture.commandClose).not.toHaveBeenCalled();
});
- it("forwards cancellation and closes a runtime that resolves after abort", async () => {
+ it("retains managed credentials until an aborted late runtime is closed", async () => {
const fixture = await hostFixture();
const runtimeAdmission = deferred<AcpxRuntimePort>();
- const lateRuntime = runtimePort();
+ const retryClose = deferred<void>();
+ const lateRuntime = runtimePort({
+ onClose: vi
+ .fn()
+ .mockRejectedValueOnce(new Error("late runtime close failed"))
+ .mockImplementationOnce(() => retryClose.promise),
+ });
let receivedSignal: AbortSignal | undefined;
+ let credentialHome = "";
+ let bridgeUrl = "";
const openRuntime = vi.fn((options) => {
receivedSignal = options.signal;
+ credentialHome = options.launchEnvironment.CODEX_HOME!;
+ bridgeUrl = options.mcpServers[0]!.url;
return runtimeAdmission.promise;
});
const controller = new AbortController();
@@ -714,6 +923,10 @@ describe("ACPX runtime host", () => {
PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}",
},
signal: controller.signal,
+ semanticTools: {
+ tools: [],
+ handler: async () => ({ ok: true }),
+ },
},
fixture.dependencies({ openRuntime }),
);
@@ -723,13 +936,93 @@ describe("ACPX runtime host", () => {
controller.abort(cancellation);
await expect(opening).rejects.toBe(cancellation);
expect(fixture.commandClose).toHaveBeenCalledOnce();
+ await expect(fetch(bridgeUrl)).rejects.toThrow();
+ const authPath = join(credentialHome, "auth.json");
+ await expect(readFile(authPath, "utf8")).resolves.toBe("{}");
+ await expect(
+ stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
+ }),
+ ).rejects.toThrow("already has an active lease");
+
runtimeAdmission.resolve(lateRuntime);
+ await vi.waitFor(() => expect(lateRuntime.close).toHaveBeenCalledTimes(2));
+ expect(lateRuntime.close).toHaveBeenNthCalledWith(1, {
+ reason: "ACPX runtime admission aborted",
+ });
+ await expect(readFile(authPath, "utf8")).resolves.toBe("{}");
+ await expect(
+ stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
+ }),
+ ).rejects.toThrow("already has an active lease");
- await vi.waitFor(() =>
- expect(lateRuntime.close).toHaveBeenCalledWith({
- reason: "ACPX runtime admission aborted",
+ retryClose.resolve(undefined);
+ await vi.waitFor(async () => {
+ await expect(readFile(authPath)).rejects.toMatchObject({
+ code: "ENOENT",
+ });
+ });
+ // File removal precedes kernel lease release. Wait for the lease itself so
+ // this assertion cannot race between those two ordered cleanup steps.
+ const contender = await vi.waitFor(() =>
+ stageManagedCodexCredential({
+ agentHomeDirectory: credentialHome,
+ environment: {
+ PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}',
+ },
}),
);
+ await contender.close();
+ });
+
+ it("scrubs credentials after rejected runtime cleanup is proven", async () => {
+ const fixture = await hostFixture();
+ const runtimeAdmission = deferred<AcpxRuntimePort>();
+ const providerCleanup = deferred<void>();
+ const credentialClose = vi.fn(async () => undefined);
+ const controller = new AbortController();
+ const cancellation = new Error("runtime admission cancelled");
+ const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => {
+ options.retainFailedAdmissionCleanup(providerCleanup.promise);
+ return runtimeAdmission.promise;
+ });
+ const opening = AcpxRuntimeHost.open(
+ {
+ ...fixture.options,
+ agent: "codex",
+ model: "gpt-5.6-sol",
+ permissionMode: "deny-all",
+ signal: controller.signal,
+ },
+ {
+ ...fixture.dependencies({ openRuntime }),
+ stageCredential: async () => ({
+ path: join(fixture.root, "auth.json"),
+ mode: "inline_json",
+ close: credentialClose,
+ }),
+ },
+ );
+ await vi.waitFor(() => expect(openRuntime).toHaveBeenCalledOnce());
+
+ controller.abort(cancellation);
+ await expect(opening).rejects.toBe(cancellation);
+ const cleanupFailure = new Error("provider survived forced cleanup");
+ runtimeAdmission.reject(new AggregateError([cancellation, cleanupFailure]));
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(credentialClose).not.toHaveBeenCalled();
+ expect(fixture.commandClose).toHaveBeenCalledOnce();
+
+ providerCleanup.resolve(undefined);
+ await vi.waitFor(() => expect(credentialClose).toHaveBeenCalledOnce());
});
});
@@ -824,10 +1117,13 @@ async function hostFixture() {
function deferred<T>(): {
promise: Promise<T>;
resolve(value: T): void;
+ reject(reason: unknown): void;
} {
let resolve!: (value: T) => void;
- const promise = new Promise<T>((settle) => {
+ let reject!: (reason: unknown) => void;
+ const promise = new Promise<T>((settle, fail) => {
resolve = settle;
+ reject = fail;
});
- return { promise, resolve };
+ return { promise, resolve, reject };
}
diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts
index 86c200744c..81e02b42c4 100644
--- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts
@@ -38,7 +38,11 @@ import {
} from "./runtime-sandbox.js";
import type { AcpxExpectedSessionIdentity } from "./sidecar-protocol.js";
-const TURN_CANCELLATION_TIMEOUT_MS = 2_000;
+export const ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS = 2_000;
+
+const ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS = 8;
+const ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS = 10;
+const ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS = 1_000;
export interface AcpxRuntimePortIdentity {
acpxRecordId: string;
@@ -83,6 +87,11 @@ export interface AcpxRuntimePortOpenOptions {
/** Abort provider admission and clean any runtime that resolves too late. */
signal?: AbortSignal;
mcpServers: readonly AcpxMcpServerBinding[];
+ /**
+ * Transfer the provider cleanup proof before a failed open settles. The host
+ * keeps credentials fenced until this exact cleanup succeeds.
+ */
+ retainFailedAdmissionCleanup(cleanup: Promise<void>): void;
}
export interface AcpxRetainedCleanupFailure {
@@ -134,6 +143,35 @@ const activeRuntimeHostCleanupOwners = new Set<Promise<unknown>>();
const RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS = 10;
const RETAINED_CLEANUP_RETRY_MAX_DELAY_MS = 1_000;
+interface RetainedRejectedRuntimeAdmission {
+ readonly credential: ManagedCodexCredentialLease;
+ cleanup: Promise<void>;
+}
+
+// A rejected provider open is not itself proof that provider processes are
+// gone. Retain the credential with the adapter's exact cleanup proof and scrub
+// it only after that proof succeeds. Terminal cleanup failure stays inert and
+// fenced instead of exposing the credential to a replacement admission.
+const retainedRejectedRuntimeAdmissions =
+ new Set<RetainedRejectedRuntimeAdmission>();
+
+interface RetainedAcpxAdmissionCleanup {
+ readonly runtime: AcpxRuntimePort | null;
+ readonly toolBridge: RunnerToolBridge | null;
+ readonly credential: ManagedCodexCredentialLease | null;
+ readonly command: VerifiedAcpxCommandLease | null;
+ readonly reason: string;
+ recovery: Promise<void> | null;
+ timer: ReturnType<typeof setTimeout> | null;
+}
+
+// An admission failure has no AcpxRuntimeHost instance for a caller to close.
+// Retain those resources here until provider shutdown succeeds and the staged
+// credential can consequently be scrubbed. Each recovery batch is finite and
+// sequential; an unref'd timer rate-limits later autonomous attempts without
+// allowing the failed admission's ownership to become unreachable.
+const retainedAcpxAdmissionCleanups = new Set<RetainedAcpxAdmissionCleanup>();
+
export class AcpxRuntimeHost {
readonly #runtime: AcpxRuntimePort;
readonly #binding: AcpxRecoveryBinding;
@@ -205,6 +243,37 @@ export class AcpxRuntimeHost {
let credential: ManagedCodexCredentialLease | null = null;
let toolBridge: RunnerToolBridge | null = null;
let runtime: AcpxRuntimePort | null = null;
+ let pendingRuntimeOwnsCredential = false;
+ let failedAdmissionCleanupTransferred = false;
+ let resolveFailedAdmissionCleanupTransfer!: () => void;
+ const failedAdmissionCleanupTransfer = new Promise<void>((resolve) => {
+ resolveFailedAdmissionCleanupTransfer = resolve;
+ });
+ const retainFailedAdmissionCleanup = (cleanup: Promise<void>): void => {
+ if (failedAdmissionCleanupTransferred) return;
+ failedAdmissionCleanupTransferred = true;
+ pendingRuntimeOwnsCredential = credential !== null;
+ resolveFailedAdmissionCleanupTransfer();
+ if (credential === null) {
+ retainRuntimeHostCleanup(cleanup);
+ return;
+ }
+ const retained: RetainedRejectedRuntimeAdmission = {
+ credential,
+ cleanup: Promise.resolve(),
+ };
+ const ownedCleanup = cleanup.then(async () => {
+ await cleanupAbortedRuntimeAdmission(
+ null,
+ retained.credential,
+ "ACPX rejected runtime admission cleanup confirmed",
+ );
+ retainedRejectedRuntimeAdmissions.delete(retained);
+ });
+ retained.cleanup = ownedCleanup;
+ retainedRejectedRuntimeAdmissions.add(retained);
+ retainRuntimeHostCleanup(ownedCleanup);
+ };
try {
const sandbox = await runAbortableAdmissionStage(options.signal, () =>
prepareAcpxRuntimeSandbox({
@@ -272,6 +341,7 @@ export class AcpxRuntimeHost {
},
]
: [],
+ retainFailedAdmissionCleanup,
}),
resource: "runtime",
releaseLate: (lateRuntime) =>
@@ -280,6 +350,19 @@ export class AcpxRuntimeHost {
}),
reportFailure: (failure) =>
dependencies.reportRetainedCleanupFailure(failure),
+ onAbortedPending: (pendingRuntime) => {
+ // A provider may already be running even though openRuntime has not
+ // returned its port. Keep its credential lease with that exact
+ // admission while the ordinary catch path revokes tools and releases
+ // the consumed command snapshot without delaying cancellation.
+ pendingRuntimeOwnsCredential = credential !== null;
+ retainAbortedRuntimeAdmissionCleanup({
+ pendingRuntime,
+ credential,
+ reason: "ACPX runtime admission aborted",
+ failedAdmissionCleanupTransfer,
+ });
+ },
});
await runAbortableAdmissionStage(options.signal, () =>
requireVerifiedAcpxModel(runtime!, profile),
@@ -316,11 +399,18 @@ export class AcpxRuntimeHost {
const cleanupError = await cleanupRuntimeResources(
runtime,
toolBridge,
- credential,
+ pendingRuntimeOwnsCredential ? null : credential,
command,
"ACPX runtime initialization failed",
);
if (cleanupError) {
+ retainFailedAcpxAdmissionCleanup({
+ runtime,
+ toolBridge,
+ credential: pendingRuntimeOwnsCredential ? null : credential,
+ command,
+ reason: "ACPX runtime initialization failed",
+ });
throw new AggregateError(
[error, ...cleanupError.errors],
"ACPX runtime initialization and cleanup failed",
@@ -346,6 +436,10 @@ export class AcpxRuntimeHost {
return Object.freeze({ ...this.#sandbox.persistedEnvironment });
}
+ async status(): Promise<AcpxModelStatus> {
+ return structuredClone(await this.#runtime.getStatus());
+ }
+
startTurn(input: AcpxRuntimeTurnInput): AcpxRuntimeTurn {
if (this.#closed || this.#closingStarted) {
throw new Error("ACPX runtime host is closing");
@@ -374,9 +468,20 @@ export class AcpxRuntimeHost {
return turn;
}
+ async interruptActiveTurn(reason: string): Promise<void> {
+ const turn = this.#activeTurn;
+ if (!turn) throw new Error("ACPX runtime host has no active turn");
+ const cancellationError = await boundedCancellation(
+ turn.cancel({ reason: boundedReason(reason) }),
+ );
+ if (cancellationError) throw cancellationError;
+ }
+
async close(input: { reason: string }): Promise<void> {
if (this.#closed) return;
- if (this.#closePromise) return await this.#closePromise;
+ if (this.#closePromise) {
+ return await this.#closePromise;
+ }
this.#closingStarted = true;
const closePromise = this.#close(boundedReason(input.reason));
this.#closePromise = closePromise;
@@ -435,6 +540,7 @@ async function acquireAbortableAdmissionResource<T>(input: {
resource: AcpxRetainedCleanupFailure["resource"];
releaseLate: (resource: T) => Promise<void>;
reportFailure: (failure: AcpxRetainedCleanupFailure) => void;
+ onAbortedPending?: (pending: Promise<T>) => void;
}): Promise<T> {
if (input.signal === undefined) return await input.acquire();
input.signal.throwIfAborted();
@@ -443,16 +549,20 @@ async function acquireAbortableAdmissionResource<T>(input: {
return await raceAdmissionWithAbort(pending, input.signal);
} catch (error) {
if (input.signal.aborted) {
- retainRuntimeHostCleanup(
- pending.then((resource) =>
- releaseRetainedAdmissionResource({
- resource,
- resourceKind: input.resource,
- release: input.releaseLate,
- reportFailure: input.reportFailure,
- }),
- ),
- );
+ if (input.onAbortedPending) {
+ input.onAbortedPending(pending);
+ } else {
+ retainRuntimeHostCleanup(
+ pending.then((resource) =>
+ releaseRetainedAdmissionResource({
+ resource,
+ resourceKind: input.resource,
+ release: input.releaseLate,
+ reportFailure: input.reportFailure,
+ }),
+ ),
+ );
+ }
}
throw error;
}
@@ -530,6 +640,123 @@ function retainRuntimeHostCleanup(cleanup: Promise<unknown>): void {
.catch(() => undefined);
}
+function retainAbortedRuntimeAdmissionCleanup(input: {
+ pendingRuntime: Promise<AcpxRuntimePort>;
+ credential: ManagedCodexCredentialLease | null;
+ reason: string;
+ failedAdmissionCleanupTransfer: Promise<void>;
+}): void {
+ const cleanup = input.pendingRuntime.then(
+ (runtime) =>
+ cleanupAbortedRuntimeAdmission(runtime, input.credential, input.reason),
+ () => input.failedAdmissionCleanupTransfer,
+ );
+ retainRuntimeHostCleanup(cleanup);
+}
+
+async function cleanupAbortedRuntimeAdmission(
+ runtime: AcpxRuntimePort | null,
+ credential: ManagedCodexCredentialLease | null,
+ reason: string,
+): Promise<void> {
+ const cleanupError = await cleanupRuntimeResources(
+ runtime,
+ null,
+ credential,
+ null,
+ reason,
+ );
+ if (!cleanupError) return;
+ retainFailedAcpxAdmissionCleanup({
+ runtime,
+ toolBridge: null,
+ credential,
+ command: null,
+ reason,
+ });
+}
+
+function retainFailedAcpxAdmissionCleanup(input: {
+ runtime: AcpxRuntimePort | null;
+ toolBridge: RunnerToolBridge | null;
+ credential: ManagedCodexCredentialLease | null;
+ command: VerifiedAcpxCommandLease | null;
+ reason: string;
+}): void {
+ const cleanup: RetainedAcpxAdmissionCleanup = {
+ ...input,
+ recovery: null,
+ timer: null,
+ };
+ retainedAcpxAdmissionCleanups.add(cleanup);
+ startRetainedAcpxAdmissionCleanup(cleanup);
+}
+
+function startRetainedAcpxAdmissionCleanup(
+ cleanup: RetainedAcpxAdmissionCleanup,
+): Promise<void> {
+ if (cleanup.recovery) return cleanup.recovery;
+ const recovery = (async () => {
+ let retryDelayMs = ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS;
+ for (
+ let attempt = 1;
+ attempt <= ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS;
+ attempt += 1
+ ) {
+ const cleanupError = await cleanupRuntimeResources(
+ cleanup.runtime,
+ cleanup.toolBridge,
+ cleanup.credential,
+ cleanup.command,
+ `${cleanup.reason} (automatic cleanup recovery ${attempt})`,
+ );
+ if (!cleanupError) {
+ retainedAcpxAdmissionCleanups.delete(cleanup);
+ if (cleanup.timer) clearTimeout(cleanup.timer);
+ cleanup.timer = null;
+ return;
+ }
+ if (attempt < ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS) {
+ await waitForAdmissionCleanupRetry(retryDelayMs);
+ retryDelayMs = Math.min(retryDelayMs * 2, 1_000);
+ }
+ }
+ })();
+ cleanup.recovery = recovery;
+ void recovery
+ .finally(() => {
+ if (cleanup.recovery === recovery) cleanup.recovery = null;
+ scheduleRetainedAcpxAdmissionCleanup(cleanup);
+ })
+ .catch(() => undefined);
+ return recovery;
+}
+
+function scheduleRetainedAcpxAdmissionCleanup(
+ cleanup: RetainedAcpxAdmissionCleanup,
+): void {
+ if (
+ !retainedAcpxAdmissionCleanups.has(cleanup) ||
+ cleanup.recovery ||
+ cleanup.timer
+ ) {
+ return;
+ }
+ cleanup.timer = setTimeout(() => {
+ cleanup.timer = null;
+ if (!retainedAcpxAdmissionCleanups.has(cleanup)) return;
+ startRetainedAcpxAdmissionCleanup(cleanup);
+ }, ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS);
+ cleanup.timer.unref?.();
+}
+
+async function waitForAdmissionCleanupRetry(delayMs: number): Promise<void> {
+ await new Promise<void>((resolve) => {
+ const timer = setTimeout(resolve, delayMs);
+ timer.unref?.();
+ });
+}
+
async function boundedCancellation(
cancellation: Promise<void>,
): Promise<unknown | null> {
@@ -547,7 +774,7 @@ async function boundedCancellation(
"ACPX turn cancellation exceeded its shutdown timeout",
),
}),
- TURN_CANCELLATION_TIMEOUT_MS,
+ ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS,
);
}),
]);
diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts
index 60c04b33ab..3cb1e0521e 100644
--- a/packages/paperclip-runner/src/native-session-runtime.test.ts
+++ b/packages/paperclip-runner/src/native-session-runtime.test.ts
@@ -40,7 +40,9 @@ const result: PrpStructuredRunResult = {
completionClaim: {
contractRevision: "1",
objectiveSatisfied: true,
- criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }],
+ criteria: [
+ { criterionId: "objective", status: "satisfied", evidenceRefs: [] },
+ ],
remainingWork: [],
},
evidence: [],
@@ -63,8 +65,16 @@ const yieldedResult: PrpStructuredRunResult = {
completionClaim: {
contractRevision: "1",
objectiveSatisfied: false,
- criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }],
- remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }],
+ criteria: [
+ {
+ criterionId: "objective",
+ status: "unknown",
+ evidenceRefs: ["interaction:pending"],
+ },
+ ],
+ remainingWork: [
+ { description: "Resume after the response.", blocksCompletion: true },
+ ],
},
evidence: [{ ref: "interaction:pending" }],
verification: [],
@@ -93,8 +103,17 @@ const input: NativeExecutionInputV1 = {
prompt: "# PAP-RECOVERY: Recover native work",
workMode: "standard",
},
- workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
- session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 },
+ workspace: {
+ cwd: "/workspace",
+ repoUrl: null,
+ repoRef: null,
+ branchName: null,
+ },
+ session: {
+ normalizedSessionId: identity.sessionId,
+ driverKind: "codex_app_server",
+ protocolVersion: 1,
+ },
provider: { kind: "codex", model: null },
completionContract: {
id: "contract-recovery",
@@ -154,24 +173,958 @@ function runnerEvent(
};
}
-function highestContiguous(events: PrpEvent[]): number {
- const sequences = new Set(events.map((event) => event.sourceSeq));
- let cursor = 0;
- while (sequences.has(cursor + 1)) cursor += 1;
- return cursor;
-}
+function highestContiguous(events: PrpEvent[]): number {
+ const sequences = new Set(events.map((event) => event.sourceSeq));
+ let cursor = 0;
+ while (sequences.has(cursor + 1)) cursor += 1;
+ return cursor;
+}
+
+describe("executeNativeSession recovery", () => {
+ it("keeps governed-wait discovery synchronous", () => {
+ type GovernedWaitResolver = NonNullable<
+ ExecuteNativeSessionOptions["resolveGovernedWait"]
+ >;
+ const resolver: GovernedWaitResolver = () => null;
+ // An async resolver could retain control-plane mutation authority after
+ // execution settles, so the public boundary rejects it at compile time.
+ // @ts-expect-error governed-wait discovery must not return a promise
+ const asynchronousResolver: GovernedWaitResolver = async () => null;
+
+ expect(resolver).toBeTypeOf("function");
+ void asynchronousResolver;
+ });
+
+ it("preserves durable success while quarantined cleanup stays bounded", async () => {
+ vi.useFakeTimers();
+ try {
+ let executionCloseCount = 0;
+ let quarantineAttempt = 0;
+ const close = vi.fn(({ reason }: { reason: string }) => {
+ if (reason === "native session quarantined cleanup recovery") {
+ quarantineAttempt += 1;
+ const attempt = quarantineAttempt;
+ return new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (attempt < 3)
+ reject(new Error("transient quarantine failure"));
+ else resolve();
+ }, 6_500);
+ });
+ }
+ if (reason === "native session execution complete") {
+ executionCloseCount += 1;
+ if (executionCloseCount > 1) return Promise.resolve();
+ }
+ return Promise.reject(new Error("persistent close failure"));
+ });
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ await expect(execute()).resolves.toMatchObject({ result });
+ expect(close).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(3_000);
+ expect(close).toHaveBeenCalledTimes(5);
+
+ // Admission inherits the already-running three-attempt recovery and
+ // waits through its bounded attempts instead of timing out after one.
+ const recoveredExecution = execute();
+ let admissionSettled = false;
+ void recoveredExecution.then(
+ () => {
+ admissionSettled = true;
+ },
+ () => {
+ admissionSettled = true;
+ },
+ );
+ await vi.advanceTimersByTimeAsync(20_000);
+ expect(admissionSettled).toBe(false);
+ expect(openSession).toHaveBeenCalledTimes(1);
+ expect(close).toHaveBeenCalledTimes(7);
+ await vi.advanceTimersByTimeAsync(2_000);
+ await expect(recoveredExecution).resolves.toMatchObject({ result });
+ expect(quarantineAttempt).toBe(3);
+ expect(close).toHaveBeenCalledTimes(8);
+ expect(openSession).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("awaits the quarantine owner that replaces an exhausted close recovery", async () => {
+ vi.useFakeTimers();
+ try {
+ let executionCloseCount = 0;
+ const close = vi.fn(({ reason }: { reason: string }) => {
+ if (reason === "native session execution complete") {
+ executionCloseCount += 1;
+ return executionCloseCount === 1
+ ? Promise.reject(new Error("initial close failed"))
+ : Promise.resolve();
+ }
+ if (
+ reason.startsWith(
+ "native session cleanup recovery after close failure",
+ )
+ ) {
+ return Promise.reject(new Error("bounded close recovery failed"));
+ }
+ if (reason === "native session quarantined cleanup recovery") {
+ return new Promise<void>((resolve) => setTimeout(resolve, 50));
+ }
+ return Promise.resolve();
+ });
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ await expect(execute()).resolves.toMatchObject({ result });
+ const admitted = execute();
+ await vi.advanceTimersByTimeAsync(3_100);
+ await expect(admitted).resolves.toMatchObject({ result });
+ expect(openSession).toHaveBeenCalledTimes(2);
+ expect(close).toHaveBeenCalledWith({
+ reason: "native session quarantined cleanup recovery",
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("covers maximum-duration retained and replacement recovery phases", async () => {
+ vi.useFakeTimers();
+ try {
+ let executionCloseCount = 0;
+ let retainedRecoveryAttempt = 0;
+ let quarantineRecoveryAttempt = 0;
+ const close = vi.fn(({ reason }: { reason: string }) => {
+ if (reason === "native session execution complete") {
+ executionCloseCount += 1;
+ if (executionCloseCount > 1) return Promise.resolve();
+ return new Promise<void>((_resolve, reject) => {
+ setTimeout(() => reject(new Error("initial close failed")), 6_900);
+ });
+ }
+ if (
+ reason.startsWith(
+ "native session cleanup recovery after close failure",
+ )
+ ) {
+ retainedRecoveryAttempt += 1;
+ return new Promise<void>((_resolve, reject) => {
+ setTimeout(
+ () => reject(new Error("bounded retained recovery failed")),
+ 6_900,
+ );
+ });
+ }
+ if (reason === "native session quarantined cleanup recovery") {
+ quarantineRecoveryAttempt += 1;
+ const attempt = quarantineRecoveryAttempt;
+ return new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (attempt < 3)
+ reject(new Error("transient quarantine recovery failure"));
+ else resolve();
+ }, 6_500);
+ });
+ }
+ return Promise.resolve();
+ });
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ const firstExecution = execute();
+ await vi.advanceTimersByTimeAsync(100);
+ await expect(firstExecution).resolves.toMatchObject({ result });
+ const admitted = execute();
+ let admissionSettled = false;
+ void admitted.then(
+ () => {
+ admissionSettled = true;
+ },
+ () => {
+ admissionSettled = true;
+ },
+ );
+
+ // The initial close plus three near-bound retries exceed the old 23s
+ // admission grace but remain within the configured 31s owner phase.
+ await vi.advanceTimersByTimeAsync(23_100);
+ expect(retainedRecoveryAttempt).toBe(2);
+ expect(quarantineRecoveryAttempt).toBe(0);
+ expect(admissionSettled).toBe(false);
+ expect(openSession).toHaveBeenCalledOnce();
+
+ await vi.advanceTimersByTimeAsync(7_600);
+ expect(retainedRecoveryAttempt).toBe(3);
+ expect(quarantineRecoveryAttempt).toBe(1);
+ expect(admissionSettled).toBe(false);
+ expect(openSession).toHaveBeenCalledOnce();
+
+ // The replacement then receives its complete three-attempt bound rather
+ // than inheriting only the remainder of the retained owner's deadline.
+ await vi.advanceTimersByTimeAsync(21_600);
+ await expect(admitted).resolves.toMatchObject({ result });
+ expect(quarantineRecoveryAttempt).toBe(3);
+ expect(openSession).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("runs a full admission batch after an inherited scheduled attempt fails", async () => {
+ vi.useFakeTimers();
+ try {
+ let executionCloseCount = 0;
+ let scheduledRecoveryAttempt = 0;
+ let admissionRecoveryAttempt = 0;
+ const close = vi.fn(({ reason }: { reason: string }) => {
+ if (reason === "native session execution complete") {
+ executionCloseCount += 1;
+ return executionCloseCount === 1
+ ? Promise.reject(new Error("initial close failed"))
+ : Promise.resolve();
+ }
+ if (
+ reason.startsWith(
+ "native session cleanup recovery after close failure",
+ )
+ ) {
+ return Promise.reject(new Error("bounded retained recovery failed"));
+ }
+ if (reason === "native session quarantined cleanup recovery") {
+ return Promise.reject(
+ new Error("bounded quarantine recovery failed"),
+ );
+ }
+ if (
+ reason === "native session scheduled quarantined cleanup recovery"
+ ) {
+ scheduledRecoveryAttempt += 1;
+ return new Promise<void>((_resolve, reject) => {
+ setTimeout(
+ () => reject(new Error("scheduled cleanup failed")),
+ 6_500,
+ );
+ });
+ }
+ if (reason === "native session quarantined admission recovery") {
+ admissionRecoveryAttempt += 1;
+ const attempt = admissionRecoveryAttempt;
+ return new Promise<void>((resolve, reject) => {
+ setTimeout(() => {
+ if (attempt === 1)
+ reject(new Error("transient admission cleanup failure"));
+ else resolve();
+ }, 6_500);
+ });
+ }
+ return Promise.resolve();
+ });
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ await expect(execute()).resolves.toMatchObject({ result });
+ // Exhaust retained and initial quarantine batches, then enter the slow
+ // autonomous one-attempt recovery scheduled sixty seconds later.
+ await vi.advanceTimersByTimeAsync(65_001);
+ expect(scheduledRecoveryAttempt).toBe(1);
+
+ const admitted = execute();
+ let admissionSettled = false;
+ void admitted.then(
+ () => {
+ admissionSettled = true;
+ },
+ () => {
+ admissionSettled = true;
+ },
+ );
+ await vi.advanceTimersByTimeAsync(6_600);
+ expect(admissionRecoveryAttempt).toBe(1);
+ expect(admissionSettled).toBe(false);
+ expect(openSession).toHaveBeenCalledOnce();
+
+ await vi.advanceTimersByTimeAsync(14_100);
+ await expect(admitted).resolves.toMatchObject({ result });
+ expect(admissionRecoveryAttempt).toBe(2);
+ expect(openSession).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("stops scheduled cleanup after the quarantine lifetime budget", async () => {
+ vi.useFakeTimers();
+ try {
+ let executionCloseCount = 0;
+ let admissionAttempt = 0;
+ let scheduledAttempt = 0;
+ const lifetimeIdentity = {
+ ...identity,
+ companyId: "company-cleanup-lifetime",
+ };
+ const close = vi.fn(({ reason }: { reason: string }) => {
+ if (reason === "native session execution complete") {
+ executionCloseCount += 1;
+ return executionCloseCount === 1
+ ? Promise.reject(new Error("initial close failed"))
+ : Promise.resolve();
+ }
+ if (
+ reason.startsWith(
+ "native session cleanup recovery after close failure",
+ )
+ ) {
+ return Promise.reject(new Error("bounded close recovery failed"));
+ }
+ if (reason === "native session quarantined cleanup recovery") {
+ return Promise.reject(new Error("quarantined close recovery failed"));
+ }
+ if (reason === "native session quarantined admission recovery") {
+ admissionAttempt += 1;
+ return Promise.reject(new Error("admission cleanup failed"));
+ }
+ if (
+ reason === "native session scheduled quarantined cleanup recovery"
+ ) {
+ scheduledAttempt += 1;
+ return Promise.reject(new Error("scheduled cleanup failed"));
+ }
+ return Promise.resolve();
+ });
+ const session: NativeSession = {
+ identity: () => lifetimeIdentity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity: lifetimeIdentity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input: {
+ ...input,
+ binding: {
+ ...input.binding,
+ companyId: lifetimeIdentity.companyId,
+ },
+ },
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ await expect(execute()).resolves.toMatchObject({ result });
+ await vi.advanceTimersByTimeAsync(6_000);
+
+ const recoveredExecution = expect(execute()).rejects.toThrow(
+ "prior session cleanup remains incomplete",
+ );
+ await vi.advanceTimersByTimeAsync(2_100);
+ await recoveredExecution;
+
+ expect(admissionAttempt).toBe(3);
+ expect(scheduledAttempt).toBe(0);
+ await vi.advanceTimersByTimeAsync(180_000);
+ expect(scheduledAttempt).toBe(3);
+ await vi.advanceTimersByTimeAsync(300_000);
+ expect(scheduledAttempt).toBe(3);
+ expect(openSession).toHaveBeenCalledOnce();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("fails admission within a bound when quarantined close never settles", async () => {
+ vi.useFakeTimers();
+ try {
+ let releaseBlockedClose = () => {};
+ let blockClose = false;
+ const close = vi.fn(() => {
+ if (blockClose) {
+ return new Promise<void>((resolve) => {
+ releaseBlockedClose = resolve;
+ });
+ }
+ return Promise.reject(new Error("persistent close failure"));
+ });
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ await expect(execute()).resolves.toMatchObject({ result });
+ await vi.advanceTimersByTimeAsync(6_000);
+ blockClose = true;
+ const blockedAdmission = execute();
+ const blockedResult = expect(blockedAdmission).rejects.toThrow(
+ "prior session cleanup exceeded the admission grace",
+ );
+ await vi.advanceTimersByTimeAsync(31_100);
+ await blockedResult;
+ expect(openSession).toHaveBeenCalledOnce();
+
+ const isolatedSession: NativeSession = {
+ ...session,
+ identity: () => ({ ...identity, companyId: "company-isolated" }),
+ close: vi.fn(async () => undefined),
+ };
+ const isolatedOpenSession = vi.fn(async () => isolatedSession);
+ await expect(
+ executeNativeSession({
+ input: {
+ ...input,
+ binding: {
+ ...input.binding,
+ companyId: "company-isolated",
+ },
+ },
+ backend: { ...backend, openSession: isolatedOpenSession },
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({ result });
+ expect(isolatedOpenSession).toHaveBeenCalledOnce();
+
+ blockClose = false;
+ releaseBlockedClose();
+ close.mockResolvedValue(undefined);
+ await vi.runAllTimersAsync();
+ await expect(execute()).resolves.toMatchObject({ result });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("quarantines a pending first close before another provider session can open", async () => {
+ vi.useFakeTimers();
+ try {
+ let releaseClose = () => {};
+ const pendingClose = new Promise<void>((resolve) => {
+ releaseClose = resolve;
+ });
+ const close = vi.fn(() => pendingClose);
+ const session: NativeSession = {
+ identity: () => identity,
+ async capabilities() {
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return {
+ backendKind: "mock",
+ sessionId: "driver-recovery",
+ identity,
+ providerSessionId: "provider-recovery",
+ cursor: null,
+ activeTurnId: null,
+ pendingRuntimeRequests: [],
+ lineage: [],
+ };
+ },
+ close,
+ };
+ const openSession = vi.fn(async () => session);
+ const backend: NativeSessionBackend = {
+ async descriptor() {
+ return {
+ kind: "mock",
+ name: "recovery-backend",
+ version: "1",
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
+ };
+ },
+ openSession,
+ };
+ const port: ControlPlanePort = {
+ async openRun() {},
+ async checkpointSession() {},
+ async appendEvent() {
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
+ async completeRun() {},
+ };
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ });
+
+ const firstExecution = execute();
+ await vi.advanceTimersByTimeAsync(100);
+ await expect(firstExecution).resolves.toMatchObject({ result });
+ expect(close).toHaveBeenCalledOnce();
-describe("executeNativeSession recovery", () => {
- it("keeps governed-wait discovery synchronous", () => {
- type GovernedWaitResolver = NonNullable<ExecuteNativeSessionOptions["resolveGovernedWait"]>;
- const resolver: GovernedWaitResolver = () => null;
- // An async resolver could retain control-plane mutation authority after
- // execution settles, so the public boundary rejects it at compile time.
- // @ts-expect-error governed-wait discovery must not return a promise
- const asynchronousResolver: GovernedWaitResolver = async () => null;
+ const blockedAdmission = execute();
+ const blockedResult = expect(blockedAdmission).rejects.toThrow(
+ "prior session cleanup exceeded the admission grace",
+ );
+ await vi.advanceTimersByTimeAsync(31_100);
+ await blockedResult;
+ expect(openSession).toHaveBeenCalledOnce();
+ expect(close).toHaveBeenCalledOnce();
- expect(resolver).toBeTypeOf("function");
- void asynchronousResolver;
+ releaseClose();
+ await vi.runAllTimersAsync();
+ } finally {
+ vi.useRealTimers();
+ }
});
it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => {
@@ -217,27 +1170,33 @@ describe("executeNativeSession recovery", () => {
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input: {
- ...input,
- schema: "paperclip.native-execution-input.v3",
- executionMode: "default",
- planningContext: null,
- runtimeContext: {
- ...context,
- aggregateDigest: canonicalNativeRuntimeContextDigest(context),
+ await expect(
+ executeNativeSession({
+ input: {
+ ...input,
+ schema: "paperclip.native-execution-input.v3",
+ executionMode: "default",
+ planningContext: null,
+ runtimeContext: {
+ ...context,
+ aggregateDigest: canonicalNativeRuntimeContextDigest(context),
+ },
},
- },
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).rejects.toThrow("does not natively realize instructions, skills, mcp");
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).rejects.toThrow("does not natively realize instructions, skills, mcp");
expect(openSession).not.toHaveBeenCalled();
});
@@ -266,31 +1225,38 @@ describe("executeNativeSession recovery", () => {
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
async replayEvents() {
return { events: [], highestContiguousSourceSeq: 0 };
},
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).rejects.toBe(providerFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).rejects.toBe(providerFailure);
- expect(openSession).toHaveBeenCalledWith(expect.objectContaining({
- identity,
- workingDirectory: input.workspace.cwd,
- signal: expect.any(AbortSignal),
- }));
+ expect(openSession).toHaveBeenCalledWith(
+ expect.objectContaining({
+ identity,
+ workingDirectory: input.workspace.cwd,
+ signal: expect.any(AbortSignal),
+ }),
+ );
expect(openRun).not.toHaveBeenCalled();
});
- it("bounds fresh session bootstrap and closes a session returned after timeout", async () => {
+ it("retains late bootstrap cleanup through the next admission", async () => {
vi.useFakeTimers();
+ let releaseClose = () => {};
try {
let resolveBootstrap = (_value: NativeSession) => {};
const stalledBootstrap = new Promise<NativeSession>((resolve) => {
@@ -304,33 +1270,57 @@ describe("executeNativeSession recovery", () => {
const closeStarted = new Promise<void>((resolve) => {
markCloseStarted = resolve;
});
- const close = vi.fn(async () => { markCloseStarted(); });
+ const closeReleased = new Promise<void>((resolve) => {
+ releaseClose = resolve;
+ });
+ const close = vi.fn(async () => {
+ markCloseStarted();
+ await closeReleased;
+ });
const lateSession: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: false, typedEvents: true, steering: false, interruption: true };
+ return {
+ resume: false,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ };
},
async *events() {},
- async startTurn() { throw new Error("late fresh session must not start"); },
- async result() { return null; },
- async snapshot() { throw new Error("late fresh session must not snapshot"); },
+ async startTurn() {
+ throw new Error("late fresh session must not start");
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw new Error("late fresh session must not snapshot");
+ },
close,
};
let bootstrapSignal: AbortSignal | undefined;
- const openSession = vi.fn((bootstrapInput: {
- identity: NativeRunIdentity;
- workingDirectory?: string;
- signal?: AbortSignal;
- }) => {
- bootstrapSignal = bootstrapInput.signal;
- bootstrapInput.signal?.addEventListener(
- "abort",
- () => resolveBootstrap(lateSession),
- { once: true },
- );
- markBootstrapStarted();
- return stalledBootstrap;
- });
+ let bootstrapCount = 0;
+ const openSession = vi.fn(
+ (bootstrapInput: {
+ identity: NativeRunIdentity;
+ workingDirectory?: string;
+ signal?: AbortSignal;
+ }) => {
+ bootstrapCount += 1;
+ if (bootstrapCount > 1) {
+ throw new Error("replacement bootstrap launched");
+ }
+ bootstrapSignal = bootstrapInput.signal;
+ bootstrapInput.signal?.addEventListener(
+ "abort",
+ () => resolveBootstrap(lateSession),
+ { once: true },
+ );
+ markBootstrapStarted();
+ return stalledBootstrap;
+ },
+ );
const openRun = vi.fn(async () => undefined);
const backend: NativeSessionBackend = {
async descriptor() {
@@ -345,19 +1335,24 @@ describe("executeNativeSession recovery", () => {
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
-
- const execution = executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- timeoutMs: 5,
- });
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ timeoutMs: 5,
+ });
+ const execution = execute();
const rejection = expect(execution).rejects.toThrow(
"native session bootstrap timed out after 5ms",
);
@@ -373,7 +1368,25 @@ describe("executeNativeSession recovery", () => {
expect(close).toHaveBeenCalledWith({
reason: "native session bootstrap timed out",
});
+
+ // Even after the detached disposer exhausts its short settlement grace,
+ // the exact close remains admission-visible until it releases provider
+ // resources. A replacement bootstrap cannot start concurrently.
+ await vi.advanceTimersByTimeAsync(101);
+ const blockedAdmission = execute();
+ const blockedAdmissionRejection = expect(blockedAdmission).rejects.toThrow(
+ "replacement bootstrap launched",
+ );
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(openSession).toHaveBeenCalledOnce();
+
+ releaseClose();
+ await vi.advanceTimersByTimeAsync(0);
+ await blockedAdmissionRejection;
+ expect(openSession).toHaveBeenCalledTimes(2);
+ expect(openRun).not.toHaveBeenCalled();
} finally {
+ releaseClose();
vi.useRealTimers();
}
});
@@ -384,12 +1397,23 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: false, typedEvents: true, steering: false, interruption: true };
+ return {
+ resume: false,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ };
},
async *events() {},
- async startTurn() { throw new Error("unexpected turn"); },
- async result() { return null; },
- async snapshot() { throw snapshotFailure; },
+ async startTurn() {
+ throw new Error("unexpected turn");
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw snapshotFailure;
+ },
close,
};
const backend: NativeSessionBackend = {
@@ -401,29 +1425,37 @@ describe("executeNativeSession recovery", () => {
capabilities: await session.capabilities(),
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
const retainedSessions: Array<NativeSession | null> = [];
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- keepSessionOpen: true,
- onSession(current) {
- retainedSessions.push(current);
- if (current === null) throw new Error("owner notification failed");
- },
- })).rejects.toBe(snapshotFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ keepSessionOpen: true,
+ onSession(current) {
+ retainedSessions.push(current);
+ if (current === null) throw new Error("owner notification failed");
+ },
+ }),
+ ).rejects.toBe(snapshotFailure);
expect(retainedSessions).toEqual([session, null]);
expect(close).toHaveBeenCalledOnce();
@@ -445,14 +1477,13 @@ describe("executeNativeSession recovery", () => {
});
let checkpointSignal: AbortSignal | undefined;
let controlPlaneCheckpointCount = 0;
- const checkpointSession: NonNullable<ControlPlanePort["checkpointSession"]> = async (
- _snapshot,
- checkpointOptions,
- ) => {
+ const checkpointSession: NonNullable<
+ ControlPlanePort["checkpointSession"]
+ > = async (_snapshot, checkpointOptions) => {
controlPlaneCheckpointCount += 1;
if (
- stalledBoundary === "control-plane checkpoint"
- && controlPlaneCheckpointCount === 2
+ stalledBoundary === "control-plane checkpoint" &&
+ controlPlaneCheckpointCount === 2
) {
checkpointSignal = checkpointOptions?.signal;
markCheckpointStalled();
@@ -460,21 +1491,22 @@ describe("executeNativeSession recovery", () => {
}
};
let ownerCheckpointCount = 0;
- const onCheckpoint: NonNullable<ExecuteNativeSessionOptions["onCheckpoint"]> = async (
- _snapshot,
- checkpointOptions,
- ) => {
+ const onCheckpoint: NonNullable<
+ ExecuteNativeSessionOptions["onCheckpoint"]
+ > = async (_snapshot, checkpointOptions) => {
ownerCheckpointCount += 1;
if (
- stalledBoundary === "owner checkpoint"
- && ownerCheckpointCount === 2
+ stalledBoundary === "owner checkpoint" &&
+ ownerCheckpointCount === 2
) {
checkpointSignal = checkpointOptions?.signal;
markCheckpointStalled();
await never;
}
};
- const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled" }));
+ const startTurn = vi.fn(async () => ({
+ turnId: "turn-checkpoint-stalled",
+ }));
const close = vi.fn(async () => {
releaseStream();
});
@@ -493,7 +1525,9 @@ describe("executeNativeSession recovery", () => {
await streamReleased;
},
startTurn,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -517,13 +1551,17 @@ describe("executeNativeSession recovery", () => {
capabilities: await session.capabilities(),
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const openRun = vi.fn(async () => undefined);
const port: ControlPlanePort = {
openRun,
checkpointSession,
- async appendEvent() { throw new Error("unexpected event"); },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
async replayEvents() {
return { events: [], highestContiguousSourceSeq: 0 };
},
@@ -596,8 +1634,12 @@ describe("executeNativeSession recovery", () => {
structuredResult: true,
};
},
- async *events() { yield runnerEvent(1, "turn.completed"); },
- async startTurn() { return { turnId: "turn-recovery" }; },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
async result() {
resultCalls += 1;
if (stalledBoundary === "provider result") {
@@ -630,7 +1672,9 @@ describe("executeNativeSession recovery", () => {
capabilities: await session.capabilities(),
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -644,8 +1688,8 @@ describe("executeNativeSession recovery", () => {
},
async appendEvent(event, operationOptions) {
if (
- stalledBoundary === "final event append"
- && (event as PrpEvent).sourceKind === "control_plane"
+ stalledBoundary === "final event append" &&
+ (event as PrpEvent).sourceKind === "control_plane"
) {
stalledSignal = operationOptions?.signal;
markFinalizationStalled();
@@ -711,10 +1755,7 @@ describe("executeNativeSession recovery", () => {
},
);
- it.each([
- "final event append",
- "run completion",
- ] as const)(
+ it.each(["final event append", "run completion"] as const)(
"confirms durable completion when %s commits before its acknowledgement stalls",
async (stalledBoundary) => {
vi.useFakeTimers();
@@ -739,9 +1780,15 @@ describe("executeNativeSession recovery", () => {
structuredResult: true,
};
},
- async *events() { yield runnerEvent(1, "turn.completed"); },
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -765,22 +1812,25 @@ describe("executeNativeSession recovery", () => {
capabilities: await session.capabilities(),
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
async appendEvent(event, operationOptions) {
const appended = structuredClone(event as PrpEvent);
- const existing = events.find((candidate) =>
- candidate.sourceInstanceId === appended.sourceInstanceId
- && candidate.sourceSeq === appended.sourceSeq
+ const existing = events.find(
+ (candidate) =>
+ candidate.sourceInstanceId === appended.sourceInstanceId &&
+ candidate.sourceSeq === appended.sourceSeq,
);
if (existing === undefined) events.push(appended);
if (
- stalledBoundary === "final event append"
- && appended.sourceKind === "control_plane"
- && !stalledOnce
+ stalledBoundary === "final event append" &&
+ appended.sourceKind === "control_plane" &&
+ !stalledOnce
) {
stalledOnce = true;
stalledSignal = operationOptions?.signal;
@@ -788,7 +1838,8 @@ describe("executeNativeSession recovery", () => {
return await never;
}
const sourceEvents = events.filter(
- (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId,
+ (candidate) =>
+ candidate.sourceInstanceId === appended.sourceInstanceId,
);
return {
cursor: events.length,
@@ -801,9 +1852,11 @@ describe("executeNativeSession recovery", () => {
(event) => event.sourceInstanceId === replay.sourceInstanceId,
);
return {
- events: structuredClone(sourceEvents.filter(
- (event) => event.sourceSeq > replay.afterSourceSeq,
- )),
+ events: structuredClone(
+ sourceEvents.filter(
+ (event) => event.sourceSeq > replay.afterSourceSeq,
+ ),
+ ),
highestContiguousSourceSeq: highestContiguous(sourceEvents),
};
},
@@ -842,8 +1895,9 @@ describe("executeNativeSession recovery", () => {
});
expect(stalledSignal?.aborted).toBe(true);
expect(durableCompletion).toMatchObject({ result, terminal });
- expect(events.filter((event) => event.sourceKind === "control_plane"))
- .toHaveLength(2);
+ expect(
+ events.filter((event) => event.sourceKind === "control_plane"),
+ ).toHaveLength(2);
} finally {
vi.useRealTimers();
}
@@ -878,9 +1932,15 @@ describe("executeNativeSession recovery", () => {
structuredResult: true,
};
},
- async *events() { yield runnerEvent(1, "turn.completed"); },
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
async snapshot(snapshotOptions) {
if (stalledBoundary === "provider snapshot" && runCompleted) {
stalledSignal = snapshotOptions?.signal;
@@ -916,7 +1976,9 @@ describe("executeNativeSession recovery", () => {
capabilities: await session.capabilities(),
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const completeRun = vi.fn(async () => {
runCompleted = true;
@@ -924,7 +1986,10 @@ describe("executeNativeSession recovery", () => {
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession(_snapshot, operationOptions) {
- if (stalledBoundary === "post-completion checkpoint" && runCompleted) {
+ if (
+ stalledBoundary === "post-completion checkpoint" &&
+ runCompleted
+ ) {
stalledSignal = operationOptions?.signal;
markEnrichmentStalled();
await never;
@@ -955,7 +2020,8 @@ describe("executeNativeSession recovery", () => {
onSession: (current) => retainedSessions.push(current),
});
await enrichmentStalled;
- if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false);
+ if (stalledSignal !== undefined)
+ expect(stalledSignal.aborted).toBe(false);
await vi.advanceTimersByTimeAsync(10);
@@ -966,7 +2032,8 @@ describe("executeNativeSession recovery", () => {
driverVersion: "1",
usage: null,
});
- if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true);
+ if (stalledSignal !== undefined)
+ expect(stalledSignal.aborted).toBe(true);
expect(completeRun).toHaveBeenCalledOnce();
expect(close).toHaveBeenCalledOnce();
expect(retainedSessions).toEqual([session, null]);
@@ -978,22 +2045,36 @@ describe("executeNativeSession recovery", () => {
it("contains a consumer rejection when starting the turn fails first", async () => {
let markAppendStarted = () => {};
- const appendStarted = new Promise<void>((resolve) => { markAppendStarted = resolve; });
+ const appendStarted = new Promise<void>((resolve) => {
+ markAppendStarted = resolve;
+ });
let releaseAppend = () => {};
- const appendReleased = new Promise<void>((resolve) => { releaseAppend = resolve; });
+ const appendReleased = new Promise<void>((resolve) => {
+ releaseAppend = resolve;
+ });
let appendCommitted = false;
const close = vi.fn(async () => undefined);
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.started");
},
- async *events() { yield runnerEvent(1, "turn.started"); },
async startTurn() {
await appendStarted;
throw new Error("start turn failed");
},
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1014,10 +2095,18 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -1027,9 +2116,13 @@ describe("executeNativeSession recovery", () => {
await Promise.race([
appendReleased,
new Promise<never>((_resolve, reject) => {
- const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted"));
+ const rejectAbort = () =>
+ reject(options?.signal.reason ?? new Error("append aborted"));
if (options?.signal.aborted) rejectAbort();
- else options?.signal.addEventListener("abort", rejectAbort, { once: true });
+ else
+ options?.signal.addEventListener("abort", rejectAbort, {
+ once: true,
+ });
}),
]);
appendCommitted = true;
@@ -1039,7 +2132,9 @@ describe("executeNativeSession recovery", () => {
disposition: "committed" as const,
};
},
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -1061,42 +2156,59 @@ describe("executeNativeSession recovery", () => {
it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => {
let markAppendStarted = () => {};
- const appendStarted = new Promise<void>((resolve) => { markAppendStarted = resolve; });
+ const appendStarted = new Promise<void>((resolve) => {
+ markAppendStarted = resolve;
+ });
let releaseAppend = () => {};
- const appendReleased = new Promise<void>((resolve) => { releaseAppend = resolve; });
+ const appendReleased = new Promise<void>((resolve) => {
+ releaseAppend = resolve;
+ });
let releaseTeardown = () => {};
- const teardownReleased = new Promise<void>((resolve) => { releaseTeardown = resolve; });
- const iteratorTeardown = vi.fn();
- let appendCommitted = false;
- const appendEvent = vi.fn(async (
- _event: PrpEvent,
- options?: { signal: AbortSignal },
- ) => {
- markAppendStarted();
- await Promise.race([
- appendReleased,
- new Promise<never>((_resolve, reject) => {
- const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted"));
- if (options?.signal.aborted) rejectAbort();
- else options?.signal.addEventListener("abort", rejectAbort, { once: true });
- }),
- ]);
- appendCommitted = true;
- return {
- cursor: 1,
- highestContiguousSourceSeq: 1,
- disposition: "committed" as const,
- };
+ const teardownReleased = new Promise<void>((resolve) => {
+ releaseTeardown = resolve;
});
+ const iteratorTeardown = vi.fn();
+ let appendCommitted = false;
+ const appendEvent = vi.fn(
+ async (_event: PrpEvent, options?: { signal: AbortSignal }) => {
+ markAppendStarted();
+ await Promise.race([
+ appendReleased,
+ new Promise<never>((_resolve, reject) => {
+ const rejectAbort = () =>
+ reject(options?.signal.reason ?? new Error("append aborted"));
+ if (options?.signal.aborted) rejectAbort();
+ else
+ options?.signal.addEventListener("abort", rejectAbort, {
+ once: true,
+ });
+ }),
+ ]);
+ appendCommitted = true;
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed" as const,
+ };
+ },
+ );
const cancel = vi.fn(() => {
releaseTeardown();
return { cleanup: Promise.resolve() };
});
- const close = vi.fn(async () => { releaseTeardown(); });
+ const close = vi.fn(async () => {
+ releaseTeardown();
+ });
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {
try {
@@ -1106,9 +2218,13 @@ describe("executeNativeSession recovery", () => {
await teardownReleased;
}
},
- async startTurn() { return { turnId: "turn-recovery" }; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
cancel,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1129,16 +2245,26 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
appendEvent,
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -1151,7 +2277,9 @@ describe("executeNativeSession recovery", () => {
timeoutMs: 1,
keepSessionOpen: true,
});
- const rejection = expect(execution).rejects.toThrow("native session timed out");
+ const rejection = expect(execution).rejects.toThrow(
+ "native session timed out",
+ );
await appendStarted;
await vi.waitFor(() => expect(iteratorTeardown).toHaveBeenCalledOnce());
await rejection;
@@ -1166,19 +2294,33 @@ describe("executeNativeSession recovery", () => {
it("closes a failed session while retaining an uncancellable event read", async () => {
let releaseStream = () => {};
- const streamReleased = new Promise<void>((resolve) => { releaseStream = resolve; });
- const close = vi.fn(async () => { releaseStream(); });
+ const streamReleased = new Promise<void>((resolve) => {
+ releaseStream = resolve;
+ });
+ const close = vi.fn(async () => {
+ releaseStream();
+ });
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ };
},
async *events() {
await streamReleased;
yield runnerEvent(1, "turn.completed");
},
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return null; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1199,56 +2341,86 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- timeoutMs: 1,
- keepSessionOpen: true,
- })).rejects.toThrow("native session timed out");
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ timeoutMs: 1,
+ keepSessionOpen: true,
+ }),
+ ).rejects.toThrow("native session timed out");
expect(close).toHaveBeenCalledOnce();
});
it("commits cancellation before bounding failed provider cleanup", async () => {
let releaseStream = () => {};
- const streamReleased = new Promise<void>((resolve) => { releaseStream = resolve; });
+ const streamReleased = new Promise<void>((resolve) => {
+ releaseStream = resolve;
+ });
let releaseCancellation = () => {};
- const cancellationReleased = new Promise<void>((resolve) => { releaseCancellation = resolve; });
+ const cancellationReleased = new Promise<void>((resolve) => {
+ releaseCancellation = resolve;
+ });
const interrupt = vi.fn(() => cancellationReleased);
let cancellationCommitted = false;
const cancel = vi.fn(() => {
cancellationCommitted = true;
return { cleanup: cancellationReleased };
});
- const close = vi.fn(async () => { releaseStream(); });
+ const close = vi.fn(async () => {
+ releaseStream();
+ });
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {
await streamReleased;
yield runnerEvent(1, "turn.completed");
},
- async startTurn() { return { turnId: "turn-recovery" }; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
interrupt,
cancel,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1269,16 +2441,28 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -1301,18 +2485,32 @@ describe("executeNativeSession recovery", () => {
it("bounds failure when iterator teardown and provider close never settle", async () => {
const never = new Promise<void>(() => undefined);
- const close = vi.fn(() => never);
+ let releaseClose = () => {};
+ const pendingClose = new Promise<void>((resolve) => {
+ releaseClose = resolve;
+ });
+ const close = vi.fn(() => pendingClose);
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ };
},
async *events() {
await never;
yield runnerEvent(1, "turn.completed");
},
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return null; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1333,43 +2531,74 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- timeoutMs: 1,
- keepSessionOpen: true,
- })).rejects.toThrow("native session timed out");
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ timeoutMs: 1,
+ keepSessionOpen: true,
+ }),
+ ).rejects.toThrow("native session timed out");
expect(close).toHaveBeenCalledOnce();
+ releaseClose();
+ await pendingClose;
});
it("preserves durable success when provider close never settles", async () => {
- const never = new Promise<void>(() => undefined);
- const close = vi.fn(() => never);
+ let releaseClose = () => {};
+ const pendingClose = new Promise<void>((resolve) => {
+ releaseClose = resolve;
+ });
+ const close = vi.fn(() => pendingClose);
const completeRun = vi.fn(async () => undefined);
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
},
- async *events() { yield runnerEvent(1, "turn.completed"); },
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
async snapshot() {
return {
backendKind: "mock",
@@ -1390,10 +2619,18 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: false,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const events: PrpEvent[] = [];
const port: ControlPlanePort = {
@@ -1407,35 +2644,53 @@ describe("executeNativeSession recovery", () => {
disposition: "committed",
};
},
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
completeRun,
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).resolves.toMatchObject({ result, terminal });
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({ result, terminal });
expect(completeRun).toHaveBeenCalledOnce();
expect(close).toHaveBeenCalledOnce();
+ releaseClose();
+ await pendingClose;
});
it("closes after a synchronous governed-wait probe returns no result", async () => {
const resolveGovernedWait = vi.fn(() => null);
const lifecycle: string[] = [];
- const close = vi.fn(async () => { lifecycle.push("closed"); });
+ const close = vi.fn(async () => {
+ lifecycle.push("closed");
+ });
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {
yield runnerEvent(1, "item.completed");
},
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return null; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1456,18 +2711,32 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
async appendEvent() {
- return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" };
+ return {
+ cursor: 1,
+ highestContiguousSourceSeq: 1,
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
},
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
async completeRun() {},
};
@@ -1494,20 +2763,32 @@ describe("executeNativeSession recovery", () => {
lifecycle.push("cancelled");
return { cleanup: new Promise<void>(() => undefined) };
});
- const close = vi.fn(async () => { lifecycle.push("closed"); });
+ const close = vi.fn(async () => {
+ lifecycle.push("closed");
+ });
const events: PrpEvent[] = [];
const retainedSessions: Array<NativeSession | null> = [];
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {
yield runnerEvent(1, "item.completed");
},
- async startTurn() { return { turnId: "turn-recovery" }; },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
cancel,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -1528,10 +2809,18 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -1544,7 +2833,9 @@ describe("executeNativeSession recovery", () => {
disposition: "committed",
};
},
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -1582,7 +2873,13 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
openSession,
@@ -1597,18 +2894,24 @@ describe("executeNativeSession recovery", () => {
};
},
checkpointSession,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).rejects.toThrow("native_session_checkpoint_binding_mismatch");
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).rejects.toThrow("native_session_checkpoint_binding_mismatch");
expect(openRun).not.toHaveBeenCalled();
expect(checkpointSession).not.toHaveBeenCalled();
expect(openSession).not.toHaveBeenCalled();
@@ -1620,13 +2923,25 @@ describe("executeNativeSession recovery", () => {
const existingSession: NativeSession = {
identity: () => ({ ...identity, companyId: "other-company" }),
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
attachRun,
async *events() {},
- async startTurn() { return { turnId: "unexpected" }; },
- async result() { return null; },
- async snapshot() { throw new Error("unexpected snapshot"); },
+ async startTurn() {
+ return { turnId: "unexpected" };
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw new Error("unexpected snapshot");
+ },
async close() {},
};
const backend: NativeSessionBackend = {
@@ -1635,26 +2950,40 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "existing-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected open"); },
+ async openSession() {
+ throw new Error("unexpected open");
+ },
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- existingSession,
- })).rejects.toThrow("native_session_attach_binding_mismatch");
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ existingSession,
+ }),
+ ).rejects.toThrow("native_session_attach_binding_mismatch");
expect(openRun).not.toHaveBeenCalled();
expect(attachRun).not.toHaveBeenCalled();
});
@@ -1673,42 +3002,66 @@ describe("executeNativeSession recovery", () => {
const existingSession: NativeSession = {
identity: () => structuredClone(retainedIdentity),
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
attachRun,
async *events() {},
startTurn,
- async result() { return null; },
- async snapshot() { throw new Error("unexpected snapshot"); },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw new Error("unexpected snapshot");
+ },
close,
};
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
- name: "existing-backend",
+ name: "attachment-failure-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected open"); },
+ async openSession() {
+ throw new Error("unexpected open");
+ },
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- existingSession,
- onSession,
- })).rejects.toBe(attachmentFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ existingSession,
+ onSession,
+ }),
+ ).rejects.toBe(attachmentFailure);
expect(attachRun).toHaveBeenCalledWith({ identity });
expect(retainedIdentity).toEqual(identity);
expect(onSession).toHaveBeenCalledOnce();
@@ -1732,42 +3085,66 @@ describe("executeNativeSession recovery", () => {
const existingSession: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
attachRun,
async *events() {},
startTurn,
- async result() { return null; },
- async snapshot() { throw new Error("unexpected snapshot"); },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw new Error("unexpected snapshot");
+ },
close,
};
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
- name: "existing-backend",
+ name: "control-plane-admission-failure-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected open"); },
+ async openSession() {
+ throw new Error("unexpected open");
+ },
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- existingSession,
- onSession,
- })).rejects.toBe(admissionFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ existingSession,
+ onSession,
+ }),
+ ).rejects.toBe(admissionFailure);
expect(attachRun).toHaveBeenCalledWith({ identity });
expect(openRun).toHaveBeenCalledOnce();
expect(onSession).toHaveBeenCalledOnce();
@@ -1787,7 +3164,13 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
openSession,
@@ -1801,21 +3184,27 @@ describe("executeNativeSession recovery", () => {
identity: { ...identity, sessionId: "other-session" },
};
},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
- await expect(executeNativeSession({
- input: {
- ...input,
- session: { ...input.session, normalizedSessionId: null },
- },
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).rejects.toThrow("native_session_checkpoint_binding_mismatch");
+ await expect(
+ executeNativeSession({
+ input: {
+ ...input,
+ session: { ...input.session, normalizedSessionId: null },
+ },
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).rejects.toThrow("native_session_checkpoint_binding_mismatch");
expect(openRun).not.toHaveBeenCalled();
expect(openSession).not.toHaveBeenCalled();
});
@@ -1838,14 +3227,22 @@ describe("executeNativeSession recovery", () => {
recovered: false as const,
reason: "provider session no longer exists",
}));
- const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); });
+ const openSession = vi.fn(async () => {
+ throw new Error("replacement is forbidden");
+ });
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
openSession,
@@ -1853,20 +3250,28 @@ describe("executeNativeSession recovery", () => {
};
const port: ControlPlanePort = {
openRun,
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
completeRun,
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).rejects.toThrow(
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).rejects.toThrow(
"native_session_recovery_failed: provider session no longer exists",
);
@@ -1921,15 +3326,25 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected replacement"); },
+ async openSession() {
+ throw new Error("unexpected replacement");
+ },
recoverSession,
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
replayEvents,
async completeRun() {},
};
@@ -1983,10 +3398,16 @@ describe("executeNativeSession recovery", () => {
pendingRuntimeRequests: [],
lineage: [],
};
- let resolveRecovery = (_value: { recovered: true; session: NativeSession }) => {};
- const stalledRecovery = new Promise<{ recovered: true; session: NativeSession }>(
- (resolve) => { resolveRecovery = resolve; },
- );
+ let resolveRecovery = (_value: {
+ recovered: true;
+ session: NativeSession;
+ }) => {};
+ const stalledRecovery = new Promise<{
+ recovered: true;
+ session: NativeSession;
+ }>((resolve) => {
+ resolveRecovery = resolve;
+ });
let markRecoveryStarted = () => {};
const recoveryStarted = new Promise<void>((resolve) => {
markRecoveryStarted = resolve;
@@ -1995,21 +3416,37 @@ describe("executeNativeSession recovery", () => {
const closeStarted = new Promise<void>((resolve) => {
markCloseStarted = resolve;
});
- const close = vi.fn(async () => { markCloseStarted(); });
+ const close = vi.fn(async () => {
+ markCloseStarted();
+ });
const lateSession: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ };
},
async *events() {},
- async startTurn() { throw new Error("late recovery session must not start"); },
- async result() { return null; },
- async snapshot() { return structuredClone(checkpoint); },
+ async startTurn() {
+ throw new Error("late recovery session must not start");
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ return structuredClone(checkpoint);
+ },
close,
};
let recoverySignal: AbortSignal | undefined;
const recoverSession = vi.fn(
- (_checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }) => {
+ (
+ _checkpoint: PersistedNativeSession,
+ recoveryOptions: { signal: AbortSignal },
+ ) => {
recoverySignal = recoveryOptions.signal;
recoveryOptions.signal.addEventListener(
"abort",
@@ -2028,16 +3465,27 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ },
};
},
- async openSession() { throw new Error("unexpected replacement"); },
+ async openSession() {
+ throw new Error("unexpected replacement");
+ },
recoverSession,
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -2099,33 +3547,48 @@ describe("executeNativeSession recovery", () => {
const closeStarted = new Promise<void>((resolve) => {
markCloseStarted = resolve;
});
- const close = vi.fn(async () => { markCloseStarted(); });
+ const close = vi.fn(async () => {
+ markCloseStarted();
+ });
const lateSession: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ };
},
async *events() {},
- async startTurn() { throw new Error("late replacement session must not start"); },
- async result() { return null; },
- async snapshot() { return structuredClone(checkpoint); },
+ async startTurn() {
+ throw new Error("late replacement session must not start");
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ return structuredClone(checkpoint);
+ },
close,
};
let replacementSignal: AbortSignal | undefined;
- const openReplacementSession = vi.fn((replacementInput: {
- identity: NativeRunIdentity;
- workingDirectory?: string;
- signal?: AbortSignal;
- }) => {
- replacementSignal = replacementInput.signal;
- replacementInput.signal?.addEventListener(
- "abort",
- () => resolveReplacement(lateSession),
- { once: true },
- );
- markReplacementStarted();
- return stalledReplacement;
- });
+ const openReplacementSession = vi.fn(
+ (replacementInput: {
+ identity: NativeRunIdentity;
+ workingDirectory?: string;
+ signal?: AbortSignal;
+ }) => {
+ replacementSignal = replacementInput.signal;
+ replacementInput.signal?.addEventListener(
+ "abort",
+ () => resolveReplacement(lateSession),
+ { once: true },
+ );
+ markReplacementStarted();
+ return stalledReplacement;
+ },
+ );
const openRun = vi.fn(async () => undefined);
const onSession = vi.fn();
const backend: NativeSessionBackend = {
@@ -2134,10 +3597,17 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "replacement-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ },
};
},
- async openSession() { throw new Error("replacement seam must be used"); },
+ async openSession() {
+ throw new Error("replacement seam must be used");
+ },
async recoverSession() {
return { recovered: false, reason: "provider session is missing" };
},
@@ -2145,8 +3615,12 @@ describe("executeNativeSession recovery", () => {
};
const port: ControlPlanePort = {
openRun,
- async appendEvent() { throw new Error("unexpected event"); },
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
+ },
async completeRun() {},
};
@@ -2198,27 +3672,41 @@ describe("executeNativeSession recovery", () => {
const openRun = vi.fn(async () => undefined);
const checkpointSession = vi.fn(async () => undefined);
const onCheckpoint = vi.fn(async () => undefined);
- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {
- expect(recoveryCheckpoint.cursor).toBe("1");
- throw recoveryFailure;
- });
+ const recoverSession = vi.fn(
+ async (recoveryCheckpoint: PersistedNativeSession) => {
+ expect(recoveryCheckpoint.cursor).toBe("1");
+ throw recoveryFailure;
+ },
+ );
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected replacement"); },
+ async openSession() {
+ throw new Error("unexpected replacement");
+ },
recoverSession,
};
const port: ControlPlanePort = {
openRun,
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
checkpointSession,
- async appendEvent() { throw new Error("unexpected event"); },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
async replayEvents(replay) {
return {
events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [],
@@ -2228,14 +3716,16 @@ describe("executeNativeSession recovery", () => {
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- onCheckpoint,
- })).rejects.toBe(recoveryFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ onCheckpoint,
+ }),
+ ).rejects.toBe(recoveryFailure);
expect(recoverSession).toHaveBeenCalledOnce();
expect(openRun).not.toHaveBeenCalled();
@@ -2256,7 +3746,9 @@ describe("executeNativeSession recovery", () => {
lineage: [],
};
const admissionFailure = new Error("control-plane admission rejected");
- const openRun = vi.fn(async () => { throw admissionFailure; });
+ const openRun = vi.fn(async () => {
+ throw admissionFailure;
+ });
const checkpointSession = vi.fn(async () => undefined);
const onCheckpoint = vi.fn(async () => undefined);
const onSession = vi.fn();
@@ -2264,35 +3756,61 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {},
- async startTurn() { throw new Error("unexpected turn"); },
- async result() { return null; },
- async snapshot() { throw new Error("unexpected snapshot"); },
+ async startTurn() {
+ throw new Error("unexpected turn");
+ },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ throw new Error("unexpected snapshot");
+ },
close,
};
- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {
- expect(recoveryCheckpoint.cursor).toBe("1");
- return { recovered: true as const, session };
- });
+ const recoverSession = vi.fn(
+ async (recoveryCheckpoint: PersistedNativeSession) => {
+ expect(recoveryCheckpoint.cursor).toBe("1");
+ return { recovered: true as const, session };
+ },
+ );
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("unexpected replacement"); },
+ async openSession() {
+ throw new Error("unexpected replacement");
+ },
recoverSession,
};
const port: ControlPlanePort = {
openRun,
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
checkpointSession,
- async appendEvent() { throw new Error("unexpected event"); },
+ async appendEvent() {
+ throw new Error("unexpected event");
+ },
async replayEvents(replay) {
return {
events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [],
@@ -2302,15 +3820,17 @@ describe("executeNativeSession recovery", () => {
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- onCheckpoint,
- onSession,
- })).rejects.toBe(admissionFailure);
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ onCheckpoint,
+ onSession,
+ }),
+ ).rejects.toBe(admissionFailure);
expect(recoverSession).toHaveBeenCalledOnce();
expect(openRun).toHaveBeenCalledOnce();
@@ -2356,16 +3876,30 @@ describe("executeNativeSession recovery", () => {
};
const bySource = new Map<string, PrpEvent[]>();
const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" }));
- const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); });
+ const openSession = vi.fn(async () => {
+ throw new Error("must recover the provider session");
+ });
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield terminalEvent;
},
- async *events() { yield terminalEvent; },
startTurn,
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
- async snapshot() { return structuredClone(providerSnapshot); },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return structuredClone(providerSnapshot);
+ },
async close() {},
};
const backend: NativeSessionBackend = {
@@ -2374,15 +3908,25 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
openSession,
- async recoverSession() { return { recovered: true, session }; },
+ async recoverSession() {
+ return { recovered: true, session };
+ },
};
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
async appendEvent(event) {
const list = bySource.get(event.sourceInstanceId) ?? [];
@@ -2397,20 +3941,27 @@ describe("executeNativeSession recovery", () => {
async replayEvents(replay) {
const list = bySource.get(replay.sourceInstanceId) ?? [];
return {
- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),
+ events: structuredClone(
+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),
+ ),
highestContiguousSourceSeq: highestContiguous(list),
};
},
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery" });
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({
+ turnId: "turn-recovery",
+ providerSessionId: "provider-recovery",
+ });
expect(openSession).not.toHaveBeenCalled();
expect(startTurn).not.toHaveBeenCalled();
});
@@ -2441,39 +3992,74 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield terminalEvent;
+ },
+ async startTurn() {
+ return { turnId: "turn-recovery" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
},
- async *events() { yield terminalEvent; },
- async startTurn() { return { turnId: "turn-recovery" }; },
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
async snapshot() {
- return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null };
+ return {
+ ...checkpoint,
+ cursor: String(terminalSequence),
+ activeTurnId: null,
+ };
},
async close() {},
};
- const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => {
- expect(recoveryCheckpoint.cursor).toBe(expectedCursor);
- return { recovered: true, session };
- });
+ const recoverSession = vi.fn(
+ async (recoveryCheckpoint: PersistedNativeSession) => {
+ expect(recoveryCheckpoint.cursor).toBe(expectedCursor);
+ return { recovered: true, session };
+ },
+ );
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("must recover the provider session"); },
+ async openSession() {
+ throw new Error("must recover the provider session");
+ },
recoverSession,
};
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
- async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
+ async checkpointSession(snapshot) {
+ checkpoints.push(structuredClone(snapshot));
+ },
async appendEvent(event) {
- const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents;
- if (target.some((existing) => existing.sourceSeq === event.sourceSeq)) {
+ const target =
+ event.sourceInstanceId === "runner-recovery"
+ ? runnerEvents
+ : controlEvents;
+ if (
+ target.some((existing) => existing.sourceSeq === event.sourceSeq)
+ ) {
throw new Error(`native_event_replay_conflict:${event.sourceSeq}`);
}
target.push(structuredClone(event));
@@ -2484,26 +4070,36 @@ describe("executeNativeSession recovery", () => {
};
},
async replayEvents(replay) {
- const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents;
+ const source =
+ replay.sourceInstanceId === "runner-recovery"
+ ? runnerEvents
+ : controlEvents;
const events = source
.filter((event) => event.sourceSeq > replay.afterSourceSeq)
.sort((left, right) => left.sourceSeq - right.sourceSeq)
.slice(0, replay.limit);
- return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source) };
+ return {
+ events: structuredClone(events),
+ highestContiguousSourceSeq: highestContiguous(source),
+ };
},
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).resolves.toMatchObject({ turnId: "turn-recovery" });
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({ turnId: "turn-recovery" });
expect(recoverSession).toHaveBeenCalledOnce();
- expect(runnerEvents.some((event) => event.sourceSeq === terminalSequence)).toBe(true);
+ expect(
+ runnerEvents.some((event) => event.sourceSeq === terminalSequence),
+ ).toBe(true);
if (checkpointCursor === "12") {
expect(checkpoints[0]).toMatchObject({ cursor: "41" });
}
@@ -2531,12 +4127,26 @@ describe("executeNativeSession recovery", () => {
const replacementSession: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield runnerEvent(1, "turn.completed");
+ },
+ async startTurn() {
+ return { turnId: "turn-replacement" };
+ },
+ async result() {
+ return { result, terminal, turnId: "turn-replacement" };
+ },
+ async snapshot() {
+ return structuredClone(replacementSnapshot);
},
- async *events() { yield runnerEvent(1, "turn.completed"); },
- async startTurn() { return { turnId: "turn-replacement" }; },
- async result() { return { result, terminal, turnId: "turn-replacement" }; },
- async snapshot() { return structuredClone(replacementSnapshot); },
async close() {},
};
const recoverSession = vi.fn(async () => ({
@@ -2551,34 +4161,52 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "replacement-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("replacement seam must be used"); },
+ async openSession() {
+ throw new Error("replacement seam must be used");
+ },
recoverSession,
openReplacementSession,
};
const events: PrpEvent[] = [];
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
async appendEvent(event) {
events.push(structuredClone(event));
- return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" };
+ return {
+ cursor: events.length,
+ highestContiguousSourceSeq: highestContiguous(events),
+ disposition: "committed",
+ };
+ },
+ async replayEvents() {
+ return { events: [], highestContiguousSourceSeq: 0 };
},
- async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-replacement",
- controlPlaneInstanceId: "control-replacement",
- onContinuityBreak,
- })).resolves.toMatchObject({ providerSessionId: "provider-new" });
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-replacement",
+ controlPlaneInstanceId: "control-replacement",
+ onContinuityBreak,
+ }),
+ ).resolves.toMatchObject({ providerSessionId: "provider-new" });
expect(recoverSession).toHaveBeenCalledOnce();
expect(openReplacementSession).toHaveBeenCalledOnce();
@@ -2599,7 +4227,9 @@ describe("executeNativeSession recovery", () => {
providerSessionId: "provider-recovery",
cursor: "1",
activeTurnId: null,
- terminalTurns: [{ turnId: "turn-work", fingerprint: "terminal-fingerprint" }],
+ terminalTurns: [
+ { turnId: "turn-work", fingerprint: "terminal-fingerprint" },
+ ],
dispositionOnlyRecoveryConsumed: true,
dispositionOnlyRecoveryTurnId: "turn-missing-disposition",
pendingRuntimeRequests: [],
@@ -2629,12 +4259,24 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield terminalEvent;
},
- async *events() { yield terminalEvent; },
startTurn,
- async result() { return { result, terminal, turnId: "turn-continuation" }; },
- async snapshot() { return structuredClone(recoveredSnapshot); },
+ async result() {
+ return { result, terminal, turnId: "turn-continuation" };
+ },
+ async snapshot() {
+ return structuredClone(recoveredSnapshot);
+ },
async close() {},
};
const backend: NativeSessionBackend = {
@@ -2643,26 +4285,42 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("must recover the provider session"); },
- async recoverSession() { return { recovered: true, session }; },
+ async openSession() {
+ throw new Error("must recover the provider session");
+ },
+ async recoverSession() {
+ return { recovered: true, session };
+ },
};
const bySource = new Map<string, PrpEvent[]>();
const replayedPages: PrpEvent[][] = [];
- const replayEvents = vi.fn(async (replay: Parameters<ControlPlanePort["replayEvents"]>[0]) => {
- const list = bySource.get(replay.sourceInstanceId) ?? [];
- const events = structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq));
- replayedPages.push(events);
- return {
- events,
- highestContiguousSourceSeq: highestContiguous(list),
- };
- });
+ const replayEvents = vi.fn(
+ async (replay: Parameters<ControlPlanePort["replayEvents"]>[0]) => {
+ const list = bySource.get(replay.sourceInstanceId) ?? [];
+ const events = structuredClone(
+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),
+ );
+ replayedPages.push(events);
+ return {
+ events,
+ highestContiguousSourceSeq: highestContiguous(list),
+ };
+ },
+ );
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
async appendEvent(event) {
const list = bySource.get(event.sourceInstanceId) ?? [];
@@ -2678,13 +4336,18 @@ describe("executeNativeSession recovery", () => {
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" });
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({
+ turnId: "turn-continuation",
+ providerSessionId: "provider-recovery",
+ });
expect(startTurn).toHaveBeenCalledOnce();
expect(replayEvents).toHaveBeenCalledWith({
runId: identity.runId,
@@ -2708,19 +4371,23 @@ describe("executeNativeSession recovery", () => {
recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined;
startTurn.mockClear();
bySource.clear();
- bySource.set("runner-recovery", [{
- ...terminalEvent,
- sourceEventId: "runner-recovery:stale-terminal",
- turnId: "turn-stale-unbound",
- }]);
+ bySource.set("runner-recovery", [
+ {
+ ...terminalEvent,
+ sourceEventId: "runner-recovery:stale-terminal",
+ turnId: "turn-stale-unbound",
+ },
+ ]);
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- })).resolves.toMatchObject({
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ }),
+ ).resolves.toMatchObject({
turnId: "turn-continuation",
providerSessionId: "provider-recovery",
});
@@ -2770,12 +4437,24 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield terminalEvent;
},
- async *events() { yield terminalEvent; },
startTurn,
- async result() { return null; },
- async snapshot() { return structuredClone(recoveredSnapshot); },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ return structuredClone(recoveredSnapshot);
+ },
async close() {},
};
const events: PrpEvent[] = [];
@@ -2785,24 +4464,42 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("must recover the provider session"); },
- async recoverSession() { return { recovered: true, session }; },
+ async openSession() {
+ throw new Error("must recover the provider session");
+ },
+ async recoverSession() {
+ return { recovered: true, session };
+ },
};
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession(snapshot) {
if (
- snapshot.terminalTurns?.some((turn) => turn.turnId === "turn-disposition")
- && !dispositionTerminalCommitted
- ) prematureDispositionCheckpoint = true;
+ snapshot.terminalTurns?.some(
+ (turn) => turn.turnId === "turn-disposition",
+ ) &&
+ !dispositionTerminalCommitted
+ )
+ prematureDispositionCheckpoint = true;
},
async appendEvent(event) {
events.push(structuredClone(event));
- if (event.eventType === "turn.completed" && event.turnId === "turn-disposition") {
+ if (
+ event.eventType === "turn.completed" &&
+ event.turnId === "turn-disposition"
+ ) {
dispositionTerminalCommitted = true;
}
return {
@@ -2813,24 +4510,29 @@ describe("executeNativeSession recovery", () => {
},
async replayEvents(replay) {
return {
- events: structuredClone(events.filter((event) =>
- event.sourceInstanceId === replay.sourceInstanceId
- && event.sourceSeq > replay.afterSourceSeq
- )),
+ events: structuredClone(
+ events.filter(
+ (event) =>
+ event.sourceInstanceId === replay.sourceInstanceId &&
+ event.sourceSeq > replay.afterSourceSeq,
+ ),
+ ),
highestContiguousSourceSeq: highestContiguous(events),
};
},
async completeRun() {},
};
- await expect(executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- resolveMissingResult: async () => result,
- })).resolves.toMatchObject({
+ await expect(
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ resolveMissingResult: async () => result,
+ }),
+ ).resolves.toMatchObject({
result,
turnId: "turn-disposition",
});
@@ -2896,11 +4598,21 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield structuredClone(terminalEvent);
},
- async *events() { yield structuredClone(terminalEvent); },
startTurn,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
...structuredClone(checkpoint),
@@ -2924,21 +4636,36 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("must recover the provider session"); },
- async recoverSession() { return { recovered: true, session }; },
+ async openSession() {
+ throw new Error("must recover the provider session");
+ },
+ async recoverSession() {
+ return { recovered: true, session };
+ },
};
const bySource = new Map<string, PrpEvent[]>([
- ["runner-recovery", [
- structuredClone(originalTaskProposal),
- structuredClone(originalTaskTerminal),
- ]],
+ [
+ "runner-recovery",
+ [
+ structuredClone(originalTaskProposal),
+ structuredClone(originalTaskTerminal),
+ ],
+ ],
]);
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
async appendEvent(event) {
const list = bySource.get(event.sourceInstanceId) ?? [];
@@ -2953,24 +4680,27 @@ describe("executeNativeSession recovery", () => {
async replayEvents(replay) {
const list = bySource.get(replay.sourceInstanceId) ?? [];
return {
- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),
+ events: structuredClone(
+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),
+ ),
highestContiguousSourceSeq: highestContiguous(list),
};
},
async completeRun() {},
};
- const execute = () => executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- resolveMissingResult: async ({ terminalEvent: replayed }) => {
- expect(replayed).toEqual(terminalEvent);
- return result;
- },
- });
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ resolveMissingResult: async ({ terminalEvent: replayed }) => {
+ expect(replayed).toEqual(terminalEvent);
+ return result;
+ },
+ });
await expect(execute()).resolves.toMatchObject({
result,
turnId: "turn-disposition",
@@ -2997,11 +4727,9 @@ describe("executeNativeSession recovery", () => {
resultProposalEvent,
terminalEvent,
]);
- expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([
- "run.result.accepted",
- "run.terminal",
- ]);
-
+ expect(
+ bySource.get("control-recovery")?.map((event) => event.eventType),
+ ).toEqual(["run.result.accepted", "run.terminal"]);
});
it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => {
@@ -3029,20 +4757,33 @@ describe("executeNativeSession recovery", () => {
pendingRuntimeRequests: [],
lineage: [],
};
- const recoveredCheckpoint: PersistedNativeSession = structuredClone(checkpoint);
+ const recoveredCheckpoint: PersistedNativeSession =
+ structuredClone(checkpoint);
const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));
- const events = vi.fn(() => (async function* () {
- throw new Error("checkpoint fallback must not consume provider events");
- })());
+ const events = vi.fn(() =>
+ (async function* () {
+ throw new Error("checkpoint fallback must not consume provider events");
+ })(),
+ );
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
events,
startTurn,
- async result() { return null; },
- async snapshot() { return structuredClone(recoveredCheckpoint); },
+ async result() {
+ return null;
+ },
+ async snapshot() {
+ return structuredClone(recoveredCheckpoint);
+ },
async close() {},
};
const backend: NativeSessionBackend = {
@@ -3051,11 +4792,21 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { throw new Error("must recover the provider session"); },
- async recoverSession() { return { recovered: true, session }; },
+ async openSession() {
+ throw new Error("must recover the provider session");
+ },
+ async recoverSession() {
+ return { recovered: true, session };
+ },
};
const bySource = new Map<string, PrpEvent[]>([
["runner-recovery", [workProposal, workTerminal]],
@@ -3063,7 +4814,9 @@ describe("executeNativeSession recovery", () => {
const completeRun = vi.fn(async () => undefined);
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
async checkpointSession() {},
async appendEvent(event) {
const list = bySource.get(event.sourceInstanceId) ?? [];
@@ -3078,36 +4831,39 @@ describe("executeNativeSession recovery", () => {
async replayEvents(replay) {
const list = bySource.get(replay.sourceInstanceId) ?? [];
return {
- events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),
+ events: structuredClone(
+ list.filter((event) => event.sourceSeq > replay.afterSourceSeq),
+ ),
highestContiguousSourceSeq: highestContiguous(list),
};
},
completeRun,
};
- const execute = () => executeNativeSession({
- input,
- backend,
- controlPlane: port,
- runnerInstanceId: "runner-recovery",
- controlPlaneInstanceId: "control-recovery",
- resolveMissingResult: async ({ turnId, terminalEvent }) => {
- expect(turnId).toBe("turn-disposition");
- expect(terminalEvent).toMatchObject({
- sourceInstanceId: "control-recovery",
- sourceKind: "control_plane",
- runId: identity.runId,
- normalizedSessionId: identity.sessionId,
- turnId: "turn-disposition",
- eventType: "turn.completed",
- payload: {
- recovery: "checkpointed_resultless_disposition",
- terminalFingerprint: "disposition-terminal",
- },
- });
- return result;
- },
- });
+ const execute = () =>
+ executeNativeSession({
+ input,
+ backend,
+ controlPlane: port,
+ runnerInstanceId: "runner-recovery",
+ controlPlaneInstanceId: "control-recovery",
+ resolveMissingResult: async ({ turnId, terminalEvent }) => {
+ expect(turnId).toBe("turn-disposition");
+ expect(terminalEvent).toMatchObject({
+ sourceInstanceId: "control-recovery",
+ sourceKind: "control_plane",
+ runId: identity.runId,
+ normalizedSessionId: identity.sessionId,
+ turnId: "turn-disposition",
+ eventType: "turn.completed",
+ payload: {
+ recovery: "checkpointed_resultless_disposition",
+ terminalFingerprint: "disposition-terminal",
+ },
+ });
+ return result;
+ },
+ });
await expect(execute()).resolves.toMatchObject({
result,
turnId: "turn-disposition",
@@ -3119,10 +4875,9 @@ describe("executeNativeSession recovery", () => {
workProposal,
workTerminal,
]);
- expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([
- "run.result.accepted",
- "run.terminal",
- ]);
+ expect(
+ bySource.get("control-recovery")?.map((event) => event.eventType),
+ ).toEqual(["run.result.accepted", "run.terminal"]);
recoveredCheckpoint.terminalTurns![1]!.fingerprint = "conflicting-terminal";
await expect(execute()).rejects.toThrow(
@@ -3143,7 +4898,9 @@ describe("executeNativeSession recovery", () => {
semanticResult: result,
terminal,
activeTurnId: "turn-recovery",
- terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }],
+ terminalTurns: [
+ { turnId: "turn-recovery", fingerprint: "terminal-fingerprint" },
+ ],
pendingRuntimeRequests: [],
lineage: [],
};
@@ -3152,17 +4909,29 @@ describe("executeNativeSession recovery", () => {
const completeRun = vi.fn(async () => undefined);
const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));
const openSession = vi.fn(async () => {
- throw new Error("a recovered run must not open a second provider session");
+ throw new Error(
+ "a recovered run must not open a second provider session",
+ );
});
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
},
async *events() {},
startTurn,
- async result() { return { result, terminal, turnId: "turn-recovery" }; },
- async snapshot() { return structuredClone(checkpoint); },
+ async result() {
+ return { result, terminal, turnId: "turn-recovery" };
+ },
+ async snapshot() {
+ return structuredClone(checkpoint);
+ },
async close() {},
};
const recoverSession = vi.fn(async () => ({ recovered: true, session }));
@@ -3172,7 +4941,13 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "recovery-backend",
version: "1",
- capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: true,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
openSession,
@@ -3180,8 +4955,12 @@ describe("executeNativeSession recovery", () => {
};
const port: ControlPlanePort = {
async openRun() {},
- async loadSessionCheckpoint() { return structuredClone(checkpoint); },
- async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },
+ async loadSessionCheckpoint() {
+ return structuredClone(checkpoint);
+ },
+ async checkpointSession(snapshot) {
+ checkpoints.push(structuredClone(snapshot));
+ },
async appendEvent(event) {
events.push(structuredClone(event as PrpEvent));
return {
@@ -3191,8 +4970,13 @@ describe("executeNativeSession recovery", () => {
};
},
async replayEvents(replay) {
- const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq);
- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) };
+ const replayed = events.filter(
+ (event) => event.sourceSeq > replay.afterSourceSeq,
+ );
+ return {
+ events: structuredClone(replayed),
+ highestContiguousSourceSeq: highestContiguous(events),
+ };
},
completeRun,
};
@@ -3208,11 +4992,20 @@ describe("executeNativeSession recovery", () => {
expect(openSession).not.toHaveBeenCalled();
expect(recoverSession).toHaveBeenCalledOnce();
expect(startTurn).not.toHaveBeenCalled();
- expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]);
+ expect(events.map((event) => event.eventType)).toEqual([
+ "run.result.accepted",
+ "run.terminal",
+ ]);
expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]);
expect(completeRun).toHaveBeenCalledOnce();
- expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 });
- expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal });
+ expect(completed).toMatchObject({
+ nativeEventCount: 1,
+ highestContiguousSourceSeq: 2,
+ });
+ expect(checkpoints.at(-1)).toMatchObject({
+ semanticResult: result,
+ terminal,
+ });
});
it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => {
@@ -3238,8 +5031,16 @@ describe("executeNativeSession recovery", () => {
completionClaim: {
contractRevision: "1",
objectiveSatisfied: false,
- criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }],
- remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }],
+ criteria: [
+ {
+ criterionId: "objective",
+ status: "unknown",
+ evidenceRefs: ["interaction:pending"],
+ },
+ ],
+ remainingWork: [
+ { description: "Resume after the response.", blocksCompletion: true },
+ ],
},
evidence: [{ ref: "interaction:pending" }],
verification: [],
@@ -3257,11 +5058,23 @@ describe("executeNativeSession recovery", () => {
const session: NativeSession = {
identity: () => identity,
async capabilities() {
- return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true };
+ return {
+ resume: false,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ };
+ },
+ async *events() {
+ yield terminalEvent;
+ },
+ async startTurn() {
+ return { turnId: "turn-waiting" };
+ },
+ async result() {
+ return null;
},
- async *events() { yield terminalEvent; },
- async startTurn() { return { turnId: "turn-waiting" }; },
- async result() { return null; },
async snapshot() {
return {
backendKind: "mock",
@@ -3282,10 +5095,18 @@ describe("executeNativeSession recovery", () => {
kind: "mock",
name: "governed-wait-backend",
version: "1",
- capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true },
+ capabilities: {
+ resume: false,
+ typedEvents: true,
+ steering: false,
+ interruption: true,
+ structuredResult: true,
+ },
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -3299,10 +5120,15 @@ describe("executeNativeSession recovery", () => {
};
},
async replayEvents(replay) {
- const replayed = events.filter((event) =>
- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq
+ const replayed = events.filter(
+ (event) =>
+ event.sourceInstanceId === replay.sourceInstanceId &&
+ event.sourceSeq > replay.afterSourceSeq,
);
- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };
+ return {
+ events: structuredClone(replayed),
+ highestContiguousSourceSeq: highestContiguous(replayed),
+ };
},
completeRun,
};
@@ -3316,10 +5142,16 @@ describe("executeNativeSession recovery", () => {
resolveMissingResult,
});
- expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent });
+ expect(resolveMissingResult).toHaveBeenCalledWith({
+ turnId: "turn-waiting",
+ terminalEvent,
+ });
expect(completed).toMatchObject({
result: yielded,
- terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" },
+ terminal: {
+ runTerminalState: "succeeded",
+ reportedWorkDisposition: "yielded",
+ },
turnId: "turn-waiting",
});
expect(completeRun).toHaveBeenCalledWith(
@@ -3501,13 +5333,18 @@ describe("executeNativeSession recovery", () => {
try {
const questionSet = {
schema: "paperclip.question_set.v1" as const,
- questions: [{
- id: "region",
- prompt: "Which region?",
- required: true,
- answerMode: "single_select" as const,
- options: [{ id: "us", label: "US" }, { id: "eu", label: "Europe" }],
- }],
+ questions: [
+ {
+ id: "region",
+ prompt: "Which region?",
+ required: true,
+ answerMode: "single_select" as const,
+ options: [
+ { id: "us", label: "US" },
+ { id: "eu", label: "Europe" },
+ ],
+ },
+ ],
};
const request = {
schema: "paperclip.runtime_request.v2",
@@ -3521,24 +5358,39 @@ describe("executeNativeSession recovery", () => {
turnId: "turn-waiting",
itemId: "input-1",
};
- const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting" };
- const expired = { ...runnerEvent(2, "runtime_request.expired", {
- requestId: "input-1",
- requestKind: "runtime",
+ const created = {
+ ...runnerEvent(1, "runtime_request.created", { request }),
turnId: "turn-waiting",
- itemId: "input-1",
- reason: "durable_handoff",
- replayAllowed: false,
- requestType: "input",
- request,
- }), turnId: "turn-waiting" };
- const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting" };
+ };
+ const expired = {
+ ...runnerEvent(2, "runtime_request.expired", {
+ requestId: "input-1",
+ requestKind: "runtime",
+ turnId: "turn-waiting",
+ itemId: "input-1",
+ reason: "durable_handoff",
+ replayAllowed: false,
+ requestType: "input",
+ request,
+ }),
+ turnId: "turn-waiting",
+ };
+ const interrupted = {
+ ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }),
+ turnId: "turn-waiting",
+ };
let releaseHandoff!: () => void;
- const handedOff = new Promise<void>((resolve) => { releaseHandoff = resolve; });
+ const handedOff = new Promise<void>((resolve) => {
+ releaseHandoff = resolve;
+ });
let releaseCancelled!: () => void;
- const cancelled = new Promise<void>((resolve) => { releaseCancelled = resolve; });
+ const cancelled = new Promise<void>((resolve) => {
+ releaseCancelled = resolve;
+ });
let releaseCreated!: () => void;
- const createdCommitted = new Promise<void>((resolve) => { releaseCreated = resolve; });
+ const createdCommitted = new Promise<void>((resolve) => {
+ releaseCreated = resolve;
+ });
const handoffRuntimeRequest = vi.fn(() => {
releaseHandoff();
return { result: "handed_off" as const, cleanup: Promise.resolve() };
@@ -3567,10 +5419,14 @@ describe("executeNativeSession recovery", () => {
await cancelled;
yield interrupted;
},
- async startTurn() { return { turnId: "turn-waiting" }; },
+ async startTurn() {
+ return { turnId: "turn-waiting" };
+ },
handoffRuntimeRequest,
cancel,
- async result() { return null; },
+ async result() {
+ return null;
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -3601,7 +5457,9 @@ describe("executeNativeSession recovery", () => {
},
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -3609,7 +5467,10 @@ describe("executeNativeSession recovery", () => {
async appendEvent(event) {
events.push(structuredClone(event as PrpEvent));
if (event.eventType === "runtime_request.created") releaseCreated();
- const sourceEvents = events.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId);
+ const sourceEvents = events.filter(
+ (candidate) =>
+ candidate.sourceInstanceId === event.sourceInstanceId,
+ );
return {
cursor: events.length,
highestContiguousSourceSeq: highestContiguous(sourceEvents),
@@ -3617,10 +5478,15 @@ describe("executeNativeSession recovery", () => {
};
},
async replayEvents(replay) {
- const replayed = events.filter((event) =>
- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq
+ const replayed = events.filter(
+ (event) =>
+ event.sourceInstanceId === replay.sourceInstanceId &&
+ event.sourceSeq > replay.afterSourceSeq,
);
- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };
+ return {
+ events: structuredClone(replayed),
+ highestContiguousSourceSeq: highestContiguous(replayed),
+ };
},
async completeRun() {},
};
@@ -3649,7 +5515,9 @@ describe("executeNativeSession recovery", () => {
signal: expect.any(AbortSignal),
});
expect(cancel).toHaveBeenCalledOnce();
- expect(events.map((event) => event.eventType)).toContain("runtime_request.expired");
+ expect(events.map((event) => event.eventType)).toContain(
+ "runtime_request.expired",
+ );
} finally {
vi.useRealTimers();
}
@@ -3665,12 +5533,14 @@ describe("executeNativeSession recovery", () => {
prompt: "Which region?",
input: {
schema: "paperclip.question_set.v1",
- questions: [{
- id: "region",
- prompt: "Which region?",
- required: true,
- answerMode: "text",
- }],
+ questions: [
+ {
+ id: "region",
+ prompt: "Which region?",
+ required: true,
+ answerMode: "text",
+ },
+ ],
},
origin: { adapter: "mock" },
turnId: "turn-stalled",
@@ -3715,7 +5585,9 @@ describe("executeNativeSession recovery", () => {
markHandoffStarted();
return {
result: "handed_off",
- cleanup: new Promise<void>((resolve) => { releaseHandoff = resolve; }),
+ cleanup: new Promise<void>((resolve) => {
+ releaseHandoff = resolve;
+ }),
};
},
cancel() {
@@ -3807,7 +5679,10 @@ describe("executeNativeSession recovery", () => {
},
async *events() {
try {
- yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal" };
+ yield {
+ ...runnerEvent(1, "turn.completed"),
+ turnId: "turn-terminal",
+ };
} finally {
teardownStarted();
await new Promise<void>((resolve) => {
@@ -3891,12 +5766,14 @@ describe("executeNativeSession recovery", () => {
prompt: "Which region?",
input: {
schema: "paperclip.question_set.v1",
- questions: [{
- id: "region",
- prompt: "Which region?",
- required: true,
- answerMode: "text",
- }],
+ questions: [
+ {
+ id: "region",
+ prompt: "Which region?",
+ required: true,
+ answerMode: "text",
+ },
+ ],
},
origin: { adapter: "mock" },
turnId: "turn-terminal",
@@ -4030,19 +5907,24 @@ describe("executeNativeSession recovery", () => {
prompt: "Which region?",
input: {
schema: "paperclip.question_set.v1",
- questions: [{
- id: "region",
- prompt: "Which region?",
- required: true,
- answerMode: "text",
- }],
+ questions: [
+ {
+ id: "region",
+ prompt: "Which region?",
+ required: true,
+ answerMode: "text",
+ },
+ ],
},
origin: { adapter: "mock" },
turnId: "turn-live",
itemId: "input-live",
};
const providerEvents = [
- { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live" },
+ {
+ ...runnerEvent(1, "runtime_request.created", { request }),
+ turnId: "turn-live",
+ },
{
...runnerEvent(2, "runtime_request.resolved", {
requestId: "input-live",
@@ -4081,10 +5963,16 @@ describe("executeNativeSession recovery", () => {
runtimeRequestHandoff: true,
};
},
- async *events() { yield* providerEvents; },
- async startTurn() { return { turnId: "turn-live" }; },
+ async *events() {
+ yield* providerEvents;
+ },
+ async startTurn() {
+ return { turnId: "turn-live" };
+ },
handoffRuntimeRequest,
- async result() { return { result, terminal, turnId: "turn-live" }; },
+ async result() {
+ return { result, terminal, turnId: "turn-live" };
+ },
async snapshot() {
return {
backendKind: "mock",
@@ -4115,7 +6003,9 @@ describe("executeNativeSession recovery", () => {
},
};
},
- async openSession() { return session; },
+ async openSession() {
+ return session;
+ },
};
const port: ControlPlanePort = {
async openRun() {},
@@ -4126,7 +6016,9 @@ describe("executeNativeSession recovery", () => {
await settlementAppendReleased;
}
appended.push(structuredClone(event as PrpEvent));
- const sourceEvents = appended.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId);
+ const sourceEvents = appended.filter(
+ (candidate) => candidate.sourceInstanceId === event.sourceInstanceId,
+ );
return {
cursor: appended.length,
highestContiguousSourceSeq: highestContiguous(sourceEvents),
@@ -4134,17 +6026,26 @@ describe("executeNativeSession recovery", () => {
};
},
async replayEvents(replay) {
- const replayed = appended.filter((event) =>
- event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq
+ const replayed = appended.filter(
+ (event) =>
+ event.sourceInstanceId === replay.sourceInstanceId &&
+ event.sourceSeq > replay.afterSourceSeq,
);
- return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };
+ return {
+ events: structuredClone(replayed),
+ highestContiguousSourceSeq: highestContiguous(replayed),
+ };
},
async completeRun() {},
};
const originalSetTimeout = globalThis.setTimeout;
let queuedHandoffCallback: (() => void) | null = null;
- const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback, delay, ...args) => {
+ const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((
+ callback,
+ delay,
+ ...args
+ ) => {
if (delay === 123_456) {
queuedHandoffCallback = () => callback(...args);
const handle = originalSetTimeout(() => undefined, 60_000);
@@ -4172,7 +6073,9 @@ describe("executeNativeSession recovery", () => {
queuedHandoffCallback?.();
await Promise.resolve();
expect(handoffRuntimeRequest).not.toHaveBeenCalled();
- expect(appended.some((event) => event.eventType === "runtime_request.expired")).toBe(false);
+ expect(
+ appended.some((event) => event.eventType === "runtime_request.expired"),
+ ).toBe(false);
} finally {
releaseSettlementAppend();
timeoutSpy.mockRestore();
diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts
index 31d89f3ddf..cf608b941d 100644
--- a/packages/paperclip-runner/src/native-session-runtime.ts
+++ b/packages/paperclip-runner/src/native-session-runtime.ts
@@ -4,17 +4,66 @@ import type {
CheckpointControlPlaneSessionOptions,
ControlPlanePort,
} from "./contracts/control-plane-port.js";
-import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contracts/native-execution.js";
-import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js";
-import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js";
+import type {
+ NativeExecutionInput,
+ NativeSessionExecutionResult,
+} from "./contracts/native-execution.js";
+import {
+ buildNativeModelEnvelope,
+ parseNativeExecutionInput,
+} from "./contracts/native-execution.js";
+import type {
+ NativeSession,
+ NativeSessionBackend,
+} from "./contracts/native-session-backend.js";
import type { PersistedNativeSession } from "./contracts/native-session-backend.js";
-import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js";
+import type {
+ PrpEvent,
+ PrpStructuredRunResult,
+ PrpTerminalState,
+} from "./protocol/replay-contract.js";
import { parsePaperclipQuestionSet } from "./contracts/question-set.js";
export const DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS = 120_000;
const OPTIONAL_SESSION_CANCELLATION_GRACE_MS = 100;
const FAILED_OPERATION_SETTLEMENT_GRACE_MS = 100;
const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000;
+type NativeSessionCleanupDomain = string;
+
+const failedSessionCleanupOwners = new Map<
+ Promise<void>,
+ NativeSessionCleanupDomain
+>();
+const FAILED_SESSION_CLOSE_RETRY_MS = 1_000;
+const MAX_FAILED_SESSION_CLOSE_RETRIES = 3;
+const MAX_QUARANTINED_SESSION_CLOSE_RETRIES = 3;
+const MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS = 9;
+const QUARANTINED_SESSION_CLOSE_RETRY_MS = 60_000;
+// Admission can inherit the initial close plus all three retained retries.
+// Keep the wait finite while covering every bounded close attempt and all
+// three production retry delays instead of timing out midway through recovery.
+const QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS = 7_000;
+const QUARANTINED_SESSION_ADMISSION_GRACE_MS =
+ (MAX_FAILED_SESSION_CLOSE_RETRIES + 1) *
+ QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS +
+ MAX_FAILED_SESSION_CLOSE_RETRIES * FAILED_SESSION_CLOSE_RETRY_MS;
+// Admission can observe a retained close recovery and then the one quarantine
+// recovery that retained owner installs when it exhausts its retries. Give
+// each owner generation a complete finite grace, while rejecting any
+// unexpected third generation instead of permitting an unbounded owner chain.
+const MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES = 2;
+
+interface QuarantinedSessionCleanup {
+ session: NativeSession;
+ domain: NativeSessionCleanupDomain;
+ automaticAttempts: number;
+ attempt: Promise<void> | null;
+ recovery: Promise<void> | null;
+ recoveryMaxAttempts: number | null;
+ timer: ReturnType<typeof setTimeout> | null;
+}
+
+const quarantinedSessionCleanups = new Set<QuarantinedSessionCleanup>();
export interface ExecuteNativeSessionOptions {
input: NativeExecutionInput;
@@ -23,6 +72,8 @@ export interface ExecuteNativeSessionOptions {
runnerInstanceId: string;
controlPlaneInstanceId: string;
timeoutMs?: number;
+ /** Abort admission while waiting for prior cleanup in the same domain. */
+ signal?: AbortSignal;
/** Internal test seam; production bounds checkpoint persistence to 30 seconds. */
checkpointTimeoutMs?: number;
/** Internal test seam; production uses the fixed 120-second platform policy. */
@@ -68,18 +119,43 @@ export interface ExecuteNativeSessionOptions {
}
function isTurnTerminal(event: PrpEvent): boolean {
- return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType);
+ return [
+ "turn.completed",
+ "turn.failed",
+ "turn.interrupted",
+ "turn.cancelled",
+ ].includes(event.eventType);
}
-function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["reportedWorkDisposition"]): PrpTerminalState {
- const states = event.eventType === "turn.completed"
- ? { turnTerminalState: "completed" as const, runTerminalState: "succeeded" as const }
- : event.eventType === "turn.failed"
- ? { turnTerminalState: "failed" as const, runTerminalState: "failed" as const }
- : event.eventType === "turn.interrupted"
- ? { turnTerminalState: "interrupted" as const, runTerminalState: "cancelled" as const }
- : { turnTerminalState: "cancelled" as const, runTerminalState: "cancelled" as const };
- return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition };
+function terminalFromEvent(
+ event: PrpEvent,
+ disposition: PrpTerminalState["reportedWorkDisposition"],
+): PrpTerminalState {
+ const states =
+ event.eventType === "turn.completed"
+ ? {
+ turnTerminalState: "completed" as const,
+ runTerminalState: "succeeded" as const,
+ }
+ : event.eventType === "turn.failed"
+ ? {
+ turnTerminalState: "failed" as const,
+ runTerminalState: "failed" as const,
+ }
+ : event.eventType === "turn.interrupted"
+ ? {
+ turnTerminalState: "interrupted" as const,
+ runTerminalState: "cancelled" as const,
+ }
+ : {
+ turnTerminalState: "cancelled" as const,
+ runTerminalState: "cancelled" as const,
+ };
+ return {
+ schema: "paperclip.prp.terminal.v1",
+ ...states,
+ reportedWorkDisposition: disposition,
+ };
}
async function attemptOptionalSessionCancellation(
@@ -104,21 +180,48 @@ async function attemptOptionalSessionCancellation(
if (await settlesWithin(attempts, OPTIONAL_SESSION_CANCELLATION_GRACE_MS)) {
return null;
}
- cancellationAbort.abort(new Error("native session cancellation grace expired"));
+ cancellationAbort.abort(
+ new Error("native session cancellation grace expired"),
+ );
return { settlement: attempts };
}
-async function settlesWithin(operation: Promise<unknown>, timeoutMs: number): Promise<boolean> {
+async function settlesWithin(
+ operation: Promise<unknown>,
+ timeoutMs: number,
+ signal?: AbortSignal,
+): Promise<boolean> {
+ signal?.throwIfAborted();
let graceTimer: ReturnType<typeof setTimeout> | undefined;
- const settled = await Promise.race([
- operation.then(() => true),
- new Promise<false>((resolve) => {
- graceTimer = setTimeout(() => resolve(false), timeoutMs);
- graceTimer.unref?.();
- }),
- ]);
- if (graceTimer !== undefined) clearTimeout(graceTimer);
- return settled;
+ let removeAbort = () => {};
+ try {
+ const aborted = signal
+ ? new Promise<never>((_resolve, reject) => {
+ const onAbort = () =>
+ reject(
+ signal.reason ??
+ new Error("native session cleanup admission aborted"),
+ );
+ if (signal.aborted) {
+ onAbort();
+ } else {
+ signal.addEventListener("abort", onAbort, { once: true });
+ removeAbort = () => signal.removeEventListener("abort", onAbort);
+ }
+ })
+ : new Promise<never>(() => undefined);
+ return await Promise.race([
+ operation.then(() => true),
+ new Promise<false>((resolve) => {
+ graceTimer = setTimeout(() => resolve(false), timeoutMs);
+ graceTimer.unref?.();
+ }),
+ aborted,
+ ]);
+ } finally {
+ if (graceTimer !== undefined) clearTimeout(graceTimer);
+ removeAbort();
+ }
}
async function runAbortableOperationWithin<T>(input: {
@@ -152,7 +255,8 @@ async function runAbortableOperationWithin<T>(input: {
operation,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
- const error = input.timeoutError?.() ?? new Error(input.timeoutMessage);
+ const error =
+ input.timeoutError?.() ?? new Error(input.timeoutMessage);
timedOut = true;
timeoutError = error;
reject(error);
@@ -174,14 +278,54 @@ async function runAbortableOperationWithin<T>(input: {
async function disposeUnadmittedSession(
session: NativeSession,
reason: string,
+ cleanupDomain: NativeSessionCleanupDomain,
): Promise<void> {
// A provider may ignore abort and return a session after its caller has
// already timed out. That session was never published through onSession, so
// close it without clearing ownership that a later execution may establish.
- const closeSettlement = Promise.allSettled([
- Promise.resolve().then(() => session.close({ reason })),
- ]);
- await settlesWithin(closeSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS);
+ // Retain slow or failed cleanup in the same admission gate as an admitted
+ // session: the absence of an onSession publication does not mean provider
+ // resources have already been released.
+ const cleanup = retainUnadmittedSessionCleanup(
+ session,
+ reason,
+ cleanupDomain,
+ );
+ await settlesWithin(
+ Promise.allSettled([cleanup]),
+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,
+ );
+}
+
+function retainUnadmittedSessionCleanup(
+ session: NativeSession,
+ reason: string,
+ cleanupDomain: NativeSessionCleanupDomain,
+): Promise<void> {
+ const cleanup = (async () => {
+ let attempt = Promise.resolve().then(() => session.close({ reason }));
+ let retryCount = 0;
+ while (true) {
+ try {
+ await attempt;
+ return;
+ } catch (error) {
+ if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
+ quarantineSessionCleanup(session, cleanupDomain);
+ throw error;
+ }
+ retryCount += 1;
+ await waitForSessionCloseRetry();
+ attempt = Promise.resolve().then(() =>
+ session.close({
+ reason: `${reason} cleanup recovery (${retryCount})`,
+ }),
+ );
+ }
+ }
+ })();
+ retainFailedSessionCleanupOwner(cleanup, cleanupDomain);
+ return cleanup;
}
class NativeSessionFinalizationTimeoutError extends Error {
@@ -198,7 +342,8 @@ async function finalizeWithin<T>(input: {
return runAbortableOperationWithin({
...input,
timeoutMessage: `native session finalization timed out after ${input.timeoutMs}ms`,
- timeoutError: () => new NativeSessionFinalizationTimeoutError(input.timeoutMs),
+ timeoutError: () =>
+ new NativeSessionFinalizationTimeoutError(input.timeoutMs),
});
}
@@ -223,13 +368,14 @@ async function quarantineRetainedSession(
session: NativeSession,
onSession: ExecuteNativeSessionOptions["onSession"],
reason: string,
+ cleanupDomain: NativeSessionCleanupDomain,
): Promise<void> {
// Eviction and provider cleanup are independent obligations. Keep both
// observed so a throwing owner callback cannot prevent close from starting,
// and a broken provider cannot keep the failed execution pending forever.
const quarantineSettlement = Promise.allSettled([
Promise.resolve().then(() => onSession?.(null)),
- Promise.resolve().then(() => session.close({ reason })),
+ retainUnadmittedSessionCleanup(session, reason, cleanupDomain),
]);
await settlesWithin(
quarantineSettlement,
@@ -251,8 +397,9 @@ async function persistCheckpointWithin(input: {
const externalAbortFailure = externalSignal
? new Promise<never>((_resolve, reject) => {
const abort = () => {
- const reason = externalSignal.reason
- ?? new Error("native session checkpoint aborted");
+ const reason =
+ externalSignal.reason ??
+ new Error("native session checkpoint aborted");
checkpointAbort.abort(reason);
reject(reason);
};
@@ -260,7 +407,8 @@ async function persistCheckpointWithin(input: {
abort();
} else {
externalSignal.addEventListener("abort", abort, { once: true });
- removeExternalAbort = () => externalSignal.removeEventListener("abort", abort);
+ removeExternalAbort = () =>
+ externalSignal.removeEventListener("abort", abort);
}
})
: new Promise<never>(() => undefined);
@@ -271,8 +419,10 @@ async function persistCheckpointWithin(input: {
checkpointOptions,
);
if (checkpointAbort.signal.aborted) {
- throw checkpointAbort.signal.reason
- ?? new Error("native session checkpoint aborted");
+ throw (
+ checkpointAbort.signal.reason ??
+ new Error("native session checkpoint aborted")
+ );
}
await input.onCheckpoint?.(input.snapshot, checkpointOptions);
})();
@@ -300,6 +450,226 @@ async function persistCheckpointWithin(input: {
}
}
+function waitForSessionCloseRetry(): Promise<void> {
+ return new Promise((resolve) => {
+ const timer = setTimeout(resolve, FAILED_SESSION_CLOSE_RETRY_MS);
+ timer.unref?.();
+ });
+}
+function retainFailedSessionCleanupOwner(
+ cleanup: Promise<void>,
+ cleanupDomain: NativeSessionCleanupDomain,
+): void {
+ failedSessionCleanupOwners.set(cleanup, cleanupDomain);
+ void cleanup
+ .finally(() => failedSessionCleanupOwners.delete(cleanup))
+ .catch(() => undefined);
+}
+
+function quarantineSessionCleanup(
+ session: NativeSession,
+ cleanupDomain: NativeSessionCleanupDomain,
+): void {
+ if (
+ [...quarantinedSessionCleanups].some((entry) => entry.session === session)
+ ) {
+ return;
+ }
+ const cleanup: QuarantinedSessionCleanup = {
+ session,
+ domain: cleanupDomain,
+ automaticAttempts: 0,
+ attempt: null,
+ recovery: null,
+ recoveryMaxAttempts: null,
+ timer: null,
+ };
+ quarantinedSessionCleanups.add(cleanup);
+ startQuarantinedSessionCleanupRecovery(
+ cleanup,
+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES,
+ "native session quarantined cleanup recovery",
+ );
+}
+
+function startQuarantinedSessionCleanupRecovery(
+ cleanup: QuarantinedSessionCleanup,
+ maxAttempts: number,
+ reason: string,
+): Promise<void> {
+ if (cleanup.recovery) return cleanup.recovery;
+ const remainingAttempts =
+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS -
+ cleanup.automaticAttempts;
+ const boundedMaxAttempts = Math.min(maxAttempts, remainingAttempts);
+ if (boundedMaxAttempts <= 0) return Promise.resolve();
+ const recovery = (async () => {
+ for (
+ let attemptCount = 0;
+ attemptCount < boundedMaxAttempts &&
+ quarantinedSessionCleanups.has(cleanup);
+ attemptCount += 1
+ ) {
+ // The first retry starts immediately so an admission-triggered recovery
+ // receives the complete close grace. Later attempts retain the bounded
+ // delay that prevents a hot retry loop.
+ if (attemptCount > 0) await waitForSessionCloseRetry();
+ cleanup.automaticAttempts += 1;
+ const attempt = Promise.resolve().then(() =>
+ cleanup.session.close({
+ reason,
+ }),
+ );
+ cleanup.attempt = attempt;
+ try {
+ await attempt;
+ quarantinedSessionCleanups.delete(cleanup);
+ } catch {
+ // Retain the quarantine after this finite, sequential retry batch.
+ } finally {
+ if (cleanup.attempt === attempt) cleanup.attempt = null;
+ }
+ }
+ })();
+ cleanup.recovery = recovery;
+ cleanup.recoveryMaxAttempts = boundedMaxAttempts;
+ failedSessionCleanupOwners.set(recovery, cleanup.domain);
+ void recovery
+ .finally(() => {
+ failedSessionCleanupOwners.delete(recovery);
+ if (cleanup.recovery === recovery) {
+ cleanup.recovery = null;
+ cleanup.recoveryMaxAttempts = null;
+ }
+ scheduleQuarantinedSessionCleanup(cleanup);
+ })
+ .catch(() => undefined);
+ return recovery;
+}
+
+function scheduleQuarantinedSessionCleanup(
+ cleanup: QuarantinedSessionCleanup,
+): void {
+ if (
+ !quarantinedSessionCleanups.has(cleanup) ||
+ cleanup.recovery ||
+ cleanup.attempt ||
+ cleanup.timer ||
+ cleanup.automaticAttempts >=
+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS
+ ) {
+ return;
+ }
+ // Keep cleanup live without a hot retry loop: one unref'd timer and one
+ // sequential close are the maximum background work owned by each entry.
+ // A later admission may accelerate, but never overlap, the scheduled try.
+ cleanup.timer = setTimeout(() => {
+ cleanup.timer = null;
+ if (!quarantinedSessionCleanups.has(cleanup)) return;
+ startQuarantinedSessionCleanupRecovery(
+ cleanup,
+ 1,
+ "native session scheduled quarantined cleanup recovery",
+ );
+ }, QUARANTINED_SESSION_CLOSE_RETRY_MS);
+ cleanup.timer.unref?.();
+}
+
+async function retryQuarantinedSessionCleanups(
+ cleanupDomain: NativeSessionCleanupDomain,
+ signal?: AbortSignal,
+): Promise<void> {
+ // A close attempt becomes admission-visible as soon as the runtime retains
+ // its exact cleanup owner. It may not have rejected yet, so it may not have
+ // entered the retry quarantine below. Observe both states through the same
+ // finite gate to prevent a later execution from opening a second provider
+ // session while the first close still owns provider resources.
+ const observedOwners = new Set<Promise<void>>();
+ const acceleratedCleanups = new Set<QuarantinedSessionCleanup>();
+ let observedOwnerPhases = 0;
+ while (true) {
+ signal?.throwIfAborted();
+ const cleanupOwners = new Set<Promise<void>>(
+ [...failedSessionCleanupOwners]
+ .filter(([, domain]) => domain === cleanupDomain)
+ .map(([owner]) => owner),
+ );
+ for (const cleanup of quarantinedSessionCleanups) {
+ if (cleanup.domain !== cleanupDomain) continue;
+ if (cleanup.timer) {
+ clearTimeout(cleanup.timer);
+ cleanup.timer = null;
+ }
+ if (cleanup.recovery) {
+ // An autonomous scheduled owner receives one attempt. Admission must
+ // observe it without mistaking it for the complete three-attempt
+ // admission batch; if that attempt fails, the next bounded owner
+ // phase accelerates one full batch. Existing full recoveries already
+ // consumed that allowance and are never duplicated.
+ if (
+ (cleanup.recoveryMaxAttempts ?? 0) >=
+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES
+ ) {
+ acceleratedCleanups.add(cleanup);
+ }
+ cleanupOwners.add(cleanup.recovery);
+ } else if (!acceleratedCleanups.has(cleanup)) {
+ acceleratedCleanups.add(cleanup);
+ if (
+ cleanup.automaticAttempts <
+ MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS
+ ) {
+ cleanupOwners.add(
+ startQuarantinedSessionCleanupRecovery(
+ cleanup,
+ MAX_QUARANTINED_SESSION_CLOSE_RETRIES,
+ "native session quarantined admission recovery",
+ ),
+ );
+ }
+ }
+ }
+ const replacementOwners = [...cleanupOwners].filter(
+ (owner) => !observedOwners.has(owner),
+ );
+ if (replacementOwners.length === 0) break;
+ replacementOwners.forEach((owner) => observedOwners.add(owner));
+ observedOwnerPhases += 1;
+ if (
+ observedOwnerPhases > MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES ||
+ !(await settlesWithin(
+ Promise.all(
+ replacementOwners.map((owner) => owner.catch(() => undefined)),
+ ),
+ QUARANTINED_SESSION_ADMISSION_GRACE_MS,
+ signal,
+ ))
+ ) {
+ throw new Error(
+ "native_session_cleanup_quarantined: prior session cleanup exceeded the admission grace",
+ );
+ }
+ }
+ if (
+ [...failedSessionCleanupOwners.values()].some(
+ (domain) => domain === cleanupDomain,
+ ) ||
+ [...quarantinedSessionCleanups].some(
+ (cleanup) => cleanup.domain === cleanupDomain,
+ )
+ ) {
+ // Admission may have pulled a scheduled retry forward and observed a
+ // complete recovery batch that still failed. Restore autonomous ownership
+ // before rejecting so cleanup cannot remain dormant until another run.
+ for (const cleanup of quarantinedSessionCleanups) {
+ if (cleanup.domain !== cleanupDomain) continue;
+ scheduleQuarantinedSessionCleanup(cleanup);
+ }
+ throw new Error(
+ "native_session_cleanup_quarantined: prior session cleanup remains incomplete",
+ );
+ }
+}
async function consumeTurn(
session: NativeSession,
controlPlane: ControlPlanePort,
@@ -328,7 +698,9 @@ async function consumeTurn(
const externalAbortFailure = externalSignal
? new Promise<never>((_resolve, reject) => {
const abort = () => {
- const reason = externalSignal.reason ?? new Error("native event consumption aborted");
+ const reason =
+ externalSignal.reason ??
+ new Error("native event consumption aborted");
stopConsumer = true;
appendAbort.abort(reason);
reject(reason);
@@ -337,7 +709,8 @@ async function consumeTurn(
abort();
} else {
externalSignal.addEventListener("abort", abort, { once: true });
- removeExternalAbort = () => externalSignal.removeEventListener("abort", abort);
+ removeExternalAbort = () =>
+ externalSignal.removeEventListener("abort", abort);
}
})
: new Promise<never>(() => undefined);
@@ -347,128 +720,155 @@ async function consumeTurn(
inputTimers.delete(requestId);
};
const consumer = (async () => {
- let eventCount = 0;
- let highestContiguousSourceSeq = 0;
- let governedResult: PrpStructuredRunResult | null = null;
- while (true) {
- const next = await eventIterator.next();
- if (stopConsumer) throw new Error("native event consumer stopped");
- if (next.done) throw new Error("native event stream closed before a turn terminal fact");
- const event = next.value;
- const payload = event.payload as Record<string, unknown>;
- const settlingRequestId =
- ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]
- .includes(event.eventType)
- && typeof payload.requestId === "string"
- ? payload.requestId
- : null;
- // Beginning settlement revokes the expiry timer's handoff authority.
- // appendEvent may remain pending across the live-window deadline; if
- // the timer stayed live until the receipt returned, both settlement
- // and a durable handoff could commit for the same request.
- if (settlingRequestId !== null) clearInputTimer(settlingRequestId);
- const receipt = await controlPlane.appendEvent(event, {
- signal: appendAbort.signal,
- });
- if (stopConsumer) throw new Error("native event consumer stopped");
- eventCount += receipt.disposition === "committed" ? 1 : 0;
- highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq);
- const request = payload.request && typeof payload.request === "object" && !Array.isArray(payload.request)
- ? payload.request as Record<string, unknown>
- : null;
- if (
- receipt.disposition === "committed"
- && event.eventType === "runtime_request.created"
- && request?.schema === "paperclip.runtime_request.v2"
- && request.type === "input"
- && typeof request.requestId === "string"
- && typeof request.turnId === "string"
- ) {
- try {
- parsePaperclipQuestionSet(request.input);
- const requestId = request.requestId;
- const turnId = request.turnId;
- clearInputTimer(requestId);
- const inputTimer = setTimeout(() => {
- // Clearing a timeout does not revoke a callback that is already
- // queued. The map entry is the per-request authority token.
- if (inputTimers.get(requestId) !== inputTimer) return;
- inputTimers.delete(requestId);
- // A timer callback can already be queued when teardown clears
- // its handle. Re-check the live-turn authority inside the
- // callback before starting or registering durable work.
- if (stopConsumer || appendAbort.signal.aborted) return;
- if (session.handoffRuntimeRequest === undefined) {
- rejectHandoff?.(new Error("native_runtime_request_handoff_unavailable"));
- return;
- }
- let handoffCleanup: Promise<unknown>;
- try {
- const handoff = session.handoffRuntimeRequest({
- requestId,
- turnId,
- reason: "durable_handoff",
- signal: appendAbort.signal,
- });
- // Durable handoff mutation is synchronous. The returned
- // promise owns provider interruption only, so it can remain
- // observed without acquiring authority to delay or reverse
- // a provider terminal fact.
- handoffCleanup = handoff.cleanup;
- } catch (error) {
- rejectHandoff?.(error);
- return;
- }
- handoffCleanupOperations.add(handoffCleanup);
- void handoffCleanup
- .catch((error) => {
- if (!stopConsumer && !appendAbort.signal.aborted) {
- rejectHandoff?.(error);
- }
- })
- .finally(() => handoffCleanupOperations.delete(handoffCleanup));
- }, runtimeInputLiveWindowMs);
- inputTimer.unref?.();
- inputTimers.set(requestId, inputTimer);
- } catch {
- // Invalid structured inputs remain rejected by the driver and never become durable questions.
- }
- }
- if (governedResult === null && resolveGovernedWait) {
- if (appendAbort.signal.aborted) {
- throw appendAbort.signal.reason ?? new Error("native event consumption aborted");
+ let eventCount = 0;
+ let highestContiguousSourceSeq = 0;
+ let governedResult: PrpStructuredRunResult | null = null;
+ while (true) {
+ const next = await eventIterator.next();
+ if (stopConsumer) throw new Error("native event consumer stopped");
+ if (next.done)
+ throw new Error(
+ "native event stream closed before a turn terminal fact",
+ );
+ const event = next.value;
+ const payload = event.payload as Record<string, unknown>;
+ const settlingRequestId =
+ [
+ "runtime_request.resolved",
+ "runtime_request.cancelled",
+ "runtime_request.expired",
+ ].includes(event.eventType) && typeof payload.requestId === "string"
+ ? payload.requestId
+ : null;
+ // Beginning settlement revokes the expiry timer's handoff authority.
+ // appendEvent may remain pending across the live-window deadline; if
+ // the timer stayed live until the receipt returned, both settlement
+ // and a durable handoff could commit for the same request.
+ if (settlingRequestId !== null) clearInputTimer(settlingRequestId);
+ const receipt = await controlPlane.appendEvent(event, {
+ signal: appendAbort.signal,
+ });
+ if (stopConsumer) throw new Error("native event consumer stopped");
+ eventCount += receipt.disposition === "committed" ? 1 : 0;
+ highestContiguousSourceSeq = Math.max(
+ highestContiguousSourceSeq,
+ receipt.highestContiguousSourceSeq,
+ );
+ const request =
+ payload.request &&
+ typeof payload.request === "object" &&
+ !Array.isArray(payload.request)
+ ? (payload.request as Record<string, unknown>)
+ : null;
+ if (
+ receipt.disposition === "committed" &&
+ event.eventType === "runtime_request.created" &&
+ request?.schema === "paperclip.runtime_request.v2" &&
+ request.type === "input" &&
+ typeof request.requestId === "string" &&
+ typeof request.turnId === "string"
+ ) {
+ try {
+ parsePaperclipQuestionSet(request.input);
+ const requestId = request.requestId;
+ const turnId = request.turnId;
+ clearInputTimer(requestId);
+ const inputTimer = setTimeout(() => {
+ // Clearing a timeout does not revoke a callback that is already
+ // queued. The map entry is the per-request authority token.
+ if (inputTimers.get(requestId) !== inputTimer) return;
+ inputTimers.delete(requestId);
+ // A timer callback can already be queued when teardown clears
+ // its handle. Re-check the live-turn authority inside the
+ // callback before starting or registering durable work.
+ if (stopConsumer || appendAbort.signal.aborted) return;
+ if (session.handoffRuntimeRequest === undefined) {
+ rejectHandoff?.(
+ new Error("native_runtime_request_handoff_unavailable"),
+ );
+ return;
}
- governedResult = resolveGovernedWait({
- turnId: event.turnId ?? null,
- event,
- });
- if (governedResult !== null && !isTurnTerminal(event)) {
- if (session.cancel === undefined) {
- throw new Error("native_governed_wait_cancellation_unavailable");
- }
- // A governed result is already durable. Commit cancellation
- // synchronously, then stop consuming provider output now rather
- // than waiting for an abort-insensitive cleanup or terminal event.
- // The returned promise owns cleanup only and remains observed in
- // finally, where its wait is bounded and the session quarantined.
- const cancellation = session.cancel({
- reason: "Paperclip parked this turn on a durable governed interaction.",
+ let handoffCleanup: Promise<unknown>;
+ try {
+ const handoff = session.handoffRuntimeRequest({
+ requestId,
+ turnId,
+ reason: "durable_handoff",
signal: appendAbort.signal,
});
- governedCancellationCommitted = true;
- const cleanup = cancellation.cleanup;
- governedCleanupOperations.add(cleanup);
- void cleanup
- .catch(() => quarantineSession())
- .finally(() => governedCleanupOperations.delete(cleanup));
- return { event, eventCount, highestContiguousSourceSeq, governedResult };
+ // Durable handoff mutation is synchronous. The returned
+ // promise owns provider interruption only, so it can remain
+ // observed without acquiring authority to delay or reverse
+ // a provider terminal fact.
+ handoffCleanup = handoff.cleanup;
+ } catch (error) {
+ rejectHandoff?.(error);
+ return;
}
+ handoffCleanupOperations.add(handoffCleanup);
+ void handoffCleanup
+ .catch((error) => {
+ if (!stopConsumer && !appendAbort.signal.aborted) {
+ rejectHandoff?.(error);
+ }
+ })
+ .finally(() => handoffCleanupOperations.delete(handoffCleanup));
+ }, runtimeInputLiveWindowMs);
+ inputTimer.unref?.();
+ inputTimers.set(requestId, inputTimer);
+ } catch {
+ // Invalid structured inputs remain rejected by the driver and never become durable questions.
+ }
+ }
+ if (governedResult === null && resolveGovernedWait) {
+ if (appendAbort.signal.aborted) {
+ throw (
+ appendAbort.signal.reason ??
+ new Error("native event consumption aborted")
+ );
+ }
+ governedResult = resolveGovernedWait({
+ turnId: event.turnId ?? null,
+ event,
+ });
+ if (governedResult !== null && !isTurnTerminal(event)) {
+ if (session.cancel === undefined) {
+ throw new Error("native_governed_wait_cancellation_unavailable");
}
- if (isTurnTerminal(event)) {
- return { event, eventCount, highestContiguousSourceSeq, governedResult };
- }
+ // A governed result is already durable. Commit cancellation
+ // synchronously, then stop consuming provider output now rather
+ // than waiting for an abort-insensitive cleanup or terminal event.
+ // The returned promise owns cleanup only and remains observed in
+ // finally, where its wait is bounded and the session quarantined.
+ const cancellation = session.cancel({
+ reason:
+ "Paperclip parked this turn on a durable governed interaction.",
+ signal: appendAbort.signal,
+ });
+ governedCancellationCommitted = true;
+ const cleanup = cancellation.cleanup;
+ governedCleanupOperations.add(cleanup);
+ void cleanup
+ .catch(() => quarantineSession())
+ .finally(() => governedCleanupOperations.delete(cleanup));
+ return {
+ event,
+ eventCount,
+ highestContiguousSourceSeq,
+ governedResult,
+ };
}
- })();
+ }
+ if (isTurnTerminal(event)) {
+ return {
+ event,
+ eventCount,
+ highestContiguousSourceSeq,
+ governedResult,
+ };
+ }
+ }
+ })();
// A timeout can win the race while an iterator is still waiting for data.
// Observe any later consumer rejection so it cannot become process-fatal.
void consumer.catch(() => undefined);
@@ -494,8 +894,15 @@ async function consumeTurn(
// cancellation has already committed synchronously. Bound only the
// authority-free provider cleanup while keeping its outcome observed.
if (governedCleanupOperations.size > 0) {
- const governedCleanupSettlement = Promise.allSettled([...governedCleanupOperations]);
- if (!(await settlesWithin(governedCleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS))) {
+ const governedCleanupSettlement = Promise.allSettled([
+ ...governedCleanupOperations,
+ ]);
+ if (
+ !(await settlesWithin(
+ governedCleanupSettlement,
+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,
+ ))
+ ) {
deferredGovernedCleanupSettlement = governedCleanupSettlement;
}
}
@@ -504,20 +911,23 @@ async function consumeTurn(
session,
"Native session event consumption failed.",
);
- deferredSessionCancellationSettlement = deferredCancellation?.settlement ?? null;
+ deferredSessionCancellationSettlement =
+ deferredCancellation?.settlement ?? null;
}
throw error;
} finally {
stopConsumer = true;
for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer);
inputTimers.clear();
- const activeHandoffCleanupSettlement = handoffCleanupOperations.size > 0
- ? Promise.allSettled([...handoffCleanupOperations])
- : null;
- const activeGovernedCleanupSettlement = deferredGovernedCleanupSettlement === null
- && governedCleanupOperations.size > 0
- ? Promise.allSettled([...governedCleanupOperations])
- : null;
+ const activeHandoffCleanupSettlement =
+ handoffCleanupOperations.size > 0
+ ? Promise.allSettled([...handoffCleanupOperations])
+ : null;
+ const activeGovernedCleanupSettlement =
+ deferredGovernedCleanupSettlement === null &&
+ governedCleanupOperations.size > 0
+ ? Promise.allSettled([...governedCleanupOperations])
+ : null;
if (!consumptionFailed && !appendAbort.signal.aborted) {
// A provider terminal or synchronous governed cancellation revokes
// live-turn authority. Handoff state was already committed; abort only
@@ -537,9 +947,15 @@ async function consumeTurn(
const passiveTeardownSettlement = Promise.allSettled([
iteratorTeardown,
consumer,
- ...(activeHandoffCleanupSettlement ? [activeHandoffCleanupSettlement] : []),
- ...(activeGovernedCleanupSettlement ? [activeGovernedCleanupSettlement] : []),
- ...(deferredGovernedCleanupSettlement ? [deferredGovernedCleanupSettlement] : []),
+ ...(activeHandoffCleanupSettlement
+ ? [activeHandoffCleanupSettlement]
+ : []),
+ ...(activeGovernedCleanupSettlement
+ ? [activeGovernedCleanupSettlement]
+ : []),
+ ...(deferredGovernedCleanupSettlement
+ ? [deferredGovernedCleanupSettlement]
+ : []),
...(deferredSessionCancellationSettlement
? [deferredSessionCancellationSettlement]
: []),
@@ -556,7 +972,10 @@ async function consumeTurn(
passiveTeardownSettlement,
Promise.resolve().then(closeFailedSession),
]);
- await settlesWithin(cleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS);
+ await settlesWithin(
+ cleanupSettlement,
+ FAILED_OPERATION_SETTLEMENT_GRACE_MS,
+ );
} else {
// Iterator and provider cleanup own no control-plane mutation authority.
// A slow subscription or cleanup remains observed and is released by the
@@ -611,7 +1030,10 @@ async function reconcileRecoveryCursor(input: {
persistedHighWater = Math.max(persistedHighWater, pageHighWater);
afterSourceSeq = pageHighWater;
}
- if (persistedHighWater === checkpointHighWater && input.checkpoint.cursor === String(checkpointHighWater)) {
+ if (
+ persistedHighWater === checkpointHighWater &&
+ input.checkpoint.cursor === String(checkpointHighWater)
+ ) {
return input.checkpoint;
}
return { ...input.checkpoint, cursor: String(persistedHighWater) };
@@ -639,9 +1061,12 @@ async function replayCheckpointedTurnTerminal(input: {
// Disposition recovery owns the newest durable provider terminal. An
// older task turn may also have a valid proposal, but selecting it would
// finalize stale work and strand the actual recovery terminal.
- const terminal = [...terminals]
- .sort((left, right) => right.sourceSeq - left.sourceSeq)[0] ?? null;
- const proposalSequence = latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0;
+ const terminal =
+ [...terminals].sort(
+ (left, right) => right.sourceSeq - left.sourceSeq,
+ )[0] ?? null;
+ const proposalSequence =
+ latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0;
return terminal === null
? null
: {
@@ -663,11 +1088,9 @@ async function replayCheckpointedTurnTerminal(input: {
if (
event.turnId &&
isTurnTerminal(event) &&
- (
- input.expectedTurnId !== undefined && input.expectedTurnId !== null
- ? event.turnId === input.expectedTurnId
- : !priorTerminalTurnIds.has(event.turnId)
- )
+ (input.expectedTurnId !== undefined && input.expectedTurnId !== null
+ ? event.turnId === input.expectedTurnId
+ : !priorTerminalTurnIds.has(event.turnId))
) {
terminals.push(structuredClone(event));
}
@@ -690,30 +1113,37 @@ function checkpointedResultlessDispositionFallback(input: {
}): PrpEvent | null {
const turnId = input.persisted.dispositionOnlyRecoveryTurnId;
if (
- !input.persisted.dispositionOnlyRecoveryConsumed
- || input.persisted.semanticResult
- || input.persisted.activeTurnId
- || typeof turnId !== "string"
- || turnId.length === 0
- ) return null;
- const persistedTerminal = input.persisted.terminalTurns?.filter(
- (terminal) => terminal.turnId === turnId,
- ) ?? [];
- if (persistedTerminal.length !== 1 || persistedTerminal[0]!.fingerprint.length === 0) {
+ !input.persisted.dispositionOnlyRecoveryConsumed ||
+ input.persisted.semanticResult ||
+ input.persisted.activeTurnId ||
+ typeof turnId !== "string" ||
+ turnId.length === 0
+ )
+ return null;
+ const persistedTerminal =
+ input.persisted.terminalTurns?.filter(
+ (terminal) => terminal.turnId === turnId,
+ ) ?? [];
+ if (
+ persistedTerminal.length !== 1 ||
+ persistedTerminal[0]!.fingerprint.length === 0
+ ) {
return null;
}
const recoveredTurnId = input.recovered.dispositionOnlyRecoveryTurnId;
- const recoveredTerminal = input.recovered.terminalTurns?.filter(
- (terminal) => terminal.turnId === recoveredTurnId,
- ) ?? [];
+ const recoveredTerminal =
+ input.recovered.terminalTurns?.filter(
+ (terminal) => terminal.turnId === recoveredTurnId,
+ ) ?? [];
if (
- !input.recovered.dispositionOnlyRecoveryConsumed
- || input.recovered.semanticResult
- || input.recovered.activeTurnId
- || recoveredTurnId !== turnId
- || recoveredTerminal.length !== 1
- || recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint
- || canonicalJson(input.recovered.identity) !== canonicalJson(input.persisted.identity)
+ !input.recovered.dispositionOnlyRecoveryConsumed ||
+ input.recovered.semanticResult ||
+ input.recovered.activeTurnId ||
+ recoveredTurnId !== turnId ||
+ recoveredTerminal.length !== 1 ||
+ recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint ||
+ canonicalJson(input.recovered.identity) !==
+ canonicalJson(input.persisted.identity)
) {
throw new Error("native_disposition_recovery_checkpoint_conflict");
}
@@ -741,43 +1171,57 @@ function checkpointedResultlessDispositionFallback(input: {
* Package-owned normalized session loop. Paperclip supplies persistence and
* authority through ControlPlanePort; provider/session behavior stays here.
*/
-export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise<NativeSessionExecutionResult> {
+export async function executeNativeSession(
+ options: ExecuteNativeSessionOptions,
+): Promise<NativeSessionExecutionResult> {
const input = parseNativeExecutionInput(options.input);
const descriptor = await options.backend.descriptor();
+ const cleanupDomain = JSON.stringify([
+ input.binding.companyId,
+ descriptor.kind,
+ descriptor.name,
+ ]);
+ await retryQuarantinedSessionCleanups(cleanupDomain, options.signal);
if ("runtimeContext" in input) {
const capabilities = descriptor.runtimeContextCapabilities;
- const unsupported = (["instructions", "skills", "mcp"] as const).filter((key) => capabilities?.[key] !== "native");
- if (unsupported.length) throw new Error(`native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`);
+ const unsupported = (["instructions", "skills", "mcp"] as const).filter(
+ (key) => capabilities?.[key] !== "native",
+ );
+ if (unsupported.length)
+ throw new Error(
+ `native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`,
+ );
}
let persistedSession = options.existingSession
? null
- : options.persistedSession ?? await options.controlPlane.loadSessionCheckpoint?.() ?? null;
+ : (options.persistedSession ??
+ (await options.controlPlane.loadSessionCheckpoint?.()) ??
+ null);
if (
- persistedSession
- && (
- persistedSession.identity.runId !== input.binding.runId
- || persistedSession.identity.companyId !== input.binding.companyId
- || persistedSession.identity.issueId !== input.binding.issueId
- || persistedSession.identity.agentId !== input.binding.agentId
- || input.session.normalizedSessionId === null
- || persistedSession.identity.sessionId !== input.session.normalizedSessionId
- )
- ) throw new Error("native_session_checkpoint_binding_mismatch");
+ persistedSession &&
+ (persistedSession.identity.runId !== input.binding.runId ||
+ persistedSession.identity.companyId !== input.binding.companyId ||
+ persistedSession.identity.issueId !== input.binding.issueId ||
+ persistedSession.identity.agentId !== input.binding.agentId ||
+ input.session.normalizedSessionId === null ||
+ persistedSession.identity.sessionId !== input.session.normalizedSessionId)
+ )
+ throw new Error("native_session_checkpoint_binding_mismatch");
const existingIdentity = options.existingSession?.identity() ?? null;
if (
- existingIdentity
- && (
- existingIdentity.companyId !== input.binding.companyId
- || existingIdentity.issueId !== input.binding.issueId
- || existingIdentity.agentId !== input.binding.agentId
- || input.session.normalizedSessionId === null
- || existingIdentity.sessionId !== input.session.normalizedSessionId
- )
- ) throw new Error("native_session_attach_binding_mismatch");
- const normalizedSessionId = persistedSession?.identity.sessionId
- ?? existingIdentity?.sessionId
- ?? input.session.normalizedSessionId
- ?? randomUUID();
+ existingIdentity &&
+ (existingIdentity.companyId !== input.binding.companyId ||
+ existingIdentity.issueId !== input.binding.issueId ||
+ existingIdentity.agentId !== input.binding.agentId ||
+ input.session.normalizedSessionId === null ||
+ existingIdentity.sessionId !== input.session.normalizedSessionId)
+ )
+ throw new Error("native_session_attach_binding_mismatch");
+ const normalizedSessionId =
+ persistedSession?.identity.sessionId ??
+ existingIdentity?.sessionId ??
+ input.session.normalizedSessionId ??
+ randomUUID();
const identity = {
runId: input.binding.runId,
sessionId: normalizedSessionId,
@@ -811,6 +1255,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
options.existingSession,
options.onSession,
"native session attachment failed",
+ cleanupDomain,
);
throw error;
}
@@ -826,13 +1271,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
persistedSession = await runAbortableOperationWithin({
timeoutMs: recoveryTimeoutMs,
timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`,
- operation: (signal) => reconcileRecoveryCursor({
- controlPlane: options.controlPlane,
- checkpoint: recoveryCheckpoint,
- runId: input.binding.runId,
- sourceInstanceId: options.runnerInstanceId,
- signal,
- }),
+ operation: (signal) =>
+ reconcileRecoveryCursor({
+ controlPlane: options.controlPlane,
+ checkpoint: recoveryCheckpoint,
+ runId: input.binding.runId,
+ sourceInstanceId: options.runnerInstanceId,
+ signal,
+ }),
});
reconciledRecoveryCheckpoint = persistedSession;
const providerRecoveryCheckpoint = persistedSession;
@@ -844,28 +1290,35 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
? await runAbortableOperationWithin({
timeoutMs: recoveryTimeoutMs,
timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`,
- operation: (signal) => options.backend.recoverSession!(
- providerRecoveryCheckpoint,
- { signal },
- ),
+ operation: (signal) =>
+ options.backend.recoverSession!(providerRecoveryCheckpoint, {
+ signal,
+ }),
onLateResolution: async (lateRecovery) => {
if (lateRecovery.session) {
await disposeUnadmittedSession(
lateRecovery.session,
"native session provider recovery timed out",
+ cleanupDomain,
);
}
},
})
- : { recovered: false as const, reason: "driver does not support recovery" };
+ : {
+ recovered: false as const,
+ reason: "driver does not support recovery",
+ };
if (!recovery.recovered || !recovery.session) {
if (!replacementAllowed) {
- throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`);
+ throw new Error(
+ `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`,
+ );
}
continuityBreak = {
reason: recovery.reason ?? "provider session is no longer recoverable",
previousDriverSessionId: providerRecoveryCheckpoint.sessionId,
- previousProviderSessionId: providerRecoveryCheckpoint.providerSessionId ?? null,
+ previousProviderSessionId:
+ providerRecoveryCheckpoint.providerSessionId ?? null,
};
const replacementInput = {
identity,
@@ -883,10 +1336,12 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
)
: options.backend.openSession(abortableReplacementInput);
},
- onLateResolution: (lateSession) => disposeUnadmittedSession(
- lateSession,
- "native session replacement bootstrap timed out",
- ),
+ onLateResolution: (lateSession) =>
+ disposeUnadmittedSession(
+ lateSession,
+ "native session replacement bootstrap timed out",
+ cleanupDomain,
+ ),
});
} else {
session = recovery.session;
@@ -905,14 +1360,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
session = await runAbortableOperationWithin({
timeoutMs: bootstrapTimeoutMs,
timeoutMessage: `native session bootstrap timed out after ${bootstrapTimeoutMs}ms`,
- operation: (signal) => options.backend.openSession({
- ...bootstrapInput,
- signal,
- }),
- onLateResolution: (lateSession) => disposeUnadmittedSession(
- lateSession,
- "native session bootstrap timed out",
- ),
+ operation: (signal) =>
+ options.backend.openSession({
+ ...bootstrapInput,
+ signal,
+ }),
+ onLateResolution: (lateSession) =>
+ disposeUnadmittedSession(
+ lateSession,
+ "native session bootstrap timed out",
+ cleanupDomain,
+ ),
});
}
try {
@@ -930,6 +1388,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
session,
options.onSession,
"native control-plane run admission failed",
+ cleanupDomain,
);
}
throw error;
@@ -945,14 +1404,66 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
// Owner notification cannot prevent provider cleanup.
}
};
+ let failedCleanupDeferred = false;
+ let sessionCloseRecoveryPromise: Promise<void> | null = null;
+ const retainFailedCleanup = (cleanup: Promise<void>) => {
+ failedCleanupDeferred = true;
+ quarantineSession();
+ retainFailedSessionCleanupOwner(cleanup, cleanupDomain);
+ };
+ const startSessionClose = (reason: string) => {
+ const attempt = session.close({ reason });
+ sessionClosePromise = attempt;
+ return attempt;
+ };
const closeSession = () => {
if (sessionClosePromise === null) {
quarantineSession();
- sessionClosePromise = session.close({
- reason: "native session execution complete",
- });
+ const firstAttempt = startSessionClose(
+ "native session execution complete",
+ );
+ // Preserve the first close outcome for its caller. If an exact attempt
+ // fails, retain one ordered, delay-bounded production recovery within a
+ // finite retry budget. A still-pending attempt is never overlapped or
+ // replaced, and repeated terminal failure cannot create an immortal loop.
+ const recovery = (async () => {
+ let attempt = firstAttempt;
+ let retryCount = 0;
+ while (true) {
+ try {
+ await attempt;
+ return;
+ } catch (error) {
+ if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
+ quarantineSessionCleanup(session, cleanupDomain);
+ throw error;
+ }
+ retryCount += 1;
+ await waitForSessionCloseRetry();
+ if (sessionClosePromise === attempt) {
+ sessionClosePromise = null;
+ }
+ attempt = startSessionClose(
+ `native session cleanup recovery after close failure (${retryCount})`,
+ );
+ }
+ }
+ })();
+ sessionCloseRecoveryPromise = recovery;
+ retainFailedCleanup(recovery);
+ void recovery
+ .finally(() => {
+ if (sessionCloseRecoveryPromise === recovery) {
+ sessionCloseRecoveryPromise = null;
+ }
+ })
+ .catch(() => undefined);
+ }
+ const activeClose = sessionClosePromise;
+ if (activeClose === null) {
+ throw new Error("native session cleanup lost its active close attempt");
}
- return sessionClosePromise;
+ return activeClose;
};
let executionSucceeded = false;
try {
@@ -960,8 +1471,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
// callback fails, the finally block below still quarantines and closes the
// provider session.
options.onSession?.(session);
- const checkpointTimeoutMs = options.checkpointTimeoutMs
- ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS;
+ const checkpointTimeoutMs =
+ options.checkpointTimeoutMs ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS;
const persistCheckpoint = (
snapshot: PersistedNativeSession,
externalSignal?: AbortSignal,
@@ -987,14 +1498,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
};
const recoveredSnapshot = await session.snapshot();
const recoveredActiveTurnId = recovered
- ? recoveredSnapshot.activeTurnId ?? null
- : persistedSession?.activeTurnId ?? null;
+ ? (recoveredSnapshot.activeTurnId ?? null)
+ : (persistedSession?.activeTurnId ?? null);
const adoptedDispositionTerminal = Boolean(
- recovered
- && recoveredSnapshot.dispositionOnlyRecoveryConsumed
- && !recoveredActiveTurnId
- && (recoveredSnapshot.terminalTurns?.length ?? 0)
- > (persistedSession?.terminalTurns?.length ?? 0)
+ recovered &&
+ recoveredSnapshot.dispositionOnlyRecoveryConsumed &&
+ !recoveredActiveTurnId &&
+ (recoveredSnapshot.terminalTurns?.length ?? 0) >
+ (persistedSession?.terminalTurns?.length ?? 0),
);
if (continuityBreak) {
await options.onContinuityBreak?.({
@@ -1020,16 +1531,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
highestContiguousSourceSeq: 0,
governedResult: null as PrpStructuredRunResult | null,
};
- const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal
- ? recoveredSnapshot
- : persistedSession;
- let completed = completionSnapshot?.semanticResult && completionSnapshot.terminal
- ? {
- result: completionSnapshot.semanticResult,
- terminal: completionSnapshot.terminal,
- turnId: completionSnapshot.activeTurnId ?? null,
- }
- : null;
+ const completionSnapshot =
+ recoveredSnapshot.semanticResult && recoveredSnapshot.terminal
+ ? recoveredSnapshot
+ : persistedSession;
+ let completed =
+ completionSnapshot?.semanticResult && completionSnapshot.terminal
+ ? {
+ result: completionSnapshot.semanticResult,
+ terminal: completionSnapshot.terminal,
+ turnId: completionSnapshot.activeTurnId ?? null,
+ }
+ : null;
if (!completed) {
// A recovered driver is authoritative about whether a provider turn is
// still active. In particular, drivers normalize the checkpoint race
@@ -1038,45 +1551,48 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
// resurrects that terminal turn and waits forever for an event that was
// already consumed.
const dispositionRecoveryWasSubmitted = Boolean(
- recovered
- && persistedSession?.dispositionOnlyRecoveryConsumed
- && !recoveredSnapshot.semanticResult
- && !recoveredActiveTurnId
+ recovered &&
+ persistedSession?.dispositionOnlyRecoveryConsumed &&
+ !recoveredSnapshot.semanticResult &&
+ !recoveredActiveTurnId,
);
const dispositionRecoveryTurnId =
- persistedSession?.dispositionOnlyRecoveryTurnId
- ?? recoveredSnapshot.dispositionOnlyRecoveryTurnId
- ?? null;
+ persistedSession?.dispositionOnlyRecoveryTurnId ??
+ recoveredSnapshot.dispositionOnlyRecoveryTurnId ??
+ null;
const recoveredDispositionTurnObserved = Boolean(
- dispositionRecoveryTurnId
- && recoveredSnapshot.terminalTurns?.some(
+ dispositionRecoveryTurnId &&
+ recoveredSnapshot.terminalTurns?.some(
(terminal) => terminal.turnId === dispositionRecoveryTurnId,
- )
+ ),
);
const dispositionRecoveryStillOwned = Boolean(
- dispositionRecoveryWasSubmitted
- && recoveredSnapshot.dispositionOnlyRecoveryConsumed
- && dispositionRecoveryTurnId !== null
- && recoveredDispositionTurnObserved
+ dispositionRecoveryWasSubmitted &&
+ recoveredSnapshot.dispositionOnlyRecoveryConsumed &&
+ dispositionRecoveryTurnId !== null &&
+ recoveredDispositionTurnObserved,
);
const replayedDisposition =
dispositionRecoveryWasSubmitted && dispositionRecoveryTurnId !== null
- ? await replayCheckpointedTurnTerminal({
- controlPlane: options.controlPlane,
- runId: input.binding.runId,
- sourceInstanceId: options.runnerInstanceId,
- priorTerminalTurnIds: (persistedSession?.terminalTurns ?? [])
- .map((terminal) => terminal.turnId),
- expectedTurnId: dispositionRecoveryTurnId,
- })
- : null;
+ ? await replayCheckpointedTurnTerminal({
+ controlPlane: options.controlPlane,
+ runId: input.binding.runId,
+ sourceInstanceId: options.runnerInstanceId,
+ priorTerminalTurnIds: (persistedSession?.terminalTurns ?? []).map(
+ (terminal) => terminal.turnId,
+ ),
+ expectedTurnId: dispositionRecoveryTurnId,
+ })
+ : null;
// Durable replay remains authoritative. If it has no terminal, an exact
// consumed marker bound to the same terminal fingerprint in both
// checkpoints proves provider completion without reconstructing provider
// output. Give only that non-provider fact to control-plane policy; a
// mismatch fails closed and a null policy result fails finalization.
const dispositionFallback =
- dispositionRecoveryWasSubmitted && replayedDisposition === null && persistedSession
+ dispositionRecoveryWasSubmitted &&
+ replayedDisposition === null &&
+ persistedSession
? checkpointedResultlessDispositionFallback({
persisted: persistedSession,
recovered: recoveredSnapshot,
@@ -1085,26 +1601,29 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
: null;
const checkpointedDispositionTerminal =
replayedDisposition !== null || dispositionFallback !== null;
- const recoveryTerminal = replayedDisposition?.terminal ?? dispositionFallback;
+ const recoveryTerminal =
+ replayedDisposition?.terminal ?? dispositionFallback;
const consumptionAbort = new AbortController();
- const consuming = recoveryTerminal === null
- ? consumeTurn(
- session,
- options.controlPlane,
- options.timeoutMs ?? 900_000,
- options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS,
- closeSession,
- quarantineSession,
- options.resolveGovernedWait,
- consumptionAbort.signal,
- )
- : Promise.resolve({
- event: recoveryTerminal,
- eventCount: 0,
- highestContiguousSourceSeq:
- replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq,
- governedResult: null,
- });
+ const consuming =
+ recoveryTerminal === null
+ ? consumeTurn(
+ session,
+ options.controlPlane,
+ options.timeoutMs ?? 900_000,
+ options.runtimeInputLiveWindowMs ??
+ DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS,
+ closeSession,
+ quarantineSession,
+ options.resolveGovernedWait,
+ consumptionAbort.signal,
+ )
+ : Promise.resolve({
+ event: recoveryTerminal,
+ eventCount: 0,
+ highestContiguousSourceSeq:
+ replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq,
+ governedResult: null,
+ });
// Event consumption must begin before startTurn so an eager provider cannot
// outrun us. Observe its rejection immediately, though: if startTurn or
// checkpointing fails first, the outer finally closes the session and the
@@ -1113,20 +1632,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
void consuming.catch(() => undefined);
try {
if (
- !recovered
- || (
- !recoveredActiveTurnId
- && !adoptedDispositionTerminal
- && !checkpointedDispositionTerminal
- && !dispositionRecoveryStillOwned
- )
+ !recovered ||
+ (!recoveredActiveTurnId &&
+ !adoptedDispositionTerminal &&
+ !checkpointedDispositionTerminal &&
+ !dispositionRecoveryStillOwned)
) {
const modelEnvelope = buildNativeModelEnvelope(input);
const dispositionOnlyRecovery = Boolean(
recovered &&
!recoveredSnapshot.semanticResult &&
(persistedSession?.terminalTurns?.length ?? 0) > 0 &&
- !recoveredActiveTurnId
+ !recoveredActiveTurnId,
);
if (dispositionOnlyRecovery) {
modelEnvelope.task.prompt = [
@@ -1138,7 +1655,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
}
await session.startTurn({
message: { role: "user", text: JSON.stringify(modelEnvelope) },
- requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default",
+ requestedCollaborationMode:
+ "executionMode" in input ? input.executionMode : "default",
});
await checkpoint();
}
@@ -1162,21 +1680,24 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
if (settledCompletion === null) {
const terminalEvent = consumed.event;
if (terminalEvent === null) {
- throw new Error("native_finalization_missing: session returned no terminal event");
+ throw new Error(
+ "native_finalization_missing: session returned no terminal event",
+ );
}
- settledCompletion = consumed.governedResult === null
- ? await session.result()
- : {
- result: consumed.governedResult,
- terminal: {
- schema: "paperclip.prp.terminal.v1",
- turnTerminalState: "completed",
- runTerminalState: "succeeded",
- reportedWorkDisposition:
- consumed.governedResult.reportedWorkDisposition,
- },
- turnId: terminalEvent.turnId ?? null,
- };
+ settledCompletion =
+ consumed.governedResult === null
+ ? await session.result()
+ : {
+ result: consumed.governedResult,
+ terminal: {
+ schema: "paperclip.prp.terminal.v1",
+ turnTerminalState: "completed",
+ runTerminalState: "succeeded",
+ reportedWorkDisposition:
+ consumed.governedResult.reportedWorkDisposition,
+ },
+ turnId: terminalEvent.turnId ?? null,
+ };
signal.throwIfAborted();
if (settledCompletion === null && options.resolveMissingResult) {
const recoveredResult = await options.resolveMissingResult({
@@ -1203,12 +1724,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
completed = settledCompletion;
}
if (settledCompletion === null) {
- throw new Error("native_finalization_missing: session returned no semantic result");
+ throw new Error(
+ "native_finalization_missing: session returned no semantic result",
+ );
}
let terminal: PrpTerminalState;
if (consumed.governedResult !== null) {
terminal = settledCompletion.terminal;
- } else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) {
+ } else if (
+ completionSnapshot?.semanticResult &&
+ completionSnapshot.terminal
+ ) {
terminal = completionSnapshot.terminal;
} else {
terminal = terminalFromEvent(
@@ -1216,10 +1742,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
settledCompletion.result.reportedWorkDisposition,
);
}
- const eventTurnId = settledCompletion.turnId
- ?? persistedSession?.activeTurnId
- ?? persistedSession?.terminalTurns?.at(-1)?.turnId
- ?? consumed.event?.turnId;
+ const eventTurnId =
+ settledCompletion.turnId ??
+ persistedSession?.activeTurnId ??
+ persistedSession?.terminalTurns?.at(-1)?.turnId ??
+ consumed.event?.turnId;
const controlEvent = (
sourceSeq: number,
eventType: PrpEvent["eventType"],
@@ -1261,22 +1788,30 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({
timeoutMs: finalizationTimeoutMs,
operation: async (signal) => {
- const controlReplay = await options.controlPlane.replayEvents({
- runId: input.binding.runId,
- sourceInstanceId: options.controlPlaneInstanceId,
- afterSourceSeq: 0,
- limit: 10,
- }, { signal });
+ const controlReplay = await options.controlPlane.replayEvents(
+ {
+ runId: input.binding.runId,
+ sourceInstanceId: options.controlPlaneInstanceId,
+ afterSourceSeq: 0,
+ limit: 10,
+ },
+ { signal },
+ );
signal.throwIfAborted();
- const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event]));
+ const replayBySequence = new Map(
+ controlReplay.events.map((event) => [event.sourceSeq, event]),
+ );
for (const existing of controlReplay.events) {
- const expected = preparedFinalization.expectedControlEvents[existing.sourceSeq - 1];
+ const expected =
+ preparedFinalization.expectedControlEvents[existing.sourceSeq - 1];
if (
- expected === undefined
- || existing.eventType !== expected.eventType
- || canonicalJson(existing.payload) !== canonicalJson(expected.payload)
+ expected === undefined ||
+ existing.eventType !== expected.eventType ||
+ canonicalJson(existing.payload) !== canonicalJson(expected.payload)
) {
- throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`);
+ throw new Error(
+ `native_control_event_replay_conflict:${existing.sourceSeq}`,
+ );
}
}
if (!baselineControlReplayCaptured) {
@@ -1287,8 +1822,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
} else {
for (const existing of controlReplay.events) {
if (
- !baselineControlEventSequences.has(existing.sourceSeq)
- && !accountedControlEventSequences.has(existing.sourceSeq)
+ !baselineControlEventSequences.has(existing.sourceSeq) &&
+ !accountedControlEventSequences.has(existing.sourceSeq)
) {
accountedControlEventSequences.add(existing.sourceSeq);
consumed.eventCount += 1;
@@ -1301,11 +1836,13 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
);
for (const event of preparedFinalization.expectedControlEvents) {
if (replayBySequence.has(event.sourceSeq)) continue;
- const receipt = await options.controlPlane.appendEvent(event, { signal });
+ const receipt = await options.controlPlane.appendEvent(event, {
+ signal,
+ });
signal.throwIfAborted();
if (
- receipt.disposition === "committed"
- && !accountedControlEventSequences.has(event.sourceSeq)
+ receipt.disposition === "committed" &&
+ !accountedControlEventSequences.has(event.sourceSeq)
) {
accountedControlEventSequences.add(event.sourceSeq);
consumed.eventCount += 1;
@@ -1315,13 +1852,16 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
receipt.highestContiguousSourceSeq,
);
}
- await options.controlPlane.completeRun({
- result: preparedFinalization.completed.result,
- terminal: preparedFinalization.terminal,
- turnId: preparedFinalization.completed.turnId,
- callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`,
- callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,
- }, { signal });
+ await options.controlPlane.completeRun(
+ {
+ result: preparedFinalization.completed.result,
+ terminal: preparedFinalization.terminal,
+ turnId: preparedFinalization.completed.turnId,
+ callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`,
+ callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,
+ },
+ { signal },
+ );
return {
result: preparedFinalization.completed.result,
terminal: preparedFinalization.terminal,
@@ -1351,13 +1891,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
};
await persistCheckpoint(completedSnapshot, signal);
signal.throwIfAborted();
- const usage = await session.usage?.() ?? null;
+ const usage = (await session.usage?.()) ?? null;
signal.throwIfAborted();
return {
providerSessionId: snapshot.providerSessionId ?? null,
- driverVersion: typeof usage?.driverVersion === "string"
- ? usage.driverVersion
- : descriptor.version,
+ driverVersion:
+ typeof usage?.driverVersion === "string"
+ ? usage.driverVersion
+ : descriptor.version,
usage,
};
},
@@ -1376,7 +1917,10 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
executionSucceeded = true;
return { ...durableExecutionResult, ...enrichment };
} finally {
- if (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) {
+ if (
+ (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) &&
+ !failedCleanupDeferred
+ ) {
// A provider that ignores close must not keep execution pending forever.
// closeSession removes it from the caller before invoking the backend;
// retain observation of the promise, but bound the final join. Provider