The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
ci(runner): split retained-settlement suite out of runnerd-codex-transport.test.ts (#13557)
Devin Foleyec40bd8bf671
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.tsnew file mode 100644index 0000000000..b760a59e09--- /dev/null+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts@@ -0,0 +1,1754 @@+// The retained-settlement maintenance suite below moved out of+// runnerd-codex-transport.test.ts. Vitest schedules whole files onto+// workers, so that single 8.9k-line file serialized ~400s of tests and was+// the wall-clock critical path of the PR "Verify Paperclip Runner (vitest)"+// lane; this family alone accounts for ~160s of it. Keeping it in its own+// file lets the worker pool and --shard run it alongside the rest of the+// transport suite instead of after it. Every case provisions its own+// mkdtemp state directory, so the file is safe to run in parallel with+// runnerd-codex-transport.test.ts.+import {+ cp,+ mkdir,+ mkdtemp,+ readFile,+ rename,+ rm,+ symlink,+ writeFile,+} from "node:fs/promises";+import { createHash, randomUUID } from "node:crypto";+import { spawn } from "node:child_process";+import { readFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";++import { expect, it, vi } from "vitest";+import type { ControlPlanePort } from "../contracts/control-plane-port.js";+import type { NativeExecutionInputV1 } from "../contracts/native-execution.js";+import type {+ NativeSession,+ NativeSessionBackend,+} from "../contracts/native-session-backend.js";+import type { PrpEvent } from "../protocol/replay-contract.js";+import { completeRetainedNativeSessionCleanup, executeNativeSession } from "../native-session-runtime.js";+import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";+import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";+import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js";++import {+ createCapabilityRunnerdCodexTransport,+ createCapabilityRunnerdProviderEnvironment,+ defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary,+ drainRetainedRunnerdMaintenanceOperations,+ runnerdRecoveryInternals,+ resolveRunnerdSessionIdentity,+ settleRetainedRunnerdSession,+ retainedRunnerdCleanupProofIsCurrent,+ retainedRunnerdMaintenanceIsIdle,+} from "./runnerd-codex-transport.js";++// Explicit private-artifact test lane; production/default dist is never changed.+const defaultCapabilityRunnerdBinary = () =>+ process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ??+ qualifiedCapabilityRunnerdBinary();++function maintenanceFixtureBackendName(fixtureId: string): string {+ return `maintenance-test-${fixtureId}`;+}++function maintenanceReplaySnapshot(directory: string) {+ const bytes = [+ "control-plane/control-plane-state.json",+ "runner/runner-state.json",+ "runner/codex-provider-state.json",+ ].map((file) => readFileSync(join(directory, file)));+ return {+ control: JSON.parse(bytes[0]!.toString("utf8")),+ runner: JSON.parse(bytes[1]!.toString("utf8")),+ provider: JSON.parse(bytes[2]!.toString("utf8")),+ providerFingerprint: createHash("sha256").update(bytes[2]!).digest("hex"),+ fingerprint: createHash("sha256")+ .update(+ JSON.stringify(+ bytes.map((value) =>+ createHash("sha256").update(value).digest("hex"),+ ),+ ),+ )+ .digest("hex"),+ };+}++it.each([+ { alreadyEnded: false, appendFailure: false },+ { alreadyEnded: true, appendFailure: false },+ { alreadyEnded: true, appendFailure: true },+ { alreadyEnded: true, appendFailure: false, bareCodex: true },+ { alreadyEnded: true, appendFailure: false, epochFailure: "launch_intent" },+ { alreadyEnded: true, appendFailure: false, epochFailure: "spawned" },+ { alreadyEnded: true, appendFailure: false, epochFailure: "retired" },+ { alreadyEnded: true, appendFailure: false, holdSpawned: true },+ { alreadyEnded: true, appendFailure: false, completedTerminalAck: "pending" },+ { alreadyEnded: true, appendFailure: false, completedTerminalAck: "completed" },+ { alreadyEnded: true, appendFailure: false, completedTerminalAck: "repeat" },+ { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "abort" },+ { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "revoked" },+ { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "during_authorize" },+ { alreadyEnded: true, appendFailure: false, homeScoped: true },+ { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true },+ { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true, unknownExit: true },+ { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true, startupFailureProof: true },+ {+ alreadyEnded: true,+ appendFailure: false,+ bareCodex: true,+ terminalReplay: true,+ },+])(+ "settles only retained control authority without starting another provider turn ($alreadyEnded/$appendFailure/$bareCodex/$epochFailure/$terminalReplay/$holdSpawned/$homeScoped/$missingHome/$unknownExit) startup-failure-proof=$startupFailureProof completed-ack=$completedTerminalAck retired-revocation=$finalRetirementRevocation",+ async ({+ alreadyEnded,+ appendFailure,+ bareCodex,+ epochFailure,+ terminalReplay,+ holdSpawned,+ completedTerminalAck,+ finalRetirementRevocation,+ homeScoped,+ missingHome,+ unknownExit,+ startupFailureProof,+ }) => {+ const replaySpyRestorers: Array<() => void> = [];+ const replayRetirements: ReturnType<typeof maintenanceReplaySnapshot>[] = [];+ const withheldTerminalFrames: Array<{+ epoch: number;+ direction: "inbound" | "outbound";+ commandId: string;+ controllerSeq: number;+ count: number;+ }> = [];+ const fixtureId = randomUUID();+ const fixtureRunner = defaultCapabilityRunnerdBinary();+ const directory = await mkdtemp(join(tmpdir(), "runnerd-maintenance-"));+ const original = join(directory, "original");+ const copy = join(directory, "copy");+ const activated = join(directory, "activated");+ const home = join(directory, "source-home");+ await mkdir(home);+ const fakeCodex = resolve(+ import.meta.dirname,+ "../../runner/target/debug/fake-codex-app-server",+ );+ const bin = join(directory, "provider-bin");+ if (bareCodex) {+ await mkdir(bin);+ await symlink(fakeCodex, join(bin, "codex"));+ await writeFile(+ join(home, "auth.json"),+ JSON.stringify({ OPENAI_API_KEY: "fixture-only-not-a-secret" }),+ );+ }+ const environment = bareCodex+ ? { PATH: bin, HOME: home, CODEX_HOME: home }+ : undefined;+ const calls = join(directory, "calls.log");+ const fakeState = homeScoped+ ? join(original, "codex-home/fake-codex-state.json")+ : join(directory, "fake.json");+ const identity = {+ runnerInstanceId: "runner-maintenance",+ environmentLeaseId: "lease-maintenance",+ runId: "run-maintenance",+ normalizedSessionId: "session-maintenance",+ turnId: "turn-maintenance",+ itemId: "item-maintenance",+ };+ const bundle = createCapabilityRunnerdCodexTransport({+ runnerBinary: fixtureRunner,+ codexCommand: bareCodex ? "codex" : fakeCodex,+ environment,+ codexArgs: [+ ...(homeScoped+ ? ["--state-file-in-codex-home", "--require-existing-resume-state"]+ : ["--state-file", fakeState]),+ "--call-log",+ calls,+ ...(startupFailureProof ? ["--record-process-start"] : []),+ "--hold-turn",+ ],+ sourceCodexHome: home,+ stateDirectory: original,+ prpIdentity: identity,+ });+ const dead = (pid: number) => {+ try {+ process.kill(pid, 0);+ return false;+ } catch (error) {+ return (error as NodeJS.ErrnoException).code === "ESRCH";+ }+ };+ let runnerPid = 0;+ let providerPid = 0;+ let retainFixtureForUnprovenExit = false;+ const stopAndJoinReplayProcess = async (+ handle: ReturnType<typeof durableControlPlane.spawnRunner>,+ ) => {+ // This helper may time out immediately after dispatching SIGKILL. Its+ // return/rejection alone is not proof that this exact child has exited.+ await durableControlPlane.waitForProcess(handle, 250).catch(() => undefined);+ let deadline: NodeJS.Timeout | undefined;+ try {+ await Promise.race([+ handle.completion,+ new Promise<never>((_resolveJoin, rejectJoin) => {+ deadline = setTimeout(() => rejectJoin(new Error(+ "startup-proof fixture could not join its exact runner child",+ )), 5_000);+ }),+ ]);+ if (handle.processGroupId && !dead(-handle.processGroupId)) {+ try { process.kill(-handle.processGroupId, "SIGKILL"); }+ catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ await vi.waitFor(() => {+ expect(handle.child.pid && dead(handle.child.pid)).toBe(true);+ expect(handle.processGroupId && dead(-handle.processGroupId)).toBe(true);+ }, { timeout: 2_000 });+ } catch (error) {+ retainFixtureForUnprovenExit = true;+ throw error;+ } finally {+ if (deadline !== undefined) clearTimeout(deadline);+ }+ };+ try {+ const thread = (await bundle.transport.request("thread/start", {+ cwd: directory,+ dynamicTools: [],+ })) as { thread: { id: string } };+ await bundle.transport.request("turn/start", {+ threadId: thread.thread.id,+ input: [{ type: "text", text: "Keep the original turn only" }],+ });+ runnerPid = bundle.evidence().runnerPid!;+ providerPid = bundle.evidence().providerPid!;+ expect(runnerPid).toBeGreaterThan(0);+ expect(providerPid).toBeGreaterThan(0);+ await bundle.detachControllerForRestart();+ // The provider can exit when its runner dies. An already-gone process+ // group satisfies teardown; still fail on other signal errors and join below.+ for (const pid of [runnerPid, providerPid]) {+ try { process.kill(-pid, "SIGKILL"); }+ catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ await vi.waitFor(() => {+ expect(dead(runnerPid)).toBe(true);+ expect(dead(providerPid)).toBe(true);+ });+ if (alreadyEnded) {+ const providerState = JSON.parse(+ await readFile(fakeState, "utf8"),+ );+ await writeFile(+ fakeState,+ JSON.stringify({ ...providerState, activeTurnId: null }),+ );+ }+ const builder = new DurablePrpControlPlane({+ stateDirectory: join(original, "control-plane"),+ identity,+ expectedRunnerVersion: "0.3.0",+ expectedRunnerDigest: `sha256:${createHash("sha256")+ .update(await readFile(fixtureRunner))+ .digest("hex")}`,+ });+ builder.queueCommand("turn.stop", {+ reason: "interrupted original close",+ });+ if (!missingHome) builder.queueCommand("runner.suspend", {});+ // Match the retained production split: runner-owned unacknowledged+ // output plus another full provider-owned prefix behind the old suspend.+ const runnerFile = join(original, "runner/runner-state.json");+ const runnerBefore = JSON.parse(await readFile(runnerFile, "utf8"));+ const template = builder.store.state.committedEvents[0]!.envelope.payload;+ for (let index = 0; index < 90; index++) {+ const sourceSeq = runnerBefore.nextSourceSeq++;+ const event = {+ ...template,+ sourceEventId: `maintenance-runner-${index}`,+ sourceSeq,+ eventType: "item.delta",+ priority: 2,+ payload: { provider: "codex", delta: `maintenance-runner-${index}` },+ };+ const envelope = {+ protocol: "paperclip.runner",+ version: 1,+ kind: "event",+ ...identity,+ payload: event,+ };+ runnerBefore.outbox.push({+ sourceSeq,+ priority: 2,+ eventType: "item.delta",+ envelope,+ byteSize: Buffer.byteLength(JSON.stringify(envelope)),+ });+ }+ runnerBefore.peakOutboxBytes = Math.max(+ runnerBefore.peakOutboxBytes,+ runnerBefore.outbox.reduce(+ (total: number, event: { byteSize: number }) =>+ total + event.byteSize,+ 0,+ ),+ );+ await writeFile(runnerFile, JSON.stringify(runnerBefore));+ const providerFile = join(original, "runner/codex-provider-state.json");+ const providerBefore = JSON.parse(await readFile(providerFile, "utf8"));+ expect(providerBefore.pendingEvents).toEqual([]);+ expect(providerBefore.queuedEvents).toEqual([]);+ for (let index = 0; index < 128; index++) {+ providerBefore.pendingEvents.push({+ executorEventId: `codex_provider_${String(providerBefore.nextProviderEventSeq++).padStart(16, "0")}`,+ eventType: "item.delta",+ priority: "p2",+ payload: {+ provider: "codex",+ delta: `maintenance-provider-${index}`,+ },+ });+ }+ await writeFile(providerFile, JSON.stringify(providerBefore));+ await cp(original, copy, { recursive: true });+ const originalProviderHome = homeScoped ? await readFile(fakeState) : null;+ if (missingHome) await rm(join(copy, "codex-home/fake-codex-state.json"));+ const files = [+ "control-plane/control-plane-state.json",+ "runner/runner-state.json",+ "runner/codex-provider-state.json",+ ];+ const bytes = await Promise.all(+ files.map((file) => readFile(join(original, file))),+ );+ const sourceFingerprint = createHash("sha256")+ .update(+ JSON.stringify(+ bytes.map((value) =>+ createHash("sha256").update(value).digest("hex"),+ ),+ ),+ )+ .digest("hex");+ const appendEvent = vi.fn(async (_event: PrpEvent) => {});+ const maintenanceAbort = new AbortController();+ let finalAuthorityRevoked = false;+ const authorize = vi.fn(async () => {+ if (finalAuthorityRevoked) {+ if (finalRetirementRevocation === "during_authorize") {+ maintenanceAbort.abort(new Error("retirement_authority_revoked"));+ return;+ }+ throw new Error("retirement_authority_revoked");+ }+ });+ const recordEpoch = vi.fn(+ async (_receipt: Record<string, unknown>) => {},+ );+ const input = {+ requestId: "maintenance-fixture-request",+ binding: {+ companyId: "company-maintenance",+ issueId: "issue-maintenance",+ agentId: "agent-maintenance",+ runId: identity.runId,+ sessionId: identity.normalizedSessionId,+ },+ backend: {+ kind: "codex",+ name: maintenanceFixtureBackendName(fixtureId),+ },+ identity,+ stateDirectory: copy,+ activationDirectory: activated,+ sourceFingerprint,+ providerSessionId: thread.thread.id,+ originalRunnerPid: runnerPid,+ originalProviderPid: providerPid,+ runnerBinary: fixtureRunner,+ sourceCodexHome: bareCodex ? undefined : home,+ environment,+ authorize,+ appendEvent,+ recordEpoch,+ signal: maintenanceAbort.signal,+ };+ const close = vi.fn(async () => {+ throw new NativeSessionCloseUnrecoverableError();+ });+ const start = vi.fn(async () => {+ throw new Error("fixture admission reached");+ });+ const capabilities = {+ resume: true,+ typedEvents: true,+ steering: false,+ interruption: true,+ structuredResult: true,+ };+ const session: NativeSession = {+ identity: () => input.binding,+ capabilities: async () => capabilities,+ events: async function* () {},+ startTurn: start,+ close,+ snapshot: async () => ({+ backendKind: "codex",+ sessionId: input.binding.sessionId,+ identity: input.binding,+ providerSessionId: thread.thread.id,+ cursor: null,+ activeTurnId: null,+ pendingRuntimeRequests: [],+ lineage: [],+ }),+ };+ const backend: NativeSessionBackend = {+ descriptor: async () => ({+ ...input.backend,+ version: "1",+ capabilities,+ }),+ openSession: async () => session,+ };+ const nativeInput: NativeExecutionInputV1 = {+ schema: "paperclip.native-execution-input.v1",+ binding: {+ companyId: input.binding.companyId,+ issueId: input.binding.issueId,+ agentId: input.binding.agentId,+ runId: input.binding.runId,+ executionWorkspaceId: "workspace-maintenance",+ },+ task: {+ identifier: "MAINT-1",+ title: "Fixture",+ description: null,+ prompt: "Fixture",+ workMode: "standard",+ },+ workspace: {+ cwd: directory,+ repoUrl: null,+ repoRef: null,+ branchName: null,+ },+ provider: { kind: "codex", model: null },+ session: {+ normalizedSessionId: input.binding.sessionId,+ driverKind: "codex_app_server",+ protocolVersion: 1,+ },+ completionContract: {+ id: "contract-maintenance",+ sha256: "contract-maintenance-sha",+ schemaVersion: "paperclip.completion-contract.v1",+ contract: {+ revision: "1",+ objective: "Fixture",+ criteria: [{ id: "objective", requirement: "Fixture" }],+ },+ },+ interactionResponses: [],+ credentialBindings: [],+ };+ const port: ControlPlanePort = {+ openRun: async () => {},+ checkpointSession: async () => {},+ completeRun: async () => {},+ replayEvents: async () => ({+ events: [],+ highestContiguousSourceSeq: 0,+ }),+ appendEvent: async () => ({+ cursor: 1,+ highestContiguousSourceSeq: 1,+ disposition: "committed",+ }),+ };+ const execute = () =>+ executeNativeSession({+ input: nativeInput,+ backend,+ controlPlane: port,+ runnerInstanceId: identity.runnerInstanceId,+ controlPlaneInstanceId: "control-maintenance",+ requireSessionCloseBeforeReturn: true,+ });+ await expect(execute()).rejects.toThrow();+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(start).toHaveBeenCalledOnce();+ if (holdSpawned) {+ // A preceding row can fail before its authenticated cleanup proof.+ // That sticky quarantine must remain, but must not contaminate the+ // next independent fixture's backend domain in the same worker.+ const independentStart = vi.fn(async () => {+ throw new Error("independent fixture admission reached");+ });+ const independentBackend: NativeSessionBackend = {+ descriptor: async () => ({+ ...(await backend.descriptor()),+ name: maintenanceFixtureBackendName(`${fixtureId}-following`),+ }),+ openSession: async () => ({+ ...session,+ startTurn: independentStart,+ close: async () => {},+ }),+ };+ await expect(executeNativeSession({+ input: nativeInput,+ backend: independentBackend,+ controlPlane: port,+ runnerInstanceId: identity.runnerInstanceId,+ controlPlaneInstanceId: "independent-control-maintenance",+ requireSessionCloseBeforeReturn: true,+ })).rejects.toThrow("independent fixture admission reached");+ expect(independentStart).toHaveBeenCalledOnce();+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(start).toHaveBeenCalledOnce();+ }+ await expect(+ settleRetainedRunnerdSession({+ ...input,+ originalProviderPid: process.pid,+ }),+ ).rejects.toThrow("native_cleanup_maintenance_unproven");+ expect(authorize).not.toHaveBeenCalled();+ const copyProvider = join(copy, "runner/codex-provider-state.json");+ const retainedProviderBytes = await readFile(copyProvider);+ for (const eventType of [+ "semantic_tool.input",+ "runtime_request.created",+ "session.resumed",+ ]) {+ const mutated = JSON.parse(retainedProviderBytes.toString("utf8"));+ mutated.pendingEvents[0] = {+ ...mutated.pendingEvents[0],+ eventType,+ payload: {+ providerSessionId: thread.thread.id,+ processId: process.pid,+ },+ };+ await writeFile(copyProvider, JSON.stringify(mutated));+ const candidateBytes = await Promise.all(+ files.map((file) => readFile(join(copy, file))),+ );+ const candidateFingerprint = createHash("sha256")+ .update(+ JSON.stringify(+ candidateBytes.map((value) =>+ createHash("sha256").update(value).digest("hex"),+ ),+ ),+ )+ .digest("hex");+ await expect(+ settleRetainedRunnerdSession({+ ...input,+ sourceFingerprint: candidateFingerprint,+ }),+ ).rejects.toThrow("native_cleanup_maintenance_unproven");+ expect(authorize).not.toHaveBeenCalled();+ }+ await writeFile(copyProvider, retainedProviderBytes);+ for (const interruption of ["abort", "timeout"] as const) {+ const abort = new AbortController();+ let releaseAuthorization!: () => void;+ const stuckAuthorization = new Promise<void>((release) => {+ releaseAuthorization = release;+ });+ let drain: Promise<void> | undefined;+ if (interruption === "timeout")+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });+ try {+ const blocked = settleRetainedRunnerdSession({+ ...input,+ signal: abort.signal,+ authorize: () => stuckAuthorization,+ });+ const observed = blocked.catch((error: unknown) => error);+ if (interruption === "abort") abort.abort();+ else await vi.advanceTimersByTimeAsync(30_000);+ expect(await observed).toMatchObject({+ message: "native_cleanup_maintenance_unproven",+ });+ expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(false);+ await expect(settleRetainedRunnerdSession(input)).rejects.toThrow(+ "native_cleanup_maintenance_unproven",+ );+ let drained = false;+ drain = drainRetainedRunnerdMaintenanceOperations().then(() => {+ drained = true;+ });+ await Promise.resolve();+ await Promise.resolve();+ // The bounded wrapper has already failed, but its original callback+ // remains owned until it actually settles. No retry/proof is granted.+ expect(drained).toBe(false);+ releaseAuthorization();+ await drain;+ expect(drained).toBe(true);+ expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(true);+ } finally {+ releaseAuthorization();+ await drain;+ if (interruption === "timeout") vi.useRealTimers();+ }+ }+ if (missingHome) {+ const startupEvents = () => appendEvent.mock.calls+ .map(([event]) => event)+ .filter((event) => event.eventType === "harness.diagnostic" &&+ event.payload.code === "provider_startup_ownership");+ let startupPhasesBeforeFailure: unknown[] | null = null;+ const launch = durableControlPlane.spawnRunner;+ const completions: Promise<unknown>[] = [];+ let releaseExit!: () => void;+ const exitGate = new Promise<void>((resolveExit) => {+ releaseExit = resolveExit;+ });+ const launchSpy = vi+ .spyOn(durableControlPlane, "spawnRunner")+ .mockImplementation((options) => {+ const handle = launch(options);+ const completion = handle.completion.then(async (result) => {+ // Model delayed delivery of the exact child's exit notification;+ // dispatching a kill is not itself a durable retirement receipt.+ if (unknownExit) await exitGate;+ else+ await new Promise((resolveExit) => setTimeout(resolveExit, 750));+ return result;+ });+ completions.push(completion);+ return { ...handle, completion };+ });+ authorize.mockImplementation(async () => {+ const current = JSON.parse(+ await readFile(join(copy, files[0]!), "utf8"),+ );+ if (+ current.commands.some(+ (command: { status: string }) => command.status === "failed",+ )+ ) {+ if (startupFailureProof && startupPhasesBeforeFailure === null)+ startupPhasesBeforeFailure = startupEvents().map((event) =>+ (event.payload.startup as Record<string, unknown>).phase);+ throw new Error("native_cleanup_maintenance_unproven");+ }+ });+ try {+ await expect(settleRetainedRunnerdSession(input)).rejects.toThrow(+ "native_cleanup_maintenance_unproven",+ );+ if (unknownExit) {+ expect(+ recordEpoch.mock.calls.some(+ ([receipt]) => receipt.phase === "retired",+ ),+ ).toBe(false);+ expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(false);+ }+ } finally {+ launchSpy.mockRestore();+ releaseExit();+ await Promise.allSettled(completions);+ await drainRetainedRunnerdMaintenanceOperations();+ }+ const receipts = recordEpoch.mock.calls.map(([receipt]) => receipt);+ const launched = receipts.filter(+ (receipt) => receipt.phase === "spawned",+ );+ expect(launched.length).toBeGreaterThan(0);+ for (const spawned of launched) {+ const retired = receipts.find(+ (receipt) =>+ receipt.phase === "retired" &&+ receipt.launchId === spawned.launchId,+ );+ if (unknownExit) expect(retired).toBeUndefined();+ else+ expect(retired).toMatchObject({+ pid: spawned.pid,+ processGroupAbsent: true,+ });+ expect(dead(Number(spawned.pid))).toBe(true);+ expect(dead(-Number(spawned.pid))).toBe(true);+ }+ const failed = JSON.parse(await readFile(join(copy, files[0]!), "utf8"));+ expect(+ failed.commands.some(+ (command: {+ type: string;+ result?: { result?: { message?: string } };+ }) =>+ command.type === "turn.stop" &&+ command.result?.result?.message?.includes("no rollout found"),+ ),+ ).toBe(true);+ expect(await readFile(fakeState)).toEqual(originalProviderHome);+ expect(+ await Promise.all(files.map((file) => readFile(join(original, file)))),+ ).toEqual(bytes);+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(start).toHaveBeenCalledOnce();+ const methods = (await readFile(calls, "utf8")).trim().split("\n");+ expect(methods.filter((method) => method === "turn/start")).toHaveLength(+ 1,+ );+ if (startupFailureProof) {+ expect(startupPhasesBeforeFailure).toEqual([+ "intent", "spawned", "initialization_failed",+ ]);+ const events = startupEvents();+ expect(events).toHaveLength(3);+ expect(events.map((event) => event.sourceSeq)).toEqual(+ events.map((event) => event.sourceSeq).sort((left, right) => left - right),+ );+ expect(new Set(events.map((event) => event.sourceEventId)).size).toBe(3);+ const facts = events.map((event) => event.payload.startup as Record<string, unknown>);+ const [intent, spawned, initializationFailed] = facts;+ expect(intent!.launchId).toMatch(/^[0-9a-f-]{36}$/);+ expect(intent!.configurationFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);+ const failedStop = failed.commands.find((command: { type: string; status: string }) =>+ command.type === "turn.stop" && command.status === "failed");+ for (const fact of facts) {+ expect(Object.keys(fact).sort()).toEqual([+ "schema", "launchId", "phase", "trigger", "attemptedProcessGeneration",+ "origin", "command", "configurationFingerprint", "requestedThreadId",+ "authenticatedThreadId", "processId", "processGroupId", "failedStage",+ "directChildExitObserved", "exitCode", "signal", "processTreeRetired",+ ].sort());+ expect(fact).toMatchObject({+ schema: "paperclip.provider_startup.v1",+ launchId: intent!.launchId,+ trigger: "restore",+ attemptedProcessGeneration: providerBefore.providerProcessGeneration + 1,+ origin: {+ runnerInstanceId: identity.runnerInstanceId,+ runId: identity.runId,+ normalizedSessionId: identity.normalizedSessionId,+ turnId: identity.turnId,+ itemId: identity.itemId,+ },+ command: {+ commandId: failedStop.commandId,+ controllerSeq: failedStop.controllerSeq,+ commandType: "turn.stop",+ },+ configurationFingerprint: intent!.configurationFingerprint,+ requestedThreadId: thread.thread.id,+ authenticatedThreadId: null,+ processTreeRetired: false,+ });+ }+ expect(intent).toMatchObject({+ phase: "intent", processId: null, processGroupId: null,+ directChildExitObserved: false, failedStage: null, exitCode: null, signal: null,+ });+ expect(spawned!.processId).toBeGreaterThan(0);+ expect(spawned).toMatchObject({+ phase: "spawned", processGroupId: spawned!.processId,+ directChildExitObserved: false, failedStage: null, exitCode: null, signal: null,+ });+ expect(initializationFailed).toMatchObject({+ phase: "initialization_failed", failedStage: "thread_open",+ processId: spawned!.processId, processGroupId: spawned!.processId,+ directChildExitObserved: true,+ });+ expect(dead(Number(spawned!.processId))).toBe(true);+ expect(appendEvent.mock.calls.filter(([event]) =>+ ["session.started", "session.resumed"].includes(event.eventType) &&+ event.payload.processId !== providerPid)).toHaveLength(0);+ expect(failed.commands.some((command: {+ type: string; result?: { result?: { providerExitConfirmed?: boolean } };+ }) => command.type === "turn.stop" &&+ command.result?.result?.providerExitConfirmed === true)).toBe(false);+ const deltas = appendEvent.mock.calls.map(([event]) => event.payload.delta)+ .filter((delta) => typeof delta === "string" && delta.startsWith("maintenance-"));+ expect(deltas).toHaveLength(218);+ expect(new Set(deltas).size).toBe(218);+ const failedProviderState = JSON.parse(await readFile(copyProvider, "utf8"));+ expect(failedProviderState.startupAttempt).toMatchObject({+ launchId: intent!.launchId,+ });+ const observedMethods = await readFile(calls, "utf8");+ expect(observedMethods.trim().split("\n").filter((method) =>+ method === "process-start")).toHaveLength(2);+ const originalFailure = structuredClone(failedStop.result);+ // Exercise the producer fence directly in this isolated fixture.+ // This does not admit the failed copy through maintenance or alter+ // its source/receipt bytes to manufacture recovery eligibility.+ for (let restart = 0; restart < 2; restart++) {+ const replayCore = new DurablePrpControlPlane({+ stateDirectory: join(copy, "control-plane"),+ identity,+ expectedRunnerVersion: "0.3.0",+ expectedRunnerDigest: `sha256:${createHash("sha256")+ .update(await readFile(fixtureRunner)).digest("hex")}`,+ onCommittedEvent: appendEvent,+ });+ const snapshot = replayCore.queueCommand("session.snapshot", {});+ const stop = replayCore.queueCommand("turn.stop", {+ reason: "startup-fence regression only",+ });+ let replayHandle: ReturnType<typeof durableControlPlane.spawnRunner> | null = null;+ let replayAssertionFailed = false;+ try {+ await replayCore.start();+ const runnerState = JSON.parse(await readFile(join(copy, files[1]!), "utf8"));+ replayHandle = durableControlPlane.spawnRunner({+ connectUrl: replayCore.connectUrl,+ stateDirectory: join(copy, "runner"),+ identity,+ ticket: replayCore.issueBootstrapTicket(),+ maxOutboxBytes: runnerState.maxOutboxBytes,+ p0ReserveBytes: runnerState.p0ReserveBytes,+ maxRuntimeMs: 2_000,+ reconnectGraceMs: 1_000,+ runnerBinaryPath: fixtureRunner,+ runnerVersion: "0.3.0",+ runnerDigest: `sha256:${createHash("sha256")+ .update(await readFile(fixtureRunner)).digest("hex")}`,+ environment: createCapabilityRunnerdProviderEnvironment({+ provider: "codex",+ options: {},+ identity,+ codexHome: join(copy, "codex-home"),+ runtimeContextPath: join(copy, "runtime-context.json"),+ hasRuntimeContext: false,+ }),+ });+ await vi.waitFor(() => {+ for (const queued of [snapshot, stop]) {+ const command = replayCore.getCommand(queued.commandId);+ expect(command?.status).toBe("failed");+ expect(command?.result).toMatchObject({+ result: {+ message: expect.stringContaining(+ "provider startup ownership remains unadmitted",+ ),+ },+ });+ }+ }, { timeout: 5_000 });+ await durableControlPlane.waitForProcess(replayHandle, 5_000);+ expect(await readFile(calls, "utf8")).toBe(observedMethods);+ expect((await readFile(calls, "utf8")).trim().split("\n")+ .filter((method) => method === "process-start")).toHaveLength(2);+ expect(replayCore.store.state.commands.find((command) =>+ command.commandId === failedStop.commandId)?.result).toEqual(originalFailure);+ expect(JSON.parse(await readFile(copyProvider, "utf8")).startupAttempt)+ .toEqual(failedProviderState.startupAttempt);+ expect(startupEvents()).toHaveLength(3);+ } catch (error) {+ replayAssertionFailed = true;+ throw error;+ } finally {+ try {+ if (replayHandle) await stopAndJoinReplayProcess(replayHandle);+ } catch (error) {+ // Keep the original assertion as the primary failure. Do not+ // delete evidence underneath an unjoined owned process.+ if (!replayAssertionFailed) throw error;+ console.error("startup-proof fixture cleanup unproven; directory retained");+ } finally {+ await replayCore.stop();+ }+ }+ }+ expect(await readFile(fakeState)).toEqual(originalProviderHome);+ expect(await Promise.all(files.map((file) => readFile(join(original, file)))))+ .toEqual(bytes);+ }+ return;+ }+ if (appendFailure) {+ const failure = new Error(+ "injected maintenance event persistence failure",+ );+ let renewedProviderPid: number | null = null;+ appendEvent.mockImplementation(async (event) => {+ if (event.eventType !== "session.resumed") return;+ const resumed = resolveRunnerdSessionIdentity(event.payload);+ if (resumed.processId === providerPid) return;+ renewedProviderPid = resumed.processId;+ throw failure;+ });+ await expect(settleRetainedRunnerdSession(input)).rejects.toBe(failure);+ expect(renewedProviderPid).not.toBeNull();+ expect(dead(renewedProviderPid!)).toBe(true);+ expect(dead(-renewedProviderPid!)).toBe(true);+ expect(+ await Promise.all(+ files.map((file) => readFile(join(original, file))),+ ),+ ).toEqual(bytes);+ await expect(+ readFile(join(activated, "runner/runner-state.json")),+ ).rejects.toMatchObject({ code: "ENOENT" });+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(start).toHaveBeenCalledOnce();+ const methods = (await readFile(calls, "utf8")).trim().split("\n");+ expect(+ methods.filter((method) => method === "turn/start"),+ ).toHaveLength(1);+ expect(+ methods.filter((method) => method === "thread/resume"),+ ).toHaveLength(1);+ return;+ }+ const assertNoProviderBeforeSpawnedReceipt = async () => {+ const callsBefore = await readFile(calls, "utf8");+ const providerBefore = await readFile(copyProvider);+ const controlBefore = JSON.parse(+ await readFile(+ join(copy, "control-plane/control-plane-state.json"), "utf8",+ ),+ );+ // Give the actual runner time to authenticate while its durable spawn+ // receipt is held. Authentication must not release even the old stop.+ await new Promise((resolveWait) => setTimeout(resolveWait, 500));+ expect(await readFile(calls, "utf8")).toBe(callsBefore);+ expect(await readFile(copyProvider)).toEqual(providerBefore);+ const controlAfter = JSON.parse(+ await readFile(+ join(copy, "control-plane/control-plane-state.json"), "utf8",+ ),+ );+ expect(controlAfter.connectionCount).toBe(+ controlBefore.connectionCount,+ );+ expect(controlAfter.commandDeliveryCounts).toEqual(+ controlBefore.commandDeliveryCounts,+ );+ expect(controlAfter.commands).toEqual(controlBefore.commands);+ };+ if (holdSpawned) {+ recordEpoch.mockImplementation(async (receipt) => {+ if (receipt.phase === "spawned")+ await assertNoProviderBeforeSpawnedReceipt();+ });+ }+ if (completedTerminalAck) {+ const originalAttach =+ DurablePrpControlPlane.prototype.attachWireConnection;+ const attachSpy = vi+ .spyOn(DurablePrpControlPlane.prototype, "attachWireConnection")+ .mockImplementation(function (this: DurablePrpControlPlane, wire) {+ const epoch = replayRetirements.length;+ const direction =+ completedTerminalAck === "completed" ? "outbound" : "inbound";+ const inject =+ this.store.path === join(copy, files[0]!) &&+ (epoch === 0 ||+ (completedTerminalAck === "repeat" && epoch === 1));+ let attachment: ReturnType<typeof originalAttach> | undefined;+ let withheld: (typeof withheldTerminalFrames)[number] | undefined;+ const shouldWithhold = (candidate: typeof direction): boolean => {+ if (+ !inject ||+ candidate !== direction ||+ !attachment?.isAuthenticated()+ )+ return false;+ if (withheld) {+ withheld.count += 1;+ return true;+ }+ const runner = JSON.parse(+ readFileSync(join(copy, files[1]!), "utf8"),+ );+ const terminal = runner.pendingTerminalDelivery;+ if (+ terminal?.commandType !== "runner.suspend" ||+ terminal.lifecycle !== "suspended"+ )+ return false;+ const result = runner.processedCommands[terminal.commandId];+ if (+ result?.status !== "completed" ||+ result.result?.status !== "completed" ||+ result.commandType !== terminal.commandType ||+ result.controllerSeq !== terminal.controllerSeq+ )+ return false;+ const control = JSON.parse(readFileSync(this.store.path, "utf8"));+ const command = control.commands.find(+ (entry: { commandId: string }) =>+ entry.commandId === terminal.commandId,+ );+ if (+ command?.type !== terminal.commandType ||+ command.controllerSeq !== terminal.controllerSeq ||+ command.status !==+ (direction === "inbound" ? "pending" : "completed")+ )+ return false;+ if (direction === "outbound")+ expect(command.result).toEqual(result);+ else expect(command.result ?? null).toBeNull();+ // Rust durably records this exact result before sending it.+ // Withhold transport delivery only after that handshake, not+ // after a guessed number of saves or an elapsed sleep. Pending+ // mode loses the result; completed mode loses its outbound ACK.+ // No retained journal/outbox bytes are removed or rewritten.+ withheld = {+ epoch,+ direction,+ commandId: terminal.commandId,+ controllerSeq: terminal.controllerSeq,+ count: 1,+ };+ withheldTerminalFrames.push(withheld);+ return true;+ };+ attachment = originalAttach.call(this, {+ sendJson(value) {+ if (!shouldWithhold("outbound")) wire.sendJson(value);+ },+ close: (code) => wire.close(code),+ onJson: (listener) =>+ wire.onJson((value) => {+ if (!shouldWithhold("inbound")) listener(value);+ }),+ onClose: (listener) => wire.onClose(listener),+ });+ return attachment;+ });+ replaySpyRestorers.push(() => attachSpy.mockRestore());+ recordEpoch.mockImplementation(async (receipt) => {+ if (receipt.phase !== "retired") return;+ const snapshot = maintenanceReplaySnapshot(copy);+ expect(snapshot.fingerprint).toBe(receipt.finalFingerprint);+ replayRetirements.push(snapshot);+ });+ }+ if (epochFailure) {+ const failure = new Error("injected epoch receipt persistence failure");+ const callsBefore = await readFile(calls, "utf8");+ recordEpoch.mockImplementation(async (receipt) => {+ if (receipt.phase === "spawned" && epochFailure === "spawned")+ await assertNoProviderBeforeSpawnedReceipt();+ if (receipt.phase === epochFailure) throw failure;+ });+ await expect(settleRetainedRunnerdSession(input)).rejects.toBe(failure);+ for (const [receipt] of recordEpoch.mock.calls) {+ if (receipt.phase !== "spawned") continue;+ expect(dead(Number(receipt.pid))).toBe(true);+ expect(dead(-Number(receipt.pid))).toBe(true);+ }+ if (epochFailure === "spawned") {+ expect(await readFile(calls, "utf8")).toBe(callsBefore);+ }+ if (epochFailure === "launch_intent") {+ expect(await readFile(calls, "utf8")).toBe(callsBefore);+ expect(+ recordEpoch.mock.calls.map(([receipt]) => receipt.phase),+ ).toEqual(["launch_intent"]);+ }+ expect(+ await Promise.all(+ files.map((file) => readFile(join(original, file))),+ ),+ ).toEqual(bytes);+ await expect(+ readFile(join(activated, "runner/runner-state.json")),+ ).rejects.toMatchObject({ code: "ENOENT" });+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(start).toHaveBeenCalledOnce();+ return;+ }+ if (finalRetirementRevocation) {+ recordEpoch.mockImplementation(async (receipt) => {+ if (receipt.phase !== "retired") return;+ const state = maintenanceReplaySnapshot(copy);+ if (+ state.runner.pendingTerminalDelivery != null ||+ state.runner.pendingProviderCleanup != null ||+ state.runner.outbox.length !== 0 ||+ state.provider.pendingEvents.length !== 0 ||+ state.provider.queuedEvents.length !== 0 ||+ state.control.commands.some(+ (command: { status: string }) => command.status === "pending",+ )+ )+ return;+ expect(state.provider.lifecycle).toBe("prepared");+ expect(state.fingerprint).toBe(receipt.finalFingerprint);+ finalAuthorityRevoked = true;+ if (finalRetirementRevocation === "abort")+ maintenanceAbort.abort(new Error("retirement_authority_revoked"));+ });+ }+ let failedAttempt: { directory: string; bytes: Buffer[] } | null = null;+ let failedStartupAttempt: {+ directory: string;+ bytes: Buffer[];+ } | null = null;+ if (terminalReplay) {+ // Forward failures now preserve a startup fence. Keep this genuinely+ // produced failed copy intact; it is NOT a legacy replay candidate.+ await expect(+ settleRetainedRunnerdSession({+ ...input,+ environment: undefined,+ sourceCodexHome: null,+ }),+ ).rejects.toThrow("native_cleanup_maintenance_unproven");+ const failedBytes = await Promise.all(+ files.map((file) => readFile(join(copy, file))),+ );+ expect(+ JSON.parse(failedBytes[2]!.toString("utf8")).startupAttempt,+ ).toMatchObject({+ schema: "paperclip.provider_startup.v1",+ phase: "initialization_failed",+ failedStage: "spawn",+ requestedThreadId: thread.thread.id,+ authenticatedThreadId: null,+ processId: null,+ directChildExitObserved: false,+ processTreeRetired: false,+ });+ expect(+ appendEvent.mock.calls+ .filter(+ ([event]) => event.payload.code === "provider_startup_ownership",+ )+ .map(+ ([event]) => (event.payload.startup as { phase: string }).phase,+ ),+ ).toEqual(["intent", "initialization_failed"]);+ expect(+ await Promise.all(+ files.map((file) => readFile(join(original, file))),+ ),+ ).toEqual(bytes);+ await expect(+ readFile(join(activated, files[1]!)),+ ).rejects.toMatchObject({ code: "ENOENT" });+ const failedStartupDirectory = join(+ original,+ "..",+ "failed-startup-attempt",+ );+ await rename(copy, failedStartupDirectory);+ failedStartupAttempt = {+ directory: failedStartupDirectory,+ bytes: failedBytes,+ };++ // Backward-compatibility fixture, synthesized ONLY from the pristine+ // original snapshots: old producers recorded terminal failure without+ // a startup-attempt field. Never delete a real generated fence above.+ const directory = join(original, "..", "legacy-failed-terminal");+ await cp(original, directory, { recursive: true });+ const legacyControl = JSON.parse(bytes[0]!.toString("utf8"));+ const legacyRunner = JSON.parse(bytes[1]!.toString("utf8"));+ expect(+ JSON.parse(bytes[2]!.toString("utf8")).startupAttempt ?? null,+ ).toBeNull();+ const legacyCommands = legacyControl.commands.slice(-2);+ expect(+ legacyCommands.map((command: { type: string }) => command.type),+ ).toEqual(["turn.stop", "runner.suspend"]);+ for (const command of legacyCommands) {+ const wire = {+ schema: command.schema,+ commandId: command.commandId,+ controllerSeq: command.controllerSeq,+ type: command.type,+ issuedAt: command.issuedAt,+ deadlineAt: null,+ precondition: null,+ payload: command.payload,+ };+ const result = {+ commandId: command.commandId,+ commandType: command.type,+ controllerSeq: command.controllerSeq,+ status: "failed",+ result: {+ code: "command_execution_failed",+ message: "legacy pre-start failure fixture",+ },+ };+ command.status = "failed";+ command.result = result;+ legacyRunner.processedCommands[command.commandId] = result;+ legacyRunner.processedCommandFingerprints[command.commandId] =+ createHash("sha256")+ .update(+ durableControlPlane.durableRecoveryInternals.canonicalJson(wire),+ )+ .digest("hex");+ legacyRunner.lastControllerCommandSeq = command.controllerSeq;+ }+ const terminal = legacyCommands[1]!;+ legacyRunner.lifecycle = "suspended";+ legacyRunner.pendingTerminalDelivery = {+ commandId: terminal.commandId,+ controllerSeq: terminal.controllerSeq,+ commandType: terminal.type,+ lifecycle: "suspended",+ };+ await writeFile(+ join(directory, files[0]!),+ JSON.stringify(legacyControl),+ );+ await writeFile(+ join(directory, files[1]!),+ JSON.stringify(legacyRunner),+ );+ const legacyBytes = await Promise.all(+ files.map((file) => readFile(join(directory, file))),+ );+ expect(legacyBytes[2]).toEqual(bytes[2]);+ expect(+ legacyControl.commands+ .slice(-2)+ .map((command: { status: string }) => command.status),+ ).toEqual(["failed", "failed"]);+ await cp(directory, copy, { recursive: true });+ failedAttempt = { directory, bytes: legacyBytes };+ input.sourceFingerprint = createHash("sha256")+ .update(+ JSON.stringify(+ legacyBytes.map((value) =>+ createHash("sha256").update(value).digest("hex"),+ ),+ ),+ )+ .digest("hex");+ input.requestId = "maintenance-fixture-continuation";+ recordEpoch.mockClear();+ appendEvent.mockClear();+ }+ const pendingProof = settleRetainedRunnerdSession(input);+ if (finalRetirementRevocation) {+ await expect(pendingProof).rejects.toThrow(+ finalRetirementRevocation === "abort"+ ? "native_cleanup_maintenance_unproven"+ : "retirement_authority_revoked",+ );+ expect(finalAuthorityRevoked).toBe(true);+ await expect(+ readFile(join(activated, "runner/runner-state.json")),+ ).rejects.toMatchObject({ code: "ENOENT" });+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(+ await Promise.all(files.map((file) => readFile(join(original, file)))),+ ).toEqual(bytes);+ const methods = (await readFile(calls, "utf8")).trim().split("\n");+ expect(methods.filter((method) => method === "turn/start")).toHaveLength(1);+ return;+ }+ if (completedTerminalAck === "repeat") {+ await expect(pendingProof).rejects.toThrow(+ "native_cleanup_maintenance_unproven",+ );+ expect(replayRetirements).toHaveLength(2);+ expect(+ withheldTerminalFrames.map(({ epoch, direction }) => ({+ epoch,+ direction,+ })),+ ).toEqual([+ { epoch: 0, direction: "inbound" },+ { epoch: 1, direction: "inbound" },+ ]);+ for (const [epoch, state] of replayRetirements.entries()) {+ expect(JSON.stringify(state.runner.diagnostics)).toContain(+ "terminal command result acknowledgement timed out",+ );+ expect(state.runner.pendingTerminalDelivery).toMatchObject({+ commandType: "runner.suspend",+ lifecycle: "suspended",+ });+ const withheld = withheldTerminalFrames[epoch]!;+ expect(withheld.count).toBeGreaterThan(0);+ expect(state.runner.pendingTerminalDelivery).toMatchObject({+ commandId: withheld.commandId,+ controllerSeq: withheld.controllerSeq,+ });+ expect(+ state.runner.processedCommands[withheld.commandId],+ ).toMatchObject({+ status: "completed",+ result: { status: "completed" },+ });+ expect(+ state.control.commands.find(+ (command: { commandId: string }) =>+ command.commandId === withheld.commandId,+ ),+ ).toMatchObject({ status: "pending" });+ }+ expect(replayRetirements[1]!.provider).toEqual(+ replayRetirements[0]!.provider,+ );+ expect(recordEpoch.mock.calls).toHaveLength(6);+ await expect(+ readFile(join(activated, "runner/runner-state.json")),+ ).rejects.toMatchObject({ code: "ENOENT" });+ await expect(execute()).rejects.toMatchObject({+ code: "native_session_cleanup_quarantined",+ });+ expect(+ await Promise.all(+ files.map((file) => readFile(join(original, file))),+ ),+ ).toEqual(bytes);+ const methods = (await readFile(calls, "utf8")).trim().split("\n");+ expect(+ methods.filter((method) => method === "turn/start"),+ ).toHaveLength(1);+ // The first cleanup epoch restored the old provider solely to stop+ // it; the failed delivery-only replay did not restore it again.+ expect(+ methods.filter((method) => method === "thread/resume"),+ ).toHaveLength(1);+ return;+ }+ const proof = await pendingProof.catch(+ async (error: unknown) => {+ const runner = JSON.parse(+ await readFile(join(copy, "runner/runner-state.json"), "utf8"),+ );+ const provider = JSON.parse(+ await readFile(+ join(copy, "runner/codex-provider-state.json"),+ "utf8",+ ),+ );+ const control = JSON.parse(+ await readFile(+ join(copy, "control-plane/control-plane-state.json"),+ "utf8",+ ),+ );+ throw new Error(+ JSON.stringify({+ runner: {+ lifecycle: runner.lifecycle,+ outbox: runner.outbox.length,+ acked: runner.ackedSourceSeq,+ next: runner.nextSourceSeq,+ terminalAckTimedOut: JSON.stringify(+ runner.diagnostics,+ ).includes("terminal command result acknowledgement timed out"),+ pendingTerminalDelivery:+ runner.pendingTerminalDelivery ?? null,+ retainedIdentityTypes: runner.outbox+ .filter((row: { eventType: string }) =>+ [+ "session.started",+ "session.resumed",+ "harness.ready",+ ].includes(row.eventType),+ )+ .map((row: { sourceSeq: number; eventType: string }) => ({+ sourceSeq: row.sourceSeq,+ eventType: row.eventType,+ })),+ },+ provider: {+ lifecycle: provider.lifecycle,+ pending: provider.pendingEvents.length,+ queued: provider.queuedEvents.length,+ generation: provider.providerProcessGeneration,+ },+ commands: control.commands.map(+ (command: { type: string; status: string }) => ({+ type: command.type,+ status: command.status,+ }),+ ),+ committedCount: appendEvent.mock.calls.length,+ epochExits: recordEpoch.mock.calls+ .map(([receipt]) => receipt)+ .filter((receipt) => receipt.phase === "retired")+ .map((receipt) => ({+ epoch: receipt.epoch,+ exitCode: receipt.exitCode,+ exitSignal: receipt.exitSignal,+ })),+ }),+ { cause: error },+ );+ },+ );+ if (failedAttempt) {+ expect(+ await Promise.all(+ files.map((file) =>+ readFile(join(failedStartupAttempt!.directory, file)),+ ),+ ),+ ).toEqual(failedStartupAttempt!.bytes);+ expect(+ await Promise.all(+ files.map((file) => readFile(join(failedAttempt!.directory, file))),+ ),+ ).toEqual(failedAttempt.bytes);+ const finalControl = JSON.parse(+ await readFile(join(copy, files[0]!), "utf8"),+ );+ const failedControl = JSON.parse(+ failedAttempt.bytes[0]!.toString("utf8"),+ );+ expect(+ finalControl.commands.slice(0, failedControl.commands.length),+ ).toEqual(failedControl.commands);+ expect(+ finalControl.commands+ .slice(failedControl.commands.length)+ .some(+ (command: {+ type: string;+ status: string;+ result: { result?: { providerExitConfirmed?: boolean } };+ }) =>+ command.type === "turn.stop" &&+ command.status === "completed" &&+ command.result.result?.providerExitConfirmed === true,+ ),+ ).toBe(true);+ }+ const epochReceipts = recordEpoch.mock.calls.map(([receipt]) => receipt);+ if (completedTerminalAck) {+ const before = replayRetirements[0]!;+ const after = replayRetirements[1]!;+ expect(withheldTerminalFrames).toHaveLength(1);+ expect(withheldTerminalFrames[0]).toMatchObject({+ epoch: 0,+ direction:+ completedTerminalAck === "completed" ? "outbound" : "inbound",+ });+ expect(withheldTerminalFrames[0]!.count).toBeGreaterThan(0);+ expect(JSON.stringify(before.runner.diagnostics)).toContain(+ "terminal command result acknowledgement timed out",+ );+ const pending = before.runner.pendingTerminalDelivery;+ expect(pending).toMatchObject({+ commandType: "runner.suspend",+ lifecycle: "suspended",+ commandId: withheldTerminalFrames[0]!.commandId,+ controllerSeq: withheldTerminalFrames[0]!.controllerSeq,+ });+ expect(before.runner.processedCommands[pending.commandId].status).toBe(+ "completed",+ );+ expect(+ before.control.commands.find(+ (command: { commandId: string }) =>+ command.commandId === pending.commandId,+ ).status,+ ).toBe(completedTerminalAck);+ expect(+ runnerdRecoveryInternals.completedMaintenanceTerminalReceipt(before),+ ).not.toBeNull();+ expect(+ runnerdRecoveryInternals.completedMaintenanceTerminalReplayMatches(+ before,+ after,+ ),+ ).toBe(true);+ expect(after.provider).toEqual(before.provider);+ expect(after.runner.pendingProviderCleanup ?? null).toBeNull();+ // An actual completed receipt copied into an INITIAL invocation is+ // not this invocation's joined retirement and cannot enable replay.+ const initialTerminal = join(directory, "unproved-initial-terminal");+ await mkdir(join(initialTerminal, "runner"), { recursive: true });+ await mkdir(join(initialTerminal, "control-plane"));+ for (const [index, value] of [+ before.control,+ before.runner,+ before.provider,+ ].entries())+ await writeFile(+ join(initialTerminal, files[index]!),+ JSON.stringify(value),+ );+ const unproved = maintenanceReplaySnapshot(initialTerminal);+ const initialEpoch = vi.fn(async () => {});+ await expect(+ settleRetainedRunnerdSession({+ ...input,+ stateDirectory: initialTerminal,+ sourceFingerprint: unproved.fingerprint,+ recordEpoch: initialEpoch,+ }),+ ).rejects.toThrow("native_cleanup_maintenance_unproven");+ expect(initialEpoch).not.toHaveBeenCalled();+ const receiptCases: Array<[string, (state: typeof before) => void]> = [+ [+ "missing result",+ (state) => {+ delete state.runner.processedCommands[pending.commandId];+ },+ ],+ ...["pending", "failed", "rejected", "indeterminate"].map(+ (status): [string, (state: typeof before) => void] => [+ `outer ${status}`,+ (state) => {+ state.runner.processedCommands[pending.commandId].status = status;+ },+ ],+ ),+ ...["failed", "rejected", "indeterminate"].map(+ (status): [string, (state: typeof before) => void] => [+ `nested ${status}`,+ (state) => {+ state.runner.processedCommands[pending.commandId].result.status =+ status;+ },+ ],+ ),+ [+ "terminal sequence",+ (state) => {+ state.runner.pendingTerminalDelivery.controllerSeq++;+ },+ ],+ [+ "terminal type",+ (state) => {+ state.runner.pendingTerminalDelivery.commandType = "runner.shutdown";+ },+ ],+ [+ "wire fingerprint",+ (state) => {+ state.control.commands.find(+ (command: { commandId: string }) =>+ command.commandId === pending.commandId,+ ).payload = { changed: true };+ },+ ],+ [+ "completed controller result",+ (state) => {+ const command = state.control.commands.find(+ (entry: { commandId: string }) =>+ entry.commandId === pending.commandId,+ );+ command.status = "completed";+ command.result = { changed: true };+ },+ ],+ [+ "earlier pending command",+ (state) => {+ const command = state.control.commands.find(+ (entry: { commandId: string }) =>+ entry.commandId === pending.commandId,+ );+ command.status = "pending";+ delete command.result;+ state.control.commands[0].status = "pending";+ },+ ],+ [+ "active provider",+ (state) => {+ state.provider.activeProviderTurnId = "another-turn";+ },+ ],+ [+ "provider cleanup",+ (state) => {+ state.runner.pendingProviderCleanup = pending;+ },+ ],+ ];+ for (const [name, mutate] of receiptCases) {+ const changed = structuredClone(before);+ mutate(changed);+ expect(+ runnerdRecoveryInternals.completedMaintenanceTerminalReceipt(changed),+ name,+ ).toBeNull();+ }+ for (const [name, mutate] of [+ [+ "other command",+ (state: typeof after) => {+ state.control.commands[0].result = { changed: true };+ },+ ],+ [+ "provider bytes",+ (state: typeof after) => {+ state.providerFingerprint = "changed";+ },+ ],+ [+ "pending delivery",+ (state: typeof after) => {+ state.runner.pendingTerminalDelivery = pending;+ },+ ],+ [+ "processed receipt",+ (state: typeof after) => {+ state.runner.processedCommands[pending.commandId].result = {+ changed: true,+ };+ },+ ],+ ] as const) {+ const changed = structuredClone(after);+ mutate(changed);+ expect(+ runnerdRecoveryInternals.completedMaintenanceTerminalReplayMatches(+ before,+ changed,+ ),+ name,+ ).toBe(false);+ }+ }+ expect(epochReceipts.length).toBeGreaterThanOrEqual(3);+ for (let index = 0; index < epochReceipts.length; index += 3) {+ const [intent, spawned, retired] = epochReceipts.slice(+ index,+ index + 3,+ );+ expect(intent).toMatchObject({+ phase: "launch_intent",+ requestId: input.requestId,+ stateDirectory: copy,+ });+ expect(spawned).toMatchObject({+ phase: "spawned",+ launchId: intent!.launchId,+ });+ expect(retired).toMatchObject({+ phase: "retired",+ launchId: intent!.launchId,+ pid: spawned!.pid,+ processGroupAbsent: true,+ });+ expect(dead(Number(retired!.pid))).toBe(true);+ expect(dead(-Number(retired!.pid))).toBe(true);+ }+ if (bareCodex) {+ expect(await readFile(join(copy, "codex-home/auth.json"), "utf8")).toBe(+ await readFile(join(home, "auth.json"), "utf8"),+ );+ }+ expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(false);+ await rename(copy, activated);+ expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(true);+ expect(retainedRunnerdCleanupProofIsCurrent({ ...proof })).toBe(false);+ expect(() =>+ completeRetainedNativeSessionCleanup({ ...proof }),+ ).toThrow();+ expect(completeRetainedNativeSessionCleanup(proof)).toBe(1);+ expect(completeRetainedNativeSessionCleanup(proof)).toBe(0);+ close.mockImplementation(async () => {});+ await expect(execute()).rejects.toThrow("fixture admission reached");+ expect(start).toHaveBeenCalledTimes(2);+ expect(+ await Promise.all(files.map((file) => readFile(join(original, file)))),+ ).toEqual(bytes);+ const methods = (await readFile(calls, "utf8")).trim().split("\n");+ expect(methods.filter((method) => method === "turn/start")).toHaveLength(+ 1,+ );+ expect(+ methods.filter((method) => method === "thread/start"),+ ).toHaveLength(1);+ expect(+ methods.filter((method) => method === "thread/resume"),+ ).toHaveLength(1);+ const finalState = JSON.parse(+ await readFile(join(activated, "runner/runner-state.json"), "utf8"),+ );+ expect(finalState).toMatchObject({+ ...identity,+ lifecycle: "suspended",+ outbox: [],+ });+ const provider = JSON.parse(+ await readFile(+ join(activated, "runner/codex-provider-state.json"),+ "utf8",+ ),+ );+ expect(provider).toMatchObject({+ threadId: thread.thread.id,+ activeProviderTurnId: null,+ pendingEvents: [],+ queuedEvents: [],+ });+ expect(+ appendEvent.mock.calls.every(+ ([event]) => event.runId === identity.runId,+ ),+ ).toBe(true);+ const deltas = appendEvent.mock.calls+ .map(([event]) => event.payload.delta)+ .filter(+ (delta) =>+ typeof delta === "string" && delta.startsWith("maintenance-"),+ );+ expect(deltas).toHaveLength(218);+ expect(new Set(deltas).size).toBe(218);+ await writeFile(+ join(activated, "runner/runner-state.json"),+ JSON.stringify({ ...finalState, lifecycle: "ready" }),+ );+ expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(false);+ } finally {+ for (const restore of replaySpyRestorers.reverse()) restore();+ await bundle.transport.close().catch(() => undefined);+ for (const pid of [runnerPid, providerPid]) {+ if (pid > 0 && !dead(pid)) {+ try {+ process.kill(-pid, "SIGKILL");+ } catch {}+ }+ }+ if (!retainFixtureForUnprovenExit)+ await rm(directory, { recursive: true, force: true });+ }+ },+ 40_000,+);+diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.tsindex 69bb413bac..5f508dafaa 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts@@ -9,7 +9,6 @@ import {rename,rm,stat,- symlink,writeFile,} from "node:fs/promises";import { createHash, randomUUID } from "node:crypto";@@ -32,7 +31,7 @@ import type {PrpStructuredRunResult,PrpTerminalState,} from "../protocol/replay-contract.js";-import { completeRetainedNativeSessionCleanup, executeNativeSession } from "../native-session-runtime.js";+import { executeNativeSession } from "../native-session-runtime.js";import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js";@@ -63,7 +62,6 @@ import {createRunnerdCodexAppServerArgs,defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary,readRunnerdArtifactBinding,- drainRetainedRunnerdMaintenanceOperations,expandRunnerdCanonicalNotifications,latestRunnerdSessionReadiness,rehydrateRunnerdGoalNotification,@@ -81,9 +79,6 @@ import {resolveRunnerdAcpxPermissionMode,resolveRunnerdSessionIdentity,resolveSourceCodexHome,- settleRetainedRunnerdSession,- retainedRunnerdCleanupProofIsCurrent,- retainedRunnerdMaintenanceIsIdle,trustedRuntimeReadOnlyRoots,unseenRunnerdCommittedEvents,unwrapRunnerdProviderNotification,@@ -261,1704 +256,11 @@ it("replaces an owned v1 runner with fresh v2 authorization before warm attachme}}, 30_000);-function maintenanceFixtureBackendName(fixtureId: string): string {- return `maintenance-test-${fixtureId}`;-}--function maintenanceReplaySnapshot(directory: string) {- const bytes = [- "control-plane/control-plane-state.json",- "runner/runner-state.json",- "runner/codex-provider-state.json",- ].map((file) => readFileSync(join(directory, file)));- return {- control: JSON.parse(bytes[0]!.toString("utf8")),- runner: JSON.parse(bytes[1]!.toString("utf8")),- provider: JSON.parse(bytes[2]!.toString("utf8")),- providerFingerprint: createHash("sha256").update(bytes[2]!).digest("hex"),- fingerprint: createHash("sha256")- .update(- JSON.stringify(- bytes.map((value) =>- createHash("sha256").update(value).digest("hex"),- ),- ),- )- .digest("hex"),- };-}--it.each([- { alreadyEnded: false, appendFailure: false },- { alreadyEnded: true, appendFailure: false },- { alreadyEnded: true, appendFailure: true },- { alreadyEnded: true, appendFailure: false, bareCodex: true },- { alreadyEnded: true, appendFailure: false, epochFailure: "launch_intent" },- { alreadyEnded: true, appendFailure: false, epochFailure: "spawned" },- { alreadyEnded: true, appendFailure: false, epochFailure: "retired" },- { alreadyEnded: true, appendFailure: false, holdSpawned: true },- { alreadyEnded: true, appendFailure: false, completedTerminalAck: "pending" },- { alreadyEnded: true, appendFailure: false, completedTerminalAck: "completed" },- { alreadyEnded: true, appendFailure: false, completedTerminalAck: "repeat" },- { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "abort" },- { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "revoked" },- { alreadyEnded: true, appendFailure: false, finalRetirementRevocation: "during_authorize" },- { alreadyEnded: true, appendFailure: false, homeScoped: true },- { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true },- { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true, unknownExit: true },- { alreadyEnded: true, appendFailure: false, homeScoped: true, missingHome: true, startupFailureProof: true },- {- alreadyEnded: true,- appendFailure: false,- bareCodex: true,- terminalReplay: true,- },-])(- "settles only retained control authority without starting another provider turn ($alreadyEnded/$appendFailure/$bareCodex/$epochFailure/$terminalReplay/$holdSpawned/$homeScoped/$missingHome/$unknownExit) startup-failure-proof=$startupFailureProof completed-ack=$completedTerminalAck retired-revocation=$finalRetirementRevocation",- async ({- alreadyEnded,- appendFailure,- bareCodex,- epochFailure,- terminalReplay,- holdSpawned,- completedTerminalAck,- finalRetirementRevocation,- homeScoped,- missingHome,- unknownExit,- startupFailureProof,- }) => {- const replaySpyRestorers: Array<() => void> = [];- const replayRetirements: ReturnType<typeof maintenanceReplaySnapshot>[] = [];- const withheldTerminalFrames: Array<{- epoch: number;- direction: "inbound" | "outbound";- commandId: string;- controllerSeq: number;- count: number;- }> = [];- const fixtureId = randomUUID();- const fixtureRunner = defaultCapabilityRunnerdBinary();- const directory = await mkdtemp(join(tmpdir(), "runnerd-maintenance-"));- const original = join(directory, "original");- const copy = join(directory, "copy");- const activated = join(directory, "activated");- const home = join(directory, "source-home");- await mkdir(home);- const fakeCodex = resolve(- import.meta.dirname,- "../../runner/target/debug/fake-codex-app-server",- );- const bin = join(directory, "provider-bin");- if (bareCodex) {- await mkdir(bin);- await symlink(fakeCodex, join(bin, "codex"));- await writeFile(- join(home, "auth.json"),- JSON.stringify({ OPENAI_API_KEY: "fixture-only-not-a-secret" }),- );- }- const environment = bareCodex- ? { PATH: bin, HOME: home, CODEX_HOME: home }- : undefined;- const calls = join(directory, "calls.log");- const fakeState = homeScoped- ? join(original, "codex-home/fake-codex-state.json")- : join(directory, "fake.json");- const identity = {- runnerInstanceId: "runner-maintenance",- environmentLeaseId: "lease-maintenance",- runId: "run-maintenance",- normalizedSessionId: "session-maintenance",- turnId: "turn-maintenance",- itemId: "item-maintenance",- };- const bundle = createCapabilityRunnerdCodexTransport({- runnerBinary: fixtureRunner,- codexCommand: bareCodex ? "codex" : fakeCodex,- environment,- codexArgs: [- ...(homeScoped- ? ["--state-file-in-codex-home", "--require-existing-resume-state"]- : ["--state-file", fakeState]),- "--call-log",- calls,- ...(startupFailureProof ? ["--record-process-start"] : []),- "--hold-turn",- ],- sourceCodexHome: home,- stateDirectory: original,- prpIdentity: identity,- });- const dead = (pid: number) => {- try {- process.kill(pid, 0);- return false;- } catch (error) {- return (error as NodeJS.ErrnoException).code === "ESRCH";- }- };- let runnerPid = 0;- let providerPid = 0;- let retainFixtureForUnprovenExit = false;- const stopAndJoinReplayProcess = async (- handle: ReturnType<typeof durableControlPlane.spawnRunner>,- ) => {- // This helper may time out immediately after dispatching SIGKILL. Its- // return/rejection alone is not proof that this exact child has exited.- await durableControlPlane.waitForProcess(handle, 250).catch(() => undefined);- let deadline: NodeJS.Timeout | undefined;- try {- await Promise.race([- handle.completion,- new Promise<never>((_resolveJoin, rejectJoin) => {- deadline = setTimeout(() => rejectJoin(new Error(- "startup-proof fixture could not join its exact runner child",- )), 5_000);- }),- ]);- if (handle.processGroupId && !dead(-handle.processGroupId)) {- try { process.kill(-handle.processGroupId, "SIGKILL"); }- catch (error) {- if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;- }- }- await vi.waitFor(() => {- expect(handle.child.pid && dead(handle.child.pid)).toBe(true);- expect(handle.processGroupId && dead(-handle.processGroupId)).toBe(true);- }, { timeout: 2_000 });- } catch (error) {- retainFixtureForUnprovenExit = true;- throw error;- } finally {- if (deadline !== undefined) clearTimeout(deadline);- }- };- try {- const thread = (await bundle.transport.request("thread/start", {- cwd: directory,- dynamicTools: [],- })) as { thread: { id: string } };- await bundle.transport.request("turn/start", {- threadId: thread.thread.id,- input: [{ type: "text", text: "Keep the original turn only" }],- });- runnerPid = bundle.evidence().runnerPid!;- providerPid = bundle.evidence().providerPid!;- expect(runnerPid).toBeGreaterThan(0);- expect(providerPid).toBeGreaterThan(0);- await bundle.detachControllerForRestart();- // The provider can exit when its runner dies. An already-gone process- // group satisfies teardown; still fail on other signal errors and join below.- for (const pid of [runnerPid, providerPid]) {- try { process.kill(-pid, "SIGKILL"); }- catch (error) {- if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;- }- }- await vi.waitFor(() => {- expect(dead(runnerPid)).toBe(true);- expect(dead(providerPid)).toBe(true);- });- if (alreadyEnded) {- const providerState = JSON.parse(- await readFile(fakeState, "utf8"),- );- await writeFile(- fakeState,- JSON.stringify({ ...providerState, activeTurnId: null }),- );- }- const builder = new DurablePrpControlPlane({- stateDirectory: join(original, "control-plane"),- identity,- expectedRunnerVersion: "0.3.0",- expectedRunnerDigest: `sha256:${createHash("sha256")- .update(await readFile(fixtureRunner))- .digest("hex")}`,- });- builder.queueCommand("turn.stop", {- reason: "interrupted original close",- });- if (!missingHome) builder.queueCommand("runner.suspend", {});- // Match the retained production split: runner-owned unacknowledged- // output plus another full provider-owned prefix behind the old suspend.- const runnerFile = join(original, "runner/runner-state.json");- const runnerBefore = JSON.parse(await readFile(runnerFile, "utf8"));- const template = builder.store.state.committedEvents[0]!.envelope.payload;- for (let index = 0; index < 90; index++) {- const sourceSeq = runnerBefore.nextSourceSeq++;- const event = {- ...template,- sourceEventId: `maintenance-runner-${index}`,- sourceSeq,- eventType: "item.delta",- priority: 2,- payload: { provider: "codex", delta: `maintenance-runner-${index}` },- };- const envelope = {- protocol: "paperclip.runner",- version: 1,- kind: "event",- ...identity,- payload: event,- };- runnerBefore.outbox.push({- sourceSeq,- priority: 2,- eventType: "item.delta",- envelope,- byteSize: Buffer.byteLength(JSON.stringify(envelope)),- });- }- runnerBefore.peakOutboxBytes = Math.max(- runnerBefore.peakOutboxBytes,- runnerBefore.outbox.reduce(- (total: number, event: { byteSize: number }) =>- total + event.byteSize,- 0,- ),- );- await writeFile(runnerFile, JSON.stringify(runnerBefore));- const providerFile = join(original, "runner/codex-provider-state.json");- const providerBefore = JSON.parse(await readFile(providerFile, "utf8"));- expect(providerBefore.pendingEvents).toEqual([]);- expect(providerBefore.queuedEvents).toEqual([]);- for (let index = 0; index < 128; index++) {- providerBefore.pendingEvents.push({- executorEventId: `codex_provider_${String(providerBefore.nextProviderEventSeq++).padStart(16, "0")}`,- eventType: "item.delta",- priority: "p2",- payload: {- provider: "codex",- delta: `maintenance-provider-${index}`,- },- });- }- await writeFile(providerFile, JSON.stringify(providerBefore));- await cp(original, copy, { recursive: true });- const originalProviderHome = homeScoped ? await readFile(fakeState) : null;- if (missingHome) await rm(join(copy, "codex-home/fake-codex-state.json"));- const files = [- "control-plane/control-plane-state.json",- "runner/runner-state.json",- "runner/codex-provider-state.json",- ];- const bytes = await Promise.all(- files.map((file) => readFile(join(original, file))),- );- const sourceFingerprint = createHash("sha256")- .update(- JSON.stringify(- bytes.map((value) =>- createHash("sha256").update(value).digest("hex"),- ),- ),- )- .digest("hex");- const appendEvent = vi.fn(async (_event: PrpEvent) => {});- const maintenanceAbort = new AbortController();- let finalAuthorityRevoked = false;- const authorize = vi.fn(async () => {- if (finalAuthorityRevoked) {- if (finalRetirementRevocation === "during_authorize") {- maintenanceAbort.abort(new Error("retirement_authority_revoked"));- return;- }- throw new Error("retirement_authority_revoked");- }- });- const recordEpoch = vi.fn(- async (_receipt: Record<string, unknown>) => {},- );- const input = {- requestId: "maintenance-fixture-request",- binding: {- companyId: "company-maintenance",- issueId: "issue-maintenance",- agentId: "agent-maintenance",- runId: identity.runId,- sessionId: identity.normalizedSessionId,- },- backend: {- kind: "codex",- name: maintenanceFixtureBackendName(fixtureId),- },- identity,- stateDirectory: copy,- activationDirectory: activated,- sourceFingerprint,- providerSessionId: thread.thread.id,- originalRunnerPid: runnerPid,- originalProviderPid: providerPid,- runnerBinary: fixtureRunner,- sourceCodexHome: bareCodex ? undefined : home,- environment,- authorize,- appendEvent,- recordEpoch,- signal: maintenanceAbort.signal,- };- const close = vi.fn(async () => {- throw new NativeSessionCloseUnrecoverableError();- });- const start = vi.fn(async () => {- throw new Error("fixture admission reached");- });- const capabilities = {- resume: true,- typedEvents: true,- steering: false,- interruption: true,- structuredResult: true,- };- const session: NativeSession = {- identity: () => input.binding,- capabilities: async () => capabilities,- events: async function* () {},- startTurn: start,- close,- snapshot: async () => ({- backendKind: "codex",- sessionId: input.binding.sessionId,- identity: input.binding,- providerSessionId: thread.thread.id,- cursor: null,- activeTurnId: null,- pendingRuntimeRequests: [],- lineage: [],- }),- };- const backend: NativeSessionBackend = {- descriptor: async () => ({- ...input.backend,- version: "1",- capabilities,- }),- openSession: async () => session,- };- const nativeInput: NativeExecutionInputV1 = {- schema: "paperclip.native-execution-input.v1",- binding: {- companyId: input.binding.companyId,- issueId: input.binding.issueId,- agentId: input.binding.agentId,- runId: input.binding.runId,- executionWorkspaceId: "workspace-maintenance",- },- task: {- identifier: "MAINT-1",- title: "Fixture",- description: null,- prompt: "Fixture",- workMode: "standard",- },- workspace: {- cwd: directory,- repoUrl: null,- repoRef: null,- branchName: null,- },- provider: { kind: "codex", model: null },- session: {- normalizedSessionId: input.binding.sessionId,- driverKind: "codex_app_server",- protocolVersion: 1,- },- completionContract: {- id: "contract-maintenance",- sha256: "contract-maintenance-sha",- schemaVersion: "paperclip.completion-contract.v1",- contract: {- revision: "1",- objective: "Fixture",- criteria: [{ id: "objective", requirement: "Fixture" }],- },- },- interactionResponses: [],- credentialBindings: [],- };- const port: ControlPlanePort = {- openRun: async () => {},- checkpointSession: async () => {},- completeRun: async () => {},- replayEvents: async () => ({- events: [],- highestContiguousSourceSeq: 0,- }),- appendEvent: async () => ({- cursor: 1,- highestContiguousSourceSeq: 1,- disposition: "committed",- }),- };- const execute = () =>- executeNativeSession({- input: nativeInput,- backend,- controlPlane: port,- runnerInstanceId: identity.runnerInstanceId,- controlPlaneInstanceId: "control-maintenance",- requireSessionCloseBeforeReturn: true,- });- await expect(execute()).rejects.toThrow();- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(start).toHaveBeenCalledOnce();- if (holdSpawned) {- // A preceding row can fail before its authenticated cleanup proof.- // That sticky quarantine must remain, but must not contaminate the- // next independent fixture's backend domain in the same worker.- const independentStart = vi.fn(async () => {- throw new Error("independent fixture admission reached");- });- const independentBackend: NativeSessionBackend = {- descriptor: async () => ({- ...(await backend.descriptor()),- name: maintenanceFixtureBackendName(`${fixtureId}-following`),- }),- openSession: async () => ({- ...session,- startTurn: independentStart,- close: async () => {},- }),- };- await expect(executeNativeSession({- input: nativeInput,- backend: independentBackend,- controlPlane: port,- runnerInstanceId: identity.runnerInstanceId,- controlPlaneInstanceId: "independent-control-maintenance",- requireSessionCloseBeforeReturn: true,- })).rejects.toThrow("independent fixture admission reached");- expect(independentStart).toHaveBeenCalledOnce();- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(start).toHaveBeenCalledOnce();- }- await expect(- settleRetainedRunnerdSession({- ...input,- originalProviderPid: process.pid,- }),- ).rejects.toThrow("native_cleanup_maintenance_unproven");- expect(authorize).not.toHaveBeenCalled();- const copyProvider = join(copy, "runner/codex-provider-state.json");- const retainedProviderBytes = await readFile(copyProvider);- for (const eventType of [- "semantic_tool.input",- "runtime_request.created",- "session.resumed",- ]) {- const mutated = JSON.parse(retainedProviderBytes.toString("utf8"));- mutated.pendingEvents[0] = {- ...mutated.pendingEvents[0],- eventType,- payload: {- providerSessionId: thread.thread.id,- processId: process.pid,- },- };- await writeFile(copyProvider, JSON.stringify(mutated));- const candidateBytes = await Promise.all(- files.map((file) => readFile(join(copy, file))),- );- const candidateFingerprint = createHash("sha256")- .update(- JSON.stringify(- candidateBytes.map((value) =>- createHash("sha256").update(value).digest("hex"),- ),- ),- )- .digest("hex");- await expect(- settleRetainedRunnerdSession({- ...input,- sourceFingerprint: candidateFingerprint,- }),- ).rejects.toThrow("native_cleanup_maintenance_unproven");- expect(authorize).not.toHaveBeenCalled();- }- await writeFile(copyProvider, retainedProviderBytes);- for (const interruption of ["abort", "timeout"] as const) {- const abort = new AbortController();- let releaseAuthorization!: () => void;- const stuckAuthorization = new Promise<void>((release) => {- releaseAuthorization = release;- });- let drain: Promise<void> | undefined;- if (interruption === "timeout")- vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });- try {- const blocked = settleRetainedRunnerdSession({- ...input,- signal: abort.signal,- authorize: () => stuckAuthorization,- });- const observed = blocked.catch((error: unknown) => error);- if (interruption === "abort") abort.abort();- else await vi.advanceTimersByTimeAsync(30_000);- expect(await observed).toMatchObject({- message: "native_cleanup_maintenance_unproven",- });- expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(false);- await expect(settleRetainedRunnerdSession(input)).rejects.toThrow(- "native_cleanup_maintenance_unproven",- );- let drained = false;- drain = drainRetainedRunnerdMaintenanceOperations().then(() => {- drained = true;- });- await Promise.resolve();- await Promise.resolve();- // The bounded wrapper has already failed, but its original callback- // remains owned until it actually settles. No retry/proof is granted.- expect(drained).toBe(false);- releaseAuthorization();- await drain;- expect(drained).toBe(true);- expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(true);- } finally {- releaseAuthorization();- await drain;- if (interruption === "timeout") vi.useRealTimers();- }- }- if (missingHome) {- const startupEvents = () => appendEvent.mock.calls- .map(([event]) => event)- .filter((event) => event.eventType === "harness.diagnostic" &&- event.payload.code === "provider_startup_ownership");- let startupPhasesBeforeFailure: unknown[] | null = null;- const launch = durableControlPlane.spawnRunner;- const completions: Promise<unknown>[] = [];- let releaseExit!: () => void;- const exitGate = new Promise<void>((resolveExit) => {- releaseExit = resolveExit;- });- const launchSpy = vi- .spyOn(durableControlPlane, "spawnRunner")- .mockImplementation((options) => {- const handle = launch(options);- const completion = handle.completion.then(async (result) => {- // Model delayed delivery of the exact child's exit notification;- // dispatching a kill is not itself a durable retirement receipt.- if (unknownExit) await exitGate;- else- await new Promise((resolveExit) => setTimeout(resolveExit, 750));- return result;- });- completions.push(completion);- return { ...handle, completion };- });- authorize.mockImplementation(async () => {- const current = JSON.parse(- await readFile(join(copy, files[0]!), "utf8"),- );- if (- current.commands.some(- (command: { status: string }) => command.status === "failed",- )- ) {- if (startupFailureProof && startupPhasesBeforeFailure === null)- startupPhasesBeforeFailure = startupEvents().map((event) =>- (event.payload.startup as Record<string, unknown>).phase);- throw new Error("native_cleanup_maintenance_unproven");- }- });- try {- await expect(settleRetainedRunnerdSession(input)).rejects.toThrow(- "native_cleanup_maintenance_unproven",- );- if (unknownExit) {- expect(- recordEpoch.mock.calls.some(- ([receipt]) => receipt.phase === "retired",- ),- ).toBe(false);- expect(retainedRunnerdMaintenanceIsIdle(copy)).toBe(false);- }- } finally {- launchSpy.mockRestore();- releaseExit();- await Promise.allSettled(completions);- await drainRetainedRunnerdMaintenanceOperations();- }- const receipts = recordEpoch.mock.calls.map(([receipt]) => receipt);- const launched = receipts.filter(- (receipt) => receipt.phase === "spawned",- );- expect(launched.length).toBeGreaterThan(0);- for (const spawned of launched) {- const retired = receipts.find(- (receipt) =>- receipt.phase === "retired" &&- receipt.launchId === spawned.launchId,- );- if (unknownExit) expect(retired).toBeUndefined();- else- expect(retired).toMatchObject({- pid: spawned.pid,- processGroupAbsent: true,- });- expect(dead(Number(spawned.pid))).toBe(true);- expect(dead(-Number(spawned.pid))).toBe(true);- }- const failed = JSON.parse(await readFile(join(copy, files[0]!), "utf8"));- expect(- failed.commands.some(- (command: {- type: string;- result?: { result?: { message?: string } };- }) =>- command.type === "turn.stop" &&- command.result?.result?.message?.includes("no rollout found"),- ),- ).toBe(true);- expect(await readFile(fakeState)).toEqual(originalProviderHome);- expect(- await Promise.all(files.map((file) => readFile(join(original, file)))),- ).toEqual(bytes);- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(start).toHaveBeenCalledOnce();- const methods = (await readFile(calls, "utf8")).trim().split("\n");- expect(methods.filter((method) => method === "turn/start")).toHaveLength(- 1,- );- if (startupFailureProof) {- expect(startupPhasesBeforeFailure).toEqual([- "intent", "spawned", "initialization_failed",- ]);- const events = startupEvents();- expect(events).toHaveLength(3);- expect(events.map((event) => event.sourceSeq)).toEqual(- events.map((event) => event.sourceSeq).sort((left, right) => left - right),- );- expect(new Set(events.map((event) => event.sourceEventId)).size).toBe(3);- const facts = events.map((event) => event.payload.startup as Record<string, unknown>);- const [intent, spawned, initializationFailed] = facts;- expect(intent!.launchId).toMatch(/^[0-9a-f-]{36}$/);- expect(intent!.configurationFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);- const failedStop = failed.commands.find((command: { type: string; status: string }) =>- command.type === "turn.stop" && command.status === "failed");- for (const fact of facts) {- expect(Object.keys(fact).sort()).toEqual([- "schema", "launchId", "phase", "trigger", "attemptedProcessGeneration",- "origin", "command", "configurationFingerprint", "requestedThreadId",- "authenticatedThreadId", "processId", "processGroupId", "failedStage",- "directChildExitObserved", "exitCode", "signal", "processTreeRetired",- ].sort());- expect(fact).toMatchObject({- schema: "paperclip.provider_startup.v1",- launchId: intent!.launchId,- trigger: "restore",- attemptedProcessGeneration: providerBefore.providerProcessGeneration + 1,- origin: {- runnerInstanceId: identity.runnerInstanceId,- runId: identity.runId,- normalizedSessionId: identity.normalizedSessionId,- turnId: identity.turnId,- itemId: identity.itemId,- },- command: {- commandId: failedStop.commandId,- controllerSeq: failedStop.controllerSeq,- commandType: "turn.stop",- },- configurationFingerprint: intent!.configurationFingerprint,- requestedThreadId: thread.thread.id,- authenticatedThreadId: null,- processTreeRetired: false,- });- }- expect(intent).toMatchObject({- phase: "intent", processId: null, processGroupId: null,- directChildExitObserved: false, failedStage: null, exitCode: null, signal: null,- });- expect(spawned!.processId).toBeGreaterThan(0);- expect(spawned).toMatchObject({- phase: "spawned", processGroupId: spawned!.processId,- directChildExitObserved: false, failedStage: null, exitCode: null, signal: null,- });- expect(initializationFailed).toMatchObject({- phase: "initialization_failed", failedStage: "thread_open",- processId: spawned!.processId, processGroupId: spawned!.processId,- directChildExitObserved: true,- });- expect(dead(Number(spawned!.processId))).toBe(true);- expect(appendEvent.mock.calls.filter(([event]) =>- ["session.started", "session.resumed"].includes(event.eventType) &&- event.payload.processId !== providerPid)).toHaveLength(0);- expect(failed.commands.some((command: {- type: string; result?: { result?: { providerExitConfirmed?: boolean } };- }) => command.type === "turn.stop" &&- command.result?.result?.providerExitConfirmed === true)).toBe(false);- const deltas = appendEvent.mock.calls.map(([event]) => event.payload.delta)- .filter((delta) => typeof delta === "string" && delta.startsWith("maintenance-"));- expect(deltas).toHaveLength(218);- expect(new Set(deltas).size).toBe(218);- const failedProviderState = JSON.parse(await readFile(copyProvider, "utf8"));- expect(failedProviderState.startupAttempt).toMatchObject({- launchId: intent!.launchId,- });- const observedMethods = await readFile(calls, "utf8");- expect(observedMethods.trim().split("\n").filter((method) =>- method === "process-start")).toHaveLength(2);- const originalFailure = structuredClone(failedStop.result);- // Exercise the producer fence directly in this isolated fixture.- // This does not admit the failed copy through maintenance or alter- // its source/receipt bytes to manufacture recovery eligibility.- for (let restart = 0; restart < 2; restart++) {- const replayCore = new DurablePrpControlPlane({- stateDirectory: join(copy, "control-plane"),- identity,- expectedRunnerVersion: "0.3.0",- expectedRunnerDigest: `sha256:${createHash("sha256")- .update(await readFile(fixtureRunner)).digest("hex")}`,- onCommittedEvent: appendEvent,- });- const snapshot = replayCore.queueCommand("session.snapshot", {});- const stop = replayCore.queueCommand("turn.stop", {- reason: "startup-fence regression only",- });- let replayHandle: ReturnType<typeof durableControlPlane.spawnRunner> | null = null;- let replayAssertionFailed = false;- try {- await replayCore.start();- const runnerState = JSON.parse(await readFile(join(copy, files[1]!), "utf8"));- replayHandle = durableControlPlane.spawnRunner({- connectUrl: replayCore.connectUrl,- stateDirectory: join(copy, "runner"),- identity,- ticket: replayCore.issueBootstrapTicket(),- maxOutboxBytes: runnerState.maxOutboxBytes,- p0ReserveBytes: runnerState.p0ReserveBytes,- maxRuntimeMs: 2_000,- reconnectGraceMs: 1_000,- runnerBinaryPath: fixtureRunner,- runnerVersion: "0.3.0",- runnerDigest: `sha256:${createHash("sha256")- .update(await readFile(fixtureRunner)).digest("hex")}`,- environment: createCapabilityRunnerdProviderEnvironment({- provider: "codex",- options: {},- identity,- codexHome: join(copy, "codex-home"),- runtimeContextPath: join(copy, "runtime-context.json"),- hasRuntimeContext: false,- }),- });- await vi.waitFor(() => {- for (const queued of [snapshot, stop]) {- const command = replayCore.getCommand(queued.commandId);- expect(command?.status).toBe("failed");- expect(command?.result).toMatchObject({- result: {- message: expect.stringContaining(- "provider startup ownership remains unadmitted",- ),- },- });- }- }, { timeout: 5_000 });- await durableControlPlane.waitForProcess(replayHandle, 5_000);- expect(await readFile(calls, "utf8")).toBe(observedMethods);- expect((await readFile(calls, "utf8")).trim().split("\n")- .filter((method) => method === "process-start")).toHaveLength(2);- expect(replayCore.store.state.commands.find((command) =>- command.commandId === failedStop.commandId)?.result).toEqual(originalFailure);- expect(JSON.parse(await readFile(copyProvider, "utf8")).startupAttempt)- .toEqual(failedProviderState.startupAttempt);- expect(startupEvents()).toHaveLength(3);- } catch (error) {- replayAssertionFailed = true;- throw error;- } finally {- try {- if (replayHandle) await stopAndJoinReplayProcess(replayHandle);- } catch (error) {- // Keep the original assertion as the primary failure. Do not- // delete evidence underneath an unjoined owned process.- if (!replayAssertionFailed) throw error;- console.error("startup-proof fixture cleanup unproven; directory retained");- } finally {- await replayCore.stop();- }- }- }- expect(await readFile(fakeState)).toEqual(originalProviderHome);- expect(await Promise.all(files.map((file) => readFile(join(original, file)))))- .toEqual(bytes);- }- return;- }- if (appendFailure) {- const failure = new Error(- "injected maintenance event persistence failure",- );- let renewedProviderPid: number | null = null;- appendEvent.mockImplementation(async (event) => {- if (event.eventType !== "session.resumed") return;- const resumed = resolveRunnerdSessionIdentity(event.payload);- if (resumed.processId === providerPid) return;- renewedProviderPid = resumed.processId;- throw failure;- });- await expect(settleRetainedRunnerdSession(input)).rejects.toBe(failure);- expect(renewedProviderPid).not.toBeNull();- expect(dead(renewedProviderPid!)).toBe(true);- expect(dead(-renewedProviderPid!)).toBe(true);- expect(- await Promise.all(- files.map((file) => readFile(join(original, file))),- ),- ).toEqual(bytes);- await expect(- readFile(join(activated, "runner/runner-state.json")),- ).rejects.toMatchObject({ code: "ENOENT" });- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(start).toHaveBeenCalledOnce();- const methods = (await readFile(calls, "utf8")).trim().split("\n");- expect(- methods.filter((method) => method === "turn/start"),- ).toHaveLength(1);- expect(- methods.filter((method) => method === "thread/resume"),- ).toHaveLength(1);- return;- }- const assertNoProviderBeforeSpawnedReceipt = async () => {- const callsBefore = await readFile(calls, "utf8");- const providerBefore = await readFile(copyProvider);- const controlBefore = JSON.parse(- await readFile(- join(copy, "control-plane/control-plane-state.json"), "utf8",- ),- );- // Give the actual runner time to authenticate while its durable spawn- // receipt is held. Authentication must not release even the old stop.- await new Promise((resolveWait) => setTimeout(resolveWait, 500));- expect(await readFile(calls, "utf8")).toBe(callsBefore);- expect(await readFile(copyProvider)).toEqual(providerBefore);- const controlAfter = JSON.parse(- await readFile(- join(copy, "control-plane/control-plane-state.json"), "utf8",- ),- );- expect(controlAfter.connectionCount).toBe(- controlBefore.connectionCount,- );- expect(controlAfter.commandDeliveryCounts).toEqual(- controlBefore.commandDeliveryCounts,- );- expect(controlAfter.commands).toEqual(controlBefore.commands);- };- if (holdSpawned) {- recordEpoch.mockImplementation(async (receipt) => {- if (receipt.phase === "spawned")- await assertNoProviderBeforeSpawnedReceipt();- });- }- if (completedTerminalAck) {- const originalAttach =- DurablePrpControlPlane.prototype.attachWireConnection;- const attachSpy = vi- .spyOn(DurablePrpControlPlane.prototype, "attachWireConnection")- .mockImplementation(function (this: DurablePrpControlPlane, wire) {- const epoch = replayRetirements.length;- const direction =- completedTerminalAck === "completed" ? "outbound" : "inbound";- const inject =- this.store.path === join(copy, files[0]!) &&- (epoch === 0 ||- (completedTerminalAck === "repeat" && epoch === 1));- let attachment: ReturnType<typeof originalAttach> | undefined;- let withheld: (typeof withheldTerminalFrames)[number] | undefined;- const shouldWithhold = (candidate: typeof direction): boolean => {- if (- !inject ||- candidate !== direction ||- !attachment?.isAuthenticated()- )- return false;- if (withheld) {- withheld.count += 1;- return true;- }- const runner = JSON.parse(- readFileSync(join(copy, files[1]!), "utf8"),- );- const terminal = runner.pendingTerminalDelivery;- if (- terminal?.commandType !== "runner.suspend" ||- terminal.lifecycle !== "suspended"- )- return false;- const result = runner.processedCommands[terminal.commandId];- if (- result?.status !== "completed" ||- result.result?.status !== "completed" ||- result.commandType !== terminal.commandType ||- result.controllerSeq !== terminal.controllerSeq- )- return false;- const control = JSON.parse(readFileSync(this.store.path, "utf8"));- const command = control.commands.find(- (entry: { commandId: string }) =>- entry.commandId === terminal.commandId,- );- if (- command?.type !== terminal.commandType ||- command.controllerSeq !== terminal.controllerSeq ||- command.status !==- (direction === "inbound" ? "pending" : "completed")- )- return false;- if (direction === "outbound")- expect(command.result).toEqual(result);- else expect(command.result ?? null).toBeNull();- // Rust durably records this exact result before sending it.- // Withhold transport delivery only after that handshake, not- // after a guessed number of saves or an elapsed sleep. Pending- // mode loses the result; completed mode loses its outbound ACK.- // No retained journal/outbox bytes are removed or rewritten.- withheld = {- epoch,- direction,- commandId: terminal.commandId,- controllerSeq: terminal.controllerSeq,- count: 1,- };- withheldTerminalFrames.push(withheld);- return true;- };- attachment = originalAttach.call(this, {- sendJson(value) {- if (!shouldWithhold("outbound")) wire.sendJson(value);- },- close: (code) => wire.close(code),- onJson: (listener) =>- wire.onJson((value) => {- if (!shouldWithhold("inbound")) listener(value);- }),- onClose: (listener) => wire.onClose(listener),- });- return attachment;- });- replaySpyRestorers.push(() => attachSpy.mockRestore());- recordEpoch.mockImplementation(async (receipt) => {- if (receipt.phase !== "retired") return;- const snapshot = maintenanceReplaySnapshot(copy);- expect(snapshot.fingerprint).toBe(receipt.finalFingerprint);- replayRetirements.push(snapshot);- });- }- if (epochFailure) {- const failure = new Error("injected epoch receipt persistence failure");- const callsBefore = await readFile(calls, "utf8");- recordEpoch.mockImplementation(async (receipt) => {- if (receipt.phase === "spawned" && epochFailure === "spawned")- await assertNoProviderBeforeSpawnedReceipt();- if (receipt.phase === epochFailure) throw failure;- });- await expect(settleRetainedRunnerdSession(input)).rejects.toBe(failure);- for (const [receipt] of recordEpoch.mock.calls) {- if (receipt.phase !== "spawned") continue;- expect(dead(Number(receipt.pid))).toBe(true);- expect(dead(-Number(receipt.pid))).toBe(true);- }- if (epochFailure === "spawned") {- expect(await readFile(calls, "utf8")).toBe(callsBefore);- }- if (epochFailure === "launch_intent") {- expect(await readFile(calls, "utf8")).toBe(callsBefore);- expect(- recordEpoch.mock.calls.map(([receipt]) => receipt.phase),- ).toEqual(["launch_intent"]);- }- expect(- await Promise.all(- files.map((file) => readFile(join(original, file))),- ),- ).toEqual(bytes);- await expect(- readFile(join(activated, "runner/runner-state.json")),- ).rejects.toMatchObject({ code: "ENOENT" });- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(start).toHaveBeenCalledOnce();- return;- }- if (finalRetirementRevocation) {- recordEpoch.mockImplementation(async (receipt) => {- if (receipt.phase !== "retired") return;- const state = maintenanceReplaySnapshot(copy);- if (- state.runner.pendingTerminalDelivery != null ||- state.runner.pendingProviderCleanup != null ||- state.runner.outbox.length !== 0 ||- state.provider.pendingEvents.length !== 0 ||- state.provider.queuedEvents.length !== 0 ||- state.control.commands.some(- (command: { status: string }) => command.status === "pending",- )- )- return;- expect(state.provider.lifecycle).toBe("prepared");- expect(state.fingerprint).toBe(receipt.finalFingerprint);- finalAuthorityRevoked = true;- if (finalRetirementRevocation === "abort")- maintenanceAbort.abort(new Error("retirement_authority_revoked"));- });- }- let failedAttempt: { directory: string; bytes: Buffer[] } | null = null;- let failedStartupAttempt: {- directory: string;- bytes: Buffer[];- } | null = null;- if (terminalReplay) {- // Forward failures now preserve a startup fence. Keep this genuinely- // produced failed copy intact; it is NOT a legacy replay candidate.- await expect(- settleRetainedRunnerdSession({- ...input,- environment: undefined,- sourceCodexHome: null,- }),- ).rejects.toThrow("native_cleanup_maintenance_unproven");- const failedBytes = await Promise.all(- files.map((file) => readFile(join(copy, file))),- );- expect(- JSON.parse(failedBytes[2]!.toString("utf8")).startupAttempt,- ).toMatchObject({- schema: "paperclip.provider_startup.v1",- phase: "initialization_failed",- failedStage: "spawn",- requestedThreadId: thread.thread.id,- authenticatedThreadId: null,- processId: null,- directChildExitObserved: false,- processTreeRetired: false,- });- expect(- appendEvent.mock.calls- .filter(- ([event]) => event.payload.code === "provider_startup_ownership",- )- .map(- ([event]) => (event.payload.startup as { phase: string }).phase,- ),- ).toEqual(["intent", "initialization_failed"]);- expect(- await Promise.all(- files.map((file) => readFile(join(original, file))),- ),- ).toEqual(bytes);- await expect(- readFile(join(activated, files[1]!)),- ).rejects.toMatchObject({ code: "ENOENT" });- const failedStartupDirectory = join(- original,- "..",- "failed-startup-attempt",- );- await rename(copy, failedStartupDirectory);- failedStartupAttempt = {- directory: failedStartupDirectory,- bytes: failedBytes,- };-- // Backward-compatibility fixture, synthesized ONLY from the pristine- // original snapshots: old producers recorded terminal failure without- // a startup-attempt field. Never delete a real generated fence above.- const directory = join(original, "..", "legacy-failed-terminal");- await cp(original, directory, { recursive: true });- const legacyControl = JSON.parse(bytes[0]!.toString("utf8"));- const legacyRunner = JSON.parse(bytes[1]!.toString("utf8"));- expect(- JSON.parse(bytes[2]!.toString("utf8")).startupAttempt ?? null,- ).toBeNull();- const legacyCommands = legacyControl.commands.slice(-2);- expect(- legacyCommands.map((command: { type: string }) => command.type),- ).toEqual(["turn.stop", "runner.suspend"]);- for (const command of legacyCommands) {- const wire = {- schema: command.schema,- commandId: command.commandId,- controllerSeq: command.controllerSeq,- type: command.type,- issuedAt: command.issuedAt,- deadlineAt: null,- precondition: null,- payload: command.payload,- };- const result = {- commandId: command.commandId,- commandType: command.type,- controllerSeq: command.controllerSeq,- status: "failed",- result: {- code: "command_execution_failed",- message: "legacy pre-start failure fixture",- },- };- command.status = "failed";- command.result = result;- legacyRunner.processedCommands[command.commandId] = result;- legacyRunner.processedCommandFingerprints[command.commandId] =- createHash("sha256")- .update(- durableControlPlane.durableRecoveryInternals.canonicalJson(wire),- )- .digest("hex");- legacyRunner.lastControllerCommandSeq = command.controllerSeq;- }- const terminal = legacyCommands[1]!;- legacyRunner.lifecycle = "suspended";- legacyRunner.pendingTerminalDelivery = {- commandId: terminal.commandId,- controllerSeq: terminal.controllerSeq,- commandType: terminal.type,- lifecycle: "suspended",- };- await writeFile(- join(directory, files[0]!),- JSON.stringify(legacyControl),- );- await writeFile(- join(directory, files[1]!),- JSON.stringify(legacyRunner),- );- const legacyBytes = await Promise.all(- files.map((file) => readFile(join(directory, file))),- );- expect(legacyBytes[2]).toEqual(bytes[2]);- expect(- legacyControl.commands- .slice(-2)- .map((command: { status: string }) => command.status),- ).toEqual(["failed", "failed"]);- await cp(directory, copy, { recursive: true });- failedAttempt = { directory, bytes: legacyBytes };- input.sourceFingerprint = createHash("sha256")- .update(- JSON.stringify(- legacyBytes.map((value) =>- createHash("sha256").update(value).digest("hex"),- ),- ),- )- .digest("hex");- input.requestId = "maintenance-fixture-continuation";- recordEpoch.mockClear();- appendEvent.mockClear();- }- const pendingProof = settleRetainedRunnerdSession(input);- if (finalRetirementRevocation) {- await expect(pendingProof).rejects.toThrow(- finalRetirementRevocation === "abort"- ? "native_cleanup_maintenance_unproven"- : "retirement_authority_revoked",- );- expect(finalAuthorityRevoked).toBe(true);- await expect(- readFile(join(activated, "runner/runner-state.json")),- ).rejects.toMatchObject({ code: "ENOENT" });- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(- await Promise.all(files.map((file) => readFile(join(original, file)))),- ).toEqual(bytes);- const methods = (await readFile(calls, "utf8")).trim().split("\n");- expect(methods.filter((method) => method === "turn/start")).toHaveLength(1);- return;- }- if (completedTerminalAck === "repeat") {- await expect(pendingProof).rejects.toThrow(- "native_cleanup_maintenance_unproven",- );- expect(replayRetirements).toHaveLength(2);- expect(- withheldTerminalFrames.map(({ epoch, direction }) => ({- epoch,- direction,- })),- ).toEqual([- { epoch: 0, direction: "inbound" },- { epoch: 1, direction: "inbound" },- ]);- for (const [epoch, state] of replayRetirements.entries()) {- expect(JSON.stringify(state.runner.diagnostics)).toContain(- "terminal command result acknowledgement timed out",- );- expect(state.runner.pendingTerminalDelivery).toMatchObject({- commandType: "runner.suspend",- lifecycle: "suspended",- });- const withheld = withheldTerminalFrames[epoch]!;- expect(withheld.count).toBeGreaterThan(0);- expect(state.runner.pendingTerminalDelivery).toMatchObject({- commandId: withheld.commandId,- controllerSeq: withheld.controllerSeq,- });- expect(- state.runner.processedCommands[withheld.commandId],- ).toMatchObject({- status: "completed",- result: { status: "completed" },- });- expect(- state.control.commands.find(- (command: { commandId: string }) =>- command.commandId === withheld.commandId,- ),- ).toMatchObject({ status: "pending" });- }- expect(replayRetirements[1]!.provider).toEqual(- replayRetirements[0]!.provider,- );- expect(recordEpoch.mock.calls).toHaveLength(6);- await expect(- readFile(join(activated, "runner/runner-state.json")),- ).rejects.toMatchObject({ code: "ENOENT" });- await expect(execute()).rejects.toMatchObject({- code: "native_session_cleanup_quarantined",- });- expect(- await Promise.all(- files.map((file) => readFile(join(original, file))),- ),- ).toEqual(bytes);- const methods = (await readFile(calls, "utf8")).trim().split("\n");- expect(- methods.filter((method) => method === "turn/start"),- ).toHaveLength(1);- // The first cleanup epoch restored the old provider solely to stop- // it; the failed delivery-only replay did not restore it again.- expect(- methods.filter((method) => method === "thread/resume"),- ).toHaveLength(1);- return;- }- const proof = await pendingProof.catch(- async (error: unknown) => {- const runner = JSON.parse(- await readFile(join(copy, "runner/runner-state.json"), "utf8"),- );- const provider = JSON.parse(- await readFile(- join(copy, "runner/codex-provider-state.json"),- "utf8",- ),- );- const control = JSON.parse(- await readFile(- join(copy, "control-plane/control-plane-state.json"),- "utf8",- ),- );- throw new Error(- JSON.stringify({- runner: {- lifecycle: runner.lifecycle,- outbox: runner.outbox.length,- acked: runner.ackedSourceSeq,- next: runner.nextSourceSeq,- terminalAckTimedOut: JSON.stringify(- runner.diagnostics,- ).includes("terminal command result acknowledgement timed out"),- pendingTerminalDelivery:- runner.pendingTerminalDelivery ?? null,- retainedIdentityTypes: runner.outbox- .filter((row: { eventType: string }) =>- [- "session.started",- "session.resumed",- "harness.ready",- ].includes(row.eventType),- )- .map((row: { sourceSeq: number; eventType: string }) => ({- sourceSeq: row.sourceSeq,- eventType: row.eventType,- })),- },- provider: {- lifecycle: provider.lifecycle,- pending: provider.pendingEvents.length,- queued: provider.queuedEvents.length,- generation: provider.providerProcessGeneration,- },- commands: control.commands.map(- (command: { type: string; status: string }) => ({- type: command.type,- status: command.status,- }),- ),- committedCount: appendEvent.mock.calls.length,- epochExits: recordEpoch.mock.calls- .map(([receipt]) => receipt)- .filter((receipt) => receipt.phase === "retired")- .map((receipt) => ({- epoch: receipt.epoch,- exitCode: receipt.exitCode,- exitSignal: receipt.exitSignal,- })),- }),- { cause: error },- );- },- );- if (failedAttempt) {- expect(- await Promise.all(- files.map((file) =>- readFile(join(failedStartupAttempt!.directory, file)),- ),- ),- ).toEqual(failedStartupAttempt!.bytes);- expect(- await Promise.all(- files.map((file) => readFile(join(failedAttempt!.directory, file))),- ),- ).toEqual(failedAttempt.bytes);- const finalControl = JSON.parse(- await readFile(join(copy, files[0]!), "utf8"),- );- const failedControl = JSON.parse(- failedAttempt.bytes[0]!.toString("utf8"),- );- expect(- finalControl.commands.slice(0, failedControl.commands.length),- ).toEqual(failedControl.commands);- expect(- finalControl.commands- .slice(failedControl.commands.length)- .some(- (command: {- type: string;- status: string;- result: { result?: { providerExitConfirmed?: boolean } };- }) =>- command.type === "turn.stop" &&- command.status === "completed" &&- command.result.result?.providerExitConfirmed === true,- ),- ).toBe(true);- }- const epochReceipts = recordEpoch.mock.calls.map(([receipt]) => receipt);- if (completedTerminalAck) {- const before = replayRetirements[0]!;- const after = replayRetirements[1]!;- expect(withheldTerminalFrames).toHaveLength(1);- expect(withheldTerminalFrames[0]).toMatchObject({- epoch: 0,- direction:- completedTerminalAck === "completed" ? "outbound" : "inbound",- });- expect(withheldTerminalFrames[0]!.count).toBeGreaterThan(0);- expect(JSON.stringify(before.runner.diagnostics)).toContain(- "terminal command result acknowledgement timed out",- );- const pending = before.runner.pendingTerminalDelivery;- expect(pending).toMatchObject({- commandType: "runner.suspend",- lifecycle: "suspended",- commandId: withheldTerminalFrames[0]!.commandId,- controllerSeq: withheldTerminalFrames[0]!.controllerSeq,- });- expect(before.runner.processedCommands[pending.commandId].status).toBe(- "completed",- );- expect(- before.control.commands.find(- (command: { commandId: string }) =>- command.commandId === pending.commandId,- ).status,- ).toBe(completedTerminalAck);- expect(- runnerdRecoveryInternals.completedMaintenanceTerminalReceipt(before),- ).not.toBeNull();- expect(- runnerdRecoveryInternals.completedMaintenanceTerminalReplayMatches(- before,- after,- ),- ).toBe(true);- expect(after.provider).toEqual(before.provider);- expect(after.runner.pendingProviderCleanup ?? null).toBeNull();- // An actual completed receipt copied into an INITIAL invocation is- // not this invocation's joined retirement and cannot enable replay.- const initialTerminal = join(directory, "unproved-initial-terminal");- await mkdir(join(initialTerminal, "runner"), { recursive: true });- await mkdir(join(initialTerminal, "control-plane"));- for (const [index, value] of [- before.control,- before.runner,- before.provider,- ].entries())- await writeFile(- join(initialTerminal, files[index]!),- JSON.stringify(value),- );- const unproved = maintenanceReplaySnapshot(initialTerminal);- const initialEpoch = vi.fn(async () => {});- await expect(- settleRetainedRunnerdSession({- ...input,- stateDirectory: initialTerminal,- sourceFingerprint: unproved.fingerprint,- recordEpoch: initialEpoch,- }),- ).rejects.toThrow("native_cleanup_maintenance_unproven");- expect(initialEpoch).not.toHaveBeenCalled();- const receiptCases: Array<[string, (state: typeof before) => void]> = [- [- "missing result",- (state) => {- delete state.runner.processedCommands[pending.commandId];- },- ],- ...["pending", "failed", "rejected", "indeterminate"].map(- (status): [string, (state: typeof before) => void] => [- `outer ${status}`,- (state) => {- state.runner.processedCommands[pending.commandId].status = status;- },- ],- ),- ...["failed", "rejected", "indeterminate"].map(- (status): [string, (state: typeof before) => void] => [- `nested ${status}`,- (state) => {- state.runner.processedCommands[pending.commandId].result.status =- status;- },- ],- ),- [- "terminal sequence",- (state) => {- state.runner.pendingTerminalDelivery.controllerSeq++;- },- ],- [- "terminal type",- (state) => {- state.runner.pendingTerminalDelivery.commandType = "runner.shutdown";- },- ],- [- "wire fingerprint",- (state) => {- state.control.commands.find(- (command: { commandId: string }) =>- command.commandId === pending.commandId,- ).payload = { changed: true };- },- ],- [- "completed controller result",- (state) => {- const command = state.control.commands.find(- (entry: { commandId: string }) =>- entry.commandId === pending.commandId,- );- command.status = "completed";- command.result = { changed: true };- },- ],- [- "earlier pending command",- (state) => {- const command = state.control.commands.find(- (entry: { commandId: string }) =>- entry.commandId === pending.commandId,- );- command.status = "pending";- delete command.result;- state.control.commands[0].status = "pending";- },- ],- [- "active provider",- (state) => {- state.provider.activeProviderTurnId = "another-turn";- },- ],- [- "provider cleanup",- (state) => {- state.runner.pendingProviderCleanup = pending;- },- ],- ];- for (const [name, mutate] of receiptCases) {- const changed = structuredClone(before);- mutate(changed);- expect(- runnerdRecoveryInternals.completedMaintenanceTerminalReceipt(changed),- name,- ).toBeNull();- }- for (const [name, mutate] of [- [- "other command",- (state: typeof after) => {- state.control.commands[0].result = { changed: true };- },- ],- [- "provider bytes",- (state: typeof after) => {- state.providerFingerprint = "changed";- },- ],- [- "pending delivery",- (state: typeof after) => {- state.runner.pendingTerminalDelivery = pending;- },- ],- [- "processed receipt",- (state: typeof after) => {- state.runner.processedCommands[pending.commandId].result = {- changed: true,- };- },- ],- ] as const) {- const changed = structuredClone(after);- mutate(changed);- expect(- runnerdRecoveryInternals.completedMaintenanceTerminalReplayMatches(- before,- changed,- ),- name,- ).toBe(false);- }- }- expect(epochReceipts.length).toBeGreaterThanOrEqual(3);- for (let index = 0; index < epochReceipts.length; index += 3) {- const [intent, spawned, retired] = epochReceipts.slice(- index,- index + 3,- );- expect(intent).toMatchObject({- phase: "launch_intent",- requestId: input.requestId,- stateDirectory: copy,- });- expect(spawned).toMatchObject({- phase: "spawned",- launchId: intent!.launchId,- });- expect(retired).toMatchObject({- phase: "retired",- launchId: intent!.launchId,- pid: spawned!.pid,- processGroupAbsent: true,- });- expect(dead(Number(retired!.pid))).toBe(true);- expect(dead(-Number(retired!.pid))).toBe(true);- }- if (bareCodex) {- expect(await readFile(join(copy, "codex-home/auth.json"), "utf8")).toBe(- await readFile(join(home, "auth.json"), "utf8"),- );- }- expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(false);- await rename(copy, activated);- expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(true);- expect(retainedRunnerdCleanupProofIsCurrent({ ...proof })).toBe(false);- expect(() =>- completeRetainedNativeSessionCleanup({ ...proof }),- ).toThrow();- expect(completeRetainedNativeSessionCleanup(proof)).toBe(1);- expect(completeRetainedNativeSessionCleanup(proof)).toBe(0);- close.mockImplementation(async () => {});- await expect(execute()).rejects.toThrow("fixture admission reached");- expect(start).toHaveBeenCalledTimes(2);- expect(- await Promise.all(files.map((file) => readFile(join(original, file)))),- ).toEqual(bytes);- const methods = (await readFile(calls, "utf8")).trim().split("\n");- expect(methods.filter((method) => method === "turn/start")).toHaveLength(- 1,- );- expect(- methods.filter((method) => method === "thread/start"),- ).toHaveLength(1);- expect(- methods.filter((method) => method === "thread/resume"),- ).toHaveLength(1);- const finalState = JSON.parse(- await readFile(join(activated, "runner/runner-state.json"), "utf8"),- );- expect(finalState).toMatchObject({- ...identity,- lifecycle: "suspended",- outbox: [],- });- const provider = JSON.parse(- await readFile(- join(activated, "runner/codex-provider-state.json"),- "utf8",- ),- );- expect(provider).toMatchObject({- threadId: thread.thread.id,- activeProviderTurnId: null,- pendingEvents: [],- queuedEvents: [],- });- expect(- appendEvent.mock.calls.every(- ([event]) => event.runId === identity.runId,- ),- ).toBe(true);- const deltas = appendEvent.mock.calls- .map(([event]) => event.payload.delta)- .filter(- (delta) =>- typeof delta === "string" && delta.startsWith("maintenance-"),- );- expect(deltas).toHaveLength(218);- expect(new Set(deltas).size).toBe(218);- await writeFile(- join(activated, "runner/runner-state.json"),- JSON.stringify({ ...finalState, lifecycle: "ready" }),- );- expect(retainedRunnerdCleanupProofIsCurrent(proof)).toBe(false);- } finally {- for (const restore of replaySpyRestorers.reverse()) restore();- await bundle.transport.close().catch(() => undefined);- for (const pid of [runnerPid, providerPid]) {- if (pid > 0 && !dead(pid)) {- try {- process.kill(-pid, "SIGKILL");- } catch {}- }- }- if (!retainFixtureForUnprovenExit)- await rm(directory, { recursive: true, force: true });- }- },- 40_000,-);+// The retained-settlement maintenance family ("settles only retained control+// authority without starting another provider turn") lives in+// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of+// sequential cases on a separate worker instead of serializing them behind+// this file's tests.it.each(["alive", "pending_liveness", "pending_registration"] as const)("bounds adopted runner authentication at its exact deadline with %s evidence",