The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test: add ACPX run lifecycle characterization baselines (#11461)
Nicky Leachcd501499a2fa
diff --git a/packages/adapter-utils/src/acpx-engine/composed-run-characterization.test.ts b/packages/adapter-utils/src/acpx-engine/composed-run-characterization.test.tsnew file mode 100644index 0000000000..b9723bebc5--- /dev/null+++ b/packages/adapter-utils/src/acpx-engine/composed-run-characterization.test.ts@@ -0,0 +1,877 @@+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { AcpRuntimeOptions } from "acpx/runtime";+import type { AdapterExecutionContext, AdapterRuntimeMcpAccess } from "@paperclipai/adapter-utils";+import {+ prepareAdapterExecutionTargetRuntime,+ startAdapterExecutionTargetPaperclipBridge,+ startAdapterExecutionTargetProcessSessionBridge,+} from "@paperclipai/adapter-utils/execution-target";++// This file is a characterization test. It pins the engine boundary's CURRENT+// behavior; it never changes production code. Each test states the observed+// contract of `executeAcpxEngine` as data, so a later refactor that alters an+// exit path's result form, its finalization set, or its warm-lane decision+// fails here.+//+// Adapter-boundary automatic CLI fallback note (File 1b): the codex, claude, and+// gemini adapters wrap this engine and fall back to their CLI lane when an+// AUTOMATIC ACP selection fails, but RETHROW when ACP was EXPLICITLY selected.+// Those three adapters live in higher packages that depend on this one, so an+// adapter-utils test cannot import them (a reverse dependency does not exist in+// package.json). That branch already has REAL coverage in each adapter package:+// - packages/adapters/codex-local/src/server/execute.acp-fallback.test.ts+// - packages/adapters/claude-local/src/server/execute.acp-fallback.test.ts+// - packages/adapters/gemini-local/src/server/execute.acp-fallback.test.ts+// and each adapter's `normalizeEngine` / `resolveXxxExecutionEngineForRun`+// classification (explicit acp/cli vs automatic acp) is pinned in each+// package's acp.test.ts. This file pins the engine-level exit paths those+// adapters wrap.++// Wrap the staging seam + both sandbox bridges in call-recording spies that+// still delegate to the real implementations (a runner-backed sandbox test+// exercises them end-to-end against a local runner). This lets the staging+// tests assert the exact `runtimeRootDir`/`workspaceLocalDir`/`assets` the+// engine threads without changing any real behavior for the other tests.+vi.mock("@paperclipai/adapter-utils/execution-target", async (importActual) => {+ const actual = await importActual<typeof import("@paperclipai/adapter-utils/execution-target")>();+ return {+ ...actual,+ prepareAdapterExecutionTargetRuntime: vi.fn(actual.prepareAdapterExecutionTargetRuntime),+ startAdapterExecutionTargetPaperclipBridge: vi.fn(actual.startAdapterExecutionTargetPaperclipBridge),+ startAdapterExecutionTargetProcessSessionBridge: vi.fn(actual.startAdapterExecutionTargetProcessSessionBridge),+ };+});+import {+ createAcpxEngineExecutor,+ type AcpxEngineExecutorOptions,+} from "./execute.js";+import { runChildProcess } from "../server-utils.js";++const tempRoots: string[] = [];++async function makeTempRoot() {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-acpx-skills-"));+ tempRoots.push(root);+ return root;+}++afterEach(async () => {+ // A remote run stages a process-session bridge whose detached event writer can+ // still be flushing a trailing event file into `.../process-sessions/<id>/events`+ // when the run's own best-effort `client.remove(sessionDir)` (which production+ // catch-wraps) has already returned. Under CI load that trailing write can land+ // between this recursive delete's directory snapshot and its `rmdir`, surfacing as+ // `ENOTEMPTY`. `maxRetries`/`retryDelay` make the cleanup ride out that window the+ // same way production tolerates it, instead of failing the just-passed test.+ await Promise.all(+ tempRoots.splice(0).map((root) =>+ fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }),+ ),+ );+});++function createLocalSandboxRunner(+ onExecute?: (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ }) => void,+) {+ let counter = 0;+ return {+ execute: async (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ stdin?: string;+ timeoutMs?: number;+ onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;+ onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ }) => {+ counter += 1;+ onExecute?.(input);+ const command = input.command === "bash" ? "/bin/bash" : input.command;+ return await runChildProcess(`acpx-sandbox-run-${counter}`, command, input.args ?? [], {+ cwd: input.cwd ?? process.cwd(),+ env: input.env ?? {},+ stdin: input.stdin,+ timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),+ graceSec: 5,+ onLog: input.onLog ?? (async () => {}),+ onSpawn: input.onSpawn+ ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt })+ : undefined,+ });+ },+ };+}++function buildRuntime(+ onSetConfigOption?: (input: { key: string; value: string }) => void,+ onEnsureSession?: (input: Record<string, unknown>) => void,+) {+ return {+ ensureSession: async (input: Record<string, unknown>) => {+ onEnsureSession?.(input);+ return ({+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ });+ },+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ }),+ setConfigOption: async (input: { key: string; value: string }) => {+ onSetConfigOption?.(input);+ },+ close: async () => {},+ };+}++async function runExecutor(+ config: Record<string, unknown>,+ options: {+ context?: Record<string, unknown>;+ executionTransport?: Record<string, unknown>;+ authToken?: string;+ executionTarget?: Record<string, unknown>;+ runtimeMcp?: AdapterRuntimeMcpAccess;+ prepareRemoteManagedHome?: AcpxEngineExecutorOptions["prepareRemoteManagedHome"];+ startupTraceContext?: AdapterExecutionContext["startupTraceContext"];+ } = {},+) {+ const runtimeOptions: Record<string, unknown>[] = [];+ const configOptions: Array<{ key: string; value: string }> = [];+ const sessionInputs: Record<string, unknown>[] = [];+ const meta: Record<string, unknown>[] = [];+ const logs: Array<{ stream: string; text: string }> = [];+ const events: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];+ const execute = createAcpxEngineExecutor({+ ...(options.prepareRemoteManagedHome+ ? { prepareRemoteManagedHome: options.prepareRemoteManagedHome }+ : {}),+ createRuntime: (options) => {+ runtimeOptions.push(options as unknown as Record<string, unknown>);+ return buildRuntime(+ ({ key, value }) => configOptions.push({ key, value }),+ (input) => sessionInputs.push(input),+ ) as never;+ },+ });++ const result = await execute({+ runId: "run-1",+ agent: {+ id: "agent-1",+ companyId: "company-1",+ },+ runtime: {},+ config,+ context: options.context ?? {},+ executionTransport: options.executionTransport,+ authToken: options.authToken,+ executionTarget: options.executionTarget,+ runtimeMcp: options.runtimeMcp,+ startupTraceContext: options.startupTraceContext,+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ onMeta: async (payload: unknown) => {+ meta.push(payload as Record<string, unknown>);+ },+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ events.push(event);+ },+ } as never);++ expect(result.exitCode).toBe(0);+ return { logs, meta, events, runtimeOptions, configOptions, sessionInputs, result };+}++// A remote sandbox setup that stages the host worktree through the real local+// runner, so a run reaches the post-build window with live bridges and a held+// staging lease.+async function setupRemoteSandbox() {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const localCwd = path.join(root, "worktree");+ const remoteCwd = path.join(root, "remote-workspace");+ await fs.mkdir(localCwd, { recursive: true });+ await fs.mkdir(remoteCwd, { recursive: true });+ await fs.writeFile(path.join(localCwd, "hello.txt"), "hi", "utf8");+ const executionTarget = {+ kind: "remote",+ transport: "sandbox",+ providerKey: "fake-plugin",+ remoteCwd,+ runner: createLocalSandboxRunner(),+ };+ return { root, stateDir, localCwd, remoteCwd, executionTarget };+}++const okHandle = {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+};++function completedTurn() {+ return {+ events: (async function* () {})(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ };+}++function throwingTurn() {+ return {+ events: (async function* () {+ throw new Error("turn upstream boom");+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ };+}++// A context whose only prompt-read field throws when read, so the prompt build+// fails after the session handshake succeeds and before the turn starts.+function throwingHandoffContext(): Record<string, unknown> {+ const context: Record<string, unknown> = {};+ Object.defineProperty(context, "paperclipSessionHandoffMarkdown", {+ enumerable: false,+ get() {+ throw new Error("prompt build boom");+ },+ });+ return context;+}++// Stub both sandbox bridges with stop spies collected per start, so a test can+// assert the bridges stopped without running the real bridge transport.+function stubBridges() {+ const paperclipStops: Array<ReturnType<typeof vi.fn>> = [];+ const processStops: Array<ReturnType<typeof vi.fn>> = [];+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementation(async () => {+ const stop = vi.fn(async () => {});+ paperclipStops.push(stop);+ return { env: {}, stop } as never;+ });+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementation(async () => {+ const stop = vi.fn(async () => {});+ processStops.push(stop);+ return { agentCommand: null, stop } as never;+ });+ const anyStopped = (stops: Array<ReturnType<typeof vi.fn>>) =>+ stops.some((stop) => stop.mock.calls.length > 0);+ const stoppedCount = (stops: Array<ReturnType<typeof vi.fn>>) =>+ stops.filter((stop) => stop.mock.calls.length > 0).length;+ return { paperclipStops, processStops, anyStopped, stoppedCount };+}++function remoteArgs(+ stateDir: string,+ localCwd: string,+ executionTarget: unknown,+ overrides: Record<string, unknown> = {},+) {+ return {+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ context: {},+ authToken: "real-run-jwt",+ executionTarget,+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ ...overrides,+ };+}++describe("composed ACPX run: engine-boundary result form per exit path", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("returns (never throws) a create_runtime error result when the post-build runtime construction fails", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () => {+ throw new Error("createRuntime boom");+ },+ });++ // The engine RETURNS a settled error result on this path; it does not throw.+ const result = await execute({+ runId: "boundary-create-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.signal).toBe(null);+ expect(result.timedOut).toBe(false);+ expect(result.resultJson?.phase).toBe("create_runtime");+ expect(result.summary).toContain("createRuntime boom");+ });++ it("returns an ensure_session error result when the handshake fails", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => {+ throw new Error("ensureSession boom");+ },+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ });++ const result = await execute({+ runId: "boundary-ensure-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("ensure_session");+ });++ it("returns a runtime-error result with no clearSession and no errorMeta when ensureSession yields no handle", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ // A runtime that returns no session handle drives the missing-handle path.+ ensureSession: async () => undefined,+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ });++ const result = await execute({+ runId: "boundary-missing-handle",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("ensure_session");+ expect(result.errorCode).toBe("acpx_runtime_error");+ // The missing-handle result carries neither clearSession nor errorMeta, unlike+ // the other pre-turn error paths.+ expect(result.clearSession).toBeUndefined();+ expect(result.errorMeta).toBeUndefined();+ });++ it("returns a configure_session error result that carries the requested session fields", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ // A custom agent with a requested model applies a session config option;+ // a throwing setConfigOption drives the configure_session failure path.+ setConfigOption: async () => {+ throw new Error("setConfigOption boom");+ },+ close: async () => {},+ }) as never,+ });++ const result = await execute({+ runId: "boundary-configure-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ model: "custom-model-x",+ stateDir: path.join(root, "state"),+ },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("configure_session");+ // The configure_session result echoes the requested session identity fields.+ expect(result.resultJson).toHaveProperty("agent");+ expect(result.resultJson?.requestedModel).toBe("custom-model-x");+ expect(result.resultJson).toHaveProperty("requestedThinkingEffort");+ expect(result.resultJson).toHaveProperty("fastMode");+ });++ it("returns a prepare_turn error result when the prompt build fails before the turn starts", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ });++ const result = await execute({+ runId: "boundary-prepare-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: throwingHandoffContext(),+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("prepare_turn");+ });++ it("returns a turn error result with acpx_turn_failed when the running turn fails", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => throwingTurn(),+ close: async () => {},+ }) as never,+ });++ const result = await execute({+ runId: "boundary-turn-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("turn");+ expect(result.errorCode).toBe("acpx_turn_failed");+ });++ it("returns the terminal turn-success shape (the one non-error-shaped return)", async () => {+ const root = await makeTempRoot();+ const execute = createAcpxEngineExecutor({+ createRuntime: () => buildRuntime() as never,+ });++ const result = await execute({+ runId: "boundary-success",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ // A completed, non-timed-out turn returns exitCode 0, no signal, no errorCode,+ // and the session identity fields, with resultJson.status = "completed".+ expect(result.exitCode).toBe(0);+ expect(result.signal).toBe(null);+ expect(result.timedOut).toBe(false);+ expect(result.errorCode).toBe(null);+ expect(result.resultJson?.status).toBe("completed");+ expect(result.sessionId).toBe("backend-session");+ expect(result.sessionDisplayId).toBe("agent-session");+ expect(result.sessionParams).toBeTruthy();+ });++ it("THROWS (does not return) when buildRuntime fails before it settles", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ // A staging failure throws inside buildRuntime before it settles, so the engine+ // rethrows through the `if (!buildRuntimeSettled) throw err` guard. This is one+ // of exactly two engine-boundary throw paths.+ vi.mocked(prepareAdapterExecutionTargetRuntime).mockImplementationOnce(async () => {+ throw new Error("staging boom");+ });+ const execute = createAcpxEngineExecutor({+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ stagingLocks: new Map(),+ createRuntime: () => buildRuntime() as never,+ });++ await expect(+ execute({+ runId: "boundary-build-throw",+ ...remoteArgs(stateDir, localCwd, executionTarget),+ } as never),+ ).rejects.toThrow("staging boom");+ });++ it("THROWS (does not return) on a partial bridge failure and stops the started sibling bridge once", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ // The paperclip bridge fails while the concurrently-started process-session+ // bridge resolves a live handle. This is the second engine-boundary throw path;+ // the abandon path must stop the started sibling so no bridge leaks.+ const stop = vi.fn(async () => {});+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementationOnce(async () => {+ throw new Error("paperclip bridge boom");+ });+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementationOnce(+ async () => ({ agentCommand: null, stop }) as never,+ );+ const execute = createAcpxEngineExecutor({+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ stagingLocks: new Map(),+ createRuntime: () => buildRuntime() as never,+ });++ await expect(+ execute({+ runId: "boundary-bridge-throw",+ ...remoteArgs(stateDir, localCwd, executionTarget),+ } as never),+ ).rejects.toThrow("paperclip bridge boom");+ expect(stop).toHaveBeenCalledTimes(1);+ });+});++describe("composed ACPX run: finalization set fires exactly once per exit path", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("stops both bridges once and releases the staging lease on every exit path", async () => {+ const scenarios: Array<{+ name: string;+ createRuntime: AcpxEngineExecutorOptions["createRuntime"];+ config?: Record<string, unknown>;+ context?: Record<string, unknown>;+ }> = [+ {+ name: "create_runtime",+ createRuntime: () => {+ throw new Error("createRuntime boom");+ },+ },+ {+ name: "ensure_session",+ createRuntime: () =>+ ({+ ensureSession: async () => {+ throw new Error("ensure boom");+ },+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ },+ {+ name: "configure_session",+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ setConfigOption: async () => {+ throw new Error("config boom");+ },+ close: async () => {},+ }) as never,+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", model: "custom-model-x" },+ },+ {+ name: "prepare_turn",+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ context: throwingHandoffContext(),+ },+ {+ name: "turn",+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => throwingTurn(),+ close: async () => {},+ }) as never,+ },+ {+ name: "success",+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close: async () => {},+ }) as never,+ },+ ];++ for (const scenario of scenarios) {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const { paperclipStops, processStops, stoppedCount } = stubBridges();+ const stagingLocks = new Map<string, Promise<unknown>>();+ const execute = createAcpxEngineExecutor({+ stagingLocks,+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ createRuntime: scenario.createRuntime,+ });++ const overrides: Record<string, unknown> = {};+ if (scenario.config) {+ overrides.config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ cwd: localCwd,+ ...scenario.config,+ };+ }+ if (scenario.context) overrides.context = scenario.context;++ await execute({+ runId: `finalize-${scenario.name}`,+ ...remoteArgs(stateDir, localCwd, executionTarget, overrides),+ } as never).catch(() => {});++ // The finalization set fires exactly once: each bridge stops once and the+ // per-session staging lease releases, so the lock map never strands the next+ // same-session run.+ expect(stoppedCount(paperclipStops), `paperclip bridge must stop once on ${scenario.name}`).toBe(1);+ expect(stoppedCount(processStops), `process-session bridge must stop once on ${scenario.name}`).toBe(1);+ expect(stagingLocks.size, `lease must release on ${scenario.name}`).toBe(0);+ }+ });++ it("does not re-run the turn teardown when the result mapping throws after the close", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const { paperclipStops, processStops, stoppedCount } = stubBridges();+ let closeCount = 0;+ const execute = createAcpxEngineExecutor({+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ stagingLocks: new Map(),+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ // A completed turn whose result mapping throws after the close is read.+ result: Promise.resolve({+ status: "completed",+ get stopReason(): string {+ throw new Error("mapping boom");+ },+ }),+ cancel: async () => {},+ }),+ close: async () => {+ closeCount += 1;+ },+ }) as never,+ });++ await execute({+ runId: "finalize-map-throw",+ ...remoteArgs(stateDir, localCwd, executionTarget),+ } as never).catch(() => {});++ // The completed turn closed the runtime once; the mapping throw did not re-run+ // the teardown through the turn catch.+ expect(closeCount).toBe(1);+ expect(stoppedCount(paperclipStops)).toBe(1);+ expect(stoppedCount(processStops)).toBe(1);+ });+});++describe("composed ACPX run: host-lane warm handle set", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("warm-saves a persistent local runtime and reuses it for the next compatible run", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let createCount = 0;+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: () => {+ createCount += 1;+ return buildRuntime() as never;+ },+ });+ const config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ };++ const first = await execute({+ runId: "warm-save-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);+ expect(first.exitCode).toBe(0);+ // A persistent local completed turn warm-saves the runtime handle.+ expect(warmHandles.size).toBe(1);+ expect(createCount).toBe(1);++ const second = await execute({+ runId: "warm-save-2",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: first.sessionParams },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);+ expect(second.exitCode).toBe(0);+ // The compatible second run reuses the warm runtime, so createRuntime is not+ // called again.+ expect(createCount).toBe(1);+ });++ it("closes (does not warm-save) a completed non-persistent runtime, so the next run re-creates", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let createCount = 0;+ let closeCount = 0;+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: () => {+ createCount += 1;+ return {+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ setConfigOption: async () => {},+ close: async () => {+ closeCount += 1;+ },+ } as never;+ },+ });+ // No persistent mode, so the completed turn closes the runtime instead of+ // warm-saving it.+ const config = { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir };++ const first = await execute({+ runId: "warm-none-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);+ expect(first.exitCode).toBe(0);+ expect(warmHandles.size).toBe(0);+ expect(closeCount).toBe(1);++ const second = await execute({+ runId: "warm-none-2",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: first.sessionParams },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);+ expect(second.exitCode).toBe(0);+ // No warm handle survived, so the second run constructs a fresh runtime.+ expect(createCount).toBe(2);+ });++ it("emits a skipped acp.handshake event on a warm-handle hit and does not re-create the runtime", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let createCount = 0;+ const warmHandles = new Map();+ const secondEvents: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: () => {+ createCount += 1;+ return buildRuntime() as never;+ },+ });+ const config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ };+ const first = await execute({+ runId: "warm-hit-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);++ await execute({+ runId: "warm-hit-2",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: (first as { sessionParams?: unknown }).sessionParams },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ secondEvents.push(event);+ },+ } as never);++ // The warm-hit skips the handshake work: it emits exactly one acp.handshake+ // event with outcome = skipped and a zero wall time, and never re-creates the+ // runtime.+ const handshakeEvents = secondEvents.filter(+ (event) => event.eventType === "run.startup.step" && event.payload?.step === "acp.handshake",+ );+ expect(handshakeEvents).toHaveLength(1);+ expect(handshakeEvents[0]!.payload?.outcome).toBe("skipped");+ expect(handshakeEvents[0]!.payload?.durationMs).toBe(0);+ expect(createCount).toBe(1);+ });+});diff --git a/packages/adapter-utils/src/acpx-engine/settlement-characterization.test.ts b/packages/adapter-utils/src/acpx-engine/settlement-characterization.test.tsnew file mode 100644index 0000000000..d5f9754a88--- /dev/null+++ b/packages/adapter-utils/src/acpx-engine/settlement-characterization.test.ts@@ -0,0 +1,1061 @@+// Characterization baselines for ACP run settlement/teardown. These tests PIN+// the current behavior of the engine's teardown orchestration (Layer A), the+// `restoreWorkspace` internal order plus native-sync selection (Layer B), and+// the per-adapter sync-back registration seam (Layer C). They never change+// production code; each expectation records what the code does today.+//+// The harness top (imports, the execution-target mock, `makeTempRoot`, the local+// sandbox runner, `buildRuntime`, `runExecutor`) is copied from+// `execute.test.ts` (lines 1-336) so this file drives the same engine the same+// way. The teardown helpers (`stubBridges`, `throwingHandoffContext`,+// `completedTurn`, `throwingTurn`, `setupRemoteSandbox`, `remoteArgs`) mirror the+// F2/F3 describes in `execute.test.ts` (~:2602, ~:4858).+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { execFile as execFileCallback } from "node:child_process";+import { promisify } from "node:util";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { AdapterExecutionContext, AdapterRuntimeMcpAccess } from "@paperclipai/adapter-utils";+import {+ startAdapterExecutionTargetPaperclipBridge,+ startAdapterExecutionTargetProcessSessionBridge,+} from "@paperclipai/adapter-utils/execution-target";++// Wrap the staging seam + both sandbox bridges in call-recording spies that+// still delegate to the real implementations. A runner-backed sandbox test+// exercises them end-to-end against a local runner, while a teardown test can+// override just the bridges with stop spies. (Copied from execute.test.ts.)+vi.mock("@paperclipai/adapter-utils/execution-target", async (importActual) => {+ const actual = await importActual<typeof import("@paperclipai/adapter-utils/execution-target")>();+ return {+ ...actual,+ prepareAdapterExecutionTargetRuntime: vi.fn(actual.prepareAdapterExecutionTargetRuntime),+ startAdapterExecutionTargetPaperclipBridge: vi.fn(actual.startAdapterExecutionTargetPaperclipBridge),+ startAdapterExecutionTargetProcessSessionBridge: vi.fn(actual.startAdapterExecutionTargetProcessSessionBridge),+ };+});+import {+ createAcpxEngineExecutor,+ type AcpxEngineExecutorOptions,+} from "./execute.js";+import { runChildProcess } from "../server-utils.js";+import {+ mirrorDirectory,+ prepareSandboxManagedRuntime,+ type SandboxManagedRuntimeClient,+ type SandboxSyncOperation,+ type SandboxSyncResult,+} from "../sandbox-managed-runtime.js";++const execFile = promisify(execFileCallback);+const tempRoots: string[] = [];++async function makeTempRoot() {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-acpx-skills-"));+ tempRoots.push(root);+ return root;+}++afterEach(async () => {+ await Promise.all(+ tempRoots.splice(0).map((root) =>+ fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }),+ ),+ );+});++function createLocalSandboxRunner(+ onExecute?: (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ }) => void,+) {+ let counter = 0;+ return {+ execute: async (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ stdin?: string;+ timeoutMs?: number;+ onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;+ onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ }) => {+ counter += 1;+ onExecute?.(input);+ const command = input.command === "bash" ? "/bin/bash" : input.command;+ return await runChildProcess(`acpx-sandbox-run-${counter}`, command, input.args ?? [], {+ cwd: input.cwd ?? process.cwd(),+ env: input.env ?? {},+ stdin: input.stdin,+ timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),+ graceSec: 5,+ onLog: input.onLog ?? (async () => {}),+ onSpawn: input.onSpawn+ ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt })+ : undefined,+ });+ },+ };+}++function buildRuntime(+ onSetConfigOption?: (input: { key: string; value: string }) => void,+ onEnsureSession?: (input: Record<string, unknown>) => void,+) {+ return {+ ensureSession: async (input: Record<string, unknown>) => {+ onEnsureSession?.(input);+ return {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ };+ },+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ }),+ setConfigOption: async (input: { key: string; value: string }) => {+ onSetConfigOption?.(input);+ },+ close: async () => {},+ };+}++async function runExecutor(+ config: Record<string, unknown>,+ options: {+ context?: Record<string, unknown>;+ executionTransport?: Record<string, unknown>;+ authToken?: string;+ executionTarget?: Record<string, unknown>;+ runtimeMcp?: AdapterRuntimeMcpAccess;+ prepareRemoteManagedHome?: AcpxEngineExecutorOptions["prepareRemoteManagedHome"];+ startupTraceContext?: AdapterExecutionContext["startupTraceContext"];+ } = {},+) {+ const runtimeOptions: Record<string, unknown>[] = [];+ const configOptions: Array<{ key: string; value: string }> = [];+ const sessionInputs: Record<string, unknown>[] = [];+ const meta: Record<string, unknown>[] = [];+ const logs: Array<{ stream: string; text: string }> = [];+ const events: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];+ const execute = createAcpxEngineExecutor({+ ...(options.prepareRemoteManagedHome+ ? { prepareRemoteManagedHome: options.prepareRemoteManagedHome }+ : {}),+ createRuntime: (createOptions) => {+ runtimeOptions.push(createOptions as unknown as Record<string, unknown>);+ return buildRuntime(+ ({ key, value }) => configOptions.push({ key, value }),+ (input) => sessionInputs.push(input),+ ) as never;+ },+ });++ const result = await execute({+ runId: "run-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: options.context ?? {},+ executionTransport: options.executionTransport,+ authToken: options.authToken,+ executionTarget: options.executionTarget,+ runtimeMcp: options.runtimeMcp,+ startupTraceContext: options.startupTraceContext,+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ onMeta: async (payload: unknown) => {+ meta.push(payload as Record<string, unknown>);+ },+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ events.push(event);+ },+ } as never);++ expect(result.exitCode).toBe(0);+ return { logs, meta, events, runtimeOptions, configOptions, sessionInputs, result };+}++// ---------------------------------------------------------------------------+// Shared teardown-test helpers (mirror execute.test.ts F3 describe, ~:4858).+// ---------------------------------------------------------------------------++const okHandle = {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+};++async function setupRemoteSandbox() {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const localCwd = path.join(root, "worktree");+ const remoteCwd = path.join(root, "remote-workspace");+ await fs.mkdir(localCwd, { recursive: true });+ await fs.mkdir(remoteCwd, { recursive: true });+ await fs.writeFile(path.join(localCwd, "hello.txt"), "hi", "utf8");+ const executionTarget = {+ kind: "remote",+ transport: "sandbox",+ providerKey: "fake-plugin",+ remoteCwd,+ runner: createLocalSandboxRunner(),+ };+ return { root, stateDir, localCwd, remoteCwd, executionTarget };+}++// Stub both sandbox bridges with stop spies collected per start, so a test can+// assert the bridges stopped without running the real bridge transport.+function stubBridges() {+ const paperclipStops: Array<ReturnType<typeof vi.fn>> = [];+ const processStops: Array<ReturnType<typeof vi.fn>> = [];+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementation(async () => {+ const stop = vi.fn(async () => {});+ paperclipStops.push(stop);+ return { env: {}, stop } as never;+ });+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementation(async () => {+ const stop = vi.fn(async () => {});+ processStops.push(stop);+ return { agentCommand: null, stop } as never;+ });+ const anyStopped = (stops: Array<ReturnType<typeof vi.fn>>) =>+ stops.some((stop) => stop.mock.calls.length > 0);+ return { paperclipStops, processStops, anyStopped };+}++function throwingHandoffContext(): Record<string, unknown> {+ const context: Record<string, unknown> = {};+ Object.defineProperty(context, "paperclipSessionHandoffMarkdown", {+ enumerable: false,+ get() {+ throw new Error("prompt build boom");+ },+ });+ return context;+}++function completedTurn() {+ return {+ events: (async function* () {})(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ };+}++function throwingTurn() {+ return {+ events: (async function* () {+ throw new Error("turn upstream boom");+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ };+}++function remoteArgs(+ stateDir: string,+ localCwd: string,+ executionTarget: unknown,+ overrides: Record<string, unknown> = {},+) {+ return {+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ context: {},+ authToken: "real-run-jwt",+ executionTarget,+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ ...overrides,+ };+}++// ===========================================================================+// Layer A — engine teardown orchestration.+// ===========================================================================+describe("ACP settlement — Layer A: engine teardown orchestration", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("test_clean_completed_remote_teardown_runs_bridge_stop_then_sync_back_then_lease_release", async () => {+ // cleanupRemoteBridges (execute.ts:2306-2326) fixes the sub-order for a clean+ // exit: stop both bridges (allSettled) → run the managed-home sync-back+ // (remoteManagedHomeTeardown) → release the per-session staging lease LAST in+ // a finally. Record the order through spies threaded into the stubbed bridges+ // and the seam teardown, and read the still-held lease during the sync-back.+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const order: string[] = [];+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementation(async () => ({+ env: {},+ stop: vi.fn(async () => {+ order.push("bridge-stop");+ }),+ }) as never);+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementation(async () => ({+ agentCommand: null,+ stop: vi.fn(async () => {+ order.push("bridge-stop");+ }),+ }) as never);++ const stagingLocks = new Map<string, Promise<unknown>>();+ let leaseSizeDuringSyncBack = -1;+ const execute = createAcpxEngineExecutor({+ stagingLocks,+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close: vi.fn(async () => {}),+ }) as never,+ prepareRemoteManagedHome: async (input) => {+ const stagedRuntime = await input.stage([]);+ return {+ stagedRuntime,+ teardown: async () => {+ order.push("sync-back");+ // The lease is still held while the sync-back runs; it releases only+ // afterward, in the cleanupRemoteBridges finally.+ leaseSizeDuringSyncBack = stagingLocks.size;+ },+ };+ },+ });++ const result = await execute({+ runId: "clean-order",+ ...remoteArgs(stateDir, localCwd, executionTarget),+ } as never);++ expect(result.exitCode).toBe(0);+ // Both bridge stops precede the sync-back; the sync-back is the last recorded+ // step before the lease release.+ expect(order).toEqual(["bridge-stop", "bridge-stop", "sync-back"]);+ // The lease was still held during the sync-back...+ expect(leaseSizeDuringSyncBack).toBe(1);+ // ...and released last, so the lock map never strands the next run.+ expect(stagingLocks.size).toBe(0);+ });++ it("test_clean_persistent_local_run_warm_saves_instead_of_closing", async () => {+ // execute.ts:3886 keeps a clean persistent local turn warm (no runtime.close)+ // when warmIdleMs>0 and there is no process-session bridge.+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ setConfigOption: async () => {},+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: "warm-save",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir: path.join(root, "state"),+ mode: "persistent",+ // A long idle window so the scheduled cleanup timer cannot fire during the+ // assertions and evict the warm handle. The warm-save decision only needs+ // warmIdleMs>0. The test clears the timer below, so it never outlives the+ // test.+ warmHandleIdleMs: 60_000,+ },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {},+ } as never);++ expect(result.exitCode).toBe(0);+ // The runtime stayed warm: no close, one warm entry.+ expect(closeSpy).not.toHaveBeenCalled();+ expect(warmHandles.size).toBe(1);++ // Clear the scheduled idle-cleanup timer so no timer outlives the test. The+ // long idle window kept the timer from racing the assertions above; clearing+ // it here makes cleanup deterministic instead of a timed drain.+ for (const entry of warmHandles.values()) {+ if (entry.cleanupTimer) clearTimeout(entry.cleanupTimer);+ }+ });++ it("test_completed_non_persistent_local_run_closes_runtime_once", async () => {+ // The default (non-persistent) completed turn closes the runtime with the+ // clean-completion reason (execute.ts:3922-3926).+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: "clean-close",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(0);+ expect(closeSpy).toHaveBeenCalledTimes(1);+ expect((closeSpy.mock.calls[0]! as unknown[])[0]).toMatchObject({+ reason: "paperclip completed turn cleanup",+ discardPersistentState: false,+ });+ });++ it("test_pre_turn_failure_after_handle_closes_runtime_except_create_runtime", async () => {+ // Once a runtime handle exists, every pre-turn failure closes the runtime —+ // missing handle (execute.ts:3633, synthesized handle), configure_session+ // (:3716), prepare_turn (:4016) — EXCEPT create_runtime, where no runtime was+ // ever constructed (:3662-3697). Each path returns a settled error with its+ // phase. (The handshake close path needs a handle obtained first via the warm+ // hit — pinned separately below, mirroring the F2 test.)+ const scenarios: Array<{+ name: string;+ phase: string;+ expectClose: boolean;+ // Builds the engine `createRuntime` given the shared close spy. The+ // create_runtime scenario ignores the spy and throws before a runtime exists.+ makeCreateRuntime: (close: () => Promise<void>) => AcpxEngineExecutorOptions["createRuntime"];+ config: Record<string, unknown>;+ context: Record<string, unknown>;+ }> = [+ {+ name: "missing_handle",+ phase: "ensure_session",+ expectClose: true,+ makeCreateRuntime: (close) => () =>+ ({+ ensureSession: async () => undefined,+ startTurn: () => completedTurn(),+ close,+ }) as never,+ config: {},+ context: {},+ },+ {+ name: "configure_session",+ phase: "configure_session",+ expectClose: true,+ makeCreateRuntime: (close) => () =>+ ({+ ensureSession: async () => okHandle,+ // A configured model on a non-claude/non-codex agent yields one session+ // config option; a throwing setConfigOption drives the configure failure.+ setConfigOption: async () => {+ throw new Error("config boom");+ },+ startTurn: () => completedTurn(),+ close,+ }) as never,+ config: { model: "custom-model-x" },+ context: {},+ },+ {+ name: "prepare_turn",+ phase: "prepare_turn",+ expectClose: true,+ makeCreateRuntime: (close) => () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => completedTurn(),+ close,+ }) as never,+ config: {},+ context: throwingHandoffContext(),+ },+ {+ name: "create_runtime",+ phase: "create_runtime",+ expectClose: false,+ makeCreateRuntime: () => () => {+ throw new Error("createRuntime boom");+ },+ config: {},+ context: {},+ },+ ];++ for (const scenario of scenarios) {+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: scenario.makeCreateRuntime(closeSpy),+ });++ const result = await execute({+ runId: `preturn-${scenario.name}`,+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir: path.join(root, "state"),+ ...scenario.config,+ },+ context: scenario.context,+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode, `${scenario.name} exit`).toBe(1);+ expect(result.resultJson?.phase, `${scenario.name} phase`).toBe(scenario.phase);+ if (scenario.expectClose) {+ expect(closeSpy, `${scenario.name} closes the runtime`).toHaveBeenCalledTimes(1);+ } else {+ // create_runtime failed before a runtime existed, so there is nothing to close.+ expect(closeSpy, `${scenario.name} never constructs a runtime`).not.toHaveBeenCalled();+ }+ }+ });++ it("test_handshake_failure_after_warm_hit_closes_runtime_and_removes_warm_entry", async () => {+ // Mirror execute.test.ts F2 test_handshake_failure_closes_runtime_and_removes_warm_entry+ // (:4666). The first run warms a handle; the second run reuses it and fails+ // while persisting process identity (onSpawn throws). Because the warm hit+ // already holds the handle, the handshake failure closes the reused runtime+ // (execute.ts:3591) and removes the warm entry.+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const startedAt = "2026-01-01T00:00:00.000Z";+ const closeSpy = vi.fn(async () => {});+ let created = 0;+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: (options) => {+ created += 1;+ const opts = options as {+ onAgentSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ };+ return {+ ensureSession: async () => {+ await opts.onAgentSpawn?.({ pid: 4242, startedAt });+ return okHandle;+ },+ startTurn: () => completedTurn(),+ close: closeSpy,+ } as never;+ },+ });+ const config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ };++ const first = await execute({+ runId: "warm-handshake-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {},+ } as never);+ expect(first.exitCode).toBe(0);+ expect(warmHandles.size).toBe(1);+ expect(created).toBe(1);++ const second = await execute({+ runId: "warm-handshake-2",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: first.sessionParams },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {+ throw new Error("onSpawn boom");+ },+ } as never);++ // The reused runtime (no new create) is closed and the warm entry removed.+ expect(created).toBe(1);+ expect(second.exitCode).toBe(1);+ expect(second.resultJson?.phase).toBe("ensure_session");+ expect(closeSpy).toHaveBeenCalledTimes(1);+ expect(warmHandles.size).toBe(0);+ });++ it("test_terminal_close_reason_and_discard_persistent_state_per_outcome", async () => {+ // The terminal block (execute.ts:3869-3928) closes with an outcome-specific+ // reason and discards persistent state only on cancel/timeout.+ const cases: Array<{+ status: "completed" | "failed" | "cancelled";+ reason: string;+ discard: boolean;+ exitCode: number;+ }> = [+ { status: "completed", reason: "paperclip completed turn cleanup", discard: false, exitCode: 0 },+ { status: "failed", reason: "paperclip turn failed", discard: false, exitCode: 1 },+ { status: "cancelled", reason: "paperclip turn cancelled", discard: true, exitCode: 1 },+ ];++ for (const testCase of cases) {+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result:+ testCase.status === "failed"+ ? Promise.resolve({ status: "failed", error: new Error("turn boom") })+ : testCase.status === "cancelled"+ ? Promise.resolve({ status: "cancelled", stopReason: "cancelled" })+ : Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ }),+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: `terminal-${testCase.status}`,+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode, `${testCase.status} exit`).toBe(testCase.exitCode);+ expect(closeSpy, `${testCase.status} closes once`).toHaveBeenCalledTimes(1);+ expect((closeSpy.mock.calls[0]! as unknown[])[0], `${testCase.status} close args`).toMatchObject({+ reason: testCase.reason,+ discardPersistentState: testCase.discard,+ });+ }+ });++ it("test_teardown_continues_after_one_step_fails_and_records_it", async () => {+ // Corrected F3 policy (execute.ts:3374-3393): a failing teardown step is+ // recorded and swallowed; later steps still run and the lease still releases.+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const { paperclipStops, processStops, anyStopped } = stubBridges();+ const stagingLocks = new Map<string, Promise<unknown>>();+ const logs: Array<{ stream: string; text: string }> = [];+ const execute = createAcpxEngineExecutor({+ stagingLocks,+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => throwingTurn(),+ close: async () => {+ throw new Error("close boom");+ },+ }) as never,+ });++ const result = await execute({+ runId: "td-continue",+ ...remoteArgs(stateDir, localCwd, executionTarget, {+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ }),+ } as never);++ expect(result.exitCode).toBe(1);+ expect(anyStopped(paperclipStops)).toBe(true);+ expect(anyStopped(processStops)).toBe(true);+ expect(stagingLocks.size).toBe(0);+ expect(+ logs.some(+ (entry) =>+ entry.stream === "stderr" && entry.text.includes('teardown step "runtime-close" failed'),+ ),+ ).toBe(true);+ });++ it("test_teardown_failure_does_not_change_external_result", async () => {+ // A teardown fault never leaks into the external result; the exit cause (the+ // turn failure) stands (execute.ts F3 policy).+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ stubBridges();+ const execute = createAcpxEngineExecutor({+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ stagingLocks: new Map(),+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ startTurn: () => throwingTurn(),+ close: async () => {+ throw new Error("close boom");+ },+ }) as never,+ });++ const result = await execute({+ runId: "td-result",+ ...remoteArgs(stateDir, localCwd, executionTarget),+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("turn");+ expect(result.errorCode).toBe("acpx_turn_failed");+ expect(result.errorMessage).toContain("turn upstream boom");+ expect(result.errorMessage).not.toContain("close boom");+ });+});++// ===========================================================================+// Layer B — restoreWorkspace internal order + native-sync selection.+// ===========================================================================++function toArrayBuffer(bytes: Buffer): ArrayBuffer {+ return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;+}++// A filesystem-backed managed-runtime client with the non-native base64-tar+// FALLBACK `syncIn` (mirrors sandbox-managed-runtime.test.ts `makeFilesystemClient`+// / `attachFallbackSyncIn`). `runCommands` records every `run` so a test can prove+// which restore transfer path the orchestrator took. When `withNativeSyncOut` is+// set, a native `syncOut` is attached so the client advertises BOTH directions.+function makeFsClient(options: {+ runCommands?: string[];+ syncOutCalls?: { count: number };+ withNativeSyncOut?: boolean;+}): SandboxManagedRuntimeClient {+ const client: SandboxManagedRuntimeClient = {+ makeDir: async (remotePath) => {+ await fs.mkdir(remotePath, { recursive: true });+ },+ writeFile: async (remotePath, bytes) => {+ await fs.mkdir(path.dirname(remotePath), { recursive: true });+ await fs.writeFile(remotePath, Buffer.from(bytes));+ },+ readFile: async (remotePath) => await fs.readFile(remotePath),+ listFiles: async (remotePath) => {+ const entries = await fs.readdir(remotePath, { withFileTypes: true }).catch(() => []);+ return entries+ .filter((entry) => entry.isFile())+ .map((entry) => entry.name)+ .sort((left, right) => left.localeCompare(right));+ },+ remove: async (remotePath) => {+ await fs.rm(remotePath, { recursive: true, force: true });+ },+ run: async (command) => {+ options.runCommands?.push(command);+ await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });+ },+ };+ client.syncIn = async (operations: SandboxSyncOperation[]): Promise<SandboxSyncResult> => {+ const resultOperations: SandboxSyncResult["operations"] = [];+ for (const operation of operations) {+ let filesTransferred = 0;+ let bytesTransferred = 0;+ for (const mapping of operation.files) {+ const bytes = await fs.readFile(mapping.sourcePath);+ await client.makeDir(path.posix.dirname(mapping.targetPath));+ await client.writeFile(mapping.targetPath, toArrayBuffer(bytes));+ filesTransferred += 1;+ bytesTransferred += bytes.byteLength;+ }+ for (const command of operation.postUploadCommands ?? []) {+ await client.run(command.command, { timeoutMs: command.timeoutMs ?? 30_000 });+ }+ resultOperations.push({ operationId: operation.operationId, filesTransferred, bytesTransferred });+ }+ return { operations: resultOperations };+ };+ if (options.withNativeSyncOut) {+ // Native outbound: materialize the sandbox workspace tree directly into the+ // host destination (a directory file mapping), never through the tarball run.+ client.syncOut = async (operations: SandboxSyncOperation[]): Promise<SandboxSyncResult> => {+ if (options.syncOutCalls) options.syncOutCalls.count += 1;+ const resultOperations: SandboxSyncResult["operations"] = [];+ for (const operation of operations) {+ for (const mapping of operation.files) {+ await fs.mkdir(mapping.targetPath, { recursive: true });+ await mirrorDirectory(mapping.sourcePath, mapping.targetPath);+ }+ resultOperations.push({ operationId: operation.operationId, filesTransferred: 1, bytesTransferred: 0 });+ }+ return { operations: resultOperations };+ };+ }+ return client;+}++async function git(cwd: string, args: string[]): Promise<string> {+ const { stdout } = await execFile("git", ["-C", cwd, ...args], { maxBuffer: 32 * 1024 * 1024 });+ return stdout.trim();+}++describe("ACP settlement — Layer B: restoreWorkspace order + native-sync selection", () => {+ it("test_non_git_workspace_restore_phase_order_and_preserve_absent_via_tarball_fallback", async () => {+ // Model on sandbox-managed-runtime.test.ts "syncs workspace and assets through+ // a provider-neutral sandbox client" (~:342). A client that exposes only the+ // fallback `syncIn` (no `syncOut`) drives the remote-tarball restore transfer+ // (sandbox-managed-runtime.ts:1192-1218) and the finalize ordering.+ const root = await makeTempRoot();+ const localWorkspaceDir = path.join(root, "local-workspace");+ const remoteWorkspaceDir = path.join(root, "remote-workspace");+ await fs.mkdir(path.join(localWorkspaceDir, ".claude"), { recursive: true });+ await fs.writeFile(path.join(localWorkspaceDir, "README.md"), "local workspace\n", "utf8");+ await fs.writeFile(path.join(localWorkspaceDir, ".claude", "settings.json"), '{"local":true}\n', "utf8");++ const runCommands: string[] = [];+ const client = makeFsClient({ runCommands });+ // A fallback-only client never advertises native outbound.+ expect(client.syncOut).toBeUndefined();++ const runtimeStatuses: string[] = [];+ const prepared = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-1",+ remoteCwd: remoteWorkspaceDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client,+ workspaceLocalDir: localWorkspaceDir,+ workspaceExclude: [".claude"],+ preserveAbsentOnRestore: [".claude"],+ onRuntimeProgress: async (status) => {+ runtimeStatuses.push(`${status.phase}:${status.message}`);+ },+ });++ // Staging excluded the host-managed `.claude` from the sandbox.+ await expect(+ fs.readFile(path.join(remoteWorkspaceDir, ".claude", "settings.json"), "utf8"),+ ).rejects.toMatchObject({ code: "ENOENT" });++ await fs.writeFile(path.join(remoteWorkspaceDir, "README.md"), "remote workspace\n", "utf8");+ await fs.writeFile(path.join(remoteWorkspaceDir, "remote-only.txt"), "sync back\n", "utf8");+ await prepared.restoreWorkspace();++ // Sync-back applied the remote edits; the preserved-absent `.claude` stayed.+ await expect(fs.readFile(path.join(localWorkspaceDir, "README.md"), "utf8")).resolves.toBe(+ "remote workspace\n",+ );+ await expect(fs.readFile(path.join(localWorkspaceDir, "remote-only.txt"), "utf8")).resolves.toBe(+ "sync back\n",+ );+ await expect(+ fs.readFile(path.join(localWorkspaceDir, ".claude", "settings.json"), "utf8"),+ ).resolves.toBe('{"local":true}\n');++ // The fallback restore built the remote workspace-download tarball via `run`.+ expect(runCommands.some((command) => command.includes("workspace-download.tar"))).toBe(true);+ // Phase order: config_sync (staging) → restore → finalize (finalize last).+ expect(runtimeStatuses).toEqual(+ expect.arrayContaining([+ "config_sync:Syncing workspace to sandbox",+ "restore:Restoring workspace from sandbox",+ "finalize:Finalizing sandbox workspace",+ ]),+ );+ expect(runtimeStatuses.at(-1)).toBe("finalize:Finalizing sandbox workspace");+ });++ it("test_git_backed_workspace_restore_phase_order", async () => {+ // Model on sandbox-managed-runtime.test.ts "syncs git-backed workspaces through+ // a shallow standalone clone…" (~:441). A git workspace adds the git_sync and+ // export phases around the same config_sync/restore/finalize spine.+ const root = await makeTempRoot();+ const sourceRepoDir = path.join(root, "source-repo");+ const localWorkspaceDir = path.join(root, "local-worktree");+ const remoteWorkspaceDir = path.join(root, "remote-workspace");++ await fs.mkdir(sourceRepoDir, { recursive: true });+ await git(sourceRepoDir, ["init"]);+ await git(sourceRepoDir, ["checkout", "-b", "main"]);+ await git(sourceRepoDir, ["config", "user.name", "Paperclip Test"]);+ await git(sourceRepoDir, ["config", "user.email", "[email protected]"]);+ await fs.writeFile(path.join(sourceRepoDir, "tracked.txt"), "base\n", "utf8");+ await git(sourceRepoDir, ["add", "tracked.txt"]);+ await git(sourceRepoDir, ["commit", "-m", "base"]);+ await git(sourceRepoDir, ["worktree", "add", "-b", "work", localWorkspaceDir, "HEAD"]);+ await fs.writeFile(path.join(localWorkspaceDir, "tracked.txt"), "dirty local\n", "utf8");++ const client = makeFsClient({});+ const phases: string[] = [];+ const prepared = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-1",+ remoteCwd: remoteWorkspaceDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client,+ workspaceLocalDir: localWorkspaceDir,+ onRuntimeProgress: async (status) => {+ phases.push(status.phase);+ },+ });++ // The sandbox holds a real git worktree seeded from the host history.+ expect((await git(remoteWorkspaceDir, ["rev-list", "--count", "HEAD"]))).toBe("1");+ await git(remoteWorkspaceDir, ["config", "user.name", "Paperclip Sandbox"]);+ await git(remoteWorkspaceDir, ["config", "user.email", "[email protected]"]);+ await git(remoteWorkspaceDir, ["add", "-A"]);+ await git(remoteWorkspaceDir, ["commit", "-m", "sandbox update"]);+ await fs.writeFile(path.join(remoteWorkspaceDir, "remote-only.txt"), "from sandbox\n", "utf8");++ await prepared.restoreWorkspace();++ // The sandbox commit imported back onto the host worktree.+ expect(await git(localWorkspaceDir, ["log", "-1", "--pretty=%s"])).toBe("sandbox update");+ await expect(fs.readFile(path.join(localWorkspaceDir, "remote-only.txt"), "utf8")).resolves.toBe(+ "from sandbox\n",+ );+ expect(phases).toEqual(+ expect.arrayContaining(["git_sync", "config_sync", "export", "restore", "finalize"]),+ );+ expect(phases.at(-1)).toBe("finalize");+ });++ it("test_native_syncOut_client_restores_through_native_transfer_not_tarball", async () => {+ // The consumer reads `nativeSyncOut = typeof client.syncOut === "function"`+ // (sandbox-managed-runtime.ts:775) and branches at :1163: with both directions+ // native it uses `client.syncOut`, never the workspace-download tarball.+ const root = await makeTempRoot();+ const localWorkspaceDir = path.join(root, "local-workspace");+ const remoteWorkspaceDir = path.join(root, "remote-workspace");+ await fs.mkdir(localWorkspaceDir, { recursive: true });+ await fs.writeFile(path.join(localWorkspaceDir, "README.md"), "local workspace\n", "utf8");++ const runCommands: string[] = [];+ const syncOutCalls = { count: 0 };+ const client = makeFsClient({ runCommands, syncOutCalls, withNativeSyncOut: true });+ // With both directions the client advertises native outbound.+ expect(client.syncOut).toBeTypeOf("function");++ const prepared = await prepareSandboxManagedRuntime({+ spec: {+ transport: "sandbox",+ provider: "test",+ sandboxId: "sandbox-1",+ remoteCwd: remoteWorkspaceDir,+ timeoutMs: 30_000,+ apiKey: null,+ },+ adapterKey: "test-adapter",+ client,+ workspaceLocalDir: localWorkspaceDir,+ });++ await fs.writeFile(path.join(remoteWorkspaceDir, "README.md"), "remote workspace\n", "utf8");+ await fs.writeFile(path.join(remoteWorkspaceDir, "remote-only.txt"), "native sync\n", "utf8");+ await prepared.restoreWorkspace();++ // The native outbound transfer ran and applied the remote edits...+ expect(syncOutCalls.count).toBeGreaterThanOrEqual(1);+ await expect(fs.readFile(path.join(localWorkspaceDir, "README.md"), "utf8")).resolves.toBe(+ "remote workspace\n",+ );+ await expect(fs.readFile(path.join(localWorkspaceDir, "remote-only.txt"), "utf8")).resolves.toBe(+ "native sync\n",+ );+ // ...and the tarball fallback was NOT used.+ expect(runCommands.some((command) => command.includes("workspace-download.tar"))).toBe(false);+ });+});++// ===========================================================================+// Layer C — sync-back registration at the engine seam.+//+// The three real adapters each register a restoreWorkspace-backed teardown via+// their managed-home seam: codex packages/adapters/codex-local/src/server/acp.ts+// (teardown ~:239-258, wired by withCodexAcpDefaults ~:277); claude+// packages/adapters/claude-local/src/server/acp.ts (registerWorkspaceSyncBack+// ~:201-215, wired by withClaudeAcpDefaults ~:292); gemini+// packages/adapters/gemini-local/src/server/acp.ts (registerWorkspaceSyncBack+// ~:163-177, wired by withGeminiAcpDefaults ~:249). Those adapter packages are+// NOT dependencies of adapter-utils and resolve only to the canonical app+// checkout (not this worktree), and the plugin-sdk env-sync-negotiation helpers+// (definePlugin/startWorkerRpcHost) are not public exports — importing them from+// an adapter-utils test is infeasible/fragile. So this pins the adapter-agnostic+// sync-back at the engine seam: `prepareRemoteManagedHome` returns a `teardown`+// (the same shape each adapter registers), and the engine fires it exactly once+// on the exit/cleanup path (mirrors execute.test.ts+// test_remote_seam_teardown_fires_once_on_exit :2556).+// ===========================================================================+describe("ACP settlement — Layer C: per-adapter sync-back teardown fires once at the engine seam", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("test_engine_fires_each_adapter_named_sync_back_teardown_exactly_once", async () => {+ for (const adapter of ["codex", "claude", "gemini"] as const) {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ let teardownCalls = 0;+ await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ {+ authToken: "real-run-jwt",+ executionTarget,+ // The seam stands in for `${adapter}`'s registerWorkspaceSyncBack: it+ // returns a teardown that would call stagedRuntime.restoreWorkspace(...).+ prepareRemoteManagedHome: async (input) => {+ const stagedRuntime = await input.stage([]);+ return {+ stagedRuntime,+ teardown: async () => {+ teardownCalls += 1;+ },+ };+ },+ },+ );++ expect(teardownCalls, `${adapter} sync-back teardown fires exactly once`).toBe(1);+ }+ });+});diff --git a/packages/adapter-utils/src/acpx-engine/startup-characterization.test.ts b/packages/adapter-utils/src/acpx-engine/startup-characterization.test.tsnew file mode 100644index 0000000000..8f55ec0ae6--- /dev/null+++ b/packages/adapter-utils/src/acpx-engine/startup-characterization.test.ts@@ -0,0 +1,913 @@+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { AcpRuntimeOptions } from "acpx/runtime";+import type { AdapterExecutionContext, AdapterRuntimeMcpAccess } from "@paperclipai/adapter-utils";+import {+ prepareAdapterExecutionTargetRuntime,+ startAdapterExecutionTargetPaperclipBridge,+ startAdapterExecutionTargetProcessSessionBridge,+} from "@paperclipai/adapter-utils/execution-target";++// Wrap the staging seam + both sandbox bridges in call-recording spies that+// still delegate to the real implementations. This copies the execute.test.ts+// harness verbatim so a startup test asserts the exact staging args and bridge+// hand-off the engine threads without changing any real behavior.+vi.mock("@paperclipai/adapter-utils/execution-target", async (importActual) => {+ const actual = await importActual<typeof import("@paperclipai/adapter-utils/execution-target")>();+ return {+ ...actual,+ prepareAdapterExecutionTargetRuntime: vi.fn(actual.prepareAdapterExecutionTargetRuntime),+ startAdapterExecutionTargetPaperclipBridge: vi.fn(actual.startAdapterExecutionTargetPaperclipBridge),+ startAdapterExecutionTargetProcessSessionBridge: vi.fn(actual.startAdapterExecutionTargetProcessSessionBridge),+ };+});+import { createAcpxEngineExecutor, type AcpxEngineExecutorOptions } from "./execute.js";+import { runChildProcess } from "../server-utils.js";++const tempRoots: string[] = [];++async function makeTempRoot() {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-acpx-skills-"));+ tempRoots.push(root);+ return root;+}++afterEach(async () => {+ // A remote run stages a process-session bridge whose detached event writer can+ // still be flushing a trailing event file into `.../process-sessions/<id>/events`+ // when the run's own best-effort `client.remove(sessionDir)` (which production+ // catch-wraps) has already returned. Under CI load that trailing write can land+ // between this recursive delete's directory snapshot and its `rmdir`, surfacing as+ // `ENOTEMPTY`. `maxRetries`/`retryDelay` make the cleanup ride out that window the+ // same way production tolerates it, instead of failing the just-passed test.+ await Promise.all(+ tempRoots.splice(0).map((root) =>+ fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }),+ ),+ );+});++async function pathExists(candidate: string): Promise<boolean> {+ return fs.access(candidate).then(() => true).catch(() => false);+}++void pathExists;++function createLocalSandboxRunner(+ onExecute?: (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ }) => void,+) {+ let counter = 0;+ return {+ execute: async (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ stdin?: string;+ timeoutMs?: number;+ onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;+ onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ }) => {+ counter += 1;+ onExecute?.(input);+ const command = input.command === "bash" ? "/bin/bash" : input.command;+ return await runChildProcess(`acpx-sandbox-run-${counter}`, command, input.args ?? [], {+ cwd: input.cwd ?? process.cwd(),+ env: input.env ?? {},+ stdin: input.stdin,+ timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),+ graceSec: 5,+ onLog: input.onLog ?? (async () => {}),+ onSpawn: input.onSpawn+ ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt })+ : undefined,+ });+ },+ };+}++function buildRuntime(+ onSetConfigOption?: (input: { key: string; value: string }) => void,+ onEnsureSession?: (input: Record<string, unknown>) => void,+) {+ return {+ ensureSession: async (input: Record<string, unknown>) => {+ onEnsureSession?.(input);+ return ({+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ });+ },+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ }),+ setConfigOption: async (input: { key: string; value: string }) => {+ onSetConfigOption?.(input);+ },+ close: async () => {},+ };+}++async function runExecutor(+ config: Record<string, unknown>,+ options: {+ context?: Record<string, unknown>;+ executionTransport?: Record<string, unknown>;+ authToken?: string;+ executionTarget?: Record<string, unknown>;+ runtimeMcp?: AdapterRuntimeMcpAccess;+ prepareRemoteManagedHome?: AcpxEngineExecutorOptions["prepareRemoteManagedHome"];+ startupTraceContext?: AdapterExecutionContext["startupTraceContext"];+ } = {},+) {+ const runtimeOptions: Record<string, unknown>[] = [];+ const configOptions: Array<{ key: string; value: string }> = [];+ const sessionInputs: Record<string, unknown>[] = [];+ const meta: Record<string, unknown>[] = [];+ const logs: Array<{ stream: string; text: string }> = [];+ const events: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];+ const execute = createAcpxEngineExecutor({+ ...(options.prepareRemoteManagedHome+ ? { prepareRemoteManagedHome: options.prepareRemoteManagedHome }+ : {}),+ createRuntime: (options) => {+ runtimeOptions.push(options as unknown as Record<string, unknown>);+ return buildRuntime(+ ({ key, value }) => configOptions.push({ key, value }),+ (input) => sessionInputs.push(input),+ ) as never;+ },+ });++ const result = await execute({+ runId: "run-1",+ agent: {+ id: "agent-1",+ companyId: "company-1",+ },+ runtime: {},+ config,+ context: options.context ?? {},+ executionTransport: options.executionTransport,+ authToken: options.authToken,+ executionTarget: options.executionTarget,+ runtimeMcp: options.runtimeMcp,+ startupTraceContext: options.startupTraceContext,+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ onMeta: async (payload: unknown) => {+ meta.push(payload as Record<string, unknown>);+ },+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ events.push(event);+ },+ } as never);++ expect(result.exitCode).toBe(0);+ return { logs, meta, events, runtimeOptions, configOptions, sessionInputs, result };+}++// The staging-seam describe helper from execute.test.ts (~:2152). It builds a+// runner-backed remote target: the local runner extracts the staged tar into+// `remoteCwd`, so the run really ships the HOST worktree into the sandbox.+async function setupRemoteSandbox() {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const localCwd = path.join(root, "worktree");+ const remoteCwd = path.join(root, "remote-workspace");+ await fs.mkdir(localCwd, { recursive: true });+ await fs.mkdir(remoteCwd, { recursive: true });+ // A file present only in the HOST worktree proves the workspace is shipped+ // into the sandbox: the local runner extracts the staged tar into remoteCwd.+ await fs.writeFile(path.join(localCwd, "hello.txt"), "hi", "utf8");+ const runner = createLocalSandboxRunner();+ const executionTarget = {+ kind: "remote",+ transport: "sandbox",+ providerKey: "fake-plugin",+ remoteCwd,+ runner,+ };+ return { root, stateDir, localCwd, remoteCwd, executionTarget };+}++// Read the `configFingerprint` off a settled run result.+function fpOf(result: { sessionParams?: unknown }): string | undefined {+ return (result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint;+}++const okHandle = {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+};++function completedTurn() {+ return {+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ };+}++describe("ACPX engine startup characterization", () => {+ // Item 1 + 8: the remote launch env and its finalization point.+ describe("remote launch environment values", () => {+ beforeEach(() => vi.clearAllMocks());++ it("mints the bridge launch env into the process-session command payload", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ // Decode the process-session LAUNCH payload (the base64 command blob). The+ // in-sandbox process env rides there, not in the exec's own `env`.+ let launchPayload: Record<string, unknown> | null = null;+ (executionTarget as { runner: unknown }).runner = createLocalSandboxRunner((input) => {+ if (input.env?.PAPERCLIP_SANDBOX_EXEC_CHANNEL === "bridge") {+ const script = input.args?.[1] ?? "";+ const match = script.match(/PAPERCLIP_PROCESS_SESSION_COMMAND_B64='([^']+)'/);+ if (match) {+ launchPayload = JSON.parse(Buffer.from(match[1]!, "base64").toString("utf8")) as Record<+ string,+ unknown+ >;+ }+ }+ });++ await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ { authToken: "real-run-jwt", executionTarget },+ );++ // The launch payload carries the MERGED paperclip bridge env: the queue+ // transport mode, a loopback bridge base URL, and a minted bridge token.+ const payloadEnv = ((launchPayload as Record<string, unknown> | null)?.env ?? {}) as Record<+ string,+ unknown+ >;+ expect(payloadEnv).toMatchObject({ PAPERCLIP_API_BRIDGE_MODE: "queue_v1" });+ expect(String(payloadEnv.PAPERCLIP_API_URL ?? "")).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);+ // The minted bridge token is present and is NOT the host run JWT.+ expect(payloadEnv.PAPERCLIP_API_KEY).toBeTruthy();+ expect(payloadEnv.PAPERCLIP_API_KEY).not.toBe("real-run-jwt");+ });++ it("finalizes the launch env at the bridge merge: the process env carries the merged bridge values", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ // Capture the launch payload AND the bridge-channel exec's own `env`.+ let launchPayload: Record<string, unknown> | null = null;+ let bridgeExecEnv: Record<string, string> | undefined;+ (executionTarget as { runner: unknown }).runner = createLocalSandboxRunner((input) => {+ if (input.env?.PAPERCLIP_SANDBOX_EXEC_CHANNEL === "bridge") {+ bridgeExecEnv = input.env;+ const script = input.args?.[1] ?? "";+ const match = script.match(/PAPERCLIP_PROCESS_SESSION_COMMAND_B64='([^']+)'/);+ if (match) {+ launchPayload = JSON.parse(Buffer.from(match[1]!, "base64").toString("utf8")) as Record<+ string,+ unknown+ >;+ }+ }+ });++ await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ { authToken: "real-run-jwt", executionTarget },+ );++ // The launch payload is the finalized carrier of the process env. It already+ // holds the merged bridge values at the point the run emits it, so no later+ // write mutates the process env after the bridge merge.+ const payloadEnv = ((launchPayload as Record<string, unknown> | null)?.env ?? {}) as Record<+ string,+ unknown+ >;+ expect(payloadEnv.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1");+ expect(payloadEnv.PAPERCLIP_API_KEY).toBeTruthy();+ // The bridge-channel exec's OWN env is the sandbox transport channel, not the+ // agent process env: it does not carry the minted agent bridge key. This pins+ // that the merged agent env lives only in the finalized launch payload.+ expect(bridgeExecEnv?.PAPERCLIP_SANDBOX_EXEC_CHANNEL).toBe("bridge");+ expect(bridgeExecEnv?.PAPERCLIP_API_KEY).toBeUndefined();+ });+ });++ // Item 2: the 17 fingerprint fields folded into `configFingerprint`, and the+ // outer session key form that embeds it.+ describe("session fingerprint and session key", () => {+ beforeEach(() => vi.clearAllMocks());++ it("forms the session key as paperclip:company:agent:taskKey:fingerprint and embeds the fingerprint", async () => {+ const root = await makeTempRoot();+ const { result } = await runExecutor({+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir: path.join(root, "state"),+ cwd: path.join(root, "workspace"),+ });++ const fp = fpOf(result);+ expect(fp).toBeTypeOf("string");+ expect(fp).toBeTruthy();+ // No taskId/issueId/workspaceId in the default context, so taskKey is "default".+ const sessionKey = (result.sessionParams as { sessionKey?: string }).sessionKey;+ expect(sessionKey).toBe(`paperclip:company-1:agent-1:default:${fp}`);+ });++ it("keeps the fingerprint stable across two identical runs and a same-config new wake", async () => {+ const root = await makeTempRoot();+ const baseConfig = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir: path.join(root, "state"),+ cwd: path.join(root, "workspace"),+ };+ const first = await runExecutor(baseConfig, {+ context: { taskId: "issue-1", wakeReason: "issue_assigned" },+ });+ const identical = await runExecutor(baseConfig, {+ context: { taskId: "issue-1", wakeReason: "issue_assigned" },+ });+ // A fresh heartbeat with a different wake reason but the same config env.+ const newWake = await runExecutor(baseConfig, {+ context: { taskId: "issue-1", wakeReason: "comment", wakeCommentId: "c-9" },+ });++ expect(fpOf(first.result)).toBeTruthy();+ expect(fpOf(identical.result)).toBe(fpOf(first.result));+ // Per-wake PAPERCLIP_* churn does not reset the session fingerprint.+ expect(fpOf(newWake.result)).toBe(fpOf(first.result));+ });++ it("busts the fingerprint when any representative folded dimension changes", async () => {+ const root = await makeTempRoot();+ const cwd = path.join(root, "workspace");+ const stateDir = path.join(root, "state");+ const context = { context: { taskId: "issue-1", wakeReason: "issue_assigned" } };+ const base = { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd };++ const baseFp = fpOf((await runExecutor(base, context)).result);+ expect(baseFp).toBeTruthy();++ // Each edit changes exactly one folded dimension and must bust the fingerprint.+ // agentCommand.+ expect(fpOf((await runExecutor({ ...base, agentCommand: "node ./other-acp.js" }, context)).result)).not.toBe(baseFp);+ // cwd.+ expect(fpOf((await runExecutor({ ...base, cwd: path.join(root, "other-cwd") }, context)).result)).not.toBe(baseFp);+ // requestedModel.+ expect(fpOf((await runExecutor({ ...base, model: "some-model" }, context)).result)).not.toBe(baseFp);+ // requestedThinkingEffort.+ expect(fpOf((await runExecutor({ ...base, thinkingEffort: "high" }, context)).result)).not.toBe(baseFp);+ // mode.+ expect(fpOf((await runExecutor({ ...base, mode: "oneshot" }, context)).result)).not.toBe(baseFp);+ // adapterEnvHash (a resolved adapter env value).+ expect(fpOf((await runExecutor({ ...base, env: { FOO: "bar" } }, context)).result)).not.toBe(baseFp);+ // mcpServers identity (injected runtime MCP set).+ expect(+ fpOf(+ (+ await runExecutor(base, {+ ...context,+ runtimeMcp: {+ getServers: () => [+ { name: "github", url: "https://x.test/mcp", connectionId: "c-1", token: "t-1" },+ ],+ },+ })+ ).result,+ ),+ ).not.toBe(baseFp);+ // secretManifestHash.+ expect(+ fpOf(+ (+ await runExecutor(base, {+ ...context,+ context: {+ taskId: "issue-1",+ wakeReason: "issue_assigned",+ paperclipSecrets: {+ manifest: [+ {+ configPath: "env.API_TOKEN",+ envKey: "API_TOKEN",+ secretId: "secret-1",+ bindingId: "binding-1",+ secretKey: "api-token",+ version: 1,+ provider: "local_encrypted",+ },+ ],+ },+ },+ })+ ).result,+ ),+ ).not.toBe(baseFp);+ // additionalSourcesIdentity (referenced-project set).+ expect(+ fpOf(+ (+ await runExecutor(base, {+ ...context,+ context: {+ taskId: "issue-1",+ wakeReason: "issue_assigned",+ paperclipWorkspace: {+ cwd,+ realization: {+ additional: [+ {+ path: "/host/project-a",+ projectId: "a",+ projectWorkspaceId: "ws-a",+ repoUrl: "https://example.test/a.git",+ repoRef: "ref-a-1",+ },+ ],+ },+ },+ },+ })+ ).result,+ ),+ ).not.toBe(baseFp);++ // fastMode folds only for codex, so pin it with a codex-vs-codex pair.+ const codexBase = { agent: "codex", agentCommand: "node ./fake-acp.js", stateDir, cwd };+ const codexFp = fpOf((await runExecutor(codexBase, context)).result);+ expect(fpOf((await runExecutor({ ...codexBase, fastMode: true }, context)).result)).not.toBe(codexFp);+ });+ });++ // Item 3: the staging seam call, its arguments, and its order (workspace then+ // assets, serial). Modeled on the PR-1 staging-seam tests.+ describe("staging seam calls, arguments, and order", () => {+ beforeEach(() => vi.clearAllMocks());++ it("stages the host workspace with no assets, exactly once, before the process launch", async () => {+ const { stateDir, localCwd, remoteCwd, executionTarget } = await setupRemoteSandbox();+ const { sessionInputs, events } = await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ { authToken: "real-run-jwt", executionTarget },+ );++ // The staging seam crossed exactly once.+ expect(vi.mocked(prepareAdapterExecutionTargetRuntime)).toHaveBeenCalledTimes(1);+ const stageArgs = vi.mocked(prepareAdapterExecutionTargetRuntime).mock.calls[0]![0];+ // The HOST worktree is shipped first; no per-adapter home asset in this lane.+ expect(stageArgs.workspaceLocalDir).toBe(localCwd);+ expect(stageArgs.assets ?? []).toEqual([]);+ expect(stageArgs.installCommand ?? null).toBeNull();+ expect(stageArgs.target).toMatchObject({ kind: "remote", transport: "sandbox" });++ // The workspace really landed in the sandbox workspace dir.+ await expect(fs.readFile(path.join(remoteCwd, "hello.txt"), "utf8")).resolves.toBe("hi");+ // A per-step timing event proves the sync ran inside its timed boundary.+ const stageEvent = events.find(+ (event) => event.eventType === "run.startup.step" && event.payload?.step === "stage.sync",+ );+ expect(stageEvent).toBeTruthy();+ // And session/new binds to the in-sandbox workspace cwd the seam returned.+ expect(sessionInputs[0]?.cwd).toBe(remoteCwd);+ });++ it("threads a managed-home asset through the same seam after the workspace", async () => {+ const { root, stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const managedHomeDir = path.join(root, "managed-home");+ await fs.mkdir(managedHomeDir, { recursive: true });+ await fs.writeFile(path.join(managedHomeDir, "config.json"), "{}", "utf8");++ await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ {+ authToken: "real-run-jwt",+ executionTarget,+ prepareRemoteManagedHome: async (input) => {+ const stagedRuntime = await input.stage([+ { key: "home", localDir: managedHomeDir, followSymlinks: true },+ ]);+ return { stagedRuntime };+ },+ },+ );++ // The seam's home asset is threaded through the SAME staging seam, keyed by+ // the workspace-local dir. Workspace ships first; the asset rides alongside it.+ const stageArgs = vi.mocked(prepareAdapterExecutionTargetRuntime).mock.calls[0]![0];+ expect(stageArgs.workspaceLocalDir).toBe(localCwd);+ expect(stageArgs.assets).toEqual([+ { key: "home", localDir: managedHomeDir, followSymlinks: true },+ ]);+ });+ });++ // Item 4: the two-bridge overlap and the ACP-initialization ordering.+ describe("two-bridge overlap and ACP initialization order", () => {+ beforeEach(() => vi.clearAllMocks());++ it("defers the process bridge env, shares one runtimeRootDir, and runs session/new on the sandbox cwd", async () => {+ const { stateDir, localCwd, remoteCwd, executionTarget } = await setupRemoteSandbox();+ const { sessionInputs, runtimeOptions } = await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ { authToken: "real-run-jwt", executionTarget },+ );++ // The process-session bridge receives its launch env as a DEFERRED thunk, the+ // seam that lets its env-independent setup overlap the paperclip bridge start.+ const processArgs = vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mock.calls[0]![0];+ expect(typeof processArgs.env).toBe("function");++ // Both bridges receive the SAME real (non-null) runtimeRootDir from staging.+ const paperclipArgs = vi.mocked(startAdapterExecutionTargetPaperclipBridge).mock.calls[0]![0];+ expect(paperclipArgs.runtimeRootDir).toBeTruthy();+ expect(String(paperclipArgs.runtimeRootDir)).toContain(".paperclip-runtime");+ expect(processArgs.runtimeRootDir).toBe(paperclipArgs.runtimeRootDir);++ // The ACP runtime + session/new both bind to the in-sandbox workspace cwd,+ // which the run resolves only after the bridges bring the sandbox up.+ expect(runtimeOptions[0]?.cwd).toBe(remoteCwd);+ expect(sessionInputs[0]?.cwd).toBe(remoteCwd);+ expect(sessionInputs[0]?.cwd).not.toBe(localCwd);+ });+ });++ // Item 5 + 6: every startup exit path, its result phase, and the cleanup-call+ // set (bridges stop / lease releases / runtime closes).+ describe("startup exit paths and cleanup", () => {+ beforeEach(() => vi.clearAllMocks());++ it("create_runtime failure: settles an error result, stops both bridges, releases the lease", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const paperclipStop = vi.fn(async () => {});+ const processStop = vi.fn(async () => {});+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementationOnce(+ async () => ({ env: {}, stop: paperclipStop }) as never,+ );+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementationOnce(+ async () => ({ agentCommand: null, stop: processStop }) as never,+ );+ const stagingLocks = new Map<string, Promise<unknown>>();+ const execute = createAcpxEngineExecutor({+ stagingLocks,+ warmHandles: new Map(),+ stagedRuntimes: new Map(),+ createRuntime: () => {+ throw new Error("createRuntime boom");+ },+ });++ const result = await execute({+ runId: "run-create-fail-remote",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ context: {},+ authToken: "real-run-jwt",+ executionTarget,+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never);++ // The post-build runtime-creation failure returns a settled error result.+ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("create_runtime");+ // Both live bridges stop exactly once and the per-session lease releases.+ expect(paperclipStop).toHaveBeenCalledTimes(1);+ expect(processStop).toHaveBeenCalledTimes(1);+ expect(stagingLocks.size).toBe(0);+ });++ it("partial-bridge failure: throws and stops the concurrently-started bridge exactly once", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ const stop = vi.fn(async () => {});+ vi.mocked(startAdapterExecutionTargetPaperclipBridge).mockImplementationOnce(async () => {+ throw new Error("paperclip bridge boom");+ });+ vi.mocked(startAdapterExecutionTargetProcessSessionBridge).mockImplementationOnce(+ async () => ({ agentCommand: null, stop }) as never,+ );++ const execute = createAcpxEngineExecutor({+ createRuntime: () => buildRuntime() as never,+ });++ // A partial bridge failure inside buildRuntime is one of the only two throw+ // paths, so the run rethrows instead of settling a result.+ await expect(+ execute({+ runId: "run-bridge-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ context: {},+ authToken: "real-run-jwt",+ executionTarget,+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ } as never),+ ).rejects.toThrow("paperclip bridge boom");++ // The concurrently-started process-session bridge was stopped exactly once.+ expect(stop).toHaveBeenCalledTimes(1);+ });++ it("cold ensure_session throw: ensure_session error, and the handshake-catch close does NOT fire (no handle yet)", async () => {+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => {+ throw new Error("ensureSession boom");+ },+ startTurn: () => completedTurn(),+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: "handshake-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("ensure_session");+ // Current behavior (execute.ts ~:3590): the handshake catch closes the runtime+ // only `if (handle)`. A cold `ensureSession` throw never assigns a handle, so+ // this close never fires. The warm-hit path below has a cached handle and does+ // close. This pins the real cold-path behavior, not an aspiration.+ expect(closeSpy).not.toHaveBeenCalled();+ });++ it("missing session handle: ensure_session error and the minimal runtime closes", async () => {+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ // A runtime that returns no session handle drives the missing-handle path.+ ensureSession: async () => undefined,+ startTurn: () => completedTurn(),+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: "missing-handle",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir: path.join(root, "state") },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("ensure_session");+ expect(result.errorCode).toBe("acpx_runtime_error");+ expect(closeSpy).toHaveBeenCalledTimes(1);+ });++ it("configure_session failure: configure_session error and the runtime closes", async () => {+ const root = await makeTempRoot();+ const closeSpy = vi.fn(async () => {});+ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ ({+ ensureSession: async () => okHandle,+ // Gemini model/effort drive a session config option; a throwing setter+ // fails the configure_session phase after the handshake succeeds.+ setConfigOption: async () => {+ throw new Error("setConfigOption boom");+ },+ startTurn: () => completedTurn(),+ close: closeSpy,+ }) as never,+ });++ const result = await execute({+ runId: "configure-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: {+ agent: "gemini",+ model: "gemini-2.5-pro",+ thinkingEffort: "high",+ stateDir: path.join(root, "state"),+ },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.resultJson?.phase).toBe("configure_session");+ expect(closeSpy).toHaveBeenCalledTimes(1);+ });++ it("warm-hit failure: reuses the runtime, fails with ensure_session, closes it, drops the warm entry", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const startedAt = "2026-01-01T00:00:00.000Z";+ const closeSpy = vi.fn(async () => {});+ let created = 0;+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: (options) => {+ created += 1;+ const opts = options as AcpRuntimeOptions & {+ onAgentSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ };+ return {+ ensureSession: async () => {+ await opts.onAgentSpawn?.({ pid: 4242, startedAt });+ return okHandle;+ },+ startTurn: () => completedTurn(),+ close: closeSpy,+ } as never;+ },+ });+ const config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ };++ const first = await execute({+ runId: "warm-1",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {},+ } as never);+ expect(first.exitCode).toBe(0);+ expect(warmHandles.size).toBe(1);+ expect(created).toBe(1);++ // The warm-hit reuses runtime #1 and fails while persisting process identity.+ const second = await execute({+ runId: "warm-2",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: first.sessionParams },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {+ throw new Error("onSpawn boom");+ },+ } as never);++ // No new runtime was created; the reused runtime is closed and the warm entry+ // removed, with the failure reported on the ensure_session phase.+ expect(created).toBe(1);+ expect(second.exitCode).toBe(1);+ expect(second.resultJson?.phase).toBe("ensure_session");+ expect(closeSpy).toHaveBeenCalledTimes(1);+ expect(warmHandles.size).toBe(0);+ });+ });++ // Item 7: the per-lane resource set.+ describe("per-lane resource set", () => {+ beforeEach(() => vi.clearAllMocks());++ it("local lane: crosses no staging seam, starts no bridge, keeps session/new on the host cwd", async () => {+ const root = await makeTempRoot();+ const localCwd = path.join(root, "worktree");+ await fs.mkdir(localCwd, { recursive: true });+ const { sessionInputs, runtimeOptions } = await runExecutor({+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir: path.join(root, "state"),+ cwd: localCwd,+ });++ expect(vi.mocked(prepareAdapterExecutionTargetRuntime)).not.toHaveBeenCalled();+ expect(vi.mocked(startAdapterExecutionTargetPaperclipBridge)).not.toHaveBeenCalled();+ expect(vi.mocked(startAdapterExecutionTargetProcessSessionBridge)).not.toHaveBeenCalled();+ expect(sessionInputs[0]?.cwd).toBe(localCwd);+ expect(runtimeOptions[0]?.cwd).toBe(localCwd);+ });++ it("persistent host lane: warm-saves the handle so a second run reuses the runtime", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let created = 0;+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ createRuntime: () => {+ created += 1;+ return buildRuntime() as never;+ },+ });+ const config = {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ };+ const base = {+ agent: { id: "agent-1", companyId: "company-1" },+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ onSpawn: async () => {},+ };++ const first = await execute({ runId: "host-warm-1", runtime: {}, ...base } as never);+ const second = await execute({+ runId: "host-warm-2",+ runtime: { sessionParams: first.sessionParams },+ ...base,+ } as never);++ expect(first.exitCode).toBe(0);+ expect(second.exitCode).toBe(0);+ // The warm handle survives, so the second run reuses runtime #1.+ expect(created).toBe(1);+ expect(warmHandles.size).toBe(1);+ });++ it("remote process-session lane: does NOT warm-save the handle, so a second run re-creates the runtime", async () => {+ const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ let created = 0;+ const ensureInputs: Array<Record<string, unknown>> = [];+ const warmHandles = new Map();+ const execute = createAcpxEngineExecutor({+ warmHandles,+ stagedRuntimes: new Map(),+ stagingLocks: new Map(),+ createRuntime: () => {+ created += 1;+ return buildRuntime(undefined, (input) => ensureInputs.push(input)) as never;+ },+ });+ const base = {+ agent: { id: "agent-1", companyId: "company-1" },+ config: {+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ cwd: localCwd,+ mode: "persistent",+ warmHandleIdleMs: 60_000,+ },+ context: {},+ authToken: "real-run-jwt",+ executionTarget,+ onLog: async () => {},+ onMeta: async () => {},+ onEvent: async () => {},+ };++ const first = await execute({ runId: "remote-warm-1", runtime: {}, ...base } as never);+ const second = await execute({+ runId: "remote-warm-2",+ runtime: { sessionParams: first.sessionParams },+ ...base,+ } as never);++ expect(first.exitCode).toBe(0);+ expect(second.exitCode).toBe(0);+ // The remote process-session lane never keeps the handle warm, so the second+ // run re-creates the runtime and runs a fresh handshake instead of reusing one.+ expect(created).toBe(2);+ expect(warmHandles.size).toBe(0);+ expect(ensureInputs).toHaveLength(2);+ });+ });+});diff --git a/packages/adapter-utils/src/acpx-engine/turn-characterization.test.ts b/packages/adapter-utils/src/acpx-engine/turn-characterization.test.tsnew file mode 100644index 0000000000..01eea58322--- /dev/null+++ b/packages/adapter-utils/src/acpx-engine/turn-characterization.test.ts@@ -0,0 +1,685 @@+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterEach, describe, expect, it, vi } from "vitest";+import type { AdapterExecutionContext, AdapterRuntimeMcpAccess } from "@paperclipai/adapter-utils";+import {+ prepareAdapterExecutionTargetRuntime,+ startAdapterExecutionTargetPaperclipBridge,+ startAdapterExecutionTargetProcessSessionBridge,+} from "@paperclipai/adapter-utils/execution-target";++// Wrap the staging seam + both sandbox bridges in call-recording spies that+// still delegate to the real implementations. This mirrors the execute.test.ts+// harness so the turn characterization tests share the same mocked module graph.+vi.mock("@paperclipai/adapter-utils/execution-target", async (importActual) => {+ const actual = await importActual<typeof import("@paperclipai/adapter-utils/execution-target")>();+ return {+ ...actual,+ prepareAdapterExecutionTargetRuntime: vi.fn(actual.prepareAdapterExecutionTargetRuntime),+ startAdapterExecutionTargetPaperclipBridge: vi.fn(actual.startAdapterExecutionTargetPaperclipBridge),+ startAdapterExecutionTargetProcessSessionBridge: vi.fn(actual.startAdapterExecutionTargetProcessSessionBridge),+ };+});+import {+ createAcpxEngineExecutor,+ summarizeAcpxTurnUsage,+ type AcpxEngineExecutorOptions,+} from "./execute.js";+import { runChildProcess } from "../server-utils.js";++const tempRoots: string[] = [];++async function makeTempRoot() {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-acpx-skills-"));+ tempRoots.push(root);+ return root;+}++afterEach(async () => {+ // A remote run stages a process-session bridge whose detached event writer can+ // still be flushing a trailing event file into `.../process-sessions/<id>/events`+ // when the run's own best-effort `client.remove(sessionDir)` (which production+ // catch-wraps) has already returned. Under CI load that trailing write can land+ // between this recursive delete's directory snapshot and its `rmdir`, surfacing as+ // `ENOTEMPTY`. `maxRetries`/`retryDelay` make the cleanup ride out that window the+ // same way production tolerates it, instead of failing the just-passed test.+ await Promise.all(+ tempRoots.splice(0).map((root) =>+ fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }),+ ),+ );+});++function createLocalSandboxRunner(+ onExecute?: (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ }) => void,+) {+ let counter = 0;+ return {+ execute: async (input: {+ command: string;+ args?: string[];+ cwd?: string;+ env?: Record<string, string>;+ stdin?: string;+ timeoutMs?: number;+ onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;+ onSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void>;+ }) => {+ counter += 1;+ onExecute?.(input);+ const command = input.command === "bash" ? "/bin/bash" : input.command;+ return await runChildProcess(`acpx-sandbox-run-${counter}`, command, input.args ?? [], {+ cwd: input.cwd ?? process.cwd(),+ env: input.env ?? {},+ stdin: input.stdin,+ timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),+ graceSec: 5,+ onLog: input.onLog ?? (async () => {}),+ onSpawn: input.onSpawn+ ? async (meta) => input.onSpawn?.({ pid: meta.pid, startedAt: meta.startedAt })+ : undefined,+ });+ },+ };+}++function buildRuntime(+ onSetConfigOption?: (input: { key: string; value: string }) => void,+ onEnsureSession?: (input: Record<string, unknown>) => void,+) {+ return {+ ensureSession: async (input: Record<string, unknown>) => {+ onEnsureSession?.(input);+ return ({+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ });+ },+ startTurn: () => ({+ events: (async function* () {+ yield { type: "done", stopReason: "end_turn" };+ })(),+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ cancel: async () => {},+ }),+ setConfigOption: async (input: { key: string; value: string }) => {+ onSetConfigOption?.(input);+ },+ close: async () => {},+ };+}++async function runExecutor(+ config: Record<string, unknown>,+ options: {+ context?: Record<string, unknown>;+ executionTransport?: Record<string, unknown>;+ authToken?: string;+ executionTarget?: Record<string, unknown>;+ runtimeMcp?: AdapterRuntimeMcpAccess;+ prepareRemoteManagedHome?: AcpxEngineExecutorOptions["prepareRemoteManagedHome"];+ startupTraceContext?: AdapterExecutionContext["startupTraceContext"];+ } = {},+) {+ const runtimeOptions: Record<string, unknown>[] = [];+ const configOptions: Array<{ key: string; value: string }> = [];+ const sessionInputs: Record<string, unknown>[] = [];+ const meta: Record<string, unknown>[] = [];+ const logs: Array<{ stream: string; text: string }> = [];+ const events: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];+ const execute = createAcpxEngineExecutor({+ ...(options.prepareRemoteManagedHome+ ? { prepareRemoteManagedHome: options.prepareRemoteManagedHome }+ : {}),+ createRuntime: (options) => {+ runtimeOptions.push(options as unknown as Record<string, unknown>);+ return buildRuntime(+ ({ key, value }) => configOptions.push({ key, value }),+ (input) => sessionInputs.push(input),+ ) as never;+ },+ });++ const result = await execute({+ runId: "run-1",+ agent: {+ id: "agent-1",+ companyId: "company-1",+ },+ runtime: {},+ config,+ context: options.context ?? {},+ executionTransport: options.executionTransport,+ authToken: options.authToken,+ executionTarget: options.executionTarget,+ runtimeMcp: options.runtimeMcp,+ startupTraceContext: options.startupTraceContext,+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ onMeta: async (payload: unknown) => {+ meta.push(payload as Record<string, unknown>);+ },+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ events.push(event);+ },+ } as never);++ expect(result.exitCode).toBe(0);+ return { logs, meta, events, runtimeOptions, configOptions, sessionInputs, result };+}++// A stub ACP runtime that yields a controlled event stream and terminal result.+// The tests supply the events/result/cancel behavior; the harness above only+// drives always-happy sessions, so the turn tests build their own runtime here.+function turnRuntime(input: {+ events: () => AsyncGenerator<Record<string, unknown>>;+ result: Promise<Record<string, unknown>>;+ onStartTurn?: (turnInput: Record<string, unknown>) => void;+ onCancel?: (reason: string) => void;+ onClose?: () => void;+ getStatus?: () => Promise<Record<string, unknown>>;+ onEnsureSession?: (session: Record<string, unknown>) => void;+ ensureSession?: (session: Record<string, unknown>) => Promise<Record<string, unknown>>;+}) {+ const runtime: Record<string, unknown> = {+ ensureSession:+ input.ensureSession ??+ (async (session: Record<string, unknown>) => {+ input.onEnsureSession?.(session);+ return {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ };+ }),+ startTurn: (turnInput: Record<string, unknown>) => {+ input.onStartTurn?.(turnInput);+ return {+ events: input.events(),+ result: input.result,+ cancel: async ({ reason }: { reason: string }) => {+ input.onCancel?.(reason);+ },+ };+ },+ close: async () => {+ input.onClose?.();+ },+ };+ if (input.getStatus) runtime.getStatus = input.getStatus;+ return runtime;+}++describe("ACPX engine turn characterization", () => {+ // The stub session handle every ensureSession returns. startTurn must receive+ // this exact handle object.+ const SESSION_HANDLE = {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ };++ it("passes exactly the six turn inputs to startTurn", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let captured: Record<string, unknown> | null = null;+ let metaPrompt = "";++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ onStartTurn: (turnInput) => {+ captured = turnInput;+ },+ events: async function* () {+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });++ const result = await execute({+ runId: "run-six-inputs",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, timeoutSec: 120 },+ context: {},+ onLog: async () => {},+ onMeta: async (payload: { prompt?: string }) => {+ metaPrompt = payload.prompt ?? "";+ },+ } as never);++ expect(result.exitCode).toBe(0);+ const input = captured!;+ // The handle is the exact session handle ensureSession returned.+ expect(input.handle).toEqual(SESSION_HANDLE);+ // The text is the built run prompt, the same string reported to onMeta.+ expect(input.text).toBe(metaPrompt);+ expect(typeof input.text).toBe("string");+ expect((input.text as string).length).toBeGreaterThan(0);+ // The turn always runs in prompt mode.+ expect(input.mode).toBe("prompt");+ // The request id is the run id.+ expect(input.requestId).toBe("run-six-inputs");+ // A positive timeoutSec becomes timeoutMs in milliseconds.+ expect(input.timeoutMs).toBe(120_000);+ // The abort signal is present and not yet aborted.+ const signal = input.signal as AbortSignal;+ expect(signal).toBeInstanceOf(AbortSignal);+ expect(signal.aborted).toBe(false);+ // Exactly the six documented keys are threaded.+ expect(Object.keys(input).sort()).toEqual(+ ["handle", "mode", "requestId", "signal", "text", "timeoutMs"].sort(),+ );+ });++ it("joins text_delta events into the trimmed result summary", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ events: async function* () {+ yield { type: "text_delta", text: " Hello, ", stream: "output", tag: "agent_message_chunk" };+ yield { type: "text_delta", text: "world ", stream: "output", tag: "agent_message_chunk" };+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });++ const result = await execute({+ runId: "run-summary",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(0);+ // The summary is the concatenation of the deltas, trimmed.+ expect(result.summary).toBe("Hello, world");+ });++ it("falls back to the stop reason for the summary when no text streams", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ events: async function* () {+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });++ const result = await execute({+ runId: "run-no-text",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(0);+ // With no streamed text, the summary is the terminal stop reason.+ expect(result.summary).toBe("end_turn");+ });++ it("pins one event sequence across the log transcript and structured event transports", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const logs: Array<{ stream: string; text: string }> = [];+ const events: Array<{ eventType: string; payload?: Record<string, unknown> }> = [];++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ events: async function* () {+ yield { type: "text_delta", text: "streamed hello", stream: "output", tag: "agent_message_chunk" };+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });++ const result = await execute({+ runId: "run-transports",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context: {},+ onLog: async (stream: "stdout" | "stderr", text: string) => {+ logs.push({ stream, text });+ },+ onMeta: async () => {},+ onEvent: async (event: { eventType: string; payload?: Record<string, unknown> }) => {+ events.push(event);+ },+ } as never);++ expect(result.exitCode).toBe(0);+ // Transport 1 — the stdout transcript carries the text_delta as an acpx record.+ expect(logs).toContainEqual({+ stream: "stdout",+ text: `${JSON.stringify({+ type: "acpx.text_delta",+ text: "streamed hello",+ channel: "output",+ tag: "agent_message_chunk",+ })}\n`,+ });+ // The joined text also lands in the result summary.+ expect(result.summary).toBe("streamed hello");+ // Transport 2 — the structured onEvent stream reflects the run bring-up+ // sequence as run.startup.step events, including the acp handshake.+ const steps = events.filter((event) => event.eventType === "run.startup.step");+ const stepNames = steps.map((event) => String(event.payload?.step));+ expect(stepNames).toContain("acp.handshake");+ expect(stepNames).toContain("workspace.resolve");+ });++ it("aborts a hung turn on the wall-clock timer and cancels with the timeout message", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const cwd = path.join(root, "worktree");+ await fs.mkdir(cwd, { recursive: true });++ const cancelReasons: string[] = [];+ let releaseTurn: (() => void) | null = null;+ const turnCancelled = new Promise<void>((resolve) => {+ releaseTurn = resolve;+ });++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ // The stream never yields on its own. Only the wall-clock timer's cancel+ // unblocks it, simulating a hung run.+ events: async function* () {+ await turnCancelled;+ },+ result: turnCancelled.then(() => ({ status: "cancelled", stopReason: "cancelled" })),+ onCancel: (reason) => {+ cancelReasons.push(reason);+ releaseTurn?.();+ },+ }) as never,+ });++ const result = await execute({+ runId: "run-timeout",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd, timeoutSec: 1 },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ const expectedMessage =+ "Run exceeded the adapter execution timeout (timeoutSec=1, configured via adapterConfig.timeoutSec). " ++ "Set adapterConfig.timeoutSec to raise it.";+ expect(result.timedOut).toBe(true);+ expect(result.signal).toBe("SIGTERM");+ expect(result.errorCode).toBe("acpx_timeout");+ expect(result.errorMessage).toBe(expectedMessage);+ // The cancel ran with the formatted timeout message.+ expect(cancelReasons).toContain(expectedMessage);+ }, 15_000);++ it("cancels the turn before closing the runtime when the turn throws", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const order: string[] = [];++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ // The event stream throws mid-turn, so the catch path runs teardown.+ events: async function* () {+ throw new Error("turn boom");+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ onCancel: () => order.push("cancel"),+ onClose: () => order.push("close"),+ }) as never,+ });++ const result = await execute({+ runId: "run-throw",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ // A failure after startTurn returned reports phase "turn".+ expect((result.resultJson as Record<string, unknown>)?.phase).toBe("turn");+ // Cancel runs before close, in that exact order.+ expect(order).toEqual(["cancel", "close"]);+ });++ it("retries the session resume once and never retries the turn", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const cwd = path.join(root, "worktree");+ await fs.mkdir(cwd, { recursive: true });+ const config = { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd };++ const ensureInputs: Array<Record<string, unknown>> = [];+ let startTurnCalls = 0;+ let resumeLogged = false;++ // A first clean run mints the session params the second run resumes from.+ const firstExecute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ onEnsureSession: (session) => ensureInputs.push(session),+ onStartTurn: () => {+ startTurnCalls += 1;+ },+ events: async function* () {+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });+ const first = await firstExecute({+ runId: "run-resume-a",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config,+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);+ expect(first.exitCode).toBe(0);++ // The second run resumes. Its first ensureSession (the resume) fails with a+ // resume-shaped error; the fresh retry (no resumeSessionId) succeeds.+ const secondExecute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ ensureSession: async (session: Record<string, unknown>) => {+ ensureInputs.push(session);+ if (session.resumeSessionId) {+ throw new Error("resume session not found");+ }+ return {+ backendSessionId: "backend-session",+ agentSessionId: "agent-session",+ runtimeSessionName: "runtime-session",+ };+ },+ onStartTurn: () => {+ startTurnCalls += 1;+ },+ events: async function* () {+ yield { type: "done", stopReason: "end_turn" };+ },+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });+ const second = await secondExecute({+ runId: "run-resume-b",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: { sessionParams: (first as { sessionParams?: unknown }).sessionParams },+ config,+ context: {},+ onLog: async (_stream: string, text: string) => {+ if (text.includes("is unavailable; retrying with a fresh session")) resumeLogged = true;+ },+ onMeta: async () => {},+ } as never);++ expect(second.exitCode).toBe(0);+ // The first run's single ensureSession, plus the second run's resume and+ // fresh retry, make three ensureSession calls total; two on the second run.+ expect(ensureInputs).toHaveLength(3);+ expect(ensureInputs[1]?.resumeSessionId).toBe(first.sessionId);+ expect(ensureInputs[2]?.resumeSessionId).toBeUndefined();+ // The turn is never retried: one startTurn per run, two across both runs.+ expect(startTurnCalls).toBe(2);+ // The engine logged the resume fallback.+ expect(resumeLogged).toBe(true);+ // The fresh retry clears the stale session for the caller.+ expect(second.clearSession).toBe(true);+ });++ it("maps a failed terminal to acpx_turn_failed and folds in the reported usage", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ events: async function* () {+ yield {+ type: "status",+ text: "usage",+ tag: "usage_update",+ cost: { amount: 0.31, currency: "USD" },+ breakdown: { inputTokens: 40, outputTokens: 700, cachedReadTokens: 60 },+ };+ yield { type: "done", stopReason: "failed" };+ },+ result: Promise.resolve({ status: "failed", error: new Error("boom") }),+ }) as never,+ });++ const result = await execute({+ runId: "run-failed",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context: {},+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ expect(result.errorCode).toBe("acpx_turn_failed");+ // The failed error message becomes the terminal stop reason and the summary.+ expect(result.summary).toBe("boom");+ expect((result.resultJson as Record<string, unknown>)?.stopReason).toBe("boom");+ // The usage math folds the usage_update event into per-run usage and cost.+ expect(result.usage).toEqual({ inputTokens: 40, outputTokens: 700, cachedInputTokens: 60 });+ expect(result.usageBasis).toBe("per_run");+ expect(result.costUsd).toBeCloseTo(0.31);+ });++ it("computes usage the same way summarizeAcpxTurnUsage does for the event fallback", () => {+ // Pin the exported helper the turn path calls: with no getStatus snapshots the+ // event breakdown and cost drive the per-run usage.+ const summary = summarizeAcpxTurnUsage({+ preStatus: null,+ postStatus: null,+ eventBreakdown: { inputTokens: 40, outputTokens: 700, cachedReadTokens: 60 },+ eventCostUsd: 0.31,+ });+ expect(summary.usage).toEqual({ inputTokens: 40, outputTokens: 700, cachedInputTokens: 60 });+ expect(summary.costUsd).toBeCloseTo(0.31);+ });++ it("returns a prepare_turn error result when the prompt build throws", async () => {+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ let startTurnCalls = 0;+ let closes = 0;++ const execute = createAcpxEngineExecutor({+ createRuntime: () =>+ turnRuntime({+ onStartTurn: () => {+ startTurnCalls += 1;+ },+ onClose: () => {+ closes += 1;+ },+ events: async function* () {},+ result: Promise.resolve({ status: "completed", stopReason: "end_turn" }),+ }) as never,+ });++ // A throwing accessor on a field only buildPrompt reads makes the prompt build+ // fail after the session handshake succeeds but before startTurn runs.+ const context: Record<string, unknown> = {};+ Object.defineProperty(context, "paperclipSessionHandoffMarkdown", {+ enumerable: false,+ get() {+ throw new Error("prompt build boom");+ },+ });++ const result = await execute({+ runId: "run-prepare-fail",+ agent: { id: "agent-1", companyId: "company-1" },+ runtime: {},+ config: { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir },+ context,+ onLog: async () => {},+ onMeta: async () => {},+ } as never);++ expect(result.exitCode).toBe(1);+ // The pre-turn failure reports phase "prepare_turn".+ expect((result.resultJson as Record<string, unknown>)?.phase).toBe("prepare_turn");+ // The turn never started, and the runtime closed once.+ expect(startTurnCalls).toBe(0);+ expect(closes).toBe(1);+ });++ it("threads the same happy-path turn through the shared runExecutor harness", async () => {+ // A smoke pin on the copied harness: the always-happy buildRuntime turn exits+ // clean with the done stop reason as the summary.+ const root = await makeTempRoot();+ const stateDir = path.join(root, "state");+ const { result } = await runExecutor({+ agent: "custom",+ agentCommand: "node ./fake-acp.js",+ stateDir,+ });+ expect(result.exitCode).toBe(0);+ expect(result.summary).toBe("end_turn");+ });+});diff --git a/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts b/server/src/__tests__/heartbeat-run-terminalize-before-release.test.tsnew file mode 100644index 0000000000..ae6af37121--- /dev/null+++ b/server/src/__tests__/heartbeat-run-terminalize-before-release.test.ts@@ -0,0 +1,312 @@+import { randomUUID } from "node:crypto";+import { eq } from "drizzle-orm";+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";+import {+ agents,+ companies,+ createDb,+ heartbeatRunEvents,+ heartbeatRuns,+ issues,+} from "@paperclipai/db";+import {+ getEmbeddedPostgresTestSupport,+ startEmbeddedPostgresTestDatabase,+} from "./helpers/embedded-postgres.js";++const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() }));+vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient }));++import {+ heartbeatService,+ leaseReleaseStatusForRunStatus,+ type HeartbeatEnvironmentRuntime,+} from "../services/heartbeat.ts";++const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;++if (!embeddedPostgresSupport.supported) {+ console.warn(+ `Skipping embedded Postgres terminalize-before-release tests on this host: ${+ embeddedPostgresSupport.reason ?? "unsupported environment"+ }`,+ );+}++// This file is a characterization test. It pins the CURRENT run-teardown order+// in server/src/services/heartbeat.ts:16586-16607: the teardown finally+// terminalizes the run FIRST, then releases the environment lease using the+// terminalized status. The production order is:+// latestRun = await terminalizeRunOnLeaseRelease(latestRun); // :16593 first+// await releaseEnvironmentLeasesForRun({ status: latestRun?.status, ... }); // :16601 second+//+// The enclosing teardown finally is not reasonably invokable in isolation: it+// lives deep in the heartbeat run body and needs a full sandbox, adapter, and+// workspace bring-up to reach. So this test drives the two real production+// functions in the same order against the embedded database:+// `terminalizeRunOnLeaseRelease` and `releaseEnvironmentLeasesForRun`. It thus+// executes the real lease-release boundary: the terminalized run status flows+// through the real run-status → lease-status mapping+// (`leaseReleaseStatusForRunStatus`) and the real environment orchestrator+// (`envOrchestrator.releaseForRun`) down to the runtime leaf. The test injects a+// fake `environmentRuntime` that records the mapped lease status at that leaf, so+// a wrong terminal state or a broken mapping fails the suite.+//+// The two additive test seams keep production behavior unchanged. The service now+// exposes `releaseEnvironmentLeasesForRun` (next to the existing+// `terminalizeRunOnLeaseRelease`), and `leaseReleaseStatusForRunStatus` is now+// exported for the direct mapping assertions below.+describeEmbeddedPostgres("heartbeat teardown terminalizes the run before releasing the lease", () => {+ let db!: ReturnType<typeof createDb>;+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;++ beforeAll(async () => {+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-terminalize-before-release-");+ db = createDb(tempDb.connectionString);+ }, 20_000);++ afterEach(async () => {+ await db.delete(heartbeatRunEvents);+ await db.delete(issues);+ await db.delete(heartbeatRuns);+ await db.delete(agents);+ await db.delete(companies);+ });++ afterAll(async () => {+ await tempDb?.cleanup();+ });++ async function seed(input: { issueStatus: string; runStatus: string }) {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const issueId = randomUUID();+ const runId = randomUUID();++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,+ requireBoardApprovalForNewAgents: false,+ });+ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "Coder",+ role: "engineer",+ status: "active",+ adapterType: "codex_local",+ adapterConfig: {},+ runtimeConfig: {},+ permissions: {},+ });+ await db.insert(issues).values({+ id: issueId,+ companyId,+ title: "Terminalize before release",+ status: input.issueStatus,+ priority: "high",+ assigneeAgentId: agentId,+ });+ await db.insert(heartbeatRuns).values({+ id: runId,+ companyId,+ agentId,+ status: input.runStatus,+ invocationSource: "manual",+ startedAt: new Date(),+ contextSnapshot: { issueId },+ });++ const run = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0]!);++ return { companyId, agentId, issueId, runId, run };+ }++ async function runStatus(runId: string) {+ return db+ .select({ status: heartbeatRuns.status })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0]?.status ?? null);+ }++ // Drive the two real production functions in the teardown order at+ // heartbeat.ts:16586-16607 and report what the real release step observes.+ // Terminalize runs first. Then `releaseEnvironmentLeasesForRun` runs with the+ // terminalized run status. That real call maps the run status through+ // `leaseReleaseStatusForRunStatus` and passes it to the real environment+ // orchestrator, which reaches the runtime leaf. The injected fake+ // `environmentRuntime` records the run id and the mapped lease status at that+ // leaf, so the test observes the actual boundary, not a reproduction.+ async function runTeardownSequenceObservingRelease(input: {+ runId: string;+ companyId: string;+ agentId: string;+ }) {+ const { runId, companyId, agentId } = input;+ const releaseLeafCalls: Array<{ runId: string; status: string }> = [];+ const fakeEnvironmentRuntime = {+ // The orchestrator's `releaseForRun` calls this leaf with the mapped lease+ // status. There are no seeded leases, so return an empty release set.+ releaseRunLeases: async (+ heartbeatRunId: string,+ status: "released" | "expired" | "failed",+ ) => {+ releaseLeafCalls.push({ runId: heartbeatRunId, status });+ return [];+ },+ } as unknown as HeartbeatEnvironmentRuntime;+ const heartbeat = heartbeatService(db, { environmentRuntime: fakeEnvironmentRuntime });++ let latestRun = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0] ?? null);+ const statusBeforeTerminalize = latestRun?.status ?? null;+ if (latestRun) latestRun = await heartbeat.terminalizeRunOnLeaseRelease(latestRun);+ // The status production passes into releaseEnvironmentLeasesForRun (:16605).+ const statusThreadedToRelease = latestRun?.status ?? null;+ // The run row as the later release step observes it in the database.+ const dbStatusAtRelease = await runStatus(runId);++ // Execute the real release step exactly as the teardown does at :16601.+ await heartbeat.releaseEnvironmentLeasesForRun({+ runId,+ companyId,+ agentId,+ status: statusThreadedToRelease,+ });+ const orchestratorObservedRunId = releaseLeafCalls.at(-1)?.runId ?? null;+ const orchestratorObservedLeaseStatus = releaseLeafCalls.at(-1)?.status ?? null;++ return {+ statusBeforeTerminalize,+ statusThreadedToRelease,+ dbStatusAtRelease,+ releaseCallCount: releaseLeafCalls.length,+ orchestratorObservedRunId,+ orchestratorObservedLeaseStatus,+ terminalRun: latestRun,+ };+ }++ it("terminalizes a running run to succeeded before release when the issue reached done", async () => {+ const { companyId, agentId, issueId, runId } = await seed({ issueStatus: "done", runStatus: "running" });++ const observed = await runTeardownSequenceObservingRelease({ runId, companyId, agentId });++ // The run was still running before terminalize, but release observes the+ // terminalized status, proving terminalize ran first.+ expect(observed.statusBeforeTerminalize).toBe("running");+ expect(observed.statusThreadedToRelease).toBe("succeeded");+ expect(observed.dbStatusAtRelease).toBe("succeeded");++ // The real orchestrator ran once and received the run id plus the mapped+ // lease status for a succeeded run.+ expect(observed.releaseCallCount).toBe(1);+ expect(observed.orchestratorObservedRunId).toBe(runId);+ expect(observed.orchestratorObservedLeaseStatus).toBe("released");++ // The issue outcome is preserved and the lifecycle event records the reason.+ const issueStatus = await db+ .select({ status: issues.status })+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0]?.status);+ expect(issueStatus).toBe("done");++ const event = await db+ .select({ message: heartbeatRunEvents.message, payload: heartbeatRunEvents.payload })+ .from(heartbeatRunEvents)+ .where(eq(heartbeatRunEvents.runId, runId))+ .then((rows) => rows[0]);+ expect(event?.message).toContain("lease release");+ expect((event?.payload as { terminalStatus?: string } | null)?.terminalStatus).toBe("succeeded");+ });++ it("terminalizes a running run to interrupted before release when the issue is not terminal", async () => {+ const { companyId, agentId, runId } = await seed({ issueStatus: "in_progress", runStatus: "running" });++ const observed = await runTeardownSequenceObservingRelease({ runId, companyId, agentId });++ expect(observed.statusBeforeTerminalize).toBe("running");+ expect(observed.statusThreadedToRelease).toBe("interrupted");+ expect(observed.dbStatusAtRelease).toBe("interrupted");++ // An interrupted run maps to a normal lease release.+ expect(observed.releaseCallCount).toBe(1);+ expect(observed.orchestratorObservedLeaseStatus).toBe("released");++ const row = await db+ .select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0]);+ expect(row?.status).toBe("interrupted");+ expect(row?.errorCode).toBe("lease_released_before_terminal");+ });++ it("terminalizes a still-queued run to interrupted before release", async () => {+ // A queued run holds a lease but never reached running. Release must observe a+ // terminal status, not the queued phantom-live status.+ const { companyId, agentId, runId } = await seed({ issueStatus: "in_progress", runStatus: "queued" });++ const observed = await runTeardownSequenceObservingRelease({ runId, companyId, agentId });++ expect(observed.statusBeforeTerminalize).toBe("queued");+ expect(observed.statusThreadedToRelease).toBe("interrupted");+ expect(observed.dbStatusAtRelease).toBe("interrupted");+ expect(observed.orchestratorObservedLeaseStatus).toBe("released");+ });++ it("threads an already-terminal run's status through unchanged and writes no new event", async () => {+ // When another path already made the run terminal, terminalize is a no-op, so+ // release still observes that authoritative terminal status.+ const { companyId, agentId, runId } = await seed({ issueStatus: "done", runStatus: "failed" });++ const observed = await runTeardownSequenceObservingRelease({ runId, companyId, agentId });++ expect(observed.statusBeforeTerminalize).toBe("failed");+ expect(observed.statusThreadedToRelease).toBe("failed");+ expect(observed.dbStatusAtRelease).toBe("failed");++ // A failed run maps to a failed lease release at the orchestrator.+ expect(observed.orchestratorObservedLeaseStatus).toBe("failed");++ const eventCount = await db+ .select({ id: heartbeatRunEvents.id })+ .from(heartbeatRunEvents)+ .where(eq(heartbeatRunEvents.runId, runId))+ .then((rows) => rows.length);+ expect(eventCount).toBe(0);+ });+});++// Pin the real run-status → lease-release-status mapping the teardown threads+// into the environment orchestrator (heartbeat.ts:16601-16606). The database+// tests above reach the "released" and "failed" branches; this direct test also+// pins the "expired" and "timed_out" branches. It needs no database, so it runs+// on every host.+describe("run-status to lease-release-status mapping", () => {+ it("maps each terminal run status to the lease-release status the orchestrator receives", () => {+ // A normal or in-progress run releases the lease.+ expect(leaseReleaseStatusForRunStatus("succeeded")).toBe("released");+ expect(leaseReleaseStatusForRunStatus("interrupted")).toBe("released");+ expect(leaseReleaseStatusForRunStatus("running")).toBe("released");+ expect(leaseReleaseStatusForRunStatus("queued")).toBe("released");+ expect(leaseReleaseStatusForRunStatus(null)).toBe("released");+ expect(leaseReleaseStatusForRunStatus(undefined)).toBe("released");+ // A failed or timed-out run marks the lease release as failed.+ expect(leaseReleaseStatusForRunStatus("failed")).toBe("failed");+ expect(leaseReleaseStatusForRunStatus("timed_out")).toBe("failed");+ // A cancelled run expires the lease.+ expect(leaseReleaseStatusForRunStatus("cancelled")).toBe("expired");+ });+});diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex a096530402..358fe7357f 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1332,7 +1332,7 @@ async function resolveRunScopedMentionedSkillKeys(input: {.filter((skillKey): skillKey is string => Boolean(skillKey));}-function leaseReleaseStatusForRunStatus(+export function leaseReleaseStatusForRunStatus(status: string | null | undefined,): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {if (status === "cancelled") return "expired";@@ -19274,6 +19274,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})terminalizeRunOnLeaseRelease,+ releaseEnvironmentLeasesForRun,+sweepStaleIssueLocks,buildIssueGraphLivenessAutoRecoveryPreview,