The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runner): require confirmed exit before remote replacement (#16835)
Dottaf3cd3f4ea895
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.mdindex 5593fd17f3..1a63195841 100644--- a/doc/execution-semantics.md+++ b/doc/execution-semantics.md@@ -1551,6 +1551,15 @@ its active readers before closing the manifest and publishes no success receipt.### Warm sandbox continuity+An active remote runner's health probe is an observation, not an exit receipt.+Timed-out or rejected read-only probes retry within the existing 30-second+budget. A successful probe keeps the same runner and turn; no provider work is+replayed. Monitoring checks the saved process generation when available.+Only a matching ownership marker and confirmed process exit permit automatic+replacement. Missing or conflicting identity evidence, failed reads, and an+exhausted probe budget retain the existing ownership and recovery safeguards.+They must not launch a competing listener or overwrite the original marker.+Healthy native warm turns retain their runner and provider processes. After acontroller crash, continuation can recover a completed remote session from itssaved checkpoint and exact sandbox process identity. The prior controller mustdiff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.mdindex a9feb5b406..9eb68f7ef3 100644--- a/maps/agent-access-map/recovery.md+++ b/maps/agent-access-map/recovery.md@@ -36,6 +36,11 @@ Older unmarked preparations do not qualify for this automatic path during anupgrade. The fencing marker comes from fresh server-owned identity creation;caller profile data cannot upgrade an existing preparation.+Remote process-health reads retry within a bounded budget. An unavailable probe+cannot authorize replacement or erase the original ownership marker. This is+internal supervision; it grants no new agent recovery action. Existing task,+company, process-identity and workspace-recovery checks remain required.+Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L9478) atthe reviewed commit above.diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.mdindex 7712f1eed9..677f40fe39 100644--- a/maps/feature-map/recovery.md+++ b/maps/feature-map/recovery.md@@ -110,6 +110,13 @@ provider restart. For that journey, interrupt a disposable deployment afterworkspace preparation and before its first executor claim, then confirm oneprovider turn starts on the original run through the normal admission gates.+Active remote process supervision distinguishes a failed health read from a+confirmed exit. The [executor tests](../../server/src/services/native-runtime/native-session-executor.test.ts)+exercise timeout recovery with a real listening process, unchanged ownership,+and rejected identity evidence. The Runner restart tests require positive exit+evidence before replacement. This coverage does not prove recovery of a+historical retained sandbox or behavior during a live provider outage.+The same database suite pauses an original preparation while recovery claimsthe remote run. It then verifies that the old preparation yields without openinga provider or changing the recovery owner, for both live and expired leases.diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.tsindex 4d44db5c05..b88959e187 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts@@ -31,6 +31,7 @@ import {SemanticToolNotDispatchedError,inspectWarmRunTransition,spawnRunner,+ waitForProcess,type RunnerProcessLaunchSpec,} from "./durable-prp-control-plane.js";import type { DurableRecoveryIdentity } from "./prp-transport-types.js";@@ -393,6 +394,29 @@ it.skipIf(process.platform === "win32")(},);+it("does not release or replace a remote runner until its exit is confirmed", async () => {+ let exited = false;+ const launch = vi.fn(() => ({+ child: { pid: 42, exitCode: null, kill: () => true },+ completion: Promise.resolve({ code: null, signal: null, stdout: "", stderr: "health probe unavailable" }),+ isExitConfirmed: () => exited,+ }));+ const handle = spawnRunner({+ connection: { mode: "connect", connectUrl: "ws://127.0.0.1:43127" },+ stateDirectory: "/tmp/paperclip-runner-exit-test", identity, ticket: "bootstrap-ticket",+ maxOutboxBytes: 256 * 1024, p0ReserveBytes: 64 * 1024,+ runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest,+ processLauncher: launch,+ });+ expect(() => handle.restart!("replacement-ticket")).toThrow("runner_process_termination_unconfirmed");+ await expect(waitForProcess(handle)).rejects.toThrow("runner_process_termination_unconfirmed");+ expect(launch).toHaveBeenCalledTimes(1);+ exited = true;+ await expect(waitForProcess(handle)).resolves.toMatchObject({ code: null });+ handle.restart!("replacement-ticket");+ expect(launch).toHaveBeenCalledTimes(2);+});+it("pins the ACPX launch profile in runner startup arguments and restarts", () => {const launches: RunnerProcessLaunchSpec[] = [];const handle = spawnRunner({diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.tsindex c671e5b778..d86aafdca7 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts@@ -289,6 +289,7 @@ export interface RunnerProcessHandle {completion: Promise<RunnerProcessResult>;processGroupId?: number | null;startedAt?: string;+ isExitConfirmed?(): boolean;/** Relaunches the same immutable process specification with a fresh ticket. */restart?(ticket: string): RunnerProcessHandle;}@@ -3654,7 +3655,12 @@ export function spawnRunner(options: {const environment = runnerEnvironment(options.ticket, options.identity.normalizedSessionId, options.environment);const withRestart = (handle: RunnerProcessHandle): RunnerProcessHandle => ({...handle,- restart: (ticket) => spawnRunner({ ...options, ticket }),+ restart: (ticket) => {+ if (handle.isExitConfirmed?.() === false) {+ throw new Error("runner_process_termination_unconfirmed");+ }+ return spawnRunner({ ...options, ticket });+ },});if (options.processLauncher !== undefined) {return withRestart(@@ -3739,7 +3745,7 @@ export async function waitForProcess(): Promise<RunnerProcessResult> {let timer: NodeJS.Timeout | undefined;try {- return await Promise.race([+ const result = await Promise.race([handle.completion,new Promise<never>((_resolve, reject) => {timer = setTimeout(() => {@@ -3760,6 +3766,10 @@ export async function waitForProcess(}, timeoutMs);}),]);+ if (handle.isExitConfirmed?.() === false) {+ throw new Error("runner_process_termination_unconfirmed");+ }+ return result;} finally {if (timer !== undefined) clearTimeout(timer);}diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.tsindex 445f6eebac..2395aa7ac4 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts@@ -288,6 +288,125 @@ it("replaces an owned v1 runner with fresh v2 authorization before warm attachme}}, 30_000);+it.each(["resolved", "rejected"] as const)(+ "retains a live authenticated runner when remote supervision is %s without exit proof",+ async outcome => {+ const directory = await mkdtemp(join(tmpdir(), "runnerd-uncertain-health-"));+ const children: ReturnType<typeof spawn>[] = [];+ const exits: Promise<void>[] = [];+ const probeError = new Error("remote health probe unavailable");+ let failObservation!: () => void;+ const bundle = createCapabilityRunnerdCodexTransport({+ runnerBinary: defaultCapabilityRunnerdBinary(),+ codexCommand: fakeCodex,+ codexArgs: fakeCodexArgs(directory),+ stateDirectory: directory,+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },+ runnerReconnectGraceMs: 1_000,+ runnerProcessLauncher: spec => {+ const child = spawn(spec.command, [...spec.args], {+ cwd: spec.cwd, env: spec.environment, stdio: "ignore",+ });+ children.push(child);+ exits.push(new Promise(resolveExit => child.once("exit", () => resolveExit())));+ const completion = new Promise<durableControlPlane.RunnerProcessResult>((resolveObservation, rejectObservation) => {+ child.once("error", rejectObservation);+ failObservation = () => {+ if (outcome === "rejected") rejectObservation(probeError);+ else resolveObservation({ code: null, signal: null, stdout: "", stderr: "remote health probe unavailable" });+ };+ });+ return { child, completion, isExitConfirmed: () => false };+ },+ });+ try {+ await bundle.transport.request("thread/start", { cwd: directory });+ const notifications = (async () => {+ for await (const _event of bundle.transport.notifications()) {}+ })();+ const failure = notifications.catch(error => error as Error);+ failObservation();+ const error = await failure;+ expect(error).toMatchObject({ message: expect.stringContaining("runner_process_termination_unconfirmed") });+ if (outcome === "rejected") expect(error).toHaveProperty("cause", probeError);+ expect(children).toHaveLength(1);+ expect(bundle.transport.processInfo()).toMatchObject({ exited: false });+ expect(() => process.kill(children[0]!.pid!, 0)).not.toThrow();+ } finally {+ const providerPid = bundle.evidence().codexPid;+ if (providerPid) {+ try { process.kill(providerPid, "SIGKILL"); } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ await bundle.transport.detachControllerForRestart!();+ for (const child of children) child.kill("SIGKILL");+ await Promise.all(exits);+ await rm(directory, { recursive: true, force: true });+ }+ },+ 15_000,+);++it("preserves the cause when a replacement runner's health cannot be verified", async () => {+ const directory = await mkdtemp(join(tmpdir(), "runnerd-replacement-health-"));+ const probeError = Object.assign(new Error("remote recovery process_verification: probe_timeout"), {+ code: "runner_remote_process_identity_unavailable",+ remoteRunnerRecovery: { stage: "process_verification", reason: "probe_timeout" },+ });+ let child: ReturnType<typeof spawn> | undefined;+ let exited: Promise<void> | undefined;+ const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => {+ if (child) return {+ child: { pid: 42, exitCode: null, kill: () => false },+ completion: Promise.reject(probeError),+ isExitConfirmed: () => false,+ };+ child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" });+ const ownedChild = child;+ exited = new Promise(resolveExit => ownedChild.once("exit", () => resolveExit()));+ return {+ child: ownedChild,+ completion: new Promise<durableControlPlane.RunnerProcessResult>((resolveExit, rejectExit) => {+ ownedChild.once("error", rejectExit);+ ownedChild.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" }));+ }),+ isExitConfirmed: () => ownedChild.exitCode !== null || ownedChild.signalCode !== null,+ };+ });+ const bundle = createCapabilityRunnerdCodexTransport({+ runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex,+ codexArgs: fakeCodexArgs(directory), stateDirectory: directory,+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },+ runnerReconnectGraceMs: 1_000, runnerProcessLauncher: launch,+ });+ try {+ await bundle.transport.request("thread/start", { cwd: directory });+ const failed = (async () => {+ for await (const _event of bundle.transport.notifications()) {}+ })().catch(error => error as Error);+ child!.kill("SIGKILL");+ const error = await failed;+ expect(error).toMatchObject({+ message: expect.stringContaining("runner_process_termination_unconfirmed"), cause: probeError,+ });+ expect(error).toHaveProperty("message", expect.stringContaining("probe_timeout"));+ expect(launch).toHaveBeenCalledTimes(2);+ expect(bundle.transport.processInfo()).toMatchObject({ exited: false });+ } finally {+ const providerPid = bundle.evidence().codexPid;+ if (providerPid) {+ try { process.kill(providerPid, "SIGKILL"); } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;+ }+ }+ await bundle.transport.detachControllerForRestart!();+ child?.kill("SIGKILL");+ await exited;+ await rm(directory, { recursive: true, force: true });+ }+}, 15_000);+// The retained-settlement maintenance family ("settles only retained control// authority without starting another provider turn") lives in// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s ofdiff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.tsindex e65db3b7ad..4d76126b57 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts@@ -6864,7 +6864,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {}async #runnerHasExited(): Promise<boolean> {- if (this.#handle) return this.#handle.child.exitCode !== null;+ if (this.#handle) return this.#handle.isExitConfirmed?.() ?? this.#handle.child.exitCode !== null;const adoptedRunner = this.options.adoptExistingRunner;if (!adoptedRunner) return true;try {@@ -6948,7 +6948,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {#watchRunner(handle: RunnerProcessHandle): void {void handle.completion.then((result) => {- this.#evidence.runnerExited = true;+ const exitConfirmed = handle.isExitConfirmed?.() ?? true;+ this.#evidence.runnerExited = exitConfirmed;this.#evidence.runnerExitCode = result.code;this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null;const detail = result.stderr.trim() || result.stdout.trim();@@ -6961,6 +6962,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {// process completion; clean terminal exit is the expected lifecycle.this.#pumpEventsSafely();const expectedPerTurnExit =+ exitConfirmed &&result.code === 0 &&(this.options.lifecyclePolicy?.mode ?? "per_turn") === "per_turn" &&(this.#core?.store.state.committedEvents.some(@@ -6968,19 +6970,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {) ??false);if (expectedPerTurnExit) return;- const code = /provider_frame_too_large|stdout frame exceeded/i.test(- detail,- )- ? "provider_frame_too_large"- : /runner_ingress_bind_conflict/i.test(detail)- ? "runner_ingress_bind_conflict"- : /transport_reconnect_grace_exceeded/i.test(detail)- ? "transport_reconnect_grace_exceeded"- : this.#startupComplete- ? "native_runner_process_exited"- : this.#startupFailureCode;+ const code = !exitConfirmed+ ? "runner_process_termination_unconfirmed"+ : /provider_frame_too_large|stdout frame exceeded/i.test(detail)+ ? "provider_frame_too_large"+ : /runner_ingress_bind_conflict/i.test(detail)+ ? "runner_ingress_bind_conflict"+ : /transport_reconnect_grace_exceeded/i.test(detail)+ ? "transport_reconnect_grace_exceeded"+ : this.#startupComplete+ ? "native_runner_process_exited"+ : this.#startupFailureCode;if (code === "native_runner_process_exited" &&+ exitConfirmed &&this.options.runnerReconnectGraceMs !== undefined &&handle.restart) {@@ -6989,7 +6992,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {}this.#failTransport(new Error(- `${code}: runnerd exited unexpectedly${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`,+ `${code}: runnerd ${exitConfirmed ? "exited unexpectedly" : "process supervision ended"}${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`,),);},@@ -6998,6 +7001,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {return;if (this.#startupComplete &&+ handle.isExitConfirmed?.() !== false &&this.options.runnerReconnectGraceMs !== undefined &&handle.restart) {@@ -7007,9 +7011,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {);return;}+ const code = handle.isExitConfirmed?.() === false+ ? "runner_process_termination_unconfirmed"+ : this.#startupComplete+ ? "native_runner_process_exited"+ : this.#startupFailureCode;this.#failTransport(new Error(- `${this.#startupComplete ? "native_runner_process_exited" : this.#startupFailureCode}: runnerd process failed: ${error instanceof Error ? error.message : String(error)}`,+ `${code}: runnerd ${handle.isExitConfirmed?.() === false ? "process supervision" : "process"} failed: ${error instanceof Error ? error.message : String(error)}`,+ { cause: error },),);},@@ -7073,15 +7083,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {this.#publish();let processSettled = false;+ let observationError: unknown;+ let observationDetail = "";const completion = recoveredHandle.completion.then((result) => {processSettled = true;- lastDetail = result.stderr.trim() || result.stdout.trim();+ observationDetail = result.stderr.trim() || result.stdout.trim();+ lastDetail = observationDetail;return false;},(error) => {processSettled = true;- lastDetail = error instanceof Error ? error.message : String(error);+ observationError = error;+ observationDetail = error instanceof Error ? error.message : String(error);+ lastDetail = observationDetail;return false;},);@@ -7107,6 +7122,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {this.#watchRunner(recoveredHandle);return;}+ if (recoveredHandle.isExitConfirmed?.() === false) {+ this.#failTransport(+ new Error(+ `runner_process_termination_unconfirmed: replacement runner health could not be verified${observationDetail ? `: ${observationDetail.slice(-1_000)}` : ""}`,+ { cause: observationError },+ ),+ );+ return;+ }attempt += 1;}if (!this.#closed) {diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.tsindex db4fba7f90..0c9a546214 100644--- a/server/src/services/native-runtime/native-session-executor.test.ts+++ b/server/src/services/native-runtime/native-session-executor.test.ts@@ -22,7 +22,7 @@ import {truncate,writeFile,} from "node:fs/promises";-import { execFileSync } from "node:child_process";+import { execFile, execFileSync, spawn } from "node:child_process";import { tmpdir } from "node:os";import { inspect } from "node:util";import { join } from "node:path";@@ -779,6 +779,143 @@ describe("remote runner launch fingerprint compatibility", () => {});describe("remote runner process supervision", () => {+ it("preserves a real listening process and its marker when a health RPC times out", async () => {+ const root = await mkdtemp(join(tmpdir(), "runner-health-listener-"));+ const marker = join(root, "identity");+ const fixture = join(root, "listener.cjs");+ await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));');+ let child: ReturnType<typeof spawn> | undefined;+ let probes = 0;+ let settled = false;+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };+ const execute = vi.fn(async (input: { command: string; args?: string[] }) => {+ const label = input.args?.[2];+ if (label === "paperclip-runner-launch") {+ child = spawn(process.execPath, [fixture, "--runner-id", "runner-health"], { stdio: ["ignore", "pipe", "pipe"] });+ await new Promise<void>((resolve, reject) => {+ child!.stdout!.once("data", () => resolve());+ child!.once("error", reject);+ child!.once("exit", code => reject(new Error(`listener fixture exited before readiness: ${code}`)));+ });+ await writeFile(marker, `${input.args![4]}\n${child.pid}\n2026-10-19T00:00:00.000Z\nrunner-health\n`);+ return ok;+ }+ if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true };+ return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {+ execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({+ exitCode: error ? typeof error.code === "number" ? error.code : null : 0,+ signal: null, timedOut: !!error?.killed, stdout, stderr,+ }));+ });+ });+ const handle = createRemoteRunnerProcessLauncher({+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root },+ runner: { execute } as never, remoteBinary: fixture, processIdentityPath: marker,+ stateDirectory: root, diagnosticsDirectory: join(root, "diagnostics"), runnerInstanceId: "runner-health",+ })({ command: fixture, args: [], cwd: root, environment: {} });+ void handle.completion.then(() => { settled = true; }, () => { settled = true; });+ try {+ await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 });+ expect(settled).toBe(false);+ expect(handle.isExitConfirmed?.()).toBe(false);+ expect(() => process.kill(child!.pid!, 0)).not.toThrow();+ expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid));+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);+ child!.kill("SIGTERM");+ await handle.completion;+ expect(handle.isExitConfirmed?.()).toBe(true);+ } finally {+ child?.kill("SIGKILL");+ await handle.completion.catch(() => undefined);+ await rm(root, { recursive: true, force: true });+ }+ });++ it.each([+ [41, "marker_missing"], [44, "process_identity_changed"], [47, "process_generation_changed"], [1, "probe_failed"],+ ] as const)("does not authorize replacement from health probe exit %s", async (exitCode, reason) => {+ let nonce = "";+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };+ const execute = vi.fn(async (input: { args?: string[] }) => {+ const label = input.args?.[2];+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; }+ if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };+ return { ...ok, exitCode };+ });+ const handle = createRemoteRunnerProcessLauncher({+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",+ })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });+ expect(await handle.completion.catch(readRemoteRunnerRecoveryError)).toEqual({ stage: "process_verification", reason });+ expect(handle.isExitConfirmed?.()).toBe(false);+ });++ it.each(["timeout", "transport"])("keeps the same runner through a transient %s health probe", async failure => {+ vi.useFakeTimers();+ let nonce = "";+ let probes = 0;+ let settled = false;+ const result = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };+ const execute = vi.fn(async (input: { args?: string[] }) => {+ const label = input.args?.[2];+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return result; }+ if (label === "paperclip-runner-process-identity") return { ...result, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };+ if (label === "paperclip-runner-monitor") {+ probes++;+ if (probes === 1) {+ if (failure === "transport") throw new Error("sandbox request timed out");+ return { ...result, timedOut: true, exitCode: null };+ }+ return { ...result, exitCode: probes === 2 ? 0 : 45 };+ }+ return { ...result, exitCode: 1 };+ });+ const launcher = createRemoteRunnerProcessLauncher({+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",+ });+ const handle = launcher({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });+ void handle.completion.then(() => { settled = true; }, () => { settled = true; });+ try {+ await vi.advanceTimersByTimeAsync(300);+ expect(settled).toBe(false);+ expect(probes).toBe(2);+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);+ expect(execute.mock.calls.some(([x]) => x.args?.[2] === "paperclip-runner-diagnostics")).toBe(false);+ await vi.advanceTimersByTimeAsync(1_000);+ await expect(handle.completion).resolves.toMatchObject({ code: null });+ } finally { vi.useRealTimers(); }+ });++ it("bounds failed health reads without confirming exit or replacing the marker", async () => {+ vi.useFakeTimers();+ let nonce = "";+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };+ const execute = vi.fn(async (input: { args?: string[]; timeoutMs?: number }) => {+ const label = input.args?.[2];+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; }+ if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };+ await new Promise(resolve => setTimeout(resolve, input.timeoutMs));+ return { ...ok, exitCode: null, timedOut: true };+ });+ const handle = createRemoteRunnerProcessLauncher({+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",+ })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });+ const failure = handle.completion.catch(readRemoteRunnerRecoveryError);+ try {+ await vi.advanceTimersByTimeAsync(30_000);+ expect(await failure).toEqual({ stage: "process_verification", reason: "probe_timeout" });+ expect(handle.isExitConfirmed?.()).toBe(false);+ const probes = execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-monitor");+ expect(probes.map(([x]) => x.timeoutMs)).toEqual([10_000, 10_000, 9_500]);+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);+ } finally { vi.useRealTimers(); }+ });+it.each(["delivered", "sandbox_missing", "logging_failed"] as const)("detaches runnerd and contains asynchronous signal failures (%s)", async (signalOutcome) => {let launchNonce = "";@@ -812,7 +949,7 @@ describe("remote runner process supervision", () => {}if (label === "paperclip-runner-monitor") {return {- exitCode: 3,+ exitCode: 45,signal: null,timedOut: false,stdout: "",diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.tsindex 4f332c53ae..0684682854 100644--- a/server/src/services/native-runtime/native-session-executor.ts+++ b/server/src/services/native-runtime/native-session-executor.ts@@ -11013,6 +11013,7 @@ export function createRemoteRunnerProcessLauncher(input: {}): (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle {const runner = input.runner;return (spec) => {+ let exitConfirmed = false;let launchedIdentity: {nonce: string;pid: number;@@ -11130,7 +11131,7 @@ export function createRemoteRunnerProcessLauncher(input: {: "runner_remote_process_launch_failed",);}- let identity: { pid: number; startedAt: string };+ let identity: Awaited<ReturnType<typeof waitForRemoteRunnerProcessIdentity>>;try {identity = await waitForRemoteRunnerProcessIdentity({runner,@@ -11177,19 +11178,27 @@ export function createRemoteRunnerProcessLauncher(input: {});while (true) {- const observed = await runner.execute({- command: "sh",- args: [- "-c",- REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT,- "paperclip-runner-monitor",- input.processIdentityPath,- identityNonce,- input.runnerInstanceId,- String(identity.pid),- ],- bypassSession: true,- timeoutMs: 10_000,+ const observed = await readRemoteRunnerRecoveryProbe({+ runner,+ stage: "process_verification",+ deadline: Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS,+ readOnly: true,+ request: {+ command: "sh",+ args: [+ "-c",+ REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + (identity.processStartFingerprint+ ? `; if ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; then :; else exit 46; fi; test "$process_fingerprint" = "$5" || exit 47`+ : ""),+ "paperclip-runner-monitor",+ input.processIdentityPath,+ identityNonce,+ input.runnerInstanceId,+ String(identity.pid),+ ...(identity.processStartFingerprint ? [identity.processStartFingerprint] : []),+ ],+ bypassSession: true,+ },});if (observed.exitCode === 0 && !observed.timedOut) {await new Promise<void>((resolve) =>@@ -11197,8 +11206,14 @@ export function createRemoteRunnerProcessLauncher(input: {);continue;}+ if (observed.exitCode !== 45) {+ throw new RemoteRunnerRecoveryError({+ stage: "process_verification",+ reason: remoteRecoveryProbeReason("process_verification", observed.exitCode, observed.timedOut),+ });+ }+ exitConfirmed = true;child.exitCode = null;- const identityMismatch = observed.exitCode === 4;const diagnostic = await runner.execute({command: "sh",@@ -11241,13 +11256,11 @@ export function createRemoteRunnerProcessLauncher(input: {code: null,signal: null,stdout: "",- stderr: identityMismatch- ? "runner_remote_process_identity_mismatch"- : diagnosticTail || stateSummary,+ stderr: diagnosticTail || stateSummary,};}})();- return { child, completion };+ return { child, completion, isExitConfirmed: () => exitConfirmed };};}