Skip to content

The open-source app everyone uses to manage agents at work

fix(runner): require confirmed exit before remote replacement (#16835)

Dottaf3cd3f4ea895
diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md
index 5593fd17f3..1a63195841 100644
--- a/doc/execution-semantics.md
+++ b/doc/execution-semantics.md
@@ -1551,6 +1551,15 @@ its active readers before closing the manifest and publishes no success receipt.
### Warm sandbox continuity
+An active remote runner's health probe is an observation, not an exit receipt.
+Timed-out or rejected read-only probes retry within the existing 30-second
+budget. A successful probe keeps the same runner and turn; no provider work is
+replayed. Monitoring checks the saved process generation when available.
+Only a matching ownership marker and confirmed process exit permit automatic
+replacement. Missing or conflicting identity evidence, failed reads, and an
+exhausted probe budget retain the existing ownership and recovery safeguards.
+They must not launch a competing listener or overwrite the original marker.
+
Healthy native warm turns retain their runner and provider processes. After a
controller crash, continuation can recover a completed remote session from its
saved checkpoint and exact sandbox process identity. The prior controller must
diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md
index a9feb5b406..9eb68f7ef3 100644
--- a/maps/agent-access-map/recovery.md
+++ b/maps/agent-access-map/recovery.md
@@ -36,6 +36,11 @@ Older unmarked preparations do not qualify for this automatic path during an
upgrade. The fencing marker comes from fresh server-owned identity creation;
caller profile data cannot upgrade an existing preparation.
+Remote process-health reads retry within a bounded budget. An unavailable probe
+cannot authorize replacement or erase the original ownership marker. This is
+internal supervision; it grants no new agent recovery action. Existing task,
+company, process-identity and workspace-recovery checks remain required.
+
Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L9478) at
the reviewed commit above.
diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md
index 7712f1eed9..677f40fe39 100644
--- a/maps/feature-map/recovery.md
+++ b/maps/feature-map/recovery.md
@@ -110,6 +110,13 @@ provider restart. For that journey, interrupt a disposable deployment after
workspace preparation and before its first executor claim, then confirm one
provider turn starts on the original run through the normal admission gates.
+Active remote process supervision distinguishes a failed health read from a
+confirmed exit. The [executor tests](../../server/src/services/native-runtime/native-session-executor.test.ts)
+exercise timeout recovery with a real listening process, unchanged ownership,
+and rejected identity evidence. The Runner restart tests require positive exit
+evidence before replacement. This coverage does not prove recovery of a
+historical retained sandbox or behavior during a live provider outage.
+
The same database suite pauses an original preparation while recovery claims
the remote run. It then verifies that the old preparation yields without opening
a provider or changing the recovery owner, for both live and expired leases.
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
index 4d44db5c05..b88959e187 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts
@@ -31,6 +31,7 @@ import {
SemanticToolNotDispatchedError,
inspectWarmRunTransition,
spawnRunner,
+ waitForProcess,
type RunnerProcessLaunchSpec,
} from "./durable-prp-control-plane.js";
import type { DurableRecoveryIdentity } from "./prp-transport-types.js";
@@ -393,6 +394,29 @@ it.skipIf(process.platform === "win32")(
},
);
+it("does not release or replace a remote runner until its exit is confirmed", async () => {
+ let exited = false;
+ const launch = vi.fn(() => ({
+ child: { pid: 42, exitCode: null, kill: () => true },
+ completion: Promise.resolve({ code: null, signal: null, stdout: "", stderr: "health probe unavailable" }),
+ isExitConfirmed: () => exited,
+ }));
+ const handle = spawnRunner({
+ connection: { mode: "connect", connectUrl: "ws://127.0.0.1:43127" },
+ stateDirectory: "/tmp/paperclip-runner-exit-test", identity, ticket: "bootstrap-ticket",
+ maxOutboxBytes: 256 * 1024, p0ReserveBytes: 64 * 1024,
+ runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest,
+ processLauncher: launch,
+ });
+ expect(() => handle.restart!("replacement-ticket")).toThrow("runner_process_termination_unconfirmed");
+ await expect(waitForProcess(handle)).rejects.toThrow("runner_process_termination_unconfirmed");
+ expect(launch).toHaveBeenCalledTimes(1);
+ exited = true;
+ await expect(waitForProcess(handle)).resolves.toMatchObject({ code: null });
+ handle.restart!("replacement-ticket");
+ expect(launch).toHaveBeenCalledTimes(2);
+});
+
it("pins the ACPX launch profile in runner startup arguments and restarts", () => {
const launches: RunnerProcessLaunchSpec[] = [];
const handle = spawnRunner({
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
index c671e5b778..d86aafdca7 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
@@ -289,6 +289,7 @@ export interface RunnerProcessHandle {
completion: Promise<RunnerProcessResult>;
processGroupId?: number | null;
startedAt?: string;
+ isExitConfirmed?(): boolean;
/** Relaunches the same immutable process specification with a fresh ticket. */
restart?(ticket: string): RunnerProcessHandle;
}
@@ -3654,7 +3655,12 @@ export function spawnRunner(options: {
const environment = runnerEnvironment(options.ticket, options.identity.normalizedSessionId, options.environment);
const withRestart = (handle: RunnerProcessHandle): RunnerProcessHandle => ({
...handle,
- restart: (ticket) => spawnRunner({ ...options, ticket }),
+ restart: (ticket) => {
+ if (handle.isExitConfirmed?.() === false) {
+ throw new Error("runner_process_termination_unconfirmed");
+ }
+ return spawnRunner({ ...options, ticket });
+ },
});
if (options.processLauncher !== undefined) {
return withRestart(
@@ -3739,7 +3745,7 @@ export async function waitForProcess(
): Promise<RunnerProcessResult> {
let timer: NodeJS.Timeout | undefined;
try {
- return await Promise.race([
+ const result = await Promise.race([
handle.completion,
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
@@ -3760,6 +3766,10 @@ export async function waitForProcess(
}, timeoutMs);
}),
]);
+ if (handle.isExitConfirmed?.() === false) {
+ throw new Error("runner_process_termination_unconfirmed");
+ }
+ return result;
} finally {
if (timer !== undefined) clearTimeout(timer);
}
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
index 445f6eebac..2395aa7ac4 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
@@ -288,6 +288,125 @@ it("replaces an owned v1 runner with fresh v2 authorization before warm attachme
}
}, 30_000);
+it.each(["resolved", "rejected"] as const)(
+ "retains a live authenticated runner when remote supervision is %s without exit proof",
+ async outcome => {
+ const directory = await mkdtemp(join(tmpdir(), "runnerd-uncertain-health-"));
+ const children: ReturnType<typeof spawn>[] = [];
+ const exits: Promise<void>[] = [];
+ const probeError = new Error("remote health probe unavailable");
+ let failObservation!: () => void;
+ const bundle = createCapabilityRunnerdCodexTransport({
+ runnerBinary: defaultCapabilityRunnerdBinary(),
+ codexCommand: fakeCodex,
+ codexArgs: fakeCodexArgs(directory),
+ stateDirectory: directory,
+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
+ runnerReconnectGraceMs: 1_000,
+ runnerProcessLauncher: spec => {
+ const child = spawn(spec.command, [...spec.args], {
+ cwd: spec.cwd, env: spec.environment, stdio: "ignore",
+ });
+ children.push(child);
+ exits.push(new Promise(resolveExit => child.once("exit", () => resolveExit())));
+ const completion = new Promise<durableControlPlane.RunnerProcessResult>((resolveObservation, rejectObservation) => {
+ child.once("error", rejectObservation);
+ failObservation = () => {
+ if (outcome === "rejected") rejectObservation(probeError);
+ else resolveObservation({ code: null, signal: null, stdout: "", stderr: "remote health probe unavailable" });
+ };
+ });
+ return { child, completion, isExitConfirmed: () => false };
+ },
+ });
+ try {
+ await bundle.transport.request("thread/start", { cwd: directory });
+ const notifications = (async () => {
+ for await (const _event of bundle.transport.notifications()) {}
+ })();
+ const failure = notifications.catch(error => error as Error);
+ failObservation();
+ const error = await failure;
+ expect(error).toMatchObject({ message: expect.stringContaining("runner_process_termination_unconfirmed") });
+ if (outcome === "rejected") expect(error).toHaveProperty("cause", probeError);
+ expect(children).toHaveLength(1);
+ expect(bundle.transport.processInfo()).toMatchObject({ exited: false });
+ expect(() => process.kill(children[0]!.pid!, 0)).not.toThrow();
+ } finally {
+ const providerPid = bundle.evidence().codexPid;
+ if (providerPid) {
+ try { process.kill(providerPid, "SIGKILL"); } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+ }
+ }
+ await bundle.transport.detachControllerForRestart!();
+ for (const child of children) child.kill("SIGKILL");
+ await Promise.all(exits);
+ await rm(directory, { recursive: true, force: true });
+ }
+ },
+ 15_000,
+);
+
+it("preserves the cause when a replacement runner's health cannot be verified", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "runnerd-replacement-health-"));
+ const probeError = Object.assign(new Error("remote recovery process_verification: probe_timeout"), {
+ code: "runner_remote_process_identity_unavailable",
+ remoteRunnerRecovery: { stage: "process_verification", reason: "probe_timeout" },
+ });
+ let child: ReturnType<typeof spawn> | undefined;
+ let exited: Promise<void> | undefined;
+ const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => {
+ if (child) return {
+ child: { pid: 42, exitCode: null, kill: () => false },
+ completion: Promise.reject(probeError),
+ isExitConfirmed: () => false,
+ };
+ child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" });
+ const ownedChild = child;
+ exited = new Promise(resolveExit => ownedChild.once("exit", () => resolveExit()));
+ return {
+ child: ownedChild,
+ completion: new Promise<durableControlPlane.RunnerProcessResult>((resolveExit, rejectExit) => {
+ ownedChild.once("error", rejectExit);
+ ownedChild.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" }));
+ }),
+ isExitConfirmed: () => ownedChild.exitCode !== null || ownedChild.signalCode !== null,
+ };
+ });
+ const bundle = createCapabilityRunnerdCodexTransport({
+ runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex,
+ codexArgs: fakeCodexArgs(directory), stateDirectory: directory,
+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
+ runnerReconnectGraceMs: 1_000, runnerProcessLauncher: launch,
+ });
+ try {
+ await bundle.transport.request("thread/start", { cwd: directory });
+ const failed = (async () => {
+ for await (const _event of bundle.transport.notifications()) {}
+ })().catch(error => error as Error);
+ child!.kill("SIGKILL");
+ const error = await failed;
+ expect(error).toMatchObject({
+ message: expect.stringContaining("runner_process_termination_unconfirmed"), cause: probeError,
+ });
+ expect(error).toHaveProperty("message", expect.stringContaining("probe_timeout"));
+ expect(launch).toHaveBeenCalledTimes(2);
+ expect(bundle.transport.processInfo()).toMatchObject({ exited: false });
+ } finally {
+ const providerPid = bundle.evidence().codexPid;
+ if (providerPid) {
+ try { process.kill(providerPid, "SIGKILL"); } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+ }
+ }
+ await bundle.transport.detachControllerForRestart!();
+ child?.kill("SIGKILL");
+ await exited;
+ await rm(directory, { recursive: true, force: true });
+ }
+}, 15_000);
+
// The retained-settlement maintenance family ("settles only retained control
// authority without starting another provider turn") lives in
// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
index e65db3b7ad..4d76126b57 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
@@ -6864,7 +6864,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
async #runnerHasExited(): Promise<boolean> {
- if (this.#handle) return this.#handle.child.exitCode !== null;
+ if (this.#handle) return this.#handle.isExitConfirmed?.() ?? this.#handle.child.exitCode !== null;
const adoptedRunner = this.options.adoptExistingRunner;
if (!adoptedRunner) return true;
try {
@@ -6948,7 +6948,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#watchRunner(handle: RunnerProcessHandle): void {
void handle.completion.then(
(result) => {
- this.#evidence.runnerExited = true;
+ const exitConfirmed = handle.isExitConfirmed?.() ?? true;
+ this.#evidence.runnerExited = exitConfirmed;
this.#evidence.runnerExitCode = result.code;
this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null;
const detail = result.stderr.trim() || result.stdout.trim();
@@ -6961,6 +6962,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
// process completion; clean terminal exit is the expected lifecycle.
this.#pumpEventsSafely();
const expectedPerTurnExit =
+ exitConfirmed &&
result.code === 0 &&
(this.options.lifecyclePolicy?.mode ?? "per_turn") === "per_turn" &&
(this.#core?.store.state.committedEvents.some(
@@ -6968,19 +6970,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
) ??
false);
if (expectedPerTurnExit) return;
- const code = /provider_frame_too_large|stdout frame exceeded/i.test(
- detail,
- )
- ? "provider_frame_too_large"
- : /runner_ingress_bind_conflict/i.test(detail)
- ? "runner_ingress_bind_conflict"
- : /transport_reconnect_grace_exceeded/i.test(detail)
- ? "transport_reconnect_grace_exceeded"
- : this.#startupComplete
- ? "native_runner_process_exited"
- : this.#startupFailureCode;
+ const code = !exitConfirmed
+ ? "runner_process_termination_unconfirmed"
+ : /provider_frame_too_large|stdout frame exceeded/i.test(detail)
+ ? "provider_frame_too_large"
+ : /runner_ingress_bind_conflict/i.test(detail)
+ ? "runner_ingress_bind_conflict"
+ : /transport_reconnect_grace_exceeded/i.test(detail)
+ ? "transport_reconnect_grace_exceeded"
+ : this.#startupComplete
+ ? "native_runner_process_exited"
+ : this.#startupFailureCode;
if (
code === "native_runner_process_exited" &&
+ exitConfirmed &&
this.options.runnerReconnectGraceMs !== undefined &&
handle.restart
) {
@@ -6989,7 +6992,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
}
this.#failTransport(
new Error(
- `${code}: runnerd exited unexpectedly${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`,
+ `${code}: runnerd ${exitConfirmed ? "exited unexpectedly" : "process supervision ended"}${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`,
),
);
},
@@ -6998,6 +7001,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
return;
if (
this.#startupComplete &&
+ handle.isExitConfirmed?.() !== false &&
this.options.runnerReconnectGraceMs !== undefined &&
handle.restart
) {
@@ -7007,9 +7011,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
);
return;
}
+ const code = handle.isExitConfirmed?.() === false
+ ? "runner_process_termination_unconfirmed"
+ : this.#startupComplete
+ ? "native_runner_process_exited"
+ : this.#startupFailureCode;
this.#failTransport(
new Error(
- `${this.#startupComplete ? "native_runner_process_exited" : this.#startupFailureCode}: runnerd process failed: ${error instanceof Error ? error.message : String(error)}`,
+ `${code}: runnerd ${handle.isExitConfirmed?.() === false ? "process supervision" : "process"} failed: ${error instanceof Error ? error.message : String(error)}`,
+ { cause: error },
),
);
},
@@ -7073,15 +7083,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#publish();
let processSettled = false;
+ let observationError: unknown;
+ let observationDetail = "";
const completion = recoveredHandle.completion.then(
(result) => {
processSettled = true;
- lastDetail = result.stderr.trim() || result.stdout.trim();
+ observationDetail = result.stderr.trim() || result.stdout.trim();
+ lastDetail = observationDetail;
return false;
},
(error) => {
processSettled = true;
- lastDetail = error instanceof Error ? error.message : String(error);
+ observationError = error;
+ observationDetail = error instanceof Error ? error.message : String(error);
+ lastDetail = observationDetail;
return false;
},
);
@@ -7107,6 +7122,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#watchRunner(recoveredHandle);
return;
}
+ if (recoveredHandle.isExitConfirmed?.() === false) {
+ this.#failTransport(
+ new Error(
+ `runner_process_termination_unconfirmed: replacement runner health could not be verified${observationDetail ? `: ${observationDetail.slice(-1_000)}` : ""}`,
+ { cause: observationError },
+ ),
+ );
+ return;
+ }
attempt += 1;
}
if (!this.#closed) {
diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts
index db4fba7f90..0c9a546214 100644
--- a/server/src/services/native-runtime/native-session-executor.test.ts
+++ b/server/src/services/native-runtime/native-session-executor.test.ts
@@ -22,7 +22,7 @@ import {
truncate,
writeFile,
} from "node:fs/promises";
-import { execFileSync } from "node:child_process";
+import { execFile, execFileSync, spawn } from "node:child_process";
import { tmpdir } from "node:os";
import { inspect } from "node:util";
import { join } from "node:path";
@@ -779,6 +779,143 @@ describe("remote runner launch fingerprint compatibility", () => {
});
describe("remote runner process supervision", () => {
+ it("preserves a real listening process and its marker when a health RPC times out", async () => {
+ const root = await mkdtemp(join(tmpdir(), "runner-health-listener-"));
+ const marker = join(root, "identity");
+ const fixture = join(root, "listener.cjs");
+ await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));');
+ let child: ReturnType<typeof spawn> | undefined;
+ let probes = 0;
+ let settled = false;
+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
+ const execute = vi.fn(async (input: { command: string; args?: string[] }) => {
+ const label = input.args?.[2];
+ if (label === "paperclip-runner-launch") {
+ child = spawn(process.execPath, [fixture, "--runner-id", "runner-health"], { stdio: ["ignore", "pipe", "pipe"] });
+ await new Promise<void>((resolve, reject) => {
+ child!.stdout!.once("data", () => resolve());
+ child!.once("error", reject);
+ child!.once("exit", code => reject(new Error(`listener fixture exited before readiness: ${code}`)));
+ });
+ await writeFile(marker, `${input.args![4]}\n${child.pid}\n2026-10-19T00:00:00.000Z\nrunner-health\n`);
+ return ok;
+ }
+ if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true };
+ return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {
+ execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({
+ exitCode: error ? typeof error.code === "number" ? error.code : null : 0,
+ signal: null, timedOut: !!error?.killed, stdout, stderr,
+ }));
+ });
+ });
+ const handle = createRemoteRunnerProcessLauncher({
+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root },
+ runner: { execute } as never, remoteBinary: fixture, processIdentityPath: marker,
+ stateDirectory: root, diagnosticsDirectory: join(root, "diagnostics"), runnerInstanceId: "runner-health",
+ })({ command: fixture, args: [], cwd: root, environment: {} });
+ void handle.completion.then(() => { settled = true; }, () => { settled = true; });
+ try {
+ await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 });
+ expect(settled).toBe(false);
+ expect(handle.isExitConfirmed?.()).toBe(false);
+ expect(() => process.kill(child!.pid!, 0)).not.toThrow();
+ expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid));
+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);
+ child!.kill("SIGTERM");
+ await handle.completion;
+ expect(handle.isExitConfirmed?.()).toBe(true);
+ } finally {
+ child?.kill("SIGKILL");
+ await handle.completion.catch(() => undefined);
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+
+ it.each([
+ [41, "marker_missing"], [44, "process_identity_changed"], [47, "process_generation_changed"], [1, "probe_failed"],
+ ] as const)("does not authorize replacement from health probe exit %s", async (exitCode, reason) => {
+ let nonce = "";
+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
+ const execute = vi.fn(async (input: { args?: string[] }) => {
+ const label = input.args?.[2];
+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; }
+ if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };
+ return { ...ok, exitCode };
+ });
+ const handle = createRemoteRunnerProcessLauncher({
+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },
+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",
+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",
+ })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });
+ expect(await handle.completion.catch(readRemoteRunnerRecoveryError)).toEqual({ stage: "process_verification", reason });
+ expect(handle.isExitConfirmed?.()).toBe(false);
+ });
+
+ it.each(["timeout", "transport"])("keeps the same runner through a transient %s health probe", async failure => {
+ vi.useFakeTimers();
+ let nonce = "";
+ let probes = 0;
+ let settled = false;
+ const result = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
+ const execute = vi.fn(async (input: { args?: string[] }) => {
+ const label = input.args?.[2];
+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return result; }
+ if (label === "paperclip-runner-process-identity") return { ...result, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };
+ if (label === "paperclip-runner-monitor") {
+ probes++;
+ if (probes === 1) {
+ if (failure === "transport") throw new Error("sandbox request timed out");
+ return { ...result, timedOut: true, exitCode: null };
+ }
+ return { ...result, exitCode: probes === 2 ? 0 : 45 };
+ }
+ return { ...result, exitCode: 1 };
+ });
+ const launcher = createRemoteRunnerProcessLauncher({
+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },
+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",
+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",
+ });
+ const handle = launcher({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });
+ void handle.completion.then(() => { settled = true; }, () => { settled = true; });
+ try {
+ await vi.advanceTimersByTimeAsync(300);
+ expect(settled).toBe(false);
+ expect(probes).toBe(2);
+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);
+ expect(execute.mock.calls.some(([x]) => x.args?.[2] === "paperclip-runner-diagnostics")).toBe(false);
+ await vi.advanceTimersByTimeAsync(1_000);
+ await expect(handle.completion).resolves.toMatchObject({ code: null });
+ } finally { vi.useRealTimers(); }
+ });
+
+ it("bounds failed health reads without confirming exit or replacing the marker", async () => {
+ vi.useFakeTimers();
+ let nonce = "";
+ const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
+ const execute = vi.fn(async (input: { args?: string[]; timeoutMs?: number }) => {
+ const label = input.args?.[2];
+ if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; }
+ if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` };
+ await new Promise(resolve => setTimeout(resolve, input.timeoutMs));
+ return { ...ok, exitCode: null, timedOut: true };
+ });
+ const handle = createRemoteRunnerProcessLauncher({
+ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" },
+ runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity",
+ stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote",
+ })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} });
+ const failure = handle.completion.catch(readRemoteRunnerRecoveryError);
+ try {
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(await failure).toEqual({ stage: "process_verification", reason: "probe_timeout" });
+ expect(handle.isExitConfirmed?.()).toBe(false);
+ const probes = execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-monitor");
+ expect(probes.map(([x]) => x.timeoutMs)).toEqual([10_000, 10_000, 9_500]);
+ expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);
+ } finally { vi.useRealTimers(); }
+ });
+
it.each(["delivered", "sandbox_missing", "logging_failed"] as const)(
"detaches runnerd and contains asynchronous signal failures (%s)", async (signalOutcome) => {
let launchNonce = "";
@@ -812,7 +949,7 @@ describe("remote runner process supervision", () => {
}
if (label === "paperclip-runner-monitor") {
return {
- exitCode: 3,
+ exitCode: 45,
signal: null,
timedOut: false,
stdout: "",
diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts
index 4f332c53ae..0684682854 100644
--- a/server/src/services/native-runtime/native-session-executor.ts
+++ b/server/src/services/native-runtime/native-session-executor.ts
@@ -11013,6 +11013,7 @@ export function createRemoteRunnerProcessLauncher(input: {
}): (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle {
const runner = input.runner;
return (spec) => {
+ let exitConfirmed = false;
let launchedIdentity: {
nonce: string;
pid: number;
@@ -11130,7 +11131,7 @@ export function createRemoteRunnerProcessLauncher(input: {
: "runner_remote_process_launch_failed",
);
}
- let identity: { pid: number; startedAt: string };
+ let identity: Awaited<ReturnType<typeof waitForRemoteRunnerProcessIdentity>>;
try {
identity = await waitForRemoteRunnerProcessIdentity({
runner,
@@ -11177,19 +11178,27 @@ export function createRemoteRunnerProcessLauncher(input: {
});
while (true) {
- const observed = await runner.execute({
- command: "sh",
- args: [
- "-c",
- REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT,
- "paperclip-runner-monitor",
- input.processIdentityPath,
- identityNonce,
- input.runnerInstanceId,
- String(identity.pid),
- ],
- bypassSession: true,
- timeoutMs: 10_000,
+ const observed = await readRemoteRunnerRecoveryProbe({
+ runner,
+ stage: "process_verification",
+ deadline: Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS,
+ readOnly: true,
+ request: {
+ command: "sh",
+ args: [
+ "-c",
+ REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + (identity.processStartFingerprint
+ ? `; if ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; then :; else exit 46; fi; test "$process_fingerprint" = "$5" || exit 47`
+ : ""),
+ "paperclip-runner-monitor",
+ input.processIdentityPath,
+ identityNonce,
+ input.runnerInstanceId,
+ String(identity.pid),
+ ...(identity.processStartFingerprint ? [identity.processStartFingerprint] : []),
+ ],
+ bypassSession: true,
+ },
});
if (observed.exitCode === 0 && !observed.timedOut) {
await new Promise<void>((resolve) =>
@@ -11197,8 +11206,14 @@ export function createRemoteRunnerProcessLauncher(input: {
);
continue;
}
+ if (observed.exitCode !== 45) {
+ throw new RemoteRunnerRecoveryError({
+ stage: "process_verification",
+ reason: remoteRecoveryProbeReason("process_verification", observed.exitCode, observed.timedOut),
+ });
+ }
+ exitConfirmed = true;
child.exitCode = null;
- const identityMismatch = observed.exitCode === 4;
const diagnostic = await runner
.execute({
command: "sh",
@@ -11241,13 +11256,11 @@ export function createRemoteRunnerProcessLauncher(input: {
code: null,
signal: null,
stdout: "",
- stderr: identityMismatch
- ? "runner_remote_process_identity_mismatch"
- : diagnosticTail || stateSummary,
+ stderr: diagnosticTail || stateSummary,
};
}
})();
- return { child, completion };
+ return { child, completion, isExitConfirmed: () => exitConfirmed };
};
}