Skip to content

The open-source app everyone uses to manage agents at work

fix: harden the sandbox bridge gateway against crashes and queue wedge (#12060)

Devin Foley87d68f476bcb
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
index a3140e9b64..7d67f87336 100644
--- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
+++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
@@ -1,5 +1,6 @@
-import { execFile as execFileCallback } from "node:child_process";
-import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { execFile as execFileCallback, spawn } from "node:child_process";
+import { createServer } from "node:http";
+import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
@@ -2706,4 +2707,237 @@ describe("sandbox callback bridge", () => {
await worker.stop({ drainTimeoutMs: 50 });
});
+
+ async function prepareGatewayFixture(prefix: string) {
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), prefix));
+ cleanupDirs.push(rootDir);
+ const localWorkspaceDir = path.join(rootDir, "local-workspace");
+ const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
+ await mkdir(localWorkspaceDir, { recursive: true });
+ await mkdir(remoteWorkspaceDir, { recursive: true });
+ await writeFile(path.join(localWorkspaceDir, "README.md"), "bridge test\n", "utf8");
+
+ const runner = createExecRunner();
+ const bridgeAsset = await createSandboxCallbackBridgeAsset();
+ cleanupFns.push(bridgeAsset.cleanup);
+ const prepared = await prepareCommandManagedRuntime({
+ runner,
+ spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 },
+ adapterKey: "codex",
+ workspaceLocalDir: localWorkspaceDir,
+ assets: [{ key: "bridge", localDir: bridgeAsset.localDir }],
+ });
+ const queueDir = path.posix.join(prepared.runtimeRootDir, "paperclip-bridge");
+ return {
+ runner,
+ remoteWorkspaceDir,
+ assetRemoteDir: prepared.assetDirs.bridge,
+ queueDir,
+ directories: sandboxCallbackBridgeDirectories(queueDir),
+ bridgeToken: createSandboxCallbackBridgeToken(),
+ };
+ }
+
+ it("cleans up a timed-out request file and keeps serving after the host recovers", async () => {
+ const fixture = await prepareGatewayFixture("paperclip-bridge-timeout-clean-");
+
+ const bridge = await startSandboxCallbackBridgeServer({
+ runner: fixture.runner,
+ remoteCwd: fixture.remoteWorkspaceDir,
+ assetRemoteDir: fixture.assetRemoteDir,
+ queueDir: fixture.queueDir,
+ bridgeToken: fixture.bridgeToken,
+ timeoutMs: 30_000,
+ responseTimeoutMs: 600,
+ pollIntervalMs: 50,
+ });
+ cleanupFns.push(async () => {
+ await bridge.stop();
+ });
+
+ // No worker runs, so the request times out at the gateway.
+ const timedOut = await fetch(`${bridge.baseUrl}/api/agents/me`, {
+ headers: { authorization: `Bearer ${fixture.bridgeToken}` },
+ });
+ expect(timedOut.status).toBe(502);
+ await expect(timedOut.json()).resolves.toMatchObject({
+ error: expect.stringContaining("Timed out"),
+ });
+
+ // The gateway cleaned its own request file, so nothing counts toward the
+ // queue-depth cap after the caller gave up.
+ const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
+ name.endsWith(".json"),
+ );
+ expect(leftover).toEqual([]);
+
+ // A worker that comes up afterwards serves the next request normally —
+ // the timeout neither wedged nor killed the gateway.
+ const worker = await startSandboxCallbackBridgeWorker({
+ client: createFileSystemSandboxCallbackBridgeQueueClient(),
+ queueDir: fixture.queueDir,
+ authorizeRequest: async () => null,
+ handleRequest: async () => ({
+ status: 200,
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ ok: true }),
+ }),
+ });
+ cleanupFns.push(async () => {
+ await worker.stop();
+ });
+
+ const recovered = await fetch(`${bridge.baseUrl}/api/agents/me`, {
+ headers: { authorization: `Bearer ${fixture.bridgeToken}` },
+ });
+ expect(recovered.status).toBe(200);
+ await expect(recovered.json()).resolves.toMatchObject({ ok: true });
+ }, 30_000);
+
+ it("sweeps stale request files before rejecting at the queue-depth cap", async () => {
+ const fixture = await prepareGatewayFixture("paperclip-bridge-stale-sweep-");
+
+ const bridge = await startSandboxCallbackBridgeServer({
+ runner: fixture.runner,
+ remoteCwd: fixture.remoteWorkspaceDir,
+ assetRemoteDir: fixture.assetRemoteDir,
+ queueDir: fixture.queueDir,
+ bridgeToken: fixture.bridgeToken,
+ timeoutMs: 30_000,
+ responseTimeoutMs: 500,
+ pollIntervalMs: 50,
+ maxQueueDepth: 1,
+ });
+ cleanupFns.push(async () => {
+ await bridge.stop();
+ });
+
+ // Plant an orphaned request file (a killed caller, or a previous gateway
+ // process's leftover) and backdate it beyond the response deadline.
+ const orphanPath = path.join(fixture.directories.requestsDir, "orphan.json");
+ await writeFile(orphanPath, bridgeRequestJson("orphan"), "utf8");
+ const staleTime = new Date(Date.now() - 60_000);
+ await utimes(orphanPath, staleTime, staleTime);
+
+ // The queue sits at the cap, but the only entry is stale: the gateway must
+ // sweep it and admit the request instead of answering 503. With no worker
+ // the admitted request then times out (502) — proof it entered the queue.
+ const response = await fetch(`${bridge.baseUrl}/api/agents/me`, {
+ headers: { authorization: `Bearer ${fixture.bridgeToken}` },
+ });
+ expect(response.status).toBe(502);
+ const body = (await response.json()) as { error?: string };
+ expect(body.error ?? "").not.toContain("queue is full");
+
+ const leftover = (await readdir(fixture.directories.requestsDir).catch(() => [])).filter((name) =>
+ name.endsWith(".json"),
+ );
+ expect(leftover).toEqual([]);
+ }, 30_000);
+
+ it("skips a request file that vanished before the read instead of escalating", async () => {
+ // The gateway deletes a request file when its caller stops waiting. The
+ // worker's read then races the deletion; a vanished file must be a quiet
+ // skip, not a worker failure with a recovery pass.
+ const queueDir = "/virtual-bridge/vanished";
+ let listCalls = 0;
+ const handled: string[] = [];
+ const responseStatuses: number[] = [];
+ const { runtimeSpan, workerErrors } = createWorkerErrorCapture();
+
+ const client: SandboxCallbackBridgeQueueClient = {
+ makeDir: async () => {},
+ makeDirs: async () => {},
+ listJsonFiles: async () => {
+ listCalls += 1;
+ return listCalls === 1 ? ["ghost.json"] : [];
+ },
+ readTextFile: async () => {
+ throw new Error("cat: ghost.json: No such file or directory");
+ },
+ writeTextFile: async () => {},
+ writeResponseFile: async (_remotePath, body) => {
+ responseStatuses.push((JSON.parse(body.trim()) as { status: number }).status);
+ return { wrote: true };
+ },
+ rename: async () => {},
+ remove: async () => {},
+ };
+
+ const worker = await startSandboxCallbackBridgeWorker({
+ client,
+ queueDir,
+ runtimeSpan,
+ authorizeRequest: async () => null,
+ handleRequest: async (request) => {
+ handled.push(request.id);
+ return { status: 200, body: "ok" };
+ },
+ });
+
+ await new Promise((resolve) => setTimeout(resolve, 250));
+ await worker.stop({ drainTimeoutMs: 50 });
+
+ expect(handled).toEqual([]);
+ expect(responseStatuses).toEqual([]);
+ expect(workerErrors).toEqual([]);
+ });
+
+ it("embeds crash handlers and queue hygiene in the generated gateway source", () => {
+ // A crashed gateway is a dead loopback port for the rest of the run, so
+ // the generated source must keep its crash handlers and its stale-queue
+ // sweep. The readiness gate matters too: survival applies only after the
+ // gateway is adoptable, so a startup fault still fails fast. This pins
+ // their presence; the behavior is proven above and below.
+ const source = getSandboxCallbackBridgeServerSource();
+ expect(source).toContain('process.on("uncaughtException"');
+ expect(source).toContain('process.on("unhandledRejection"');
+ expect(source).toContain("gatewayReady");
+ expect(source).toContain("sweepStaleRequests");
+ });
+
+ it("exits fast when the gateway cannot bind its port instead of lingering un-ready", async () => {
+ // Before readiness, the crash handlers must not keep the process alive: a
+ // failed bind means the gateway can never serve, and surviving would only
+ // leave an un-ready zombie while the host waits out its readiness poll.
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-bind-fail-"));
+ cleanupDirs.push(rootDir);
+ const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs");
+ await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
+ const queueDir = path.join(rootDir, "queue");
+ await mkdir(queueDir, { recursive: true });
+
+ const blocker = createServer();
+ await new Promise<void>((resolve) => {
+ blocker.listen(0, "127.0.0.1", resolve);
+ });
+ cleanupFns.push(
+ () =>
+ new Promise<void>((resolve) => {
+ blocker.close(() => resolve());
+ }),
+ );
+ const blockedPort = (blocker.address() as { port: number }).port;
+
+ const child = spawn(process.execPath, [entrypoint], {
+ env: {
+ ...process.env,
+ PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
+ PAPERCLIP_BRIDGE_TOKEN: "test-token",
+ PAPERCLIP_BRIDGE_PORT: String(blockedPort),
+ },
+ stdio: ["ignore", "ignore", "pipe"],
+ });
+ let stderr = "";
+ child.stderr.on("data", (chunk) => {
+ stderr += chunk;
+ });
+ const exitCode = await new Promise<number | null>((resolve) => {
+ child.on("close", resolve);
+ });
+
+ expect(exitCode).toBe(1);
+ expect(stderr).toContain("[paperclip-bridge] server error");
+ expect(stderr).toContain("EADDRINUSE");
+ }, 15_000);
});
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts
index b198d6d9cb..42e5888e2b 100644
--- a/packages/adapter-utils/src/sandbox-callback-bridge.ts
+++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts
@@ -983,7 +983,22 @@ export async function startSandboxCallbackBridgeWorker(input: {
await writeAbortedHandlerBackstop(fileName, guard, lastWriteError);
};
try {
- const raw = await input.client.readTextFile(requestPath);
+ let raw: string;
+ try {
+ raw = await input.client.readTextFile(requestPath);
+ } catch (error) {
+ // The gateway deletes a request file when its caller stops waiting
+ // (client-side timeout cleanup). A read that fails because the file is
+ // gone is that benign race, not a channel fault: confirm the file
+ // vanished and skip quietly instead of escalating into a recovery
+ // pass. A file that is still listed rethrows, so a real read fault
+ // keeps its existing handling.
+ const remaining = await input.client.listJsonFiles(directories.requestsDir).catch(() => null);
+ if (remaining !== null && !remaining.includes(fileName)) {
+ return;
+ }
+ throw error;
+ }
let request: SandboxCallbackBridgeRequest;
try {
request = JSON.parse(raw) as SandboxCallbackBridgeRequest;
@@ -2118,6 +2133,32 @@ if (bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" && !queueDir) {
throw new Error("PAPERCLIP_BRIDGE_QUEUE_DIR and PAPERCLIP_BRIDGE_TOKEN are required.");
}
+// A crashed gateway is a dead loopback port for the rest of the run: nothing
+// inside the sandbox respawns this process, and every later agent API call
+// then fails at the connection level. Once the gateway is ready, log an
+// uncaught fault to stderr (the host redirects it into logs/bridge.log) and
+// keep serving — the relay holds no state a fault can corrupt beyond the one
+// request it interrupted. Before readiness the same fault means the gateway
+// can never become usable (a failed bind, a failed readiness write), so exit
+// instead: surviving there only leaves an un-ready zombie behind while the
+// host waits out its readiness poll.
+let gatewayReady = false;
+process.on("uncaughtException", (error) => {
+ process.stderr.write(
+ "[paperclip-bridge] uncaught exception: " + (error && error.stack ? error.stack : String(error)) + "\\n",
+ );
+ if (!gatewayReady) {
+ process.exit(1);
+ }
+});
+process.on("unhandledRejection", (reason) => {
+ const detail = reason && typeof reason === "object" && "stack" in reason ? reason.stack : String(reason);
+ process.stderr.write("[paperclip-bridge] unhandled rejection: " + detail + "\\n");
+ if (!gatewayReady) {
+ process.exit(1);
+ }
+});
+
// The embedded zero-dependency frame codec. The duplex gateway uses it; the file
// gateway ignores it.
${DUPLEX_GATEWAY_CODEC_SOURCE}
@@ -2177,6 +2218,24 @@ async function runFileGateway() {
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json")).length;
}
+ // Delete request files older than the response deadline. Every live caller
+ // cleans its own request file when it times out, so a file this old is an
+ // orphan: its writer was killed mid-wait, or a previous gateway process died
+ // and left its queue behind. Orphans otherwise count toward the queue-depth
+ // cap forever and wedge the gateway at a permanent 503.
+ async function sweepStaleRequests() {
+ const staleBefore = Date.now() - responseTimeoutMs - 2000;
+ const entries = await fs.readdir(requestsDir, { withFileTypes: true }).catch(() => []);
+ for (const entry of entries) {
+ if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
+ const filePath = path.posix.join(requestsDir, entry.name);
+ const stats = await fs.stat(filePath).catch(() => null);
+ if (stats && stats.mtimeMs < staleBefore) {
+ await fs.rm(filePath, { force: true }).catch(() => undefined);
+ }
+ }
+ }
+
async function waitForResponse(requestId) {
const responsePath = path.posix.join(responsesDir, \`\${requestId}.json\`);
const deadline = Date.now() + responseTimeoutMs;
@@ -2201,8 +2260,13 @@ async function runFileGateway() {
}
if (await queueDepth() >= maxQueueDepth) {
- writeJsonResponse(res, 503, { error: "Bridge request queue is full." });
- return;
+ // Reclaim orphaned request files before rejecting; only a queue that
+ // is genuinely full of live requests gets the 503.
+ await sweepStaleRequests();
+ if (await queueDepth() >= maxQueueDepth) {
+ writeJsonResponse(res, 503, { error: "Bridge request queue is full." });
+ return;
+ }
}
const url = new URL(req.url || "/", "http://127.0.0.1");
@@ -2227,7 +2291,19 @@ async function runFileGateway() {
await fs.writeFile(tempPath, \`\${JSON.stringify(payload)}\\n\`, "utf8");
await fs.rename(tempPath, requestPath);
- const response = await waitForResponse(requestId);
+ let response;
+ try {
+ response = await waitForResponse(requestId);
+ } catch (error) {
+ // The host never delivered a response inside the deadline. Remove this
+ // request's file so it cannot pile up toward the queue-depth cap. The
+ // host's normal response write is guarded on the request file, so the
+ // removal also tells the host that no caller waits anymore. Without
+ // this cleanup a stalled host wedges the gateway at the cap and every
+ // later request gets an immediate 503 until run end.
+ await fs.rm(requestPath, { force: true }).catch(() => undefined);
+ throw error;
+ }
const responseHeaders = response.headers || {};
// The host marks a possibly-committed mutation with an indeterminate outcome.
// The host cannot cancel a host operation that is in flight, so the mutation
@@ -2265,7 +2341,24 @@ async function runFileGateway() {
await fs.mkdir(responsesDir, { recursive: true });
await fs.mkdir(logsDir, { recursive: true });
+ // Newer Node runtimes do not reliably surface a failed bind through
+ // uncaughtException here: with nothing else keeping the event loop alive,
+ // the process can drain and exit 0 before the error event is delivered
+ // (observed on Node 24/25; Node 22 delivered it). Attach an explicit error
+ // listener and pin the loop with a keepalive until the bind settles, so a
+ // startup failure exits 1 with the fault on stderr on every runtime.
+ const bindKeepalive = setInterval(() => {}, 1000);
+ server.once("error", (error) => {
+ clearInterval(bindKeepalive);
+ process.stderr.write(
+ "[paperclip-bridge] server error: " + (error && error.stack ? error.stack : String(error)) + "\\n",
+ );
+ if (!gatewayReady) {
+ process.exit(1);
+ }
+ });
server.listen(port, host, async () => {
+ clearInterval(bindKeepalive);
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("Bridge server did not expose a TCP address.");
@@ -2280,6 +2373,9 @@ async function runFileGateway() {
const tempReadyFile = \`\${readyFile}.tmp\`;
await fs.writeFile(tempReadyFile, JSON.stringify(ready), "utf8");
await fs.rename(tempReadyFile, readyFile);
+ // The readiness file is on disk, so the host will adopt this process.
+ // From here on an uncaught fault must not kill the listener.
+ gatewayReady = true;
});
}
@@ -2621,6 +2717,9 @@ function runDuplexGateway() {
type: "ready",
nonce: bridgeNonce,
});
+ // READY is on the wire, so the host will adopt this process. From here on
+ // an uncaught fault must not kill the listener.
+ gatewayReady = true;
});
}