Skip to content

The open-source app everyone uses to manage agents at work

fix: survive transient sandbox exec failures in the callback bridge worker (#12052)

Devin Foleyc7f4bc1300d5
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
index b0300d6d7d..a3140e9b64 100644
--- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
+++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts
@@ -475,6 +475,68 @@ describe("sandbox callback bridge", () => {
}
});
+ it("recovers from transient queue polling failures and keeps relaying requests", async () => {
+ // A single reset or slow sandbox exec used to unwind the poll loop into its
+ // terminal catch, killing the relay for the rest of the run. The loop must
+ // instead back off, retry, and still deliver requests queued after the
+ // failure window.
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-transient-"));
+ cleanupDirs.push(rootDir);
+
+ const queueDir = path.posix.join(rootDir, "queue");
+ const directories = sandboxCallbackBridgeDirectories(queueDir);
+ const baseClient = createFileSystemSandboxCallbackBridgeQueueClient();
+ let listCalls = 0;
+ const client: SandboxCallbackBridgeQueueClient = {
+ ...baseClient,
+ listJsonFiles: async (dirPath: string) => {
+ listCalls += 1;
+ if (listCalls <= 3) {
+ throw new Error("list requests failed: kex_exchange_identification: read: Connection reset by peer");
+ }
+ return baseClient.listJsonFiles(dirPath);
+ },
+ };
+
+ const seenPaths: string[] = [];
+ const worker = await startSandboxCallbackBridgeWorker({
+ client,
+ queueDir,
+ authorizeRequest: async () => null,
+ handleRequest: async (request) => {
+ seenPaths.push(request.path);
+ return {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ ok: true }),
+ };
+ },
+ });
+ cleanupFns.push(async () => {
+ await worker.stop();
+ });
+
+ const requestId = "transient-recovery-1";
+ await writeFile(
+ path.join(directories.requestsDir, `${requestId}.json`),
+ JSON.stringify({
+ id: requestId,
+ method: "GET",
+ path: "/api/agents/me",
+ query: "",
+ headers: {},
+ body: "",
+ }),
+ "utf8",
+ );
+
+ const responseFile = await waitForJsonFile(directories.responsesDir, 10_000);
+ const raw = await readFile(path.join(directories.responsesDir, responseFile), "utf8");
+ expect(JSON.parse(raw)).toMatchObject({ id: requestId, status: 200 });
+ expect(seenPaths).toEqual(["/api/agents/me"]);
+ expect(listCalls).toBeGreaterThan(3);
+ });
+
it("keeps the queue-directory setup on the startup step but resets the poll loop store", async () => {
// The worker starts inside the measured `bridge.paperclip` step. Its awaited
// queue-directory setup is startup work, so a `makeDir` `sandbox.exec` span
@@ -1510,7 +1572,7 @@ describe("sandbox callback bridge", () => {
})}\n`;
}
- it("times out a stalled poll, writes a 503, and surfaces a run-level error", async () => {
+ it("times out a stalled poll, surfaces a run-level error, and recovers to deliver the request", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-hang-"));
cleanupDirs.push(rootDir);
@@ -1526,9 +1588,11 @@ describe("sandbox callback bridge", () => {
const client: SandboxCallbackBridgeQueueClient = {
...base,
// The first poll never resolves — a silently unresponsive sandbox channel.
- // The per-iteration timeout must convert the hang into a caught error. The
- // request never reaches the handler, so the recovery path can safely 503
- // it. Later calls (the recovery's failPendingRequests) resolve.
+ // The per-iteration timeout must convert the hang into a caught error, and
+ // the loop must then back off and retry rather than die: the request never
+ // reached the handler, so the retry delivers the real response. A
+ // sustained outage is the watchdog's job (proven separately below), not a
+ // reason to fail a request one transient hang could still serve.
listJsonFiles: async (dir) => {
listCalls += 1;
if (listCalls === 1) {
@@ -1550,7 +1614,7 @@ describe("sandbox callback bridge", () => {
const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);
const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");
- expect(JSON.parse(responseBody).status).toBe(503);
+ expect(JSON.parse(responseBody).status).toBe(200);
expect(workerErrors.length).toBeGreaterThan(0);
expect(workerErrors[0]).toContain("timed out");
@@ -2261,10 +2325,13 @@ describe("sandbox callback bridge", () => {
});
it("retries a recovery 503 write that fails transiently, delivers the 503, and removes the request", async () => {
- // The poll times out, so the recovery path aborts the queued request with a
- // 503. The first 503 write fails, so the recovery must retry it inside the
- // same pass. It then delivers the 503 and removes the request. The request is
- // unclaimed, so its host mutation never ran; the 503 stays retry-safe.
+ // A request attempt times out (its first read hangs), so the loop's request
+ // catch runs the recovery pass, which aborts the queued request with a 503.
+ // The first 503 write fails, so the recovery must retry it inside the same
+ // pass. It then delivers the 503 and removes the request. The request is
+ // unclaimed, so its host mutation never ran; the 503 stays retry-safe. (A
+ // hung poll no longer triggers this pass — the loop backs off and retries
+ // the poll instead, and a sustained hang is the watchdog's job.)
const waitFor = async (predicate: () => boolean, timeoutMs: number) => {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
@@ -2286,27 +2353,26 @@ describe("sandbox callback bridge", () => {
requestBodies.set(requestPath, bridgeRequestJson("req-503-retry"));
const responseWrites: Array<{ path: string; status: number; body: string }> = [];
const requestRemovals: string[] = [];
- let listCalls = 0;
+ let readCalls = 0;
let writeAttempts = 0;
const client: SandboxCallbackBridgeQueueClient = {
makeDir: async () => {},
makeDirs: async () => {},
- // The first poll never resolves — a silently unresponsive sandbox channel.
- // The per-iteration timeout converts the hang into a caught error, so the
- // loop `catch` runs the recovery path. Later listings resolve, so the
- // recovery enumerates and aborts the request.
- listJsonFiles: async (dir) => {
- if (dir !== directories.requestsDir) {
- return [];
- }
- listCalls += 1;
- if (listCalls === 1) {
- return await new Promise<string[]>(() => {});
- }
- return [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort();
- },
+ listJsonFiles: async (dir) =>
+ dir === directories.requestsDir
+ ? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort()
+ : [],
+ // The first read never resolves — a silently unresponsive sandbox channel
+ // hit mid-request, before the handler claim. The per-iteration timeout
+ // converts the hang into a caught error, so the loop's request catch runs
+ // the recovery pass. The recovery's own read resolves, so it can build and
+ // deliver the 503.
readTextFile: async (remotePath) => {
+ readCalls += 1;
+ if (readCalls === 1) {
+ return await new Promise<string>(() => {});
+ }
const body = requestBodies.get(remotePath);
if (body === undefined) {
throw new Error(`missing request ${remotePath}`);
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts
index 5d8907ce04..b198d6d9cb 100644
--- a/packages/adapter-utils/src/sandbox-callback-bridge.ts
+++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts
@@ -34,7 +34,8 @@ const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024;
// (PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS), so the host loop fails fast and writes
// 503 responses before the in-sandbox client gives up. A silently unresponsive
// sandbox channel makes a client call hang with no reject; this timeout turns
-// that hang into a caught error, so the loop `catch` runs `failPendingRequests`.
+// that hang into a caught error, so the poll loop can back off and retry while
+// the watchdog below decides when to fail the queued requests.
const DEFAULT_BRIDGE_ITERATION_TIMEOUT_MS = 10_000;
// Watchdog backstop for a hang that the per-iteration timeout does not catch
// (for example many slow-but-under-timeout calls, or a stall outside the awaited
@@ -58,6 +59,12 @@ const MAX_BACKSTOP_WRITE_ATTEMPTS = 3;
// The delay between two 504 backstop write attempts. It is short, so all retries
// finish well under the in-sandbox 30s response deadline.
const BACKSTOP_WRITE_RETRY_MS = 50;
+// Backoff cap between poll-loop retries after a transient iteration failure.
+// The cap keeps a recovering loop probing often enough to resume before the
+// in-sandbox 30s response deadline strands queued callers, while the
+// exponential ramp below it keeps a hard-down channel from burning an exec
+// call every poll interval.
+const MAX_TRANSIENT_ITERATION_BACKOFF_MS = 5_000;
const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024;
export const SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT = "paperclip-bridge-server.mjs";
const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL";
@@ -1380,13 +1387,54 @@ export async function startSandboxCallbackBridgeWorker(input: {
watchdogTimer.unref();
}
try {
+ // Consecutive transient poll failures. A single failed list call — one
+ // reset or slow sandbox exec — must not end the relay for the rest of the
+ // run: the in-sandbox gateway keeps queueing requests, so a dead loop
+ // strands every later API call from the agent (its status writes then look
+ // like connection failures and the issue loses its disposition). Back off
+ // and retry the poll instead. The watchdog stays the escalation path for a
+ // sustained outage — it fires after `watchdogTimeoutMs` without a
+ // successful iteration and fails the queued requests fast, while this loop
+ // keeps probing for recovery.
+ let consecutivePollFailures = 0;
while (true) {
- const fileNames = await withTimeout(
- input.client.listJsonFiles(directories.requestsDir),
- iterationTimeoutMs,
- "Sandbox callback bridge list requests",
- );
- if (fileNames.length === 0) {
+ let fileNames: string[];
+ try {
+ fileNames = await withTimeout(
+ input.client.listJsonFiles(directories.requestsDir),
+ iterationTimeoutMs,
+ "Sandbox callback bridge list requests",
+ );
+ consecutivePollFailures = 0;
+ } catch (error) {
+ if (stopping) {
+ break;
+ }
+ consecutivePollFailures += 1;
+ const message = `${buildWorkerFailureMessage(error)} (transient poll failure ${consecutivePollFailures}; retrying)`;
+ if (consecutivePollFailures === 1) {
+ // Put the first failure of a streak on the run trace; later repeats
+ // only warn, so a flapping channel does not spam failed spans.
+ await surfaceRunError(new Error(message));
+ } else {
+ console.warn(`[paperclip] ${message}`);
+ }
+ const backoffMs = Math.min(
+ pollIntervalMs * 2 ** consecutivePollFailures,
+ MAX_TRANSIENT_ITERATION_BACKOFF_MS,
+ );
+ await new Promise((resolve) => setTimeout(resolve, backoffMs));
+ continue;
+ }
+ // A file whose attempt is still in flight (or waiting on its 504
+ // backstop) is not actionable: `processRequestFile` would skip it via
+ // the guard map. Treat an all-guarded listing like an empty one and
+ // sleep a poll interval. Re-listing immediately would spin the loop —
+ // an exec storm against a real sandbox channel, and with an in-memory
+ // client a pure-microtask loop that starves every timer in the process
+ // (including the guard's own backstop and abort timers).
+ const actionableFileNames = fileNames.filter((fileName) => !inFlightRequestGuards.has(fileName));
+ if (actionableFileNames.length === 0) {
lastSuccessfulIterationAt = Date.now();
if (stopping) {
break;
@@ -1394,7 +1442,7 @@ export async function startSandboxCallbackBridgeWorker(input: {
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
continue;
}
- for (const fileName of fileNames) {
+ for (const fileName of actionableFileNames) {
if (stopping && Date.now() >= stopDeadline) break;
inFlight += 1;
try {
@@ -1406,7 +1454,7 @@ export async function startSandboxCallbackBridgeWorker(input: {
// `task.run` after it. Without a runner, the request runs under the
// run parent with no wrapper span, exactly like the earlier behavior.
// The per-iteration timeout wraps the whole request, so a hung
- // request rejects and the loop `catch` runs `failPendingRequests`.
+ // request rejects and the catch below runs the recovery pass.
await withTimeout(
input.runtimeSpan
? input.runtimeSpan(CALLBACK_BRIDGE_RELAY_REQUEST_SPAN, () =>
@@ -1419,6 +1467,23 @@ export async function startSandboxCallbackBridgeWorker(input: {
`Sandbox callback bridge process request ${fileName}`,
);
lastSuccessfulIterationAt = Date.now();
+ } catch (error) {
+ // A single request attempt failed or hung. Run the same recovery
+ // pass the loop previously died on — abort the in-flight handler
+ // (its 504 backstop keeps the caller from stranding) and 503 the
+ // unclaimed queued requests — but keep the loop alive afterward. A
+ // caller that sees the retry-safe 503 re-queues, and the recovered
+ // loop serves the retry; the old terminal catch left every later
+ // request to strand instead.
+ const message = buildWorkerFailureMessage(error);
+ await surfaceRunError(new Error(message));
+ try {
+ await failPendingRequests(message, { abandonInFlight: true });
+ } catch (failPendingError) {
+ console.warn(
+ `[paperclip] sandbox callback bridge failed to abort queued requests after a request failure: ${failPendingError instanceof Error ? failPendingError.message : String(failPendingError)}`,
+ );
+ }
} finally {
inFlight -= 1;
}