The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: survive transient sandbox exec failures in the callback bridge worker (#12052)
Devin Foleyc7f4bc1300d5
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.tsindex b0300d6d7d..a3140e9b64 100644--- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts+++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts@@ -475,6 +475,68 @@ describe("sandbox callback bridge", () => {}});+ it("recovers from transient queue polling failures and keeps relaying requests", async () => {+ // A single reset or slow sandbox exec used to unwind the poll loop into its+ // terminal catch, killing the relay for the rest of the run. The loop must+ // instead back off, retry, and still deliver requests queued after the+ // failure window.+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-transient-"));+ cleanupDirs.push(rootDir);++ const queueDir = path.posix.join(rootDir, "queue");+ const directories = sandboxCallbackBridgeDirectories(queueDir);+ const baseClient = createFileSystemSandboxCallbackBridgeQueueClient();+ let listCalls = 0;+ const client: SandboxCallbackBridgeQueueClient = {+ ...baseClient,+ listJsonFiles: async (dirPath: string) => {+ listCalls += 1;+ if (listCalls <= 3) {+ throw new Error("list requests failed: kex_exchange_identification: read: Connection reset by peer");+ }+ return baseClient.listJsonFiles(dirPath);+ },+ };++ const seenPaths: string[] = [];+ const worker = await startSandboxCallbackBridgeWorker({+ client,+ queueDir,+ authorizeRequest: async () => null,+ handleRequest: async (request) => {+ seenPaths.push(request.path);+ return {+ status: 200,+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ ok: true }),+ };+ },+ });+ cleanupFns.push(async () => {+ await worker.stop();+ });++ const requestId = "transient-recovery-1";+ await writeFile(+ path.join(directories.requestsDir, `${requestId}.json`),+ JSON.stringify({+ id: requestId,+ method: "GET",+ path: "/api/agents/me",+ query: "",+ headers: {},+ body: "",+ }),+ "utf8",+ );++ const responseFile = await waitForJsonFile(directories.responsesDir, 10_000);+ const raw = await readFile(path.join(directories.responsesDir, responseFile), "utf8");+ expect(JSON.parse(raw)).toMatchObject({ id: requestId, status: 200 });+ expect(seenPaths).toEqual(["/api/agents/me"]);+ expect(listCalls).toBeGreaterThan(3);+ });+it("keeps the queue-directory setup on the startup step but resets the poll loop store", async () => {// The worker starts inside the measured `bridge.paperclip` step. Its awaited// queue-directory setup is startup work, so a `makeDir` `sandbox.exec` span@@ -1510,7 +1572,7 @@ describe("sandbox callback bridge", () => {})}\n`;}- it("times out a stalled poll, writes a 503, and surfaces a run-level error", async () => {+ it("times out a stalled poll, surfaces a run-level error, and recovers to deliver the request", async () => {const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-hang-"));cleanupDirs.push(rootDir);@@ -1526,9 +1588,11 @@ describe("sandbox callback bridge", () => {const client: SandboxCallbackBridgeQueueClient = {...base,// The first poll never resolves — a silently unresponsive sandbox channel.- // The per-iteration timeout must convert the hang into a caught error. The- // request never reaches the handler, so the recovery path can safely 503- // it. Later calls (the recovery's failPendingRequests) resolve.+ // The per-iteration timeout must convert the hang into a caught error, and+ // the loop must then back off and retry rather than die: the request never+ // reached the handler, so the retry delivers the real response. A+ // sustained outage is the watchdog's job (proven separately below), not a+ // reason to fail a request one transient hang could still serve.listJsonFiles: async (dir) => {listCalls += 1;if (listCalls === 1) {@@ -1550,7 +1614,7 @@ describe("sandbox callback bridge", () => {const responseFile = await waitForJsonFile(directories.responsesDir, 3_000);const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8");- expect(JSON.parse(responseBody).status).toBe(503);+ expect(JSON.parse(responseBody).status).toBe(200);expect(workerErrors.length).toBeGreaterThan(0);expect(workerErrors[0]).toContain("timed out");@@ -2261,10 +2325,13 @@ describe("sandbox callback bridge", () => {});it("retries a recovery 503 write that fails transiently, delivers the 503, and removes the request", async () => {- // The poll times out, so the recovery path aborts the queued request with a- // 503. The first 503 write fails, so the recovery must retry it inside the- // same pass. It then delivers the 503 and removes the request. The request is- // unclaimed, so its host mutation never ran; the 503 stays retry-safe.+ // A request attempt times out (its first read hangs), so the loop's request+ // catch runs the recovery pass, which aborts the queued request with a 503.+ // The first 503 write fails, so the recovery must retry it inside the same+ // pass. It then delivers the 503 and removes the request. The request is+ // unclaimed, so its host mutation never ran; the 503 stays retry-safe. (A+ // hung poll no longer triggers this pass — the loop backs off and retries+ // the poll instead, and a sustained hang is the watchdog's job.)const waitFor = async (predicate: () => boolean, timeoutMs: number) => {const deadline = Date.now() + timeoutMs;while (Date.now() < deadline) {@@ -2286,27 +2353,26 @@ describe("sandbox callback bridge", () => {requestBodies.set(requestPath, bridgeRequestJson("req-503-retry"));const responseWrites: Array<{ path: string; status: number; body: string }> = [];const requestRemovals: string[] = [];- let listCalls = 0;+ let readCalls = 0;let writeAttempts = 0;const client: SandboxCallbackBridgeQueueClient = {makeDir: async () => {},makeDirs: async () => {},- // The first poll never resolves — a silently unresponsive sandbox channel.- // The per-iteration timeout converts the hang into a caught error, so the- // loop `catch` runs the recovery path. Later listings resolve, so the- // recovery enumerates and aborts the request.- listJsonFiles: async (dir) => {- if (dir !== directories.requestsDir) {- return [];- }- listCalls += 1;- if (listCalls === 1) {- return await new Promise<string[]>(() => {});- }- return [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort();- },+ listJsonFiles: async (dir) =>+ dir === directories.requestsDir+ ? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort()+ : [],+ // The first read never resolves — a silently unresponsive sandbox channel+ // hit mid-request, before the handler claim. The per-iteration timeout+ // converts the hang into a caught error, so the loop's request catch runs+ // the recovery pass. The recovery's own read resolves, so it can build and+ // deliver the 503.readTextFile: async (remotePath) => {+ readCalls += 1;+ if (readCalls === 1) {+ return await new Promise<string>(() => {});+ }const body = requestBodies.get(remotePath);if (body === undefined) {throw new Error(`missing request ${remotePath}`);diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.tsindex 5d8907ce04..b198d6d9cb 100644--- a/packages/adapter-utils/src/sandbox-callback-bridge.ts+++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts@@ -34,7 +34,8 @@ const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024;// (PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS), so the host loop fails fast and writes// 503 responses before the in-sandbox client gives up. A silently unresponsive// sandbox channel makes a client call hang with no reject; this timeout turns-// that hang into a caught error, so the loop `catch` runs `failPendingRequests`.+// that hang into a caught error, so the poll loop can back off and retry while+// the watchdog below decides when to fail the queued requests.const DEFAULT_BRIDGE_ITERATION_TIMEOUT_MS = 10_000;// Watchdog backstop for a hang that the per-iteration timeout does not catch// (for example many slow-but-under-timeout calls, or a stall outside the awaited@@ -58,6 +59,12 @@ const MAX_BACKSTOP_WRITE_ATTEMPTS = 3;// The delay between two 504 backstop write attempts. It is short, so all retries// finish well under the in-sandbox 30s response deadline.const BACKSTOP_WRITE_RETRY_MS = 50;+// Backoff cap between poll-loop retries after a transient iteration failure.+// The cap keeps a recovering loop probing often enough to resume before the+// in-sandbox 30s response deadline strands queued callers, while the+// exponential ramp below it keeps a hard-down channel from burning an exec+// call every poll interval.+const MAX_TRANSIENT_ITERATION_BACKOFF_MS = 5_000;const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024;export const SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT = "paperclip-bridge-server.mjs";const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL";@@ -1380,13 +1387,54 @@ export async function startSandboxCallbackBridgeWorker(input: {watchdogTimer.unref();}try {+ // Consecutive transient poll failures. A single failed list call — one+ // reset or slow sandbox exec — must not end the relay for the rest of the+ // run: the in-sandbox gateway keeps queueing requests, so a dead loop+ // strands every later API call from the agent (its status writes then look+ // like connection failures and the issue loses its disposition). Back off+ // and retry the poll instead. The watchdog stays the escalation path for a+ // sustained outage — it fires after `watchdogTimeoutMs` without a+ // successful iteration and fails the queued requests fast, while this loop+ // keeps probing for recovery.+ let consecutivePollFailures = 0;while (true) {- const fileNames = await withTimeout(- input.client.listJsonFiles(directories.requestsDir),- iterationTimeoutMs,- "Sandbox callback bridge list requests",- );- if (fileNames.length === 0) {+ let fileNames: string[];+ try {+ fileNames = await withTimeout(+ input.client.listJsonFiles(directories.requestsDir),+ iterationTimeoutMs,+ "Sandbox callback bridge list requests",+ );+ consecutivePollFailures = 0;+ } catch (error) {+ if (stopping) {+ break;+ }+ consecutivePollFailures += 1;+ const message = `${buildWorkerFailureMessage(error)} (transient poll failure ${consecutivePollFailures}; retrying)`;+ if (consecutivePollFailures === 1) {+ // Put the first failure of a streak on the run trace; later repeats+ // only warn, so a flapping channel does not spam failed spans.+ await surfaceRunError(new Error(message));+ } else {+ console.warn(`[paperclip] ${message}`);+ }+ const backoffMs = Math.min(+ pollIntervalMs * 2 ** consecutivePollFailures,+ MAX_TRANSIENT_ITERATION_BACKOFF_MS,+ );+ await new Promise((resolve) => setTimeout(resolve, backoffMs));+ continue;+ }+ // A file whose attempt is still in flight (or waiting on its 504+ // backstop) is not actionable: `processRequestFile` would skip it via+ // the guard map. Treat an all-guarded listing like an empty one and+ // sleep a poll interval. Re-listing immediately would spin the loop —+ // an exec storm against a real sandbox channel, and with an in-memory+ // client a pure-microtask loop that starves every timer in the process+ // (including the guard's own backstop and abort timers).+ const actionableFileNames = fileNames.filter((fileName) => !inFlightRequestGuards.has(fileName));+ if (actionableFileNames.length === 0) {lastSuccessfulIterationAt = Date.now();if (stopping) {break;@@ -1394,7 +1442,7 @@ export async function startSandboxCallbackBridgeWorker(input: {await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));continue;}- for (const fileName of fileNames) {+ for (const fileName of actionableFileNames) {if (stopping && Date.now() >= stopDeadline) break;inFlight += 1;try {@@ -1406,7 +1454,7 @@ export async function startSandboxCallbackBridgeWorker(input: {// `task.run` after it. Without a runner, the request runs under the// run parent with no wrapper span, exactly like the earlier behavior.// The per-iteration timeout wraps the whole request, so a hung- // request rejects and the loop `catch` runs `failPendingRequests`.+ // request rejects and the catch below runs the recovery pass.await withTimeout(input.runtimeSpan? input.runtimeSpan(CALLBACK_BRIDGE_RELAY_REQUEST_SPAN, () =>@@ -1419,6 +1467,23 @@ export async function startSandboxCallbackBridgeWorker(input: {`Sandbox callback bridge process request ${fileName}`,);lastSuccessfulIterationAt = Date.now();+ } catch (error) {+ // A single request attempt failed or hung. Run the same recovery+ // pass the loop previously died on — abort the in-flight handler+ // (its 504 backstop keeps the caller from stranding) and 503 the+ // unclaimed queued requests — but keep the loop alive afterward. A+ // caller that sees the retry-safe 503 re-queues, and the recovered+ // loop serves the retry; the old terminal catch left every later+ // request to strand instead.+ const message = buildWorkerFailureMessage(error);+ await surfaceRunError(new Error(message));+ try {+ await failPendingRequests(message, { abandonInFlight: true });+ } catch (failPendingError) {+ console.warn(+ `[paperclip] sandbox callback bridge failed to abort queued requests after a request failure: ${failPendingError instanceof Error ? failPendingError.message : String(failPendingError)}`,+ );+ }} finally {inFlight -= 1;}