Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(sandbox): stop killing runs whose engine is busy for 45s (#16698)

Mo AbuAboud4339dde2ceed
diff --git a/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md b/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md
index 7b54537da6b..2b153be60a0 100644
--- a/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md
+++ b/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md
@@ -27,3 +27,5 @@ Throughput = replica count. The worker image must carry the execution toolchain
**Headroom belongs where the budget is *derived*, not where it's applied.** `--max-old-space-size` must sit below the container limit, because V8's old space is only part of RSS (new space, external buffers, isolated-vm heaps at 128 MB each, native allocations, plus the worker process itself). But the reserve must NOT be a blanket fraction inside the node-args helper: at the default `AP_WORKER_CONCURRENCY=5`, `SANDBOX_MEMORY_LIMIT` is an operator-configured number (default 1024 MB) with no relationship to container size, so shaving 25% off it just steals working memory from self-hosters who had no problem. The reserve lives in `sandbox-config.ts` `primeFullContainerMemory()` — the concurrency-1 path that derives the budget *from* `memory.max`, and the only place where 100% is provably unusable. An explicit `SANDBOX_MEMORY_LIMIT` is honored exactly.
`--no-node-snapshot` **is mandatory wherever isolated-vm loads** (isolated-vm#424) — that means the isolate path too, since `SANDBOX_CODE_AND_PROCESS` runs code steps in isolated-vm *inside* the isolate sandbox.
+
+**The worker↔engine socket.io heartbeat must outlast the flow timeout.** The engine exits on any socket disconnect (`worker-socket.ts`), and socket.io's default heartbeat (25s ping + 20s timeout) fires whenever the engine's event loop is blocked that long — any synchronous step, e.g. text-helper's Strip HTML spending 31s on a 3 MB page. The run then died as `SANDBOX_INTERNAL_ERROR` with `Worker socket disconnected (ping timeout)` in stderr. `sandbox.ts` sets `pingTimeout` to the sandbox time limit, capped so `pingInterval + pingTimeout` stays under Node's 2^31-1 ms timer limit; process death is already caught by the child's `close` event, so the heartbeat adds nothing on a loopback socket.
diff --git a/packages/server/sandbox/src/lib/sandbox/sandbox.ts b/packages/server/sandbox/src/lib/sandbox/sandbox.ts
index bbc7494d74d..d417b170ac8 100644
--- a/packages/server/sandbox/src/lib/sandbox/sandbox.ts
+++ b/packages/server/sandbox/src/lib/sandbox/sandbox.ts
@@ -10,6 +10,13 @@ import { cacheUtils } from '../cache/cache-paths'
import { assertSafeCodeNamespace, assertSafePathSegment } from '../utils/path-safety'
import { Sandbox, SandboxInitOptions, SandboxLogger, SandboxMount, SandboxOptions, SandboxProcessMaker, SandboxResult } from './types'
+const PING_INTERVAL_MS = 25_000
+const MAX_NODE_TIMER_MS = 2 ** 31 - 1
+
+function heartbeatTimeoutMs(timeLimitSeconds: number): number {
+ return Math.min(timeLimitSeconds * 1000, MAX_NODE_TIMER_MS - PING_INTERVAL_MS)
+}
+
function assertSandboxPathUnderRoot(mount: SandboxMount): void {
const normalized = path.posix.normalize(mount.sandboxPath)
if (!normalized.startsWith('/root/') && normalized !== '/root') {
@@ -86,6 +93,8 @@ export function createSandbox(
const ioServer = new SocketIOServer(server, {
path: '/worker/ws',
maxHttpBufferSize: options.maxHttpBufferSizeBytes,
+ pingInterval: PING_INTERVAL_MS,
+ pingTimeout: heartbeatTimeoutMs(options.timeLimitSeconds),
cors: { origin: '*' },
})
wireConnectionHandler(ioServer)
diff --git a/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts b/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts
index 7ae22031f3b..8f1481d6ad4 100644
--- a/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts
+++ b/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts
@@ -53,6 +53,7 @@ function createMockLogger(): ApLogger {
function createTestProcessMaker() {
let client: ClientSocket | null = null
let child: (ChildProcess & EventEmitter) | null = null
+ let handshake: { pingTimeout: number, pingInterval: number } | null = null
const maker: SandboxProcessMaker = {
create: vi.fn(async (params) => {
@@ -71,6 +72,9 @@ function createTestProcessMaker() {
reconnection: false,
auth: { sandboxId: params.sandboxId, connectionToken: token },
})
+ client.io.engine.once('handshake', (data) => {
+ handshake = data
+ })
return child as ChildProcess
}),
@@ -80,6 +84,7 @@ function createTestProcessMaker() {
maker,
getClient: () => client!,
getChild: () => child!,
+ getHandshake: () => handshake,
}
}
@@ -415,6 +420,29 @@ describe('createSandbox', () => {
}
})
+ it('keeps the heartbeat alive while the engine event loop is busy for the whole time limit', async () => {
+ testPM = createTestProcessMaker()
+ sandbox = createSandbox(createMockLogger(), 'sb-heartbeat', defaultOptions, testPM.maker)
+
+ await sandbox.start(startOptions)
+
+ const handshake = testPM.getHandshake()
+ expect(handshake).not.toBeNull()
+ expect(handshake!.pingTimeout).toBeGreaterThanOrEqual(defaultOptions.timeLimitSeconds * 1000)
+ })
+
+ it('keeps the heartbeat timer within what Node can schedule when the time limit is huge', async () => {
+ testPM = createTestProcessMaker()
+ const thirtyDaysSeconds = 30 * 24 * 60 * 60
+ sandbox = createSandbox(createMockLogger(), 'sb-heartbeat-cap', { ...defaultOptions, timeLimitSeconds: thirtyDaysSeconds }, testPM.maker)
+
+ await sandbox.start(startOptions)
+
+ const handshake = testPM.getHandshake()!
+ expect(handshake.pingInterval + handshake.pingTimeout).toBeLessThanOrEqual(2 ** 31 - 1)
+ expect(sandbox.isReady()).toBe(true)
+ })
+
it('is idempotent when already connected', async () => {
const log = createMockLogger()
testPM = createTestProcessMaker()