AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(sandbox): stop killing runs whose engine is busy for 45s (#16698)
Mo AbuAboud4339dde2ceed
diff --git a/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md b/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.mdindex 7b54537da6b..2b153be60a0 100644--- a/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md+++ b/brain/knowledge/decisions/000001-worker-is-the-sandbox-one-job-per-worker-scale-by-replicas.md@@ -27,3 +27,5 @@ Throughput = replica count. The worker image must carry the execution toolchain**Headroom belongs where the budget is *derived*, not where it's applied.** `--max-old-space-size` must sit below the container limit, because V8's old space is only part of RSS (new space, external buffers, isolated-vm heaps at 128 MB each, native allocations, plus the worker process itself). But the reserve must NOT be a blanket fraction inside the node-args helper: at the default `AP_WORKER_CONCURRENCY=5`, `SANDBOX_MEMORY_LIMIT` is an operator-configured number (default 1024 MB) with no relationship to container size, so shaving 25% off it just steals working memory from self-hosters who had no problem. The reserve lives in `sandbox-config.ts` `primeFullContainerMemory()` — the concurrency-1 path that derives the budget *from* `memory.max`, and the only place where 100% is provably unusable. An explicit `SANDBOX_MEMORY_LIMIT` is honored exactly.`--no-node-snapshot` **is mandatory wherever isolated-vm loads** (isolated-vm#424) — that means the isolate path too, since `SANDBOX_CODE_AND_PROCESS` runs code steps in isolated-vm *inside* the isolate sandbox.++**The worker↔engine socket.io heartbeat must outlast the flow timeout.** The engine exits on any socket disconnect (`worker-socket.ts`), and socket.io's default heartbeat (25s ping + 20s timeout) fires whenever the engine's event loop is blocked that long — any synchronous step, e.g. text-helper's Strip HTML spending 31s on a 3 MB page. The run then died as `SANDBOX_INTERNAL_ERROR` with `Worker socket disconnected (ping timeout)` in stderr. `sandbox.ts` sets `pingTimeout` to the sandbox time limit, capped so `pingInterval + pingTimeout` stays under Node's 2^31-1 ms timer limit; process death is already caught by the child's `close` event, so the heartbeat adds nothing on a loopback socket.diff --git a/packages/server/sandbox/src/lib/sandbox/sandbox.ts b/packages/server/sandbox/src/lib/sandbox/sandbox.tsindex bbc7494d74d..d417b170ac8 100644--- a/packages/server/sandbox/src/lib/sandbox/sandbox.ts+++ b/packages/server/sandbox/src/lib/sandbox/sandbox.ts@@ -10,6 +10,13 @@ import { cacheUtils } from '../cache/cache-paths'import { assertSafeCodeNamespace, assertSafePathSegment } from '../utils/path-safety'import { Sandbox, SandboxInitOptions, SandboxLogger, SandboxMount, SandboxOptions, SandboxProcessMaker, SandboxResult } from './types'+const PING_INTERVAL_MS = 25_000+const MAX_NODE_TIMER_MS = 2 ** 31 - 1++function heartbeatTimeoutMs(timeLimitSeconds: number): number {+ return Math.min(timeLimitSeconds * 1000, MAX_NODE_TIMER_MS - PING_INTERVAL_MS)+}+function assertSandboxPathUnderRoot(mount: SandboxMount): void {const normalized = path.posix.normalize(mount.sandboxPath)if (!normalized.startsWith('/root/') && normalized !== '/root') {@@ -86,6 +93,8 @@ export function createSandbox(const ioServer = new SocketIOServer(server, {path: '/worker/ws',maxHttpBufferSize: options.maxHttpBufferSizeBytes,+ pingInterval: PING_INTERVAL_MS,+ pingTimeout: heartbeatTimeoutMs(options.timeLimitSeconds),cors: { origin: '*' },})wireConnectionHandler(ioServer)diff --git a/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts b/packages/server/sandbox/test/lib/sandbox/sandbox.test.tsindex 7ae22031f3b..8f1481d6ad4 100644--- a/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts+++ b/packages/server/sandbox/test/lib/sandbox/sandbox.test.ts@@ -53,6 +53,7 @@ function createMockLogger(): ApLogger {function createTestProcessMaker() {let client: ClientSocket | null = nulllet child: (ChildProcess & EventEmitter) | null = null+ let handshake: { pingTimeout: number, pingInterval: number } | null = nullconst maker: SandboxProcessMaker = {create: vi.fn(async (params) => {@@ -71,6 +72,9 @@ function createTestProcessMaker() {reconnection: false,auth: { sandboxId: params.sandboxId, connectionToken: token },})+ client.io.engine.once('handshake', (data) => {+ handshake = data+ })return child as ChildProcess}),@@ -80,6 +84,7 @@ function createTestProcessMaker() {maker,getClient: () => client!,getChild: () => child!,+ getHandshake: () => handshake,}}@@ -415,6 +420,29 @@ describe('createSandbox', () => {}})+ it('keeps the heartbeat alive while the engine event loop is busy for the whole time limit', async () => {+ testPM = createTestProcessMaker()+ sandbox = createSandbox(createMockLogger(), 'sb-heartbeat', defaultOptions, testPM.maker)++ await sandbox.start(startOptions)++ const handshake = testPM.getHandshake()+ expect(handshake).not.toBeNull()+ expect(handshake!.pingTimeout).toBeGreaterThanOrEqual(defaultOptions.timeLimitSeconds * 1000)+ })++ it('keeps the heartbeat timer within what Node can schedule when the time limit is huge', async () => {+ testPM = createTestProcessMaker()+ const thirtyDaysSeconds = 30 * 24 * 60 * 60+ sandbox = createSandbox(createMockLogger(), 'sb-heartbeat-cap', { ...defaultOptions, timeLimitSeconds: thirtyDaysSeconds }, testPM.maker)++ await sandbox.start(startOptions)++ const handshake = testPM.getHandshake()!+ expect(handshake.pingInterval + handshake.pingTimeout).toBeLessThanOrEqual(2 ** 31 - 1)+ expect(sandbox.isReady()).toBe(true)+ })+it('is idempotent when already connected', async () => {const log = createMockLogger()testPM = createTestProcessMaker()