🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(assistant): fix tool races in sandbox runtime (#15970)
chelsealongf22e44cd84a1
diff --git a/packages/in-app-agent-sandbox-runtime/src/server.ts b/packages/in-app-agent-sandbox-runtime/src/server.tsindex 4f65cde48a..3abf03a849 100644--- a/packages/in-app-agent-sandbox-runtime/src/server.ts+++ b/packages/in-app-agent-sandbox-runtime/src/server.ts@@ -27,7 +27,18 @@ const WORKSPACE_ROOT = "/workspace";const TOOL_CALLS_ROOT = path.join(WORKSPACE_ROOT, "tool_calls");const MICROVM_RUNTIME_HOOKS_ROOT = "/aws/lambda-microvms/runtime/v1";const MAX_REQUEST_BODY_BYTES = 10 * 1024 * 1024;+const COMMAND_TERMINATION_GRACE_MS = 1_000;let requestCounter = 0;+let sandboxOperationTail: Promise<unknown> = Promise.resolve();++function runSandboxOperationExclusive<T>(task: () => Promise<T>): Promise<T> {+ const result = sandboxOperationTail.then(task, task);+ sandboxOperationTail = result.then(+ () => undefined,+ () => undefined,+ );+ return result;+}const server = createServer(async (request, response) => {const requestId = `req-${++requestCounter}`;@@ -142,21 +153,27 @@ async function routeRequest(request: IncomingMessage, requestId: string) {requestId,operation: summarizeOperation(body),});- await syncToolCallFiles(body.toolCallFiles, requestId);- if (body.operation === "read") {- return { statusCode: 200, body: await readOperation(body, requestId) };- }+ return runSandboxOperationExclusive(async () => {+ await syncToolCallFiles(body.toolCallFiles, requestId);- if (body.operation === "write") {- return { statusCode: 200, body: await writeOperation(body, requestId) };- }+ if (body.operation === "read") {+ return { statusCode: 200, body: await readOperation(body, requestId) };+ }- if (body.operation === "edit") {- return { statusCode: 200, body: await editOperation(body, requestId) };- }+ if (body.operation === "write") {+ return {+ statusCode: 200,+ body: await writeOperation(body, requestId),+ };+ }- return { statusCode: 200, body: await bashOperation(body, requestId) };+ if (body.operation === "edit") {+ return { statusCode: 200, body: await editOperation(body, requestId) };+ }++ return { statusCode: 200, body: await bashOperation(body, requestId) };+ });}return { statusCode: 404, body: { error: "Not found" } };@@ -317,9 +334,32 @@ function runCommand(let stdout = "";let stderr = "";let settled = false;+ let timedOut = false;+ let terminationGraceTimeoutId: NodeJS.Timeout | undefined;const startedAt = new Date().toISOString();const startedAtMs = Date.now();+ const resolveTimedOutCommand = () => {+ if (settled) {+ return;+ }++ settled = true;+ if (timeoutId) {+ clearTimeout(timeoutId);+ }+ if (terminationGraceTimeoutId) {+ clearTimeout(terminationGraceTimeoutId);+ }+ resolve({+ stdout,+ stderr: `${stderr}Sandbox command timed out after ${timeoutMs}ms`,+ exitCode: 124,+ startedAt,+ completedAt: new Date().toISOString(),+ });+ };+logSandboxServer("bash.start", {requestId,pid: child.pid ?? null,@@ -356,7 +396,7 @@ function runCommand(return;}- settled = true;+ timedOut = true;if (child.pid) {try {// Process-group termination is best-effort:@@ -374,13 +414,21 @@ function runCommand(} else {child.kill("SIGKILL");}- resolve({- stdout,- stderr: `${stderr}Sandbox command timed out after ${timeoutMs}ms`,- exitCode: 124,- startedAt,- completedAt: new Date().toISOString(),- });++ terminationGraceTimeoutId = setTimeout(() => {+ if (settled) {+ return;+ }++ logSandboxServer("bash.terminationGraceExceeded", {+ requestId,+ pid: child.pid ?? null,+ graceMs: COMMAND_TERMINATION_GRACE_MS,+ });+ child.stdout.destroy();+ child.stderr.destroy();+ resolveTimedOutCommand();+ }, COMMAND_TERMINATION_GRACE_MS);}, timeoutMs);child.on("close", (code) => {@@ -388,11 +436,6 @@ function runCommand(return;}- settled = true;- if (timeoutId) {- clearTimeout(timeoutId);- }-logSandboxServer("bash.processComplete", {requestId,pid: child.pid ?? null,@@ -402,6 +445,15 @@ function runCommand(stderrBytes: Buffer.byteLength(stderr, "utf8"),});+ if (timedOut) {+ resolveTimedOutCommand();+ return;+ }++ settled = true;+ if (timeoutId) {+ clearTimeout(timeoutId);+ }resolve({stdout,stderr,diff --git a/packages/in-app-agent-sandbox-runtime/tests/sandbox.test.ts b/packages/in-app-agent-sandbox-runtime/tests/sandbox.test.tsindex 49f40ca4fe..b3bd77193d 100644--- a/packages/in-app-agent-sandbox-runtime/tests/sandbox.test.ts+++ b/packages/in-app-agent-sandbox-runtime/tests/sandbox.test.ts@@ -4,7 +4,7 @@ import path from "node:path";import { fileURLToPath } from "node:url";import Docker from "dockerode";-import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)),@@ -21,12 +21,14 @@ type JsonResponse = {describe("sandbox runtime docker container", () => {const docker = new Docker();- let container: Docker.Container | undefined;+ let container: Docker.Container;let baseUrl = "";beforeAll(async () => {await buildSandboxImage(docker);+ }, 300_000);+ beforeEach(async () => {container = await docker.createContainer({Image: IMAGE_TAG,name: `langfuse-in-app-agent-sandbox-e2e-${randomUUID().slice(0, 8)}`,@@ -47,10 +49,10 @@ describe("sandbox runtime docker container", () => {baseUrl = `http://127.0.0.1:${hostPort}`;await waitForHealth(baseUrl, container);- }, 300_000);+ });- afterAll(async () => {- await container?.remove({ force: true }).catch(() => undefined);+ afterEach(async () => {+ await container.remove({ force: true }).catch(() => undefined);});it(@@ -297,6 +299,110 @@ describe("sandbox runtime docker container", () => {}},);++ it(+ "queues concurrent sandbox operations in request order",+ { timeout: 60_000 },+ async () => {+ const first = requestJson(baseUrl, "/sandbox", {+ method: "POST",+ body: JSON.stringify({+ operation: "bash",+ command: [+ "rm -f queue-active queue-release queue-order.txt",+ "touch queue-active",+ "while [ ! -e queue-release ]; do sleep 0.05; done",+ "printf 'first\\n' >> queue-order.txt",+ "rm queue-active",+ ].join("; "),+ }),+ });++ await waitForContainerFile(container, "/workspace/queue-active");++ const second = requestJson(baseUrl, "/sandbox", {+ method: "POST",+ body: JSON.stringify({+ operation: "bash",+ command: [+ "test ! -e queue-active",+ "printf 'second\\n' >> queue-order.txt",+ ].join("; "),+ }),+ });++ const releaseExec = await container.exec({+ Cmd: ["touch", "/workspace/queue-release"],+ });+ await releaseExec.start({});++ const [firstResponse, secondResponse] = await Promise.all([+ first,+ second,+ ]);+ expect(firstResponse).toEqual({+ status: 200,+ body: { result: expect.objectContaining({ exitCode: 0 }) },+ });+ expect(secondResponse).toEqual({+ status: 200,+ body: { result: expect.objectContaining({ exitCode: 0 }) },+ });++ const order = await requestJson(baseUrl, "/sandbox", {+ method: "POST",+ body: JSON.stringify({+ operation: "read",+ path: "queue-order.txt",+ }),+ });+ expect(order).toEqual({+ status: 200,+ body: {+ result: {+ path: "/workspace/queue-order.txt",+ content: "first\nsecond\n",+ },+ },+ });++ const timedOut = requestJson(baseUrl, "/sandbox", {+ method: "POST",+ body: JSON.stringify({+ operation: "bash",+ command:+ "setsid sh -c 'echo $$ > timeout-process.pid; sleep 10' & wait",+ timeoutMs: 200,+ }),+ });++ await waitForContainerFile(container, "/workspace/timeout-process.pid");++ const afterTimeout = requestJson(baseUrl, "/sandbox", {+ method: "POST",+ body: JSON.stringify({+ operation: "bash",+ command: [+ "test -d /proc/$(cat timeout-process.pid)",+ "printf 'continued\\n' > after-timeout.txt",+ ].join(" && "),+ }),+ });++ const [timedOutResponse, afterTimeoutResponse] = await Promise.all([+ timedOut,+ afterTimeout,+ ]);+ expect(timedOutResponse).toEqual({+ status: 200,+ body: { result: expect.objectContaining({ exitCode: 124 }) },+ });+ expect(afterTimeoutResponse).toEqual({+ status: 200,+ body: { result: expect.objectContaining({ exitCode: 0 }) },+ });+ },+ );});async function buildSandboxImage(docker: Docker) {@@ -378,6 +484,25 @@ async function waitForHealth(baseUrl: string, container: Docker.Container) {);}+async function waitForContainerFile(+ container: Docker.Container,+ filePath: string,+) {+ const startedAt = Date.now();++ while (Date.now() - startedAt < HEALTH_TIMEOUT_MS) {+ const exec = await container.exec({ Cmd: ["test", "-e", filePath] });+ await exec.start({});+ if ((await exec.inspect()).ExitCode === 0) {+ return;+ }++ await new Promise((resolve) => setTimeout(resolve, 25));+ }++ throw new Error(`Container file did not appear: ${filePath}`);+}+async function getContainerLogs(container: Docker.Container) {const logs = await container.logs({ stdout: true, stderr: true, tail: 50 })