The Open Framework for Building Agents
fix(eve): shut down eval sandbox handles (#557)
JJ Kasper0b42ba1fc631
diff --git a/.changeset/eval-sandbox-shutdown.md b/.changeset/eval-sandbox-shutdown.mdnew file mode 100644index 000000000..3194423b5--- /dev/null+++ b/.changeset/eval-sandbox-shutdown.md@@ -0,0 +1,5 @@+---+"eve": patch+---++`eve eval` now shuts down tracked sandbox handles after a local one-shot eval run completes. This prevents local sandbox compute, including microsandbox sessions, from outliving the eval process.diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.tsindex 45dff3f62..d2e4a9941 100644--- a/packages/eve/src/evals/cli/eval.ts+++ b/packages/eve/src/evals/cli/eval.ts@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";import { basename, join } from "node:path";import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";+import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";import { resolveApplicationRoot } from "#internal/application/paths.js";import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";import { createEvalClient } from "#evals/cli/eval-client.js";@@ -166,6 +167,9 @@ export async function runEvalCommand(} finally {if (devServer) {await devServer.close();+ await shutdownActiveSandboxHandles({+ log: (message) => logger.error(message),+ });}}diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.tsindex d4db77e9f..78544f061 100644--- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts+++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts@@ -4,9 +4,14 @@ import { join } from "node:path";import { afterEach, describe, expect, it, vi } from "vitest";import { runCli } from "../../src/cli/run.js";+import {+ clearActiveSandboxHandlesForTest,+ trackActiveSandboxHandle,+} from "../../src/execution/sandbox/active-handles.js";import { useTemporaryDirectories } from "../../src/internal/testing/use-temporary-app-roots.js";const mockedEvalDependencies = vi.hoisted(() => ({+ createDevelopmentServer: vi.fn(),discoverAndImportEvals: vi.fn(),discoverEvalConfig: vi.fn(),executeEval: vi.fn(),@@ -26,6 +31,10 @@ vi.mock("../../src/evals/target.js", () => ({resolveEvalTargetHandle: mockedEvalDependencies.resolveEvalTargetHandle,}));+vi.mock("../../src/internal/nitro/host.js", () => ({+ createDevelopmentServer: mockedEvalDependencies.createDevelopmentServer,+}));+const createScratchDirectory = useTemporaryDirectories();const DEVELOPMENT_ENV_KEYS = [@@ -74,6 +83,8 @@ afterEach(() => {clearDevelopmentEnvironment();process.exitCode = undefined;vi.restoreAllMocks();+ clearActiveSandboxHandlesForTest();+ mockedEvalDependencies.createDevelopmentServer.mockReset();mockedEvalDependencies.discoverAndImportEvals.mockReset();mockedEvalDependencies.discoverEvalConfig.mockReset();mockedEvalDependencies.executeEval.mockReset();@@ -139,6 +150,51 @@ describe("eve eval environment loading", () => {expect(exit).toHaveBeenCalledWith(0);});+ it("shuts down tracked sandbox handles after a local eval run", async () => {+ const fixtureRoot = await createEnvironmentFixture();+ const previousCwd = process.cwd();+ const logger = {+ error: vi.fn(),+ log: vi.fn(),+ };+ const exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);+ const close = vi.fn(async () => {});+ const handle = { shutdown: vi.fn(async () => {}) };+ const evaluation = makeEvaluation("local");++ trackActiveSandboxHandle({ backendName: "microsandbox", handle, sessionKey: "session-1" });+ process.chdir(fixtureRoot);+ mockedEvalDependencies.createDevelopmentServer.mockReturnValue({+ close,+ start: vi.fn(async () => ({+ appRoot: fixtureRoot,+ kind: "started" as const,+ url: "http://127.0.0.1:43123",+ })),+ });+ mockedEvalDependencies.discoverAndImportEvals.mockResolvedValue([evaluation]);+ mockedEvalDependencies.discoverEvalConfig.mockResolvedValue(TEST_CONFIG);+ mockedEvalDependencies.resolveEvalTargetHandle.mockResolvedValue({+ attachSession: vi.fn(),+ capabilities: { devRoutes: true },+ dispatchSchedule: vi.fn(),+ fetch: vi.fn(),+ kind: "local",+ url: "http://127.0.0.1:43123",+ });+ mockedEvalDependencies.executeEval.mockResolvedValue(makeEvalResult(evaluation.id));++ try {+ await runCli(["eval"], logger);+ } finally {+ process.chdir(previousCwd);+ }++ expect(close).toHaveBeenCalledTimes(1);+ expect(handle.shutdown).toHaveBeenCalledTimes(1);+ expect(exit).toHaveBeenCalledWith(0);+ });+it("writes all evals to one JUnit file", async () => {const fixtureRoot = await createEnvironmentFixture();const previousCwd = process.cwd();