Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): shut down eval sandbox handles (#557)

JJ Kasper0b42ba1fc631
diff --git a/.changeset/eval-sandbox-shutdown.md b/.changeset/eval-sandbox-shutdown.md
new file mode 100644
index 000000000..3194423b5
--- /dev/null
+++ b/.changeset/eval-sandbox-shutdown.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`eve eval` now shuts down tracked sandbox handles after a local one-shot eval run completes. This prevents local sandbox compute, including microsandbox sessions, from outliving the eval process.
diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts
index 45dff3f62..d2e4a9941 100644
--- a/packages/eve/src/evals/cli/eval.ts
+++ b/packages/eve/src/evals/cli/eval.ts
@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";
+import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";
import { resolveApplicationRoot } from "#internal/application/paths.js";
import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
import { createEvalClient } from "#evals/cli/eval-client.js";
@@ -166,6 +167,9 @@ export async function runEvalCommand(
} finally {
if (devServer) {
await devServer.close();
+ await shutdownActiveSandboxHandles({
+ log: (message) => logger.error(message),
+ });
}
}
diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
index d4db77e9f..78544f061 100644
--- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
+++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
@@ -4,9 +4,14 @@ import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { runCli } from "../../src/cli/run.js";
+import {
+ clearActiveSandboxHandlesForTest,
+ trackActiveSandboxHandle,
+} from "../../src/execution/sandbox/active-handles.js";
import { useTemporaryDirectories } from "../../src/internal/testing/use-temporary-app-roots.js";
const mockedEvalDependencies = vi.hoisted(() => ({
+ createDevelopmentServer: vi.fn(),
discoverAndImportEvals: vi.fn(),
discoverEvalConfig: vi.fn(),
executeEval: vi.fn(),
@@ -26,6 +31,10 @@ vi.mock("../../src/evals/target.js", () => ({
resolveEvalTargetHandle: mockedEvalDependencies.resolveEvalTargetHandle,
}));
+vi.mock("../../src/internal/nitro/host.js", () => ({
+ createDevelopmentServer: mockedEvalDependencies.createDevelopmentServer,
+}));
+
const createScratchDirectory = useTemporaryDirectories();
const DEVELOPMENT_ENV_KEYS = [
@@ -74,6 +83,8 @@ afterEach(() => {
clearDevelopmentEnvironment();
process.exitCode = undefined;
vi.restoreAllMocks();
+ clearActiveSandboxHandlesForTest();
+ mockedEvalDependencies.createDevelopmentServer.mockReset();
mockedEvalDependencies.discoverAndImportEvals.mockReset();
mockedEvalDependencies.discoverEvalConfig.mockReset();
mockedEvalDependencies.executeEval.mockReset();
@@ -139,6 +150,51 @@ describe("eve eval environment loading", () => {
expect(exit).toHaveBeenCalledWith(0);
});
+ it("shuts down tracked sandbox handles after a local eval run", async () => {
+ const fixtureRoot = await createEnvironmentFixture();
+ const previousCwd = process.cwd();
+ const logger = {
+ error: vi.fn(),
+ log: vi.fn(),
+ };
+ const exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);
+ const close = vi.fn(async () => {});
+ const handle = { shutdown: vi.fn(async () => {}) };
+ const evaluation = makeEvaluation("local");
+
+ trackActiveSandboxHandle({ backendName: "microsandbox", handle, sessionKey: "session-1" });
+ process.chdir(fixtureRoot);
+ mockedEvalDependencies.createDevelopmentServer.mockReturnValue({
+ close,
+ start: vi.fn(async () => ({
+ appRoot: fixtureRoot,
+ kind: "started" as const,
+ url: "http://127.0.0.1:43123",
+ })),
+ });
+ mockedEvalDependencies.discoverAndImportEvals.mockResolvedValue([evaluation]);
+ mockedEvalDependencies.discoverEvalConfig.mockResolvedValue(TEST_CONFIG);
+ mockedEvalDependencies.resolveEvalTargetHandle.mockResolvedValue({
+ attachSession: vi.fn(),
+ capabilities: { devRoutes: true },
+ dispatchSchedule: vi.fn(),
+ fetch: vi.fn(),
+ kind: "local",
+ url: "http://127.0.0.1:43123",
+ });
+ mockedEvalDependencies.executeEval.mockResolvedValue(makeEvalResult(evaluation.id));
+
+ try {
+ await runCli(["eval"], logger);
+ } finally {
+ process.chdir(previousCwd);
+ }
+
+ expect(close).toHaveBeenCalledTimes(1);
+ expect(handle.shutdown).toHaveBeenCalledTimes(1);
+ expect(exit).toHaveBeenCalledWith(0);
+ });
+
it("writes all evals to one JUnit file", async () => {
const fixtureRoot = await createEnvironmentFixture();
const previousCwd = process.cwd();