The Open Framework for Building Agents
feat(eve): add eval setup, teardown, and shared context (#3453)
Matan Kushnerd88aedeef375
diff --git a/.changeset/quiet-evals-setup.md b/.changeset/quiet-evals-setup.mdnew file mode 100644index 000000000..cfddb4b5b--- /dev/null+++ b/.changeset/quiet-evals-setup.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Add eval setup and teardown callbacks with typed context shared by reference across evals and cleanup. Setup returns the context directly before the local agent starts, and teardown runs after shutdown even when setup or the run fails.diff --git a/docs/evals/overview.mdx b/docs/evals/overview.mdxindex e95c6e382..8a32a0dd1 100644--- a/docs/evals/overview.mdx+++ b/docs/evals/overview.mdx@@ -58,6 +58,49 @@ export default defineEvalConfig({Everything is optional. `judge` sets the default model for [LLM-as-judge](./judge) assertions (`t.judge(...)`); omit it to use the default evaluator, `typesafe-ai/jev`. Deterministic evals make no judge requests. `reporters`, `maxConcurrency`, and `timeoutMs` round out the defaults. Config `reporters` observe every eval in the run, so set one `Braintrust()` here instead of adding it to each eval. CLI flags (`--max-concurrency`, `--timeout`) and per-eval values take precedence over the config defaults.+### Run setup and teardown++Use `setup` to start shared resources or run migrations before eve starts the local agent server. Return shared context directly from `setup`, then release resources in `teardown`:++```ts title="evals/evals.config.ts"+import { defineEvalConfig } from "eve/evals";+import { startTestDatabase } from "./database.js";++export default defineEvalConfig({+ async setup() {+ const database = await startTestDatabase();+ return { database };+ },+ async teardown(context) {+ await context?.database.close();+ },+});+```++`context` is inferred from the setup return value. To use that type in an eval file, pass the config's type to `defineEval`:++```ts title="evals/database.eval.ts"+import { defineEval } from "eve/evals";+import { equals } from "eve/evals/expect";+import type config from "./evals.config.js";++export default defineEval<typeof config>({+ async test(t) {+ const result = await t.context.database.query("SELECT 1 AS value");+ await t.require(result.rows[0]?.value, equals(1));+ },+});+```++Setup, eval bodies, and teardown share the same context object in the runner process. Class instances and functions keep their identity; context is neither serialized nor sent to the agent server. Concurrent evals share these resources, so isolate any database records they modify.++`setup` can also return nothing.++- Setup runs once per `eve eval` invocation, after discovery and config loading, before target startup.+- Teardown runs after the local server and its sandbox handles shut down, including after setup, startup, or eval failures. Cleanup failures fail the command.++`teardown` receives `undefined` when setup returns no context or throws before returning. In that case, setup must clean up resources that only it can access. Teardown can also be used without setup. The per-eval `timeoutMs` does not apply to setup or teardown.+## Deterministic fixture modelsUse `mockModel` when an eval fixture needs to exercise eve's runtime without calling a model provider. A static fixture can be one line:diff --git a/e2e/fixtures/extensions/evals/evals.config.ts b/e2e/fixtures/extensions/evals/evals.config.tsindex 7a61a5b55..24b37312b 100644--- a/e2e/fixtures/extensions/evals/evals.config.ts+++ b/e2e/fixtures/extensions/evals/evals.config.ts@@ -1,6 +1,26 @@import { e2eJudgeModel } from "@eve-e2e/config";import { defineEvalConfig } from "eve/evals";+/** Resource shared by setup, evals, and teardown. */+export class SetupResource {+ #closed = false;++ read() {+ if (this.#closed) throw new Error("Eval setup resource is closed.");+ return "ready";+ }++ close() {+ this.#closed = true;+ }+}+export default defineEvalConfig({judge: { model: e2eJudgeModel() },+ setup() {+ return { resource: new SetupResource() };+ },+ teardown(context) {+ context?.resource.close();+ },});diff --git a/e2e/fixtures/extensions/evals/setup.eval.ts b/e2e/fixtures/extensions/evals/setup.eval.tsnew file mode 100644index 000000000..e7f3c2221--- /dev/null+++ b/e2e/fixtures/extensions/evals/setup.eval.ts@@ -0,0 +1,11 @@+import { defineEval } from "eve/evals";+import { equals } from "eve/evals/expect";++import type config from "./evals.config.js";++export default defineEval<typeof config>({+ description: "Eval setup supplies a live resource to evals.",+ async test(t) {+ await t.require(t.context.resource.read(), equals("ready"));+ },+});diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.tsindex 822c08053..7fb52269b 100644--- a/packages/eve/src/evals/cli/eval.ts+++ b/packages/eve/src/evals/cli/eval.ts@@ -130,8 +130,11 @@ export async function runEvalCommand(let devServer: DevelopmentServer | undefined;let target: EveEvalTargetHandle;let client: Awaited<ReturnType<typeof createEvalClient>>;+ let setupContext: unknown;try {+ setupContext = await config.setup?.();+if (options.url) {client = await createEvalClient({ kind: "remote", url: options.url },@@ -167,6 +170,7 @@ export async function runEvalCommand(const summary = await runEvals({evaluations,config,+ setupContext,target,client,appRoot,@@ -193,11 +197,19 @@ export async function runEvalCommand(process.exitCode = 1;}} finally {- if (devServer) {- await devServer.close();- await shutdownActiveSandboxHandles({- log: (message) => logger.error(message),- });+ for (const cleanup of [+ () => devServer?.close(),+ () => devServer && shutdownActiveSandboxHandles({ log: (message) => logger.error(message) }),+ () => config.teardown?.(setupContext),+ ]) {+ try {+ await cleanup();+ } catch (error) {+ logger.error(+ `Eval cleanup failed: ${error instanceof Error ? error.message : String(error)}`,+ );+ process.exitCode = 1;+ }}}diff --git a/packages/eve/src/evals/context.ts b/packages/eve/src/evals/context.tsindex c2e829860..af1ab0d2c 100644--- a/packages/eve/src/evals/context.ts+++ b/packages/eve/src/evals/context.ts@@ -19,6 +19,7 @@ import type {* AssertionCollector.finalize} it against the completed task result.*/export function createEvalContext(deps: {+ readonly setupContext?: unknown;readonly manager: EvalSessionManager;readonly collector: AssertionCollector;readonly target: EveEvalTargetHandle;@@ -40,6 +41,7 @@ export function createEvalContext(deps: {send: (message, options) => deps.manager.send(message, options),// Run context.+ context: deps.setupContext,signal: deps.signal,target: deps.target,log: deps.log,diff --git a/packages/eve/src/evals/define-eval-config.test.ts b/packages/eve/src/evals/define-eval-config.test.tsindex 021bb39d3..97f898d73 100644--- a/packages/eve/src/evals/define-eval-config.test.ts+++ b/packages/eve/src/evals/define-eval-config.test.ts@@ -1,6 +1,7 @@-import { describe, expect, it } from "vitest";+import { describe, expect, expectTypeOf, it } from "vitest";import { defineEvalConfig } from "#evals/define-eval-config.js";+import { defineEval } from "#evals/define-eval.js";import type { EveEvalConfigInput } from "#evals/types.js";const TEST_MODEL = "openai/gpt-5.4-mini";@@ -10,6 +11,58 @@ function defineInvalidConfig(input: Partial<EveEvalConfigInput> & Record<string,}describe("defineEvalConfig", () => {+ it("infers setup context for teardown and evals", () => {+ class Database {+ query() {+ return 42;+ }+ }+ const config = defineEvalConfig({+ async setup() {+ return { database: new Database() };+ },+ teardown(context) {+ expectTypeOf(context).toEqualTypeOf<{ database: Database } | undefined>();+ },+ });+ const evaluation = defineEval<typeof config>({+ test(t) {+ expectTypeOf(t.context).toEqualTypeOf<{ database: Database }>();+ expectTypeOf(t.context.database.query()).toEqualTypeOf<number>();+ },+ });+ expect(evaluation._tag).toBe("EveEval");+ });++ it("infers undefined without setup", () => {+ const config = defineEvalConfig({+ teardown(context) {+ expectTypeOf(context).toEqualTypeOf<undefined>();+ },+ });+ defineEval<typeof config>({+ test(t) {+ expectTypeOf(t.context).toEqualTypeOf<undefined>();+ },+ });+ });++ it("preserves optional context when setup may return nothing", () => {+ const config = defineEvalConfig({+ setup(): Map<string, number> | undefined {+ return undefined;+ },+ teardown(context) {+ expectTypeOf(context).toEqualTypeOf<Map<string, number> | undefined>();+ },+ });+ defineEval<typeof config>({+ test(t) {+ expectTypeOf(t.context).toEqualTypeOf<Map<string, number> | undefined>();+ },+ });+ });+it("returns a tagged config from valid input", () => {const config = defineEvalConfig({judge: { model: TEST_MODEL },@@ -56,4 +109,14 @@ describe("defineEvalConfig", () => {"`reporters` must be an array",);});++ it("rejects a non-function teardown", () => {+ expect(() => defineInvalidConfig({ teardown: {} as never })).toThrow(+ "`teardown` must be a function",+ );+ });++ it("rejects a non-function setup", () => {+ expect(() => defineInvalidConfig({ setup: {} as never })).toThrow("`setup` must be a function");+ });});diff --git a/packages/eve/src/evals/define-eval-config.ts b/packages/eve/src/evals/define-eval-config.tsindex 0c32a8931..de14e83cd 100644--- a/packages/eve/src/evals/define-eval-config.ts+++ b/packages/eve/src/evals/define-eval-config.ts@@ -6,15 +6,17 @@ import type { EveEvalConfig, EveEvalConfigInput } from "#evals/types.js";** Exactly one `evals.config.ts` is required. It supplies the optional default* `judge` model for `t.judge(...)` assertions (so individual evals need not- * repeat it), optional run-level `reporters`, a default `maxConcurrency`, and a- * default `timeoutMs`. CLI flags (`--max-concurrency`, `--timeout`) and- * per-eval values take precedence over the config defaults.+ * repeat it), run-level `reporters`, `maxConcurrency` and `timeoutMs` defaults,+ * and optional run-wide `setup` and `teardown`. CLI flags (`--max-concurrency`,+ * `--timeout`) and per-eval values take precedence over the config defaults.*- * Throws on invalid input: a non-positive or- * non-integer `maxConcurrency`, a negative or non-finite `timeoutMs`, or a- * non-array `reporters`.+ * Throws on invalid input: a non-positive or non-integer `maxConcurrency`,+ * a negative or non-finite `timeoutMs`, non-array `reporters`, or a+ * non-function `setup` or `teardown`.*/-export function defineEvalConfig(input: EveEvalConfigInput): EveEvalConfig {+export function defineEvalConfig<TContext = undefined>(+ input: EveEvalConfigInput<TContext>,+): EveEvalConfig<TContext> {validateEvalConfigInput(input);return {@@ -24,6 +26,14 @@ export function defineEvalConfig(input: EveEvalConfigInput): EveEvalConfig {}function validateEvalConfigInput(input: EveEvalConfigInput): void {+ if (input.setup !== undefined && typeof input.setup !== "function") {+ throw new Error("Eval config `setup` must be a function.");+ }++ if (input.teardown !== undefined && typeof input.teardown !== "function") {+ throw new Error("Eval config `teardown` must be a function.");+ }+if (input.maxConcurrency !== undefined &&(!Number.isInteger(input.maxConcurrency) || input.maxConcurrency < 1)diff --git a/packages/eve/src/evals/define-eval.ts b/packages/eve/src/evals/define-eval.tsindex 1cf8b45b6..aa48f2747 100644--- a/packages/eve/src/evals/define-eval.ts+++ b/packages/eve/src/evals/define-eval.ts@@ -1,4 +1,9 @@-import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";+import type {+ EveEvalConfig,+ EveEvalConfigContext,+ EveEvalDefinition,+ EveEvalInput,+} from "#evals/types.js";/*** Defines one eve eval. Each eval file is exactly one case: an imperative@@ -8,6 +13,9 @@ import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";* `evals/`, or default-export an array of evals to fan one file out over a* dataset.*+ * Pass `defineEval<typeof config>(...)` with a type-only config import to+ * infer the setup context available as `t.context`.+ ** A `judge` is optional: `t.judge(...)` assertions fall back to the `judge`* declared in `evals.config.ts`, then the shared evaluation default. The judge model* is used solely for scoring, never for the agent under test. Eval identity is@@ -18,7 +26,9 @@ import type { EveEvalDefinition, EveEvalInput } from "#evals/types.js";* (`input`/`run`/`checks`/`scores`/`expected`/`thresholds`/`parseOutput`/* `model`/`requires`), or a negative or non-finite `timeoutMs`.*/-export function defineEval(input: EveEvalInput): EveEvalDefinition {+export function defineEval<TConfig extends EveEvalConfig = EveEvalConfig>(+ input: EveEvalInput<EveEvalConfigContext<TConfig>>,+): EveEvalDefinition<EveEvalConfigContext<TConfig>> {validateEvalInput(input);return {diff --git a/packages/eve/src/evals/index.ts b/packages/eve/src/evals/index.tsindex a97416699..34f87fb43 100644--- a/packages/eve/src/evals/index.ts+++ b/packages/eve/src/evals/index.ts@@ -44,6 +44,7 @@ export type {EveEval,EveEvalConfig,EveEvalConfigInput,+ EveEvalConfigContext,EveEvalDefinition,EveEvalInput,EveEvalLiveTurn,diff --git a/packages/eve/src/evals/runner/execute-eval.ts b/packages/eve/src/evals/runner/execute-eval.tsindex 1b3fa5c3a..f85e4d03c 100644--- a/packages/eve/src/evals/runner/execute-eval.ts+++ b/packages/eve/src/evals/runner/execute-eval.ts@@ -21,6 +21,8 @@ export interface ExecuteEvalOptions {readonly onLog?: (message: string) => void;/** Receives the first trace context observed for each session. */readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;+ /** Shared setup context; stays in the runner process. */+ readonly setupContext?: unknown;readonly target: EveEvalTargetHandle;/** Overrides the eval's own `timeoutMs` when set (CLI `--timeout`). */readonly timeoutMs?: number;@@ -51,6 +53,7 @@ export async function executeEval(options: ExecuteEvalOptions): Promise<EveEvalRconst outcome = await executeTask({client,evaluation,+ setupContext: options.setupContext,onLog: options.onLog,onSessionStart: options.onSessionStart,target,diff --git a/packages/eve/src/evals/runner/execute-task.ts b/packages/eve/src/evals/runner/execute-task.tsindex 296b85be4..54402c729 100644--- a/packages/eve/src/evals/runner/execute-task.ts+++ b/packages/eve/src/evals/runner/execute-task.ts@@ -32,6 +32,8 @@ interface ExecuteTaskOptions {readonly onLog?: (message: string) => void;/** Receives the first trace context observed for each session. */readonly onSessionStart?: (event: EvalSessionStartedEvent) => void;+ /** Shared setup context; stays in the runner process. */+ readonly setupContext?: unknown;readonly target: EveEvalTargetHandle;readonly timeoutMs?: number;}@@ -69,6 +71,7 @@ export async function executeTask(options: ExecuteTaskOptions): Promise<ExecuteTconst logs: string[] = [];const { context } = createEvalContext({+ setupContext: options.setupContext,collector,manager,target: targetForRun,diff --git a/packages/eve/src/evals/runner/run-evals.ts b/packages/eve/src/evals/runner/run-evals.tsindex 99b43f0b8..2a6c4759d 100644--- a/packages/eve/src/evals/runner/run-evals.ts+++ b/packages/eve/src/evals/runner/run-evals.ts@@ -19,6 +19,8 @@ export interface RunEvalsOptions {readonly evaluations: readonly EveEval[];/** Run-wide configuration from `evals.config.ts` (defaults shared by every eval). */readonly config: EveEvalConfig;+ /** Shared setup context; stays in the runner process. */+ readonly setupContext?: unknown;readonly target: EveEvalTargetHandle;readonly client: Client;readonly appRoot: string;@@ -118,6 +120,7 @@ export async function runEvals(options: RunEvalsOptions): Promise<EveEvalRunSummconst result = await executeEval({client,evaluation,+ setupContext: options.setupContext,onLog:options.onEvalLog === undefined? undefineddiff --git a/packages/eve/src/evals/types.ts b/packages/eve/src/evals/types.tsindex 9413a172e..6dd858727 100644--- a/packages/eve/src/evals/types.ts+++ b/packages/eve/src/evals/types.ts@@ -423,7 +423,9 @@ export interface JudgeContext {* Scoped assertions (`succeeded`, `calledTool`, …) record an entry evaluated* after the test body; `check`, `require`, and `judge` evaluate explicit values.*/-export interface EveEvalContext extends EveEvalAssertions {+export interface EveEvalContext<TContext = unknown> extends EveEvalAssertions {+ /** Run-wide setup context, shared by reference across evals and teardown. */+ readonly context: TContext;/** Eval timeout signal. */readonly signal: AbortSignal;/** Current target under test. */@@ -543,9 +545,9 @@ export interface EveEvalInputFields extends EveEvalBase {* drives the agent and asserts on what it produced. Eval identity is derived* from the file path, so authors do not specify an `id` or `name`.*/-export interface EveEvalInput extends EveEvalBase {+export interface EveEvalInput<TContext = unknown> extends EveEvalBase {/** Imperative interaction-and-assertion script. */- test(t: EveEvalContext): void | Promise<void>;+ test(t: EveEvalContext<TContext>): void | Promise<void>;}/**@@ -554,7 +556,7 @@ export interface EveEvalInput extends EveEvalBase {* `_tag` literal (`"EveEval"`) brands the value so discovery and the runner* can recognize a defined eval.*/-export type EveEvalDefinition = EveEvalInput & {+export type EveEvalDefinition<TContext = unknown> = EveEvalInput<TContext> & {readonly _tag: "EveEval";};@@ -629,7 +631,19 @@ export interface EveEvalRunSummary {* Exactly one `evals.config.ts` is required at the root of the `evals/`* directory; it supplies the defaults every eval in the run shares.*/-export interface EveEvalConfigInput {+export interface EveEvalConfigInput<TContext = unknown> {+ /**+ * Runs once before target startup, except for `--list` or an empty selection.+ * The return value is shared with evals and teardown by reference in the runner process.+ */+ readonly setup?: () => TContext | Promise<TContext>;+ /**+ * Runs once after server and sandbox shutdown, even if setup or eval execution fails.+ * Receives undefined if setup returns no context or throws before returning.+ * Can be used without setup.+ * Does not run for `--list` or an empty selection.+ */+ teardown?(context: TContext | undefined): void | Promise<void>;/*** Default judge model for `t.judge(...)` assertions across every eval.* Optional: omission uses the shared evaluation default. Individual evals@@ -658,6 +672,11 @@ export interface EveEvalConfigInput {* Validated eval run configuration returned by `defineEvalConfig()`. The* `_tag` literal brands the value so discovery can recognize it.*/-export type EveEvalConfig = EveEvalConfigInput & {+export type EveEvalConfig<TContext = unknown> = EveEvalConfigInput<TContext> & {readonly _tag: "EveEvalConfig";};++/** Setup context type associated with an authored eval config. */+export type EveEvalConfigContext<TConfig extends EveEvalConfig> = Awaited<+ ReturnType<NonNullable<TConfig["setup"]>>+>;diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.tsindex ff99968f0..a6011b68a 100644--- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts+++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts@@ -3,6 +3,9 @@ import { join } from "node:path";import { afterEach, describe, expect, it, vi } from "vitest";+import { defineEvalConfig } from "../../src/evals/define-eval-config.js";+import { defineEval } from "../../src/evals/define-eval.js";+import { runCli } from "../../src/cli/run.js";import {clearActiveSandboxHandlesForTest,@@ -27,7 +30,8 @@ vi.mock("../../src/evals/runner/execute-eval.js", () => ({executeEval: mockedEvalDependencies.executeEval,}));-vi.mock("../../src/evals/target.js", () => ({+vi.mock("../../src/evals/target.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../../src/evals/target.js")>()),resolveEvalTargetHandle: mockedEvalDependencies.resolveEvalTargetHandle,}));@@ -110,6 +114,199 @@ const TEST_CONFIG = {};describe("eve eval environment loading", () => {+ it("shares live setup resources with every eval and teardown", async () => {+ const fixture = await createEvalSetupFixture();+ class Database {+ #queries = 0;+ readonly marker = 1n;+ closed = false;+ query() {+ if (this.closed) throw new Error("Database is closed");+ return ++this.#queries;+ }+ close() {+ this.closed = true;+ }+ }+ const database = new Database();+ const config = defineEvalConfig({+ async setup() {+ await fixture.setup();+ return database;+ },+ teardown(context) {+ expect(context).toBe(database);+ expect(database.query()).toBe(3);+ context?.close();+ },+ });+ const evaluation = defineEval<typeof config>({+ test(t) {+ expect(t.context).toBe(database);+ expect(t.context.query()).toBeGreaterThan(0);+ },+ });+ mockedEvalDependencies.discoverEvalConfig.mockResolvedValue(config);+ mockedEvalDependencies.discoverAndImportEvals.mockResolvedValue([+ { ...evaluation, id: "first" },+ { ...evaluation, id: "second" },+ ]);+ const { executeEval } = await vi.importActual<+ typeof import("../../src/evals/runner/execute-eval.js")+ >("../../src/evals/runner/execute-eval.js");+ mockedEvalDependencies.executeEval.mockImplementation(executeEval);++ await fixture.run();++ expect(database.closed).toBe(true);+ expect(fixture.exit).toHaveBeenCalledWith(0);+ expect(fixture.close.mock.invocationCallOrder[0]).toBeLessThan(+ fixture.exit.mock.invocationCallOrder[0]!,+ );+ });++ it("runs setup once before local startup and tears down after server shutdown", async () => {+ const fixture = await createEvalSetupFixture();++ await fixture.run();++ expect(fixture.setup).toHaveBeenCalledTimes(1);+ expect(mockedEvalDependencies.executeEval).toHaveBeenCalledTimes(2);+ expect(fixture.close).toHaveBeenCalledTimes(1);+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.setup.mock.invocationCallOrder[0]).toBeLessThan(+ fixture.start.mock.invocationCallOrder[0]!,+ );+ expect(fixture.close.mock.invocationCallOrder[0]).toBeLessThan(+ fixture.teardown.mock.invocationCallOrder[0]!,+ );+ expect(fixture.exit).toHaveBeenCalledWith(0);+ });++ it("runs setup and teardown locally for a remote target", async () => {+ const fixture = await createEvalSetupFixture();+ mockedEvalDependencies.resolveEvalTargetHandle.mockResolvedValue({+ kind: "remote",+ url: "https://example.com",+ });++ await fixture.run(["--url", "https://example.com"]);++ expect(fixture.setup).toHaveBeenCalledTimes(1);+ expect(fixture.setup.mock.invocationCallOrder[0]).toBeLessThan(+ mockedEvalDependencies.resolveEvalTargetHandle.mock.invocationCallOrder[0]!,+ );+ expect(mockedEvalDependencies.createDevelopmentServer).not.toHaveBeenCalled();+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.exit).toHaveBeenCalledWith(0);+ });++ it("tears down resources when local startup fails", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.start.mockRejectedValueOnce(new Error("fixture startup failed"));++ await expect(fixture.run()).rejects.toThrow("fixture startup failed");++ expect(mockedEvalDependencies.executeEval).not.toHaveBeenCalled();+ expect(fixture.close).toHaveBeenCalledTimes(1);+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.close.mock.invocationCallOrder[0]).toBeLessThan(+ fixture.teardown.mock.invocationCallOrder[0]!,+ );+ });++ it("tears down resources and fails the command when an eval fails", async () => {+ const fixture = await createEvalSetupFixture();+ mockedEvalDependencies.executeEval.mockResolvedValueOnce(makeEvalResult("beta"));++ await fixture.run();++ expect(fixture.close).toHaveBeenCalledTimes(1);+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.exit).toHaveBeenCalledWith(1);+ });++ it("still tears down resources when server shutdown fails", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.close.mockRejectedValueOnce(new Error("fixture close failed"));++ await fixture.run();++ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.logger.error).toHaveBeenCalledWith("Eval cleanup failed: fixture close failed");+ expect(fixture.exit).toHaveBeenCalledWith(1);+ });++ it("fails the command when teardown fails", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.teardown.mockRejectedValueOnce(new Error("fixture teardown failed"));++ await fixture.run();++ expect(fixture.close).toHaveBeenCalledTimes(1);+ expect(fixture.logger.error).toHaveBeenCalledWith(+ "Eval cleanup failed: fixture teardown failed",+ );+ expect(fixture.exit).toHaveBeenCalledWith(1);+ });++ it("runs teardown without starting a target when setup fails", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.setup.mockRejectedValueOnce(new Error("fixture setup failed"));+ fixture.teardown.mockResolvedValueOnce(undefined);++ await expect(fixture.run()).rejects.toThrow("fixture setup failed");++ expect(mockedEvalDependencies.createDevelopmentServer).not.toHaveBeenCalled();+ expect(mockedEvalDependencies.resolveEvalTargetHandle).not.toHaveBeenCalled();+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.teardown).toHaveBeenCalledWith(undefined);+ });++ it("preserves the setup error when teardown also fails", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.setup.mockRejectedValueOnce(new Error("fixture setup failed"));+ fixture.teardown.mockRejectedValueOnce(new Error("fixture teardown failed"));++ await expect(fixture.run()).rejects.toThrow("fixture setup failed");++ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.logger.error).toHaveBeenCalledWith(+ "Eval cleanup failed: fixture teardown failed",+ );+ });++ it("runs teardown without a setup callback", async () => {+ const fixture = await createEvalSetupFixture();+ fixture.teardown.mockResolvedValueOnce(undefined);+ mockedEvalDependencies.discoverEvalConfig.mockResolvedValue({+ ...TEST_CONFIG,+ teardown: fixture.teardown,+ });+ mockedEvalDependencies.executeEval.mockResolvedValue(makeEvalResult("first"));++ await fixture.run(["--url", "https://example.com"]);++ expect(fixture.setup).not.toHaveBeenCalled();+ expect(mockedEvalDependencies.executeEval).toHaveBeenCalledTimes(2);+ expect(fixture.teardown).toHaveBeenCalledTimes(1);+ expect(fixture.exit).toHaveBeenCalledWith(0);+ });++ it.each([+ { name: "listing evals", args: ["--list"] },+ { name: "excluding all evals", args: ["--exclude-tag", "setup"] },+ ])("skips setup when $name", async ({ args }) => {+ const fixture = await createEvalSetupFixture();++ await fixture.run(args);++ expect(fixture.setup).not.toHaveBeenCalled();+ expect(mockedEvalDependencies.createDevelopmentServer).not.toHaveBeenCalled();+ expect(mockedEvalDependencies.resolveEvalTargetHandle).not.toHaveBeenCalled();+ expect(fixture.teardown).not.toHaveBeenCalled();+ });+it("loads local env files before resolving a remote target", async () => {const fixtureRoot = await createEnvironmentFixture();const resolvedFixtureRoot = await realpath(fixtureRoot);@@ -263,6 +460,51 @@ describe("eve eval environment loading", () => {});});+async function createEvalSetupFixture() {+ const appRoot = await realpath(await createEnvironmentFixture());+ const logger = { error: vi.fn(), log: vi.fn() };+ const exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);+ const teardown = vi.fn(async () => {});+ const setup = vi.fn(async () => {});+ const start = vi.fn(async () => ({ url: "http://127.0.0.1:43123" }));+ const close = vi.fn(async () => {});+ process.env.EVE_DEV_SHELL_ONLY = "from-shell";+ mockedEvalDependencies.discoverAndImportEvals.mockResolvedValue(+ [makeEvaluation("first"), makeEvaluation("second")].map((evaluation) => ({+ ...evaluation,+ tags: ["setup"],+ })),+ );+ mockedEvalDependencies.discoverEvalConfig.mockResolvedValue({ ...TEST_CONFIG, setup, teardown });+ mockedEvalDependencies.createDevelopmentServer.mockReturnValue({ start, close });+ mockedEvalDependencies.resolveEvalTargetHandle.mockResolvedValue({+ kind: "local",+ url: "http://127.0.0.1:43123",+ });+ mockedEvalDependencies.executeEval.mockImplementation(async ({ evaluation }) => {+ return makeEvalResult(evaluation.id);+ });++ return {+ appRoot,+ logger,+ exit,+ setup,+ start,+ close,+ teardown,+ async run(args: string[] = []) {+ const previousCwd = process.cwd();+ process.chdir(appRoot);+ try {+ await runCli(["eval", "--skip-report", ...args], logger);+ } finally {+ process.chdir(previousCwd);+ }+ },+ };+}+function makeEvaluation(id: string) {return {_tag: "EveEval" as const,