The Open Framework for Building Agents
fix(eve): clear error when eval files are nested inside agent/ (#525)
Casey Gowrie7699e98a2b39
diff --git a/.changeset/eval-nested-dir-error.md b/.changeset/eval-nested-dir-error.mdnew file mode 100644index 000000000..a1cce5dde--- /dev/null+++ b/.changeset/eval-nested-dir-error.md@@ -0,0 +1,5 @@+---+"eve": patch+---++`eve eval` now prints a clear, actionable message when it finds no evals but detects `*.eval.ts` files placed inside `agent/`. Instead of the generic "No evals found", it names the offending directories and reminds you that eval files belong in the top-level `evals/` directory (a sibling of `agent/`).diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.tsindex 4bc1016e5..45dff3f62 100644--- a/packages/eve/src/evals/cli/eval.ts+++ b/packages/eve/src/evals/cli/eval.ts@@ -5,7 +5,11 @@ import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";import { resolveApplicationRoot } from "#internal/application/paths.js";import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";import { createEvalClient } from "#evals/cli/eval-client.js";-import { discoverAndImportEvals, discoverEvalConfig } from "#evals/runner/discover.js";+import {+ discoverAndImportEvals,+ discoverEvalConfig,+ findMisplacedEvalDirs,+} from "#evals/runner/discover.js";import { runEvals } from "#evals/runner/run-evals.js";import { ConsoleReporter } from "#evals/runner/reporters/console.js";import { JUnit } from "#evals/runner/reporters/junit.js";@@ -48,7 +52,15 @@ export async function runEvalCommand(const discovered = await discoverAndImportEvals(appRoot, requestedEvalIds);if (discovered.length === 0) {- if (requestedEvalIds) {+ const misplaced = await findMisplacedEvalDirs(appRoot);+ if (misplaced.length > 0) {+ logger.error(+ "No evals found under evals/, but eval files are present inside agent/:\n" ++ misplaced.map((dir) => ` - ${dir}`).join("\n") ++ "\neve eval only scans the top-level evals/ directory (a sibling of agent/). " ++ "Move these files there.",+ );+ } else if (requestedEvalIds) {logger.error(`No evals found matching: ${requestedEvalIds.join(", ")}`);} else {logger.error("No evals found. Create files under evals/ with the *.eval.ts extension.");diff --git a/packages/eve/src/evals/runner/discover.integration.test.ts b/packages/eve/src/evals/runner/discover.integration.test.tsindex 7aeb13e0b..448d8384e 100644--- a/packages/eve/src/evals/runner/discover.integration.test.ts+++ b/packages/eve/src/evals/runner/discover.integration.test.ts@@ -7,6 +7,7 @@ import {discoverAndImportEvals,discoverEvalConfig,discoverEvalFiles,+ findMisplacedEvalDirs,matchesEvalFilter,} from "#evals/runner/discover.js";@@ -233,6 +234,57 @@ describe("discoverEvalFiles", () => {});});+describe("findMisplacedEvalDirs", () => {+ let tempDir: string;++ beforeEach(async () => {+ tempDir = await mkdtemp(join(tmpdir(), "eve-eval-misplaced-"));+ });++ afterEach(async () => {+ await rm(tempDir, { recursive: true, force: true });+ });++ it("returns an empty array when agent/ does not exist", async () => {+ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);+ });++ it("returns an empty array when agent/ has no eval files", async () => {+ await mkdir(join(tempDir, "agent", "skills"), { recursive: true });+ await writeFile(join(tempDir, "agent", "agent.ts"), "export default {}");++ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);+ });++ it("detects an evals directory nested directly in agent/", async () => {+ await mkdir(join(tempDir, "agent", "evals"), { recursive: true });+ await writeFile(join(tempDir, "agent", "evals", "weather.eval.ts"), "export default {}");++ expect(await findMisplacedEvalDirs(tempDir)).toEqual(["agent/evals"]);+ });++ it("detects eval files nested deeper in the agent tree", async () => {+ await mkdir(join(tempDir, "agent", "skills", "weather"), { recursive: true });+ await writeFile(+ join(tempDir, "agent", "skills", "weather", "forecast.eval.ts"),+ "export default {}",+ );+ await mkdir(join(tempDir, "agent", "evals"), { recursive: true });+ await writeFile(join(tempDir, "agent", "evals", "alpha.eval.ts"), "export default {}");++ expect(await findMisplacedEvalDirs(tempDir)).toEqual(["agent/evals", "agent/skills/weather"]);+ });++ it("does not flag the legitimate top-level evals/ directory", async () => {+ await mkdir(join(tempDir, "agent"), { recursive: true });+ await writeFile(join(tempDir, "agent", "agent.ts"), "export default {}");+ await mkdir(join(tempDir, "evals"), { recursive: true });+ await writeFile(join(tempDir, "evals", "weather.eval.ts"), "export default {}");++ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);+ });+});+describe("discoverEvalConfig", () => {let tempDir: string;diff --git a/packages/eve/src/evals/runner/discover.ts b/packages/eve/src/evals/runner/discover.tsindex 8c4994066..dc61b1b8d 100644--- a/packages/eve/src/evals/runner/discover.ts+++ b/packages/eve/src/evals/runner/discover.ts@@ -35,6 +35,31 @@ export async function discoverEvalFiles(appRoot: string): Promise<string[]> {return files;}+/**+ * Finds directories under `<appRoot>/agent/` that directly contain+ * `*.eval.ts` files. Used to produce a clear error when eval files are placed+ * inside the agent tree instead of the top-level `evals/` directory that+ * {@link discoverEvalFiles} scans.+ *+ * Returns app-root-relative, POSIX-normalized directory paths, sorted and+ * de-duplicated. Returns `[]` when `agent/` is absent.+ */+export async function findMisplacedEvalDirs(appRoot: string): Promise<string[]> {+ const agentDir = join(appRoot, "agent");+ const dirs = new Set<string>();++ try {+ await collectMisplacedEvalDirs(agentDir, appRoot, dirs);+ } catch (error) {+ if (isNoEntryError(error)) {+ return [];+ }+ throw error;+ }++ return [...dirs].sort((a, b) => a.localeCompare(b));+}+/*** Derives the canonical eval id from one absolute eval file path.*@@ -205,6 +230,22 @@ async function collectEvalFiles(dir: string, files: string[]): Promise<void> {}}+async function collectMisplacedEvalDirs(+ dir: string,+ appRoot: string,+ dirs: Set<string>,+): Promise<void> {+ const entries = await readdir(dir, { withFileTypes: true });++ for (const entry of entries) {+ if (entry.isDirectory()) {+ await collectMisplacedEvalDirs(join(dir, entry.name), appRoot, dirs);+ } else if (entry.isFile() && entry.name.endsWith(EVAL_FILE_SUFFIX)) {+ dirs.add(relative(appRoot, dir).split(/[\\/]/u).join("/"));+ }+ }+}+function isNoEntryError(error: unknown): boolean {return (typeof error === "object" &&