Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): clear error when eval files are nested inside agent/ (#525)

Casey Gowrie7699e98a2b39
diff --git a/.changeset/eval-nested-dir-error.md b/.changeset/eval-nested-dir-error.md
new file mode 100644
index 000000000..a1cce5dde
--- /dev/null
+++ b/.changeset/eval-nested-dir-error.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`eve eval` now prints a clear, actionable message when it finds no evals but detects `*.eval.ts` files placed inside `agent/`. Instead of the generic "No evals found", it names the offending directories and reminds you that eval files belong in the top-level `evals/` directory (a sibling of `agent/`).
diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts
index 4bc1016e5..45dff3f62 100644
--- a/packages/eve/src/evals/cli/eval.ts
+++ b/packages/eve/src/evals/cli/eval.ts
@@ -5,7 +5,11 @@ import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";
import { resolveApplicationRoot } from "#internal/application/paths.js";
import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
import { createEvalClient } from "#evals/cli/eval-client.js";
-import { discoverAndImportEvals, discoverEvalConfig } from "#evals/runner/discover.js";
+import {
+ discoverAndImportEvals,
+ discoverEvalConfig,
+ findMisplacedEvalDirs,
+} from "#evals/runner/discover.js";
import { runEvals } from "#evals/runner/run-evals.js";
import { ConsoleReporter } from "#evals/runner/reporters/console.js";
import { JUnit } from "#evals/runner/reporters/junit.js";
@@ -48,7 +52,15 @@ export async function runEvalCommand(
const discovered = await discoverAndImportEvals(appRoot, requestedEvalIds);
if (discovered.length === 0) {
- if (requestedEvalIds) {
+ const misplaced = await findMisplacedEvalDirs(appRoot);
+ if (misplaced.length > 0) {
+ logger.error(
+ "No evals found under evals/, but eval files are present inside agent/:\n" +
+ misplaced.map((dir) => ` - ${dir}`).join("\n") +
+ "\neve eval only scans the top-level evals/ directory (a sibling of agent/). " +
+ "Move these files there.",
+ );
+ } else if (requestedEvalIds) {
logger.error(`No evals found matching: ${requestedEvalIds.join(", ")}`);
} else {
logger.error("No evals found. Create files under evals/ with the *.eval.ts extension.");
diff --git a/packages/eve/src/evals/runner/discover.integration.test.ts b/packages/eve/src/evals/runner/discover.integration.test.ts
index 7aeb13e0b..448d8384e 100644
--- a/packages/eve/src/evals/runner/discover.integration.test.ts
+++ b/packages/eve/src/evals/runner/discover.integration.test.ts
@@ -7,6 +7,7 @@ import {
discoverAndImportEvals,
discoverEvalConfig,
discoverEvalFiles,
+ findMisplacedEvalDirs,
matchesEvalFilter,
} from "#evals/runner/discover.js";
@@ -233,6 +234,57 @@ describe("discoverEvalFiles", () => {
});
});
+describe("findMisplacedEvalDirs", () => {
+ let tempDir: string;
+
+ beforeEach(async () => {
+ tempDir = await mkdtemp(join(tmpdir(), "eve-eval-misplaced-"));
+ });
+
+ afterEach(async () => {
+ await rm(tempDir, { recursive: true, force: true });
+ });
+
+ it("returns an empty array when agent/ does not exist", async () => {
+ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);
+ });
+
+ it("returns an empty array when agent/ has no eval files", async () => {
+ await mkdir(join(tempDir, "agent", "skills"), { recursive: true });
+ await writeFile(join(tempDir, "agent", "agent.ts"), "export default {}");
+
+ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);
+ });
+
+ it("detects an evals directory nested directly in agent/", async () => {
+ await mkdir(join(tempDir, "agent", "evals"), { recursive: true });
+ await writeFile(join(tempDir, "agent", "evals", "weather.eval.ts"), "export default {}");
+
+ expect(await findMisplacedEvalDirs(tempDir)).toEqual(["agent/evals"]);
+ });
+
+ it("detects eval files nested deeper in the agent tree", async () => {
+ await mkdir(join(tempDir, "agent", "skills", "weather"), { recursive: true });
+ await writeFile(
+ join(tempDir, "agent", "skills", "weather", "forecast.eval.ts"),
+ "export default {}",
+ );
+ await mkdir(join(tempDir, "agent", "evals"), { recursive: true });
+ await writeFile(join(tempDir, "agent", "evals", "alpha.eval.ts"), "export default {}");
+
+ expect(await findMisplacedEvalDirs(tempDir)).toEqual(["agent/evals", "agent/skills/weather"]);
+ });
+
+ it("does not flag the legitimate top-level evals/ directory", async () => {
+ await mkdir(join(tempDir, "agent"), { recursive: true });
+ await writeFile(join(tempDir, "agent", "agent.ts"), "export default {}");
+ await mkdir(join(tempDir, "evals"), { recursive: true });
+ await writeFile(join(tempDir, "evals", "weather.eval.ts"), "export default {}");
+
+ expect(await findMisplacedEvalDirs(tempDir)).toEqual([]);
+ });
+});
+
describe("discoverEvalConfig", () => {
let tempDir: string;
diff --git a/packages/eve/src/evals/runner/discover.ts b/packages/eve/src/evals/runner/discover.ts
index 8c4994066..dc61b1b8d 100644
--- a/packages/eve/src/evals/runner/discover.ts
+++ b/packages/eve/src/evals/runner/discover.ts
@@ -35,6 +35,31 @@ export async function discoverEvalFiles(appRoot: string): Promise<string[]> {
return files;
}
+/**
+ * Finds directories under `<appRoot>/agent/` that directly contain
+ * `*.eval.ts` files. Used to produce a clear error when eval files are placed
+ * inside the agent tree instead of the top-level `evals/` directory that
+ * {@link discoverEvalFiles} scans.
+ *
+ * Returns app-root-relative, POSIX-normalized directory paths, sorted and
+ * de-duplicated. Returns `[]` when `agent/` is absent.
+ */
+export async function findMisplacedEvalDirs(appRoot: string): Promise<string[]> {
+ const agentDir = join(appRoot, "agent");
+ const dirs = new Set<string>();
+
+ try {
+ await collectMisplacedEvalDirs(agentDir, appRoot, dirs);
+ } catch (error) {
+ if (isNoEntryError(error)) {
+ return [];
+ }
+ throw error;
+ }
+
+ return [...dirs].sort((a, b) => a.localeCompare(b));
+}
+
/**
* Derives the canonical eval id from one absolute eval file path.
*
@@ -205,6 +230,22 @@ async function collectEvalFiles(dir: string, files: string[]): Promise<void> {
}
}
+async function collectMisplacedEvalDirs(
+ dir: string,
+ appRoot: string,
+ dirs: Set<string>,
+): Promise<void> {
+ const entries = await readdir(dir, { withFileTypes: true });
+
+ for (const entry of entries) {
+ if (entry.isDirectory()) {
+ await collectMisplacedEvalDirs(join(dir, entry.name), appRoot, dirs);
+ } else if (entry.isFile() && entry.name.endsWith(EVAL_FILE_SUFFIX)) {
+ dirs.add(relative(appRoot, dir).split(/[\\/]/u).join("/"));
+ }
+ }
+}
+
function isNoEntryError(error: unknown): boolean {
return (
typeof error === "object" &&