Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(evals): classify missing Vitest environments as setup errors

Louis Beaumont49f60beb78ed
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
index 9fc7fdc19..6e0d2e9c4 100644
--- a/evals/coding-agent/grader-outcome.mjs
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -29,6 +29,16 @@ export function classifyGraderError(grader) {
return "playwright_ct_build_error";
}
const summary = [...stdout.matchAll(/^\s*Tests\s+(.+)$/gm)].at(-1)?.[1]?.trim();
+ // A missing Vitest environment can abort before any suite or test runs.
+ // Match the dependency in both diagnostics and require the final empty
+ // summaries; executed outcomes and quoted assertions remain behavior results.
+ const fileSummary = [...stdout.matchAll(/^\s*Test Files\s+(.+)$/gm)].at(-1)?.[1]?.trim();
+ const missingEnvironment = stderr.match(/^\s*MISSING DEPENDENCY\s+Cannot find dependency ['"]([^'"\r\n]+)['"]\s*$/m)?.[1];
+ const missingPackage = stderr.match(/^Error: Cannot find package ['"]([^'"\r\n]+)['"] imported from [^\r\n]*[/\\]vitest[/\\]dist[/\\][^\r\n]+$/m)?.[1];
+ if (summary === "no tests" && fileSummary === "no tests" && noAssertion &&
+ /^\s*RUN\s+v\d+\./m.test(stdout) && /^\s*Errors\s+[1-9]\d* errors?\s*$/m.test(stdout) &&
+ missingEnvironment && missingEnvironment === missingPackage &&
+ /^Serialized Error: \{ code: ['"]ERR_MODULE_NOT_FOUND['"] \}\s*$/m.test(stderr)) return "vitest_environment_error";
// A failed suite can collect zero tests while neighboring suites pass or skip.
// Only summaries without executed failures qualify; diagnostic and assertion
// guards still apply to both streams, including quoted setup errors.
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 3ea449878..804731ef6 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -28,6 +28,9 @@ test("process and setup failures cannot prove a regression or count as scored fa
git("commit", "-qam", "Synthetic reference state");
const fix = git("rev-parse", "HEAD");
const controls = [
+ { id: "baseline-vitest-environment", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-vitest-environment", baseline: "fail", oracle: "error", valid: false },
+ { id: "vitest-environment-assertion", baseline: "fail", oracle: "pass", valid: true },
{ id: "baseline-vitest-transform", baseline: "error", oracle: "pass", valid: false },
{ id: "reference-vitest-transform", baseline: "fail", oracle: "error", valid: false },
{ id: "vitest-transform-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },
@@ -100,6 +103,12 @@ import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
+if (((id === "baseline-vitest-environment" || id === "reference-vitest-environment") && affected) || (id === "vitest-environment-assertion" && broken)) {
+ process.stdout.write(" RUN v2.1.9 /synthetic\\n Test Files no tests\\n Tests no tests\\n Errors 1 error\\n");
+ process.stderr.write(" MISSING DEPENDENCY Cannot find dependency 'jsdom'\\nError: Cannot find package 'jsdom' imported from /synthetic/node_modules/vitest/dist/chunks/environment.js\\nSerialized Error: { code: 'ERR_MODULE_NOT_FOUND' }\\n");
+ if (id === "vitest-environment-assertion") process.stderr.write("AssertionError: preserved behavior failed\\n");
+ process.exit(1);
+} else if (id === "reference-vitest-environment" && broken) assert.fail("synthetic broken behavior");
if ((id.includes("vitest-transform") && affected && id !== "vitest-transform-quoted-assertion") || (id === "vitest-transform-quoted-assertion" && broken)) {
process.stdout.write(" Test Files 1 failed (1)\\n Tests " + (id === "vitest-transform-quoted-assertion" ? "1 failed (1)" : "no tests") + "\\n");
process.stderr.write("Failed Suites 1\\nError: Transform failed with 1 error:\\n/synthetic/source.ts:2:5: ERROR: Unexpected token\\n");
@@ -248,7 +257,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi,baseline-vitest-mixed-collection");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-vitest-environment,baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi,baseline-vitest-mixed-collection");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));
@@ -323,3 +332,23 @@ test("Vitest transform diagnostics require unexecuted failure evidence", async (
expect(classifyGraderError({...error,stderr})).toBeNull();
expect(classifyGraderError({...error,status:0})).toBeNull();
});
+
+
+test("Vitest missing environments require matching diagnostics and no executed tests", async () => {
+ const { classifyGraderError } = await import("./grader-outcome.mjs");
+ const setup = {
+ status: 1,
+ stdout: " RUN v2.1.9 /synthetic\n Test Files no tests\n Tests no tests\n Errors 1 error\n",
+ stderr: " MISSING DEPENDENCY Cannot find dependency 'jsdom'\nError: Cannot find package 'jsdom' imported from /synthetic/node_modules/vitest/dist/chunks/environment.js\nSerialized Error: { code: 'ERR_MODULE_NOT_FOUND' }\n",
+ };
+ expect(classifyGraderError(setup)).toBe("vitest_environment_error");
+ expect(classifyGraderError({ ...setup, stderr: setup.stderr.replaceAll("jsdom", "happy-dom") })).toBe("vitest_environment_error");
+ expect(classifyGraderError({ ...setup, stdout: "\x1b[31m" + setup.stdout + "\x1b[0m", stderr: "\x1b[31m" + setup.stderr + "\x1b[0m" })).toBe("vitest_environment_error");
+ for (const summary of ["1 failed (1)", "1 passed (1)", "1 skipped (1)", "no tests\n Tests 1 failed (1)"])
+ expect(classifyGraderError({ ...setup, stdout: setup.stdout.replace(" Tests no tests", " Tests " + summary) })).toBeNull();
+ for (const stdout of [setup.stdout.replace(" RUN v2.1.9 /synthetic\n", ""), setup.stdout.replace(" Errors 1 error\n", ""), setup.stdout.replace(" Test Files no tests", " Test Files 1 failed (1)"), setup.stdout + " Test Files 1 passed (1)\n", setup.stdout + "AssertionError: actual differs\n"])
+ expect(classifyGraderError({ ...setup, stdout })).toBeNull();
+ for (const stderr of ["", setup.stderr.replace("MISSING DEPENDENCY", "unknown"), setup.stderr.replace("package 'jsdom'", "package 'different'"), setup.stderr.replace("/vitest/", "/application/"), setup.stderr.replace("ERR_MODULE_NOT_FOUND", "OTHER_ERROR"), setup.stderr + "AssertionError: real failure\n"])
+ expect(classifyGraderError({ ...setup, stderr })).toBeNull();
+ expect(classifyGraderError({ ...setup, status: 0 })).toBeNull();
+});