Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(evals): classify Vite import resolution failures

louisc6e34740b244
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 8fe5f76a8..c2f4d967a 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -73,7 +73,7 @@ Grader timeouts and terminating signals cannot establish a failing baseline:
`--verify` requires a behavioral `fail` followed by an oracle `pass`.
Known command-unavailable exits (126/127), Rust compiler failures before test execution,
Bun unhandled test-load errors, and
-Node missing-module/syntax diagnostics and Vitest URL-load or imported-module collection failures
+Node missing-module/syntax diagnostics and Vitest/Vite import-resolution, URL-load or imported-module collection failures
with zero tests are also reported as `error`, with a
`grader_error_kind` in the result. A missing import followed by an oracle pass
therefore cannot certify a regression. Plain failed assertions and successful
@@ -87,9 +87,9 @@ change, caused it. Report error counts and inspect candidate-caused errors befor
comparing model success rates. No isolation or model-quality claim follows.
Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
-Forty-three end-to-end controls cover baseline/reference timeouts and signals,
+Fifty end-to-end controls cover baseline/reference timeouts and signals,
missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitest
-collection failures (including alias imports), missing PostCSS-plugin startup
+collection failures (including alias imports and Vite resolve-import diagnostics), missing PostCSS-plugin startup
failures, Rust/Cargo compilation failures and assertions quoting compiler output,
quoted diagnostics followed by real assertions, unavailable or non-executable
commands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
index 35aaf47c4..39a46d83f 100644
--- a/evals/coding-agent/grader-outcome.mjs
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -17,7 +17,9 @@ export function classifyGraderError(grader) {
const summary = [...stdout.matchAll(/^\s*Tests\s+(.+)$/gm)].at(-1)?.[1]?.trim();
if (summary === "no tests" && /^\s*Test Files\s+\d+ failed/m.test(stdout) &&
/Failed Suites [1-9]/.test(stderr) &&
- (/^Error: Failed to load url /m.test(stderr) ||
+ ((/^Error: Failed to resolve import ['"][^\r\n]+['"] from ['"][^\r\n]+['"]\. Does the file exist\?\s*$/m.test(stderr) &&
+ !/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) ||
+ /^Error: Failed to load url /m.test(stderr) ||
/^Error: Cannot find module ['"][^\n]+['"] imported from ['"][^\n]+['"]\.\s*$/m.test(stderr))) return "vitest_collection_error";
// A beforeAll build failure can register tests but skip every affected test.
// Passing neighboring suites do not turn a compile failure into an assertion.
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 8706afe73..9b29ec6dc 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -28,6 +28,13 @@ test("process and setup failures cannot prove a regression or count as scored fa
git("commit", "-qam", "Synthetic reference state");
const fix = git("rev-parse", "HEAD");
const controls = [
+ { id: "baseline-vitest-resolve", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-vitest-resolve", baseline: "fail", oracle: "error", valid: false },
+ { id: "baseline-vitest-resolve-ansi", baseline: "error", oracle: "pass", valid: false },
+ { id: "vitest-assertion-with-quoted-resolve", baseline: "fail", oracle: "pass", valid: true },
+ { id: "node-assertion-with-quoted-resolve", baseline: "fail", oracle: "pass", valid: true },
+ { id: "both-pass-resolve-diagnostic", baseline: "pass", oracle: "pass", valid: false },
+ { id: "resolve-without-test-summary", baseline: "fail", oracle: "pass", valid: true },
{ id: "baseline-vitest-bun-build", baseline: "error", oracle: "pass", valid: false },
{ id: "reference-vitest-bun-build", baseline: "fail", oracle: "error", valid: false },
{ id: "baseline-vitest-bun-build-with-pass", baseline: "error", oracle: "pass", valid: false },
@@ -79,7 +86,18 @@ import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
-if ((id.endsWith("vitest-bun-build") && affected) || (id === "baseline-vitest-bun-build-with-pass" && broken) ||
+if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||
+ (["vitest-assertion-with-quoted-resolve", "node-assertion-with-quoted-resolve", "resolve-without-test-summary"].includes(id) && broken) || id === "both-pass-resolve-diagnostic") {
+ const summary = "Test Files 1 failed (1)\\nTests no tests\\n";
+ if (id !== "resolve-without-test-summary") process.stdout.write(id.endsWith("-ansi") ? "\\x1b[31m" + summary + "\\x1b[0m" : summary);
+ process.stderr.write('Failed Suites 1\\nError: Failed to resolve import "./missing-synthetic" from "fixture.test.ts". Does the file exist?\\n');
+ if (id === "vitest-assertion-with-quoted-resolve") {
+ process.stdout.write("Test Files 1 failed (1)\\nTests 1 failed (1)\\n");
+ process.stderr.write("AssertionError: intended synthetic outcome failed\\n");
+ }
+ if (id === "node-assertion-with-quoted-resolve") assert.fail("behavior failure after quoted collection diagnostic");
+ process.exit(id === "both-pass-resolve-diagnostic" ? 0 : 1);
+} else if ((id.endsWith("vitest-bun-build") && affected) || (id === "baseline-vitest-bun-build-with-pass" && broken) ||
(["vitest-build-quoted-assertion", "vitest-build-hook-assertion", "node-build-quoted-assertion", "unknown-build-command"].includes(id) && broken) || id === "both-pass-build-diagnostic") {
const summary = id === "vitest-build-quoted-assertion" ? "1 failed (1)" : id === "baseline-vitest-bun-build-with-pass" ? "1 passed | 1 skipped (2)" : "1 skipped (1)";
process.stdout.write(" RUN v4.0.18 /synthetic\\n Test Files 1 failed (1)\\n Tests " + summary + "\\n");
@@ -149,7 +167,7 @@ else if (id === "baseline-syntax" && affected) {
assert.fail("\\nError [ERR_MODULE_NOT_FOUND]: quoted diagnostic\\nSyntaxError: quoted diagnostic\\n at quotedFixture");
} else if (id === "both-pass-diagnostic-words") {
console.error("Cannot find module; SyntaxError; command not found are fixture words");
-} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile" || id === "reference-vitest-bun-build") && broken) assert.fail("synthetic broken behavior");
+} else if ((id === "reference-vitest-resolve" || id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile" || id === "reference-vitest-bun-build") && broken) assert.fail("synthetic broken behavior");
else process.exit(id === "intended-failure" && broken ? 1 : 0);
`);
const manifest = join(repo, "cases.json");
@@ -188,7 +206,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));