Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: reject Vitest Bun build hook setup failures

louisfacb8b4dd91e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 5315e0c62..7d35feed8 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -66,7 +66,7 @@ change, caused it. Report error counts and inspect candidate-caused errors befor
comparing model success rates. No isolation or model-quality claim follows.
Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
-Thirty-five end-to-end controls cover baseline/reference timeouts and signals,
+Forty-three end-to-end controls cover baseline/reference timeouts and signals,
missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitest
collection failures (including alias imports), missing PostCSS-plugin startup
failures, Rust/Cargo compilation failures and assertions quoting compiler output,
@@ -217,3 +217,15 @@ The fixture tests sequential saved-file changes and real local process/protocol
behavior. It does not verify concurrent file writes, native desktop persistence,
external enterprise gateways, token cryptography or the harness's filesystem
isolation. Dependencies are exposed only during grading, after the trajectory.
+
+
+Vitest can register tests and then skip them because a `beforeAll` Bun build
+failed. A bounded missing-input compiler diagnostic, failed suite and skipped
+summary with no test failures is now `vitest_bun_build_setup_error`; passing
+neighboring tests do not turn the build failure into regression evidence.
+Assertions quoting compiler diagnostics and successful commands retain their
+outcomes. The runner controls cover both baseline/reference build-hook failure,
+neighboring passes, quoted assertion diagnostics and error exclusion from scores.
+An actual Bun compiler/Vitest setup failure previously certified a false valid
+contrast; it now becomes an error and invalid verification. This is bounded
+recognition, not general compiler attribution or proof of execution isolation.
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
index 2d3decf38..35aaf47c4 100644
--- a/evals/coding-agent/grader-outcome.mjs
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -19,6 +19,15 @@ export function classifyGraderError(grader) {
/Failed Suites [1-9]/.test(stderr) &&
(/^Error: Failed to load url /m.test(stderr) ||
/^Error: Cannot find module ['"][^\n]+['"] imported from ['"][^\n]+['"]\.\s*$/m.test(stderr))) return "vitest_collection_error";
+ // A beforeAll build failure can register tests but skip every affected test.
+ // Passing neighboring suites do not turn a compile failure into an assertion.
+ // Preserve executed failures and assertion headers, including quoted diagnostics.
+ const buildHookSummary = summary?.match(/^(?:(?:\d+ (?:passed|skipped))(?: \| )?)+ \(\d+\)$/);
+ if (buildHookSummary && /[1-9]\d* skipped/.test(summary) &&
+ /^\s*Test Files\s+.*[1-9]\d* failed/m.test(stdout) && /Failed Suites [1-9]/.test(stderr) &&
+ /^Error: Command failed: [^\n]*\bbun(?:\.exe)? (?:build|run build)\b/m.test(stderr) &&
+ /^error: Could not resolve: ["'][^\n]+["']\s*$/m.test(stderr) &&
+ !/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) return "vitest_bun_build_setup_error";
// Vite can reject PostCSS startup before printing a test-count summary.
// Require the specific missing-plugin diagnostic and startup banner; an
// executed-test summary or assertion header must keep its behavioral result.
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 179b65c07..8706afe73 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -28,6 +28,14 @@ test("process and setup failures cannot prove a regression or count as scored fa
git("commit", "-qam", "Synthetic reference state");
const fix = git("rev-parse", "HEAD");
const controls = [
+ { id: "baseline-vitest-bun-build", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-vitest-bun-build", baseline: "fail", oracle: "error", valid: false },
+ { id: "baseline-vitest-bun-build-with-pass", baseline: "error", oracle: "pass", valid: false },
+ { id: "vitest-build-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "vitest-build-hook-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "node-build-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "both-pass-build-diagnostic", baseline: "pass", oracle: "pass", valid: false },
+ { id: "unknown-build-command", baseline: "fail", oracle: "pass", valid: true },
{ id: "baseline-rust-compile", baseline: "error", oracle: "pass", valid: false },
{ id: "reference-rust-compile", baseline: "fail", oracle: "error", valid: false },
{ id: "baseline-cargo-compile", baseline: "error", oracle: "pass", valid: false },
@@ -71,7 +79,15 @@ import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
-if ((id.endsWith("-rust-compile") && affected) || ((id === "baseline-cargo-compile" || id === "baseline-rust-passed-before-compile") && broken) ||
+if ((id.endsWith("vitest-bun-build") && affected) || (id === "baseline-vitest-bun-build-with-pass" && broken) ||
+ (["vitest-build-quoted-assertion", "vitest-build-hook-assertion", "node-build-quoted-assertion", "unknown-build-command"].includes(id) && broken) || id === "both-pass-build-diagnostic") {
+ const summary = id === "vitest-build-quoted-assertion" ? "1 failed (1)" : id === "baseline-vitest-bun-build-with-pass" ? "1 passed | 1 skipped (2)" : "1 skipped (1)";
+ process.stdout.write(" RUN v4.0.18 /synthetic\\n Test Files 1 failed (1)\\n Tests " + summary + "\\n");
+ process.stderr.write(" Failed Suites 1\\nError: Command failed: " + (id === "unknown-build-command" ? "other-compiler" : "bun build") + " entry.ts\\nerror: Could not resolve: \\"./missing-input\\"\\n");
+ if (id === "vitest-build-quoted-assertion" || id === "vitest-build-hook-assertion") process.stderr.write("AssertionError: intended synthetic behavior failed\\n");
+ if (id === "node-build-quoted-assertion") assert.fail("synthetic assertion after quoted build output");
+ process.exit(id === "both-pass-build-diagnostic" ? 0 : 1);
+} else if ((id.endsWith("-rust-compile") && affected) || ((id === "baseline-cargo-compile" || id === "baseline-rust-passed-before-compile") && broken) ||
((id === "rust-assertion-with-quoted-compile" || id === "node-assertion-with-quoted-compile" || id === "rust-panic-with-quoted-compile") && broken) || id === "both-pass-compile-diagnostic") {
process.stderr.write("error[E0425]: cannot find value synthetic_fixture in this scope\\n");
process.stderr.write(id === "baseline-cargo-compile" ? "error: could not compile \`synthetic-fixture\` (lib test) due to 1 previous error\\n" : "error: aborting due to 1 previous error\\n");
@@ -133,7 +149,7 @@ else if (id === "baseline-syntax" && affected) {
assert.fail("\\nError [ERR_MODULE_NOT_FOUND]: quoted diagnostic\\nSyntaxError: quoted diagnostic\\n at quotedFixture");
} else if (id === "both-pass-diagnostic-words") {
console.error("Cannot find module; SyntaxError; command not found are fixture words");
-} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile") && broken) assert.fail("synthetic broken behavior");
+} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile" || id === "reference-vitest-bun-build") && broken) assert.fail("synthetic broken behavior");
else process.exit(id === "intended-failure" && broken ? 1 : 0);
`);
const manifest = join(repo, "cases.json");
@@ -172,7 +188,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));