YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: reject Vitest Bun build hook setup failures
louisfacb8b4dd91e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 5315e0c62..7d35feed8 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -66,7 +66,7 @@ change, caused it. Report error counts and inspect candidate-caused errors beforcomparing model success rates. No isolation or model-quality claim follows.Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.-Thirty-five end-to-end controls cover baseline/reference timeouts and signals,+Forty-three end-to-end controls cover baseline/reference timeouts and signals,missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitestcollection failures (including alias imports), missing PostCSS-plugin startupfailures, Rust/Cargo compilation failures and assertions quoting compiler output,@@ -217,3 +217,15 @@ The fixture tests sequential saved-file changes and real local process/protocolbehavior. It does not verify concurrent file writes, native desktop persistence,external enterprise gateways, token cryptography or the harness's filesystemisolation. Dependencies are exposed only during grading, after the trajectory.+++Vitest can register tests and then skip them because a `beforeAll` Bun build+failed. A bounded missing-input compiler diagnostic, failed suite and skipped+summary with no test failures is now `vitest_bun_build_setup_error`; passing+neighboring tests do not turn the build failure into regression evidence.+Assertions quoting compiler diagnostics and successful commands retain their+outcomes. The runner controls cover both baseline/reference build-hook failure,+neighboring passes, quoted assertion diagnostics and error exclusion from scores.+An actual Bun compiler/Vitest setup failure previously certified a false valid+contrast; it now becomes an error and invalid verification. This is bounded+recognition, not general compiler attribution or proof of execution isolation.diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjsindex 2d3decf38..35aaf47c4 100644--- a/evals/coding-agent/grader-outcome.mjs+++ b/evals/coding-agent/grader-outcome.mjs@@ -19,6 +19,15 @@ export function classifyGraderError(grader) {/Failed Suites [1-9]/.test(stderr) &&(/^Error: Failed to load url /m.test(stderr) ||/^Error: Cannot find module ['"][^\n]+['"] imported from ['"][^\n]+['"]\.\s*$/m.test(stderr))) return "vitest_collection_error";+ // A beforeAll build failure can register tests but skip every affected test.+ // Passing neighboring suites do not turn a compile failure into an assertion.+ // Preserve executed failures and assertion headers, including quoted diagnostics.+ const buildHookSummary = summary?.match(/^(?:(?:\d+ (?:passed|skipped))(?: \| )?)+ \(\d+\)$/);+ if (buildHookSummary && /[1-9]\d* skipped/.test(summary) &&+ /^\s*Test Files\s+.*[1-9]\d* failed/m.test(stdout) && /Failed Suites [1-9]/.test(stderr) &&+ /^Error: Command failed: [^\n]*\bbun(?:\.exe)? (?:build|run build)\b/m.test(stderr) &&+ /^error: Could not resolve: ["'][^\n]+["']\s*$/m.test(stderr) &&+ !/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) return "vitest_bun_build_setup_error";// Vite can reject PostCSS startup before printing a test-count summary.// Require the specific missing-plugin diagnostic and startup banner; an// executed-test summary or assertion header must keep its behavioral result.diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.tsindex 179b65c07..8706afe73 100644--- a/evals/coding-agent/run.test.ts+++ b/evals/coding-agent/run.test.ts@@ -28,6 +28,14 @@ test("process and setup failures cannot prove a regression or count as scored fagit("commit", "-qam", "Synthetic reference state");const fix = git("rev-parse", "HEAD");const controls = [+ { id: "baseline-vitest-bun-build", baseline: "error", oracle: "pass", valid: false },+ { id: "reference-vitest-bun-build", baseline: "fail", oracle: "error", valid: false },+ { id: "baseline-vitest-bun-build-with-pass", baseline: "error", oracle: "pass", valid: false },+ { id: "vitest-build-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },+ { id: "vitest-build-hook-assertion", baseline: "fail", oracle: "pass", valid: true },+ { id: "node-build-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },+ { id: "both-pass-build-diagnostic", baseline: "pass", oracle: "pass", valid: false },+ { id: "unknown-build-command", baseline: "fail", oracle: "pass", valid: true },{ id: "baseline-rust-compile", baseline: "error", oracle: "pass", valid: false },{ id: "reference-rust-compile", baseline: "fail", oracle: "error", valid: false },{ id: "baseline-cargo-compile", baseline: "error", oracle: "pass", valid: false },@@ -71,7 +79,15 @@ import assert from "node:assert/strict";const broken = readFileSync("state.txt", "utf8") === "broken";const id = process.env.SCREENPIPE_EVAL_CASE_ID;const affected = id.startsWith("baseline-") ? broken : !broken;-if ((id.endsWith("-rust-compile") && affected) || ((id === "baseline-cargo-compile" || id === "baseline-rust-passed-before-compile") && broken) ||+if ((id.endsWith("vitest-bun-build") && affected) || (id === "baseline-vitest-bun-build-with-pass" && broken) ||+ (["vitest-build-quoted-assertion", "vitest-build-hook-assertion", "node-build-quoted-assertion", "unknown-build-command"].includes(id) && broken) || id === "both-pass-build-diagnostic") {+ const summary = id === "vitest-build-quoted-assertion" ? "1 failed (1)" : id === "baseline-vitest-bun-build-with-pass" ? "1 passed | 1 skipped (2)" : "1 skipped (1)";+ process.stdout.write(" RUN v4.0.18 /synthetic\\n Test Files 1 failed (1)\\n Tests " + summary + "\\n");+ process.stderr.write(" Failed Suites 1\\nError: Command failed: " + (id === "unknown-build-command" ? "other-compiler" : "bun build") + " entry.ts\\nerror: Could not resolve: \\"./missing-input\\"\\n");+ if (id === "vitest-build-quoted-assertion" || id === "vitest-build-hook-assertion") process.stderr.write("AssertionError: intended synthetic behavior failed\\n");+ if (id === "node-build-quoted-assertion") assert.fail("synthetic assertion after quoted build output");+ process.exit(id === "both-pass-build-diagnostic" ? 0 : 1);+} else if ((id.endsWith("-rust-compile") && affected) || ((id === "baseline-cargo-compile" || id === "baseline-rust-passed-before-compile") && broken) ||((id === "rust-assertion-with-quoted-compile" || id === "node-assertion-with-quoted-compile" || id === "rust-panic-with-quoted-compile") && broken) || id === "both-pass-compile-diagnostic") {process.stderr.write("error[E0425]: cannot find value synthetic_fixture in this scope\\n");process.stderr.write(id === "baseline-cargo-compile" ? "error: could not compile \`synthetic-fixture\` (lib test) due to 1 previous error\\n" : "error: aborting due to 1 previous error\\n");@@ -133,7 +149,7 @@ else if (id === "baseline-syntax" && affected) {assert.fail("\\nError [ERR_MODULE_NOT_FOUND]: quoted diagnostic\\nSyntaxError: quoted diagnostic\\n at quotedFixture");} else if (id === "both-pass-diagnostic-words") {console.error("Cannot find module; SyntaxError; command not found are fixture words");-} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile") && broken) assert.fail("synthetic broken behavior");+} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile" || id === "reference-vitest-bun-build") && broken) assert.fail("synthetic broken behavior");else process.exit(id === "intended-failure" && broken ? 1 : 0);`);const manifest = join(repo, "cases.json");@@ -172,7 +188,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");}}- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile");+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build");expect(scored.error).toBeUndefined();expect(scored.status).toBe(0);const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));