YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(evals): classify partial Vitest collection failures as infrastructure
Louis Beaumontc41a2ff131d2
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 29a019e34..b53105b8c 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -18,13 +18,17 @@ and the status banner with synthetic settings and native command ports. Fourteenoutcomes cover audio-off policy, loading, policy changes during delayed completion,unmount and ordinary audio-on requests. The parent fails eight outcomes andpreserves six; the historical fix and current source pass fourteen. Run-`bun test evals/coding-agent/calibrate-audio-policy.test.js` for nine controls,+`bun test evals/coding-agent/calibrate-audio-policy.test.js` for ten controls,including an equivalent policy implementation, unused correct interfaces andblanket microphone requirements or suppression. Explicit DOM cleanup keeps testsindependent. Fixtures and dependency links appear only at grading. Native pauseand recording behavior, OS permission enforcement, execution isolation and model-improvement are outside this evidence. Partial collection failures still need log-inspection; a nonzero process exit alone does not establish a behavior regression.+improvement are outside this evidence. Recognized Vitest collection failures stay+infrastructure errors when neighboring suites pass or skip. Executed test failures+and assertion headers retain their behavior classification. The shared runner's+`bun test evals/coding-agent/run.test.ts` controls cover both verification arms+and exclusion from scored trials. Unknown diagnostics still need log inspection;+a nonzero process exit alone does not establish a behavior regression.`app-model-catalog-session-freshness` runs the actual model-catalog hook withsynthetic settings and HTTP ports. Ten outcomes cover hydration, token changes,diff --git a/evals/coding-agent/calibrate-audio-policy.test.js b/evals/coding-agent/calibrate-audio-policy.test.jsindex d276c957f..b5e28b456 100644--- a/evals/coding-agent/calibrate-audio-policy.test.js+++ b/evals/coding-agent/calibrate-audio-policy.test.js@@ -33,3 +33,5 @@ test('equivalent audio policy implementation passes',()=>passes(grade('equivalentest('always requiring microphone is rejected',()=>fails(grade('always-required',item.oracle_ref,cwd=>helperBody(cwd,'return true;'))),60000);test('never requiring microphone violates preserved audio-on behavior',()=>fails(grade('never-required',item.oracle_ref,cwd=>helperBody(cwd,'return false;'))),60000);test('missing interfaces are an infrastructure error',()=>{const r=grade('missing',item.oracle_ref,cwd=>{for(const p of item.oracle_paths.slice(0,3))rmSync(join(cwd,p));});expect(r.status).toBe(1);expect(classifyGraderError(r)).toBe('vitest_collection_error')},60000);++test('one missing interface remains infrastructure when neighboring suites pass',()=>{const r=grade('missing-one',item.oracle_ref,cwd=>rmSync(join(cwd,item.oracle_paths[0])));expect(r.status).toBe(1);expect(r.stdout).toContain('7 passed');expect(r.stdout).not.toMatch(/Tests\s+.*[1-9]\d* failed/);expect(classifyGraderError(r)).toBe('vitest_collection_error')},60000);diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjsindex e763ba291..6982d1160 100644--- a/evals/coding-agent/grader-outcome.mjs+++ b/evals/coding-agent/grader-outcome.mjs@@ -29,13 +29,18 @@ export function classifyGraderError(grader) {return "playwright_ct_build_error";}const summary = [...stdout.matchAll(/^\s*Tests\s+(.+)$/gm)].at(-1)?.[1]?.trim();- if (summary === "no tests" && /^\s*Test Files\s+\d+ failed/m.test(stdout) &&+ // A failed suite can collect zero tests while neighboring suites pass or skip.+ // Only summaries without executed failures qualify; diagnostic and assertion+ // guards still apply to both streams, including quoted setup errors.+ const collectionSummary = summary === "no tests" ||+ /^(?:\d+ (?:passed|skipped))(?: \| \d+ (?:passed|skipped))* \(\d+\)$/.test(summary ?? "");+ if (collectionSummary && noAssertion && /^\s*Test Files\s+\d+ failed/m.test(stdout) &&/Failed Suites [1-9]/.test(stderr) &&((/^Error: Failed to resolve import ['"][^\r\n]+['"] from ['"][^\r\n]+['"]\. Does the file exist\?\s*$/m.test(stderr) &&!/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) ||/^Error: Failed to load url /m.test(stderr) ||// Vitest wraps failures inside vi.mock factories. Require its wrapper,- // the caused-by missing URL, zero executed tests, and no assertions.+ // the caused-by missing URL, no failed tests, and no assertions.(/^Error: \[vitest\] There was an error when mocking a module\./m.test(stderr) &&/^Caused by: Error: Failed to load url [^\n]+Does the file exist\?\s*$/m.test(stderr) &&!/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) ||diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.tsindex 5fbb2d2da..96e23b6a6 100644--- a/evals/coding-agent/run.test.ts+++ b/evals/coding-agent/run.test.ts@@ -35,6 +35,10 @@ test("process and setup failures cannot prove a regression or count as scored fa{ id: "playwright-partial-run", baseline: "fail", oracle: "pass", valid: true },{ id: "playwright-incomplete-diagnostic", baseline: "fail", oracle: "pass", valid: true },{ id: "both-pass-playwright-diagnostic", baseline: "pass", oracle: "pass", valid: false },+ { id: "baseline-vitest-mixed-collection", baseline: "error", oracle: "pass", valid: false },+ { id: "reference-vitest-mixed-collection", baseline: "fail", oracle: "error", valid: false },+ { id: "vitest-mixed-assertion", baseline: "fail", oracle: "pass", valid: true },+ { id: "both-pass-vitest-mixed", baseline: "pass", oracle: "pass", valid: false },{ id: "baseline-vitest-resolve", baseline: "error", oracle: "pass", valid: false },{ id: "reference-vitest-resolve", baseline: "fail", oracle: "error", valid: false },{ id: "baseline-vitest-resolve-ansi", baseline: "error", oracle: "pass", valid: false },@@ -106,6 +110,15 @@ if (((id === "baseline-playwright-load" || id === "reference-playwright-export"if (id !== "playwright-incomplete-diagnostic") process.stderr.write("✗ Build failed in 1.2s\\n");process.exit(id === "both-pass-playwright-diagnostic" ? 0 : 1);} else if (id === "reference-playwright-export" && broken) assert.fail("synthetic broken behavior");+else if ((id.endsWith("vitest-mixed-collection") && affected) || (id === "vitest-mixed-assertion" && broken) || id === "both-pass-vitest-mixed") {+ process.stdout.write(" RUN v1.6.1 /synthetic\\nTest Files 1 failed | 2 passed (3)\\nTests 7 passed (7)\\n");+ process.stderr.write('Failed Suites 1\\nError: Failed to resolve import "./missing" from "fixture.test.ts". Does the file exist?\\n');+ if (id === "vitest-mixed-assertion") {+ process.stdout.write("Test Files 2 failed | 1 passed (3)\\nTests 1 failed | 6 passed (7)\\n");+ process.stderr.write("AssertionError: preserved behavior failed\\n");+ }+ process.exit(id === "both-pass-vitest-mixed" ? 0 : 1);+} else if (id === "reference-vitest-mixed-collection" && broken) assert.fail("synthetic broken behavior");else if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||(["vitest-assertion-with-quoted-resolve", "node-assertion-with-quoted-resolve", "resolve-without-test-summary"].includes(id) && broken) || id === "both-pass-resolve-diagnostic") {const summary = "Test Files 1 failed (1)\\nTests no tests\\n";@@ -226,7 +239,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");}}- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi");+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi,baseline-vitest-mixed-collection");expect(scored.error).toBeUndefined();expect(scored.status).toBe(0);const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));@@ -257,3 +270,35 @@ test("Playwright build classification requires complete unexecuted evidence", asexpect(classifyGraderError({ ...build, stderr: build.stderr + "AssertionError: real behavior failed\n" })).toBeNull();expect(classifyGraderError({ ...build, status: 0 })).toBeNull();});+++test("Vitest collection classification preserves executed failures and requires diagnostics", async () => {+ const { classifyGraderError } = await import("./grader-outcome.mjs");+ const collection = {+ status: 1,+ stdout: " RUN v1.6.1 /synthetic\n Test Files 1 failed | 2 passed (3)\n Tests 7 passed (7)\n",+ stderr: 'Failed Suites 1\nError: Failed to resolve import "./missing" from "fixture.test.ts". Does the file exist?\n',+ };+ expect(classifyGraderError(collection)).toBe("vitest_collection_error");+ for (const summary of ["no tests", "7 passed | 2 skipped (9)", "2 skipped (2)"]) {+ expect(classifyGraderError({ ...collection, stdout: collection.stdout.replace("7 passed (7)", summary) })).toBe("vitest_collection_error");+ }+ for (const diagnostic of [+ "Error: Failed to load url ./missing (resolved id: ./missing). Does the file exist?",+ "Error: Cannot find module '@/missing' imported from '/synthetic/fixture.test.ts'.",+ "Error: [vitest] There was an error when mocking a module.\nCaused by: Error: Failed to load url ./missing. Does the file exist?",+ ]) expect(classifyGraderError({ ...collection, stderr: "Failed Suites 1\n" + diagnostic + "\n" })).toBe("vitest_collection_error");+ expect(classifyGraderError({ ...collection, stdout: "\x1b[31m" + collection.stdout + "\x1b[0m" })).toBe("vitest_collection_error");+ for (const stdout of [+ collection.stdout + "Tests 1 failed | 6 passed (7)\n",+ collection.stdout.replace("Tests 7 passed (7)", "Tests 1 failed | 6 passed (7)"),+ collection.stdout.replace("Tests 7 passed (7)", "Tests unknown"),+ collection.stdout.replace("Tests 7 passed (7)", ""),+ collection.stdout.replace("Test Files 1 failed | 2 passed (3)", "Test Files 3 passed (3)"),+ collection.stdout + "AssertionError: actual did not equal expected\n",+ ]) expect(classifyGraderError({ ...collection, stdout })).toBeNull();+ for (const stderr of ["", "Failed Suites 1\nError: arbitrary failure\n", collection.stderr.replace("Failed Suites 1", ""), collection.stderr + "AssertionError: actual did not equal expected\n"]) {+ expect(classifyGraderError({ ...collection, stderr })).toBeNull();+ }+ expect(classifyGraderError({ ...collection, status: 0 })).toBeNull();+});