YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(evals): classify unexecuted Playwright builds as errors
louisebc79066e438
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 18fcd9d88..ee6281ba9 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -201,6 +201,12 @@ with zero tests are also reported as `error`, with atherefore cannot certify a regression. Plain failed assertions and successfulcommands containing diagnostic words retain their previous outcomes.+Playwright component-test missing-input and missing-export builds are+`playwright_ct_build_error` when the planned and unrun counts match and the build+failure diagnostic is complete. Assertions, partial execution, successful exits+and incomplete diagnostics retain their previous outcomes. The controls use+synthetic paths and symbols; no private application source is embedded.+This is bounded diagnostic recognition, not universal error attribution. Unknownsetup/compiler failures, including other test frameworks, may still be ordinarynonzero exits. Inspect logs before promoting a case; preserve command exit codes@@ -209,10 +215,11 @@ change, caused it. Report error counts and inspect candidate-caused errors beforcomparing model success rates. No isolation or model-quality claim follows.Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.-Fifty end-to-end controls cover baseline/reference timeouts and signals,+Fifty-seven end-to-end controls cover baseline/reference timeouts and signals,missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitestcollection failures (including alias imports and Vite resolve-import diagnostics), missing PostCSS-plugin startupfailures, Rust/Cargo compilation failures and assertions quoting compiler output,+Playwright component-test Vite builds that leave all discovered tests unrun,quoted diagnostics followed by real assertions, unavailable or non-executablecommands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,already-passing baselines and exclusion of known setup/process errors from scores.diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjsindex 39a46d83f..d2095b2ba 100644--- a/evals/coding-agent/grader-outcome.mjs+++ b/evals/coding-agent/grader-outcome.mjs@@ -14,6 +14,20 @@ export function classifyGraderError(grader) {// stdout. Use the last summary so diagnostic text printed by a test cannot// hide a later genuine assertion failure. Unknown formats still need review.const stdout = (grader.stdout ?? "").replace(/\x1b\[[0-9;]*m/g, "");+ // Playwright CT can discover tests and then fail its Vite build before any+ // test runs. Require matching planned/unrun counts, a known build diagnostic+ // and the build terminator. Executed outcomes or assertion headers must win+ // even when their messages quote these diagnostics.+ const planned = stdout.match(/^Running ([1-9]\d*) tests? using [1-9]\d* workers?\s*$/m)?.[1];+ const unrun = stdout.match(/^\s*([1-9]\d*) did not run\s*$/m)?.[1];+ const noExecutedPlaywright = !/^\s*[1-9]\d* (?:passed|failed|skipped|flaky|timed out)\b/m.test(stdout);+ const noAssertion = !/^(?:\s*Error: expect\(|\s*AssertionError(?: \[[^\]]+\])?:)/m.test(stdout + "\n" + stderr);+ if (planned && unrun === planned && noExecutedPlaywright && noAssertion &&+ /^✗ Build failed in /m.test(stderr) &&+ (/^Error: \[vite:load-fallback\] Could not load [^\n]+: ENOENT:/m.test(stdout) ||+ /^RollupError: [^\n]+ is not exported by [^\n]+, imported by /m.test(stdout))) {+ return "playwright_ct_build_error";+ }const summary = [...stdout.matchAll(/^\s*Tests\s+(.+)$/gm)].at(-1)?.[1]?.trim();if (summary === "no tests" && /^\s*Test Files\s+\d+ failed/m.test(stdout) &&/Failed Suites [1-9]/.test(stderr) &&diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.tsindex 9b29ec6dc..5fbb2d2da 100644--- a/evals/coding-agent/run.test.ts+++ b/evals/coding-agent/run.test.ts@@ -28,6 +28,13 @@ test("process and setup failures cannot prove a regression or count as scored fagit("commit", "-qam", "Synthetic reference state");const fix = git("rev-parse", "HEAD");const controls = [+ { id: "baseline-playwright-load", baseline: "error", oracle: "pass", valid: false },+ { id: "reference-playwright-export", baseline: "fail", oracle: "error", valid: false },+ { id: "baseline-playwright-export-ansi", baseline: "error", oracle: "pass", valid: false },+ { id: "playwright-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },+ { id: "playwright-partial-run", baseline: "fail", oracle: "pass", valid: true },+ { id: "playwright-incomplete-diagnostic", baseline: "fail", oracle: "pass", valid: true },+ { id: "both-pass-playwright-diagnostic", baseline: "pass", oracle: "pass", valid: false },{ id: "baseline-vitest-resolve", baseline: "error", oracle: "pass", valid: false },{ id: "reference-vitest-resolve", baseline: "fail", oracle: "error", valid: false },{ id: "baseline-vitest-resolve-ansi", baseline: "error", oracle: "pass", valid: false },@@ -86,7 +93,20 @@ import assert from "node:assert/strict";const broken = readFileSync("state.txt", "utf8") === "broken";const id = process.env.SCREENPIPE_EVAL_CASE_ID;const affected = id.startsWith("baseline-") ? broken : !broken;-if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||+if (((id === "baseline-playwright-load" || id === "reference-playwright-export" || id === "baseline-playwright-export-ansi") && affected) ||+ (["playwright-quoted-assertion", "playwright-partial-run", "playwright-incomplete-diagnostic"].includes(id) && broken) || id === "both-pass-playwright-diagnostic") {+ let output = "Running 7 tests using 1 worker\\n";+ output += id === "baseline-playwright-load"+ ? "Error: [vite:load-fallback] Could not load /synthetic/missing: ENOENT: no such file or directory\\n"+ : 'RollupError: component.tsx (1:2): "MissingIcon" is not exported by "icons/index.mjs", imported by "component.tsx".\\n';+ output += " 7 did not run\\n";+ if (id === "playwright-partial-run") output += " 1 failed\\n";+ if (id === "playwright-quoted-assertion") output += "Error: expect(locator).toBeVisible() failed\\n";+ process.stdout.write(id.endsWith("-ansi") ? "\\x1b[31m" + output + "\\x1b[0m" : output);+ if (id !== "playwright-incomplete-diagnostic") process.stderr.write("✗ Build failed in 1.2s\\n");+ process.exit(id === "both-pass-playwright-diagnostic" ? 0 : 1);+} else if (id === "reference-playwright-export" && broken) assert.fail("synthetic broken behavior");+else if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||(["vitest-assertion-with-quoted-resolve", "node-assertion-with-quoted-resolve", "resolve-without-test-summary"].includes(id) && broken) || id === "both-pass-resolve-diagnostic") {const summary = "Test Files 1 failed (1)\\nTests no tests\\n";if (id !== "resolve-without-test-summary") process.stdout.write(id.endsWith("-ansi") ? "\\x1b[31m" + summary + "\\x1b[0m" : summary);@@ -206,7 +226,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");}}- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve");+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi");expect(scored.error).toBeUndefined();expect(scored.status).toBe(0);const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));@@ -215,3 +235,25 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);rmSync(repo, { recursive: true, force: true });}}, 90_000);++test("Playwright build classification requires complete unexecuted evidence", async () => {+ const { classifyGraderError } = await import("./grader-outcome.mjs");+ const build = {+ status: 1,+ stdout: 'Running 7 tests using 1 worker\nRollupError: widget.tsx (1:2): "MissingIcon" is not exported by "icons/index.mjs", imported by "widget.tsx".\n 7 did not run\n',+ stderr: "✗ Build failed in 1.2s\n",+ };+ expect(classifyGraderError(build)).toBe("playwright_ct_build_error");+ for (const stdout of [+ build.stdout.replace("7 did not run", "6 did not run"),+ build.stdout.replace("Running 7 tests using 1 worker\n", ""),+ build.stdout.replace(" 7 did not run\n", ""),+ build.stdout + " 1 passed (1s)\n",+ build.stdout + " 1 failed\n",+ build.stdout + " 1 timed out\n",+ build.stdout + "Error: expect(locator).toBeVisible() failed\n",+ ]) expect(classifyGraderError({ ...build, stdout })).toBeNull();+ expect(classifyGraderError({ ...build, stderr: "" })).toBeNull();+ expect(classifyGraderError({ ...build, stderr: build.stderr + "AssertionError: real behavior failed\n" })).toBeNull();+ expect(classifyGraderError({ ...build, status: 0 })).toBeNull();+});