Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(evals): classify unexecuted Playwright builds as errors

louisebc79066e438
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 18fcd9d88..ee6281ba9 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -201,6 +201,12 @@ with zero tests are also reported as `error`, with a
therefore cannot certify a regression. Plain failed assertions and successful
commands containing diagnostic words retain their previous outcomes.
+Playwright component-test missing-input and missing-export builds are
+`playwright_ct_build_error` when the planned and unrun counts match and the build
+failure diagnostic is complete. Assertions, partial execution, successful exits
+and incomplete diagnostics retain their previous outcomes. The controls use
+synthetic paths and symbols; no private application source is embedded.
+
This is bounded diagnostic recognition, not universal error attribution. Unknown
setup/compiler failures, including other test frameworks, may still be ordinary
nonzero exits. Inspect logs before promoting a case; preserve command exit codes
@@ -209,10 +215,11 @@ change, caused it. Report error counts and inspect candidate-caused errors befor
comparing model success rates. No isolation or model-quality claim follows.
Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
-Fifty end-to-end controls cover baseline/reference timeouts and signals,
+Fifty-seven end-to-end controls cover baseline/reference timeouts and signals,
missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitest
collection failures (including alias imports and Vite resolve-import diagnostics), missing PostCSS-plugin startup
failures, Rust/Cargo compilation failures and assertions quoting compiler output,
+Playwright component-test Vite builds that leave all discovered tests unrun,
quoted diagnostics followed by real assertions, unavailable or non-executable
commands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,
already-passing baselines and exclusion of known setup/process errors from scores.
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
index 39a46d83f..d2095b2ba 100644
--- a/evals/coding-agent/grader-outcome.mjs
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -14,6 +14,20 @@ export function classifyGraderError(grader) {
// stdout. Use the last summary so diagnostic text printed by a test cannot
// hide a later genuine assertion failure. Unknown formats still need review.
const stdout = (grader.stdout ?? "").replace(/\x1b\[[0-9;]*m/g, "");
+ // Playwright CT can discover tests and then fail its Vite build before any
+ // test runs. Require matching planned/unrun counts, a known build diagnostic
+ // and the build terminator. Executed outcomes or assertion headers must win
+ // even when their messages quote these diagnostics.
+ const planned = stdout.match(/^Running ([1-9]\d*) tests? using [1-9]\d* workers?\s*$/m)?.[1];
+ const unrun = stdout.match(/^\s*([1-9]\d*) did not run\s*$/m)?.[1];
+ const noExecutedPlaywright = !/^\s*[1-9]\d* (?:passed|failed|skipped|flaky|timed out)\b/m.test(stdout);
+ const noAssertion = !/^(?:\s*Error: expect\(|\s*AssertionError(?: \[[^\]]+\])?:)/m.test(stdout + "\n" + stderr);
+ if (planned && unrun === planned && noExecutedPlaywright && noAssertion &&
+ /^✗ Build failed in /m.test(stderr) &&
+ (/^Error: \[vite:load-fallback\] Could not load [^\n]+: ENOENT:/m.test(stdout) ||
+ /^RollupError: [^\n]+ is not exported by [^\n]+, imported by /m.test(stdout))) {
+ return "playwright_ct_build_error";
+ }
const summary = [...stdout.matchAll(/^\s*Tests\s+(.+)$/gm)].at(-1)?.[1]?.trim();
if (summary === "no tests" && /^\s*Test Files\s+\d+ failed/m.test(stdout) &&
/Failed Suites [1-9]/.test(stderr) &&
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 9b29ec6dc..5fbb2d2da 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -28,6 +28,13 @@ test("process and setup failures cannot prove a regression or count as scored fa
git("commit", "-qam", "Synthetic reference state");
const fix = git("rev-parse", "HEAD");
const controls = [
+ { id: "baseline-playwright-load", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-playwright-export", baseline: "fail", oracle: "error", valid: false },
+ { id: "baseline-playwright-export-ansi", baseline: "error", oracle: "pass", valid: false },
+ { id: "playwright-quoted-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "playwright-partial-run", baseline: "fail", oracle: "pass", valid: true },
+ { id: "playwright-incomplete-diagnostic", baseline: "fail", oracle: "pass", valid: true },
+ { id: "both-pass-playwright-diagnostic", baseline: "pass", oracle: "pass", valid: false },
{ id: "baseline-vitest-resolve", baseline: "error", oracle: "pass", valid: false },
{ id: "reference-vitest-resolve", baseline: "fail", oracle: "error", valid: false },
{ id: "baseline-vitest-resolve-ansi", baseline: "error", oracle: "pass", valid: false },
@@ -86,7 +93,20 @@ import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
-if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||
+if (((id === "baseline-playwright-load" || id === "reference-playwright-export" || id === "baseline-playwright-export-ansi") && affected) ||
+ (["playwright-quoted-assertion", "playwright-partial-run", "playwright-incomplete-diagnostic"].includes(id) && broken) || id === "both-pass-playwright-diagnostic") {
+ let output = "Running 7 tests using 1 worker\\n";
+ output += id === "baseline-playwright-load"
+ ? "Error: [vite:load-fallback] Could not load /synthetic/missing: ENOENT: no such file or directory\\n"
+ : 'RollupError: component.tsx (1:2): "MissingIcon" is not exported by "icons/index.mjs", imported by "component.tsx".\\n';
+ output += " 7 did not run\\n";
+ if (id === "playwright-partial-run") output += " 1 failed\\n";
+ if (id === "playwright-quoted-assertion") output += "Error: expect(locator).toBeVisible() failed\\n";
+ process.stdout.write(id.endsWith("-ansi") ? "\\x1b[31m" + output + "\\x1b[0m" : output);
+ if (id !== "playwright-incomplete-diagnostic") process.stderr.write("✗ Build failed in 1.2s\\n");
+ process.exit(id === "both-pass-playwright-diagnostic" ? 0 : 1);
+} else if (id === "reference-playwright-export" && broken) assert.fail("synthetic broken behavior");
+else if ((id.endsWith("vitest-resolve") && affected) || (id === "baseline-vitest-resolve-ansi" && broken) ||
(["vitest-assertion-with-quoted-resolve", "node-assertion-with-quoted-resolve", "resolve-without-test-summary"].includes(id) && broken) || id === "both-pass-resolve-diagnostic") {
const summary = "Test Files 1 failed (1)\\nTests no tests\\n";
if (id !== "resolve-without-test-summary") process.stdout.write(id.endsWith("-ansi") ? "\\x1b[31m" + summary + "\\x1b[0m" : summary);
@@ -206,7 +226,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile,baseline-vitest-bun-build,baseline-vitest-resolve,baseline-playwright-load,baseline-playwright-export-ansi");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));
@@ -215,3 +235,25 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
rmSync(repo, { recursive: true, force: true });
}
}, 90_000);
+
+test("Playwright build classification requires complete unexecuted evidence", async () => {
+ const { classifyGraderError } = await import("./grader-outcome.mjs");
+ const build = {
+ status: 1,
+ stdout: 'Running 7 tests using 1 worker\nRollupError: widget.tsx (1:2): "MissingIcon" is not exported by "icons/index.mjs", imported by "widget.tsx".\n 7 did not run\n',
+ stderr: "✗ Build failed in 1.2s\n",
+ };
+ expect(classifyGraderError(build)).toBe("playwright_ct_build_error");
+ for (const stdout of [
+ build.stdout.replace("7 did not run", "6 did not run"),
+ build.stdout.replace("Running 7 tests using 1 worker\n", ""),
+ build.stdout.replace(" 7 did not run\n", ""),
+ build.stdout + " 1 passed (1s)\n",
+ build.stdout + " 1 failed\n",
+ build.stdout + " 1 timed out\n",
+ build.stdout + "Error: expect(locator).toBeVisible() failed\n",
+ ]) expect(classifyGraderError({ ...build, stdout })).toBeNull();
+ expect(classifyGraderError({ ...build, stderr: "" })).toBeNull();
+ expect(classifyGraderError({ ...build, stderr: build.stderr + "AssertionError: real behavior failed\n" })).toBeNull();
+ expect(classifyGraderError({ ...build, status: 0 })).toBeNull();
+});