Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: exclude Rust compilation errors from regression evidence

Louis Beaumont60b2e4ff5d30
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 4690450a0..26944dea2 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -50,7 +50,8 @@ Agent-process, grader-process and harness failures are reported as `error` and e
the success denominator. They are never silently converted into model failures.
Grader timeouts and terminating signals cannot establish a failing baseline:
`--verify` requires a behavioral `fail` followed by an oracle `pass`.
-Known command-unavailable exits (126/127), Bun unhandled test-load errors, and
+Known command-unavailable exits (126/127), Rust compiler failures before test execution,
+Bun unhandled test-load errors, and
Node missing-module/syntax diagnostics and Vitest URL-load or imported-module collection failures
with zero tests are also reported as `error`, with a
`grader_error_kind` in the result. A missing import followed by an oracle pass
@@ -65,11 +66,12 @@ change, caused it. Report error counts and inspect candidate-caused errors befor
comparing model success rates. No isolation or model-quality claim follows.
Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
-Twenty-seven end-to-end controls cover baseline/reference timeouts and signals,
+Thirty-five end-to-end controls cover baseline/reference timeouts and signals,
missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, Vitest
collection failures (including alias imports), missing PostCSS-plugin startup
-failures, quoted diagnostics followed by real assertions, unavailable
-or non-executable commands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,
+failures, Rust/Cargo compilation failures and assertions quoting compiler output,
+quoted diagnostics followed by real assertions, unavailable or non-executable
+commands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,
already-passing baselines and exclusion of known setup/process errors from scores.
The controls use temporary local Git fixtures and invoke no model or provider.
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
index 50b67de91..2d3decf38 100644
--- a/evals/coding-agent/grader-outcome.mjs
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -28,6 +28,18 @@ export function classifyGraderError(grader) {
/Loading PostCSS Plugin failed: Cannot find module ['"][^\n]+['"]/.test(stderr) &&
/[/\\]postcss\.config\.[cm]?[jt]s/.test(stderr) &&
!/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) return "vitest_postcss_setup_error";
+ // Rust compilation ends before its test executable can run. Require both
+ // a compiler diagnostic and a terminating compiler summary, and preserve
+ // executed test/panic/assertion outcomes even if they quote those lines.
+ const rustTestsFailed = /^test result: FAILED\. \d+ passed; [1-9]\d* failed;/m.test(stdout);
+ const assertionHeader = /^AssertionError(?: \[[^\]]+\])?:/m.test(stderr) ||
+ /^thread ['"][^\n]+['"] panicked at /m.test(stderr);
+ if (!rustTestsFailed && !assertionHeader &&
+ /^error(?:\[E\d{4}\])?: (?!aborting due to |could not compile )/m.test(stderr) &&
+ (/^error: aborting due to \d+ previous errors?\s*$/m.test(stderr) ||
+ /^error: could not compile [`'"][^\n]+[`'"](?: \([^\n]*\))? due to \d+ previous errors?/m.test(stderr))) {
+ return "rust_compile_error";
+ }
// Only the first thrown-error header classifies a Node failure. Assertion
// messages may quote complete setup diagnostics on later lines.
const header = stderr.match(/^([A-Za-z]*Error)(?: \[([A-Z_0-9]+)\])?:[^\n]*/m);
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 68fc4f9e6..179b65c07 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -28,6 +28,14 @@ test("process and setup failures cannot prove a regression or count as scored fa
git("commit", "-qam", "Synthetic reference state");
const fix = git("rev-parse", "HEAD");
const controls = [
+ { id: "baseline-rust-compile", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-rust-compile", baseline: "fail", oracle: "error", valid: false },
+ { id: "baseline-cargo-compile", baseline: "error", oracle: "pass", valid: false },
+ { id: "rust-assertion-with-quoted-compile", baseline: "fail", oracle: "pass", valid: true },
+ { id: "node-assertion-with-quoted-compile", baseline: "fail", oracle: "pass", valid: true },
+ { id: "rust-panic-with-quoted-compile", baseline: "fail", oracle: "pass", valid: true },
+ { id: "baseline-rust-passed-before-compile", baseline: "error", oracle: "pass", valid: false },
+ { id: "both-pass-compile-diagnostic", baseline: "pass", oracle: "pass", valid: false },
{ id: "baseline-timeout", baseline: "error", oracle: "pass", valid: false },
{ id: "intended-failure", baseline: "fail", oracle: "pass", valid: true },
{ id: "both-pass", baseline: "pass", oracle: "pass", valid: false },
@@ -63,7 +71,20 @@ import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
-if (id.endsWith("-timeout") && affected) setInterval(() => {}, 1000);
+if ((id.endsWith("-rust-compile") && affected) || ((id === "baseline-cargo-compile" || id === "baseline-rust-passed-before-compile") && broken) ||
+ ((id === "rust-assertion-with-quoted-compile" || id === "node-assertion-with-quoted-compile" || id === "rust-panic-with-quoted-compile") && broken) || id === "both-pass-compile-diagnostic") {
+ process.stderr.write("error[E0425]: cannot find value synthetic_fixture in this scope\\n");
+ process.stderr.write(id === "baseline-cargo-compile" ? "error: could not compile \`synthetic-fixture\` (lib test) due to 1 previous error\\n" : "error: aborting due to 1 previous error\\n");
+ if (id === "rust-assertion-with-quoted-compile") {
+ process.stdout.write("running 1 test\\ntest synthetic ... FAILED\\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\\n");
+ process.stderr.write("thread 'synthetic' panicked at main.rs:2: assertion failed\\n");
+ }
+ if (id === "baseline-rust-passed-before-compile") process.stdout.write("test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\\n");
+ if (id === "rust-panic-with-quoted-compile") process.stderr.write("thread 'synthetic' panicked at main.rs:2: assertion failed\\n");
+ if (id === "node-assertion-with-quoted-compile") assert.fail("synthetic behavior assertion after quoted compiler text");
+ process.exit(id === "both-pass-compile-diagnostic" ? 0 : 1);
+}
+else if (id.endsWith("-timeout") && affected) setInterval(() => {}, 1000);
else if (id.endsWith("-signal") && affected) process.kill(process.pid, "SIGTERM");
else if ((id.endsWith("vitest-postcss") && affected) || ((id === "vitest-assertion-with-quoted-postcss" || id === "node-assertion-with-quoted-postcss") && broken) || id === "both-pass-postcss-diagnostic") {
process.stdout.write(" RUN v2.1.9 /synthetic\\n");
@@ -112,7 +133,7 @@ else if (id === "baseline-syntax" && affected) {
assert.fail("\\nError [ERR_MODULE_NOT_FOUND]: quoted diagnostic\\nSyntaxError: quoted diagnostic\\n at quotedFixture");
} else if (id === "both-pass-diagnostic-words") {
console.error("Cannot find module; SyntaxError; command not found are fixture words");
-} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss") && broken) assert.fail("synthetic broken behavior");
+} else if ((id === "reference-vitest-alias" || id === "reference-vitest-postcss" || id === "reference-rust-compile") && broken) assert.fail("synthetic broken behavior");
else process.exit(id === "intended-failure" && broken ? 1 : 0);
`);
const manifest = join(repo, "cases.json");
@@ -151,7 +172,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module,baseline-rust-compile");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));