Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: reject known setup errors as regression evidence

Louis Beaumonte3da3eabaf7c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 448d4f0e5..1a3129ec7 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -50,12 +50,24 @@ Agent-process, grader-process and harness failures are reported as `error` and e
the success denominator. They are never silently converted into model failures.
Grader timeouts and terminating signals cannot establish a failing baseline:
`--verify` requires a behavioral `fail` followed by an oracle `pass`.
-Ordinary nonzero grader exits are still treated as behavioral failures, so
-inspect grader logs to distinguish assertion failures from setup failures.
+Known command-unavailable exits (126/127), Bun unhandled test-load errors, and
+Node missing-module/syntax diagnostics are also reported as `error`, with a
+`grader_error_kind` in the result. A missing import followed by an oracle pass
+therefore cannot certify a regression. Plain failed assertions and successful
+commands containing diagnostic words retain their previous outcomes.
+
+This is bounded diagnostic recognition, not universal error attribution. Unknown
+setup/compiler failures, including other test frameworks, may still be ordinary
+nonzero exits. Inspect logs before promoting a case; preserve command exit codes
+in wrappers. An error is not proof that infrastructure, rather than a candidate
+change, caused it. Report error counts and inspect candidate-caused errors before
+comparing model success rates. No isolation or model-quality claim follows.
Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
-They cover baseline/reference timeouts and signals, a genuine failing baseline,
-an already-passing baseline, and exclusion of process errors from scored trials.
+Seventeen end-to-end controls cover baseline/reference timeouts and signals,
+missing ESM/CommonJS modules, Node/Bun syntax and Bun import errors, unavailable
+or non-executable commands, genuine failures, assertions quoting diagnostic words or complete diagnostic blocks,
+already-passing baselines and exclusion of known setup/process errors from scores.
The controls use temporary local Git fixtures and invoke no model or provider.
The same runner can score another checkout and manifest with `--repo` and
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjs
new file mode 100644
index 000000000..ac613b2f0
--- /dev/null
+++ b/evals/coding-agent/grader-outcome.mjs
@@ -0,0 +1,22 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+// Recognize bounded, known setup diagnostics. This is not a general proof that
+// an unrecognized exit is an assertion failure; promotion still needs log review.
+export function classifyGraderError(grader) {
+ if (grader.error || grader.signal || grader.status === null) return "process";
+ if (grader.status === 0) return null;
+ if (grader.status === 126 || grader.status === 127) return "command_unavailable";
+ const stderr = (grader.stderr ?? "").replace(/\x1b\[[0-9;]*m/g, "");
+ if (/^# Unhandled error between tests\s*$/m.test(stderr) &&
+ /^\s*\d+ errors?\s*$/m.test(stderr)) return "bun_unhandled_error";
+ // Only the first thrown-error header classifies a Node failure. Assertion
+ // messages may quote complete setup diagnostics on later lines.
+ const header = stderr.match(/^([A-Za-z]*Error)(?: \[([A-Z_0-9]+)\])?:[^\n]*/m);
+ if (header?.[1] === "Error" &&
+ (header[2] === "ERR_MODULE_NOT_FOUND" ||
+ (header[0].startsWith("Error: Cannot find module ") &&
+ /^\s*code: ['"]MODULE_NOT_FOUND['"]/m.test(stderr)))) return "node_missing_module";
+ if (header?.[1] === "SyntaxError" && /^\s+at /m.test(stderr)) return "node_syntax_error";
+ return null;
+}
diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjs
index 22ad40ce8..d21c5b697 100644
--- a/evals/coding-agent/run.mjs
+++ b/evals/coding-agent/run.mjs
@@ -18,6 +18,7 @@ import { dirname, join, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import process from "node:process";
+import { classifyGraderError } from "./grader-outcome.mjs";
const HERE = dirname(new URL(import.meta.url).pathname);
let REPO = resolve(HERE, "../..");
@@ -370,7 +371,8 @@ function runTrial(evalCase, mode, trial, options, runDir) {
writeFileSync(join(trialDir, "grader.stdout.log"), grader.stdout);
writeFileSync(join(trialDir, "grader.stderr.log"), grader.stderr);
const agentOk = !["agent", "regrade"].includes(mode) || (agent.status === 0 && !agent.error);
- const graderErrored = Boolean(grader.error || grader.signal || grader.status === null);
+ const graderErrorKind = classifyGraderError(grader);
+ const graderErrored = graderErrorKind !== null;
const passed = agentOk && !graderErrored && grader.status === 0;
const outcome = !agentOk || graderErrored ? "error" : passed ? "pass" : "fail";
const result = {
@@ -385,6 +387,7 @@ function runTrial(evalCase, mode, trial, options, runDir) {
grader_exit: grader.status,
grader_signal: grader.signal,
grader_error: grader.error,
+ grader_error_kind: graderErrorKind,
changed_files: candidate.changedFiles,
workspace: options.keep ? workspace : undefined,
};
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
index 762af3f61..a9a22025e 100644
--- a/evals/coding-agent/run.test.ts
+++ b/evals/coding-agent/run.test.ts
@@ -7,7 +7,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
-test("process failures cannot prove a regression or count as scored failures", () => {
+test("process and setup failures cannot prove a regression or count as scored failures", () => {
const repo = mkdtempSync(join(tmpdir(), "eval-runner-control-"));
const runner = new URL("./run.mjs", import.meta.url).pathname;
const git = (...args: string[]) => {
@@ -34,15 +34,52 @@ test("process failures cannot prove a regression or count as scored failures", (
{ id: "reference-timeout", baseline: "pass", oracle: "error", valid: false },
{ id: "baseline-signal", baseline: "error", oracle: "pass", valid: false },
{ id: "reference-signal", baseline: "pass", oracle: "error", valid: false },
+ { id: "baseline-missing-module", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-missing-module", baseline: "pass", oracle: "error", valid: false },
+ { id: "baseline-syntax", baseline: "error", oracle: "pass", valid: false },
+ { id: "baseline-command-missing", baseline: "error", oracle: "pass", valid: false },
+ { id: "baseline-bun-import", baseline: "error", oracle: "pass", valid: false },
+ { id: "baseline-commonjs", baseline: "error", oracle: "pass", valid: false },
+ { id: "baseline-bun-syntax", baseline: "error", oracle: "pass", valid: false },
+ { id: "baseline-permission", baseline: "error", oracle: "pass", valid: false },
+ { id: "diagnostic-words-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "diagnostic-block-assertion", baseline: "fail", oracle: "pass", valid: true },
+ { id: "both-pass-diagnostic-words", baseline: "pass", oracle: "pass", valid: false },
];
writeFileSync(join(repo, "grade.mjs"), `
-import { readFileSync } from "node:fs";
+import { readFileSync, writeFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import assert from "node:assert/strict";
const broken = readFileSync("state.txt", "utf8") === "broken";
const id = process.env.SCREENPIPE_EVAL_CASE_ID;
const affected = id.startsWith("baseline-") ? broken : !broken;
if (id.endsWith("-timeout") && affected) setInterval(() => {}, 1000);
else if (id.endsWith("-signal") && affected) process.kill(process.pid, "SIGTERM");
-else process.exit(id === "intended-failure" && broken ? 1 : 0);
+else if (id.endsWith("-missing-module") && affected) await import("./absent-synthetic-module.mjs");
+else if (id === "baseline-syntax" && affected) {
+ writeFileSync("invalid.mjs", "export const value = ;");
+ await import("./invalid.mjs");
+} else if (id.endsWith("-command-missing") && affected) {
+ const result = spawnSync("/bin/sh", ["-c", "exec synthetic_eval_command_that_does_not_exist"], { encoding: "utf8" });
+ process.stderr.write(result.stderr); process.exit(result.status);
+} else if ((id.endsWith("-bun-import") || id.endsWith("-bun-syntax")) && affected) {
+ writeFileSync("missing.test.js", id.endsWith("-bun-import") ? 'import "./absent-synthetic-module.mjs";' : 'export const value = ;');
+ const result = spawnSync(process.env.SYNTHETIC_BUN_BIN, ["test", "missing.test.js"], { encoding: "utf8" });
+ process.stdout.write(result.stdout); process.stderr.write(result.stderr); process.exit(result.status);
+} else if (id.endsWith("-commonjs") && affected) {
+ writeFileSync("missing.cjs", 'require("./absent-synthetic-module.cjs");');
+ await import("./missing.cjs");
+} else if (id.endsWith("-permission") && affected) {
+ writeFileSync("not-executable", "exit 0", { mode: 0o644 });
+ const result = spawnSync("/bin/sh", ["-c", "exec ./not-executable"], { encoding: "utf8" });
+ process.stderr.write(result.stderr); process.exit(result.status);
+} else if (id === "diagnostic-words-assertion" && broken) {
+ assert.equal("actual", "expected", "Cannot find module; SyntaxError; command not found are fixture words");
+} else if (id === "diagnostic-block-assertion" && broken) {
+ assert.fail("\\nError [ERR_MODULE_NOT_FOUND]: quoted diagnostic\\nSyntaxError: quoted diagnostic\\n at quotedFixture");
+} else if (id === "both-pass-diagnostic-words") {
+ console.error("Cannot find module; SyntaxError; command not found are fixture words");
+} else process.exit(id === "intended-failure" && broken ? 1 : 0);
`);
const manifest = join(repo, "cases.json");
writeFileSync(manifest, JSON.stringify({
@@ -57,7 +94,7 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
}));
const invoke = (directory: string, ...args: string[]) => spawnSync("node", [
runner, "--repo", repo, "--manifest", manifest, "--results-dir", join(repo, directory), ...args,
- ], { cwd: repo, encoding: "utf8", timeout: 60_000 });
+ ], { cwd: repo, encoding: "utf8", timeout: 60_000, env: { ...process.env, SYNTHETIC_BUN_BIN: process.execPath } });
const verified = invoke("verification", "--verify");
expect(verified.error).toBeUndefined();
expect(verified.status).toBe(1); // Invalid controls must make the CLI reject verification.
@@ -72,18 +109,19 @@ else process.exit(id === "intended-failure" && broken ? 1 : 0);
expect(result.valid).toBe(control.valid);
const errored = control.baseline === "error" ? result.baseline :
control.oracle === "error" ? result.oracle : null;
- if (errored) {
+ if (errored) expect(errored.grader_error_kind).toBeString();
+ if (errored && (control.id.endsWith("timeout") || control.id.endsWith("signal"))) {
expect(errored.passed).toBe(false);
expect(errored.grader_exit).toBeNull();
expect(errored.grader_signal).toBe("SIGTERM");
if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
}
}
- const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal");
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal,baseline-missing-module");
expect(scored.error).toBeUndefined();
expect(scored.status).toBe(0);
const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));
- expect(summary.cases[0]).toMatchObject({ errors: 1, scored_trials: 0, success_rate: null });
+ for (const result of summary.cases) expect(result).toMatchObject({ errors: 1, scored_trials: 0, success_rate: null });
} finally {
rmSync(repo, { recursive: true, force: true });
}