Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: reject grader process errors as regression evidence

louis90ac5aa20dab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 7c0e98c51..ef9074ea5 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -46,8 +46,17 @@ The agent receives only the task and archived broken tree. It does not receive t
Results are written under `evals/coding-agent/results/` unless `--results-dir` is supplied. Use `--keep` only for debugging a failed trial.
-Agent-process and harness failures are reported as `error` and excluded from
+Agent-process, grader-process and harness failures are reported as `error` and excluded from
the success denominator. They are never silently converted into model failures.
+Grader timeouts and terminating signals cannot establish a failing baseline:
+`--verify` requires a behavioral `fail` followed by an oracle `pass`.
+Ordinary nonzero grader exits are still treated as behavioral failures, so
+inspect grader logs to distinguish assertion failures from setup failures.
+
+Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.
+They cover baseline/reference timeouts and signals, a genuine failing baseline,
+an already-passing baseline, and exclusion of process errors from scored trials.
+The controls use temporary local Git fixtures and invoke no model or provider.
The same runner can score another checkout and manifest with `--repo` and
`--manifest`; the website corpus uses this so both repositories share exactly
diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjs
index 9b144a259..22ad40ce8 100644
--- a/evals/coding-agent/run.mjs
+++ b/evals/coding-agent/run.mjs
@@ -370,8 +370,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {
writeFileSync(join(trialDir, "grader.stdout.log"), grader.stdout);
writeFileSync(join(trialDir, "grader.stderr.log"), grader.stderr);
const agentOk = !["agent", "regrade"].includes(mode) || (agent.status === 0 && !agent.error);
- const passed = agentOk && grader.status === 0;
- const outcome = !agentOk ? "error" : passed ? "pass" : "fail";
+ const graderErrored = Boolean(grader.error || grader.signal || grader.status === null);
+ const passed = agentOk && !graderErrored && grader.status === 0;
+ const outcome = !agentOk || graderErrored ? "error" : passed ? "pass" : "fail";
const result = {
case_id: evalCase.id,
trial,
@@ -382,6 +383,8 @@ function runTrial(evalCase, mode, trial, options, runDir) {
agent_exit: agent.status,
agent_usage: extractAgentUsage(agent.stdout),
grader_exit: grader.status,
+ grader_signal: grader.signal,
+ grader_error: grader.error,
changed_files: candidate.changedFiles,
workspace: options.keep ? workspace : undefined,
};
@@ -487,7 +490,7 @@ function main() {
for (const evalCase of cases) {
const baseline = runTrial(evalCase, "baseline", 1, options, join(runDir, "baseline"));
const oracle = runTrial(evalCase, "oracle", 1, options, join(runDir, "oracle"));
- const valid = !baseline.passed && oracle.passed && !baseline.harness_error && !oracle.harness_error;
+ const valid = baseline.outcome === "fail" && oracle.outcome === "pass";
verification.push({ case_id: evalCase.id, valid, baseline, oracle });
console.log(`${evalCase.id}: broken=${baseline.passed ? "PASS" : "FAIL"} oracle=${oracle.passed ? "PASS" : "FAIL"}`);
}
diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.ts
new file mode 100644
index 000000000..762af3f61
--- /dev/null
+++ b/evals/coding-agent/run.test.ts
@@ -0,0 +1,90 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { expect, test } from "bun:test";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+test("process failures cannot prove a regression or count as scored failures", () => {
+ const repo = mkdtempSync(join(tmpdir(), "eval-runner-control-"));
+ const runner = new URL("./run.mjs", import.meta.url).pathname;
+ const git = (...args: string[]) => {
+ const result = spawnSync("git", args, { cwd: repo, encoding: "utf8", timeout: 10_000 });
+ if (result.status !== 0) throw new Error(result.stderr || String(result.error));
+ return result.stdout.trim();
+ };
+ try {
+ git("init", "-q");
+ git("config", "user.name", "Synthetic eval control");
+ git("config", "user.email", "[email protected]");
+ git("config", "commit.gpgsign", "false");
+ writeFileSync(join(repo, "state.txt"), "broken");
+ git("add", "state.txt");
+ git("commit", "-qm", "Synthetic broken state");
+ const base = git("rev-parse", "HEAD");
+ writeFileSync(join(repo, "state.txt"), "fixed");
+ git("commit", "-qam", "Synthetic reference state");
+ const fix = git("rev-parse", "HEAD");
+ const controls = [
+ { id: "baseline-timeout", baseline: "error", oracle: "pass", valid: false },
+ { id: "intended-failure", baseline: "fail", oracle: "pass", valid: true },
+ { id: "both-pass", baseline: "pass", oracle: "pass", valid: false },
+ { id: "reference-timeout", baseline: "pass", oracle: "error", valid: false },
+ { id: "baseline-signal", baseline: "error", oracle: "pass", valid: false },
+ { id: "reference-signal", baseline: "pass", oracle: "error", valid: false },
+ ];
+ writeFileSync(join(repo, "grade.mjs"), `
+import { readFileSync } from "node:fs";
+const broken = readFileSync("state.txt", "utf8") === "broken";
+const id = process.env.SCREENPIPE_EVAL_CASE_ID;
+const affected = id.startsWith("baseline-") ? broken : !broken;
+if (id.endsWith("-timeout") && affected) setInterval(() => {}, 1000);
+else if (id.endsWith("-signal") && affected) process.kill(process.pid, "SIGTERM");
+else process.exit(id === "intended-failure" && broken ? 1 : 0);
+`);
+ const manifest = join(repo, "cases.json");
+ writeFileSync(manifest, JSON.stringify({
+ schema_version: 1, suite: "synthetic-runner-controls", dataset_version: "1",
+ cases: controls.map(({ id }) => ({
+ id, name: id, base_ref: base, oracle_ref: fix, oracle_paths: ["state.txt"],
+ source: { kind: "git_regression", fix_commit: fix },
+ prompt: "Synthetic harness control; no model is invoked.",
+ grader: { timeout_seconds: 1, command: "exec node grade.mjs",
+ fixtures: [{ local_path: "grade.mjs", destination_path: "grade.mjs" }] },
+ })),
+ }));
+ const invoke = (directory: string, ...args: string[]) => spawnSync("node", [
+ runner, "--repo", repo, "--manifest", manifest, "--results-dir", join(repo, directory), ...args,
+ ], { cwd: repo, encoding: "utf8", timeout: 60_000 });
+ const verified = invoke("verification", "--verify");
+ expect(verified.error).toBeUndefined();
+ expect(verified.status).toBe(1); // Invalid controls must make the CLI reject verification.
+ const results = JSON.parse(readFileSync(join(repo, "verification/verification.json"), "utf8"));
+ expect(results).toHaveLength(controls.length);
+ for (const control of controls) {
+ const result = results.find((item: { case_id: string }) => item.case_id === control.id);
+ expect(result.baseline.harness_error).toBeUndefined();
+ expect(result.oracle.harness_error).toBeUndefined();
+ expect(result.baseline.outcome).toBe(control.baseline);
+ expect(result.oracle.outcome).toBe(control.oracle);
+ expect(result.valid).toBe(control.valid);
+ const errored = control.baseline === "error" ? result.baseline :
+ control.oracle === "error" ? result.oracle : null;
+ if (errored) {
+ expect(errored.passed).toBe(false);
+ expect(errored.grader_exit).toBeNull();
+ expect(errored.grader_signal).toBe("SIGTERM");
+ if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");
+ }
+ }
+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal");
+ expect(scored.error).toBeUndefined();
+ expect(scored.status).toBe(0);
+ const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));
+ expect(summary.cases[0]).toMatchObject({ errors: 1, scored_trials: 0, success_rate: null });
+ } finally {
+ rmSync(repo, { recursive: true, force: true });
+ }
+}, 90_000);