YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: reject grader process errors as regression evidence
louis90ac5aa20dab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 7c0e98c51..ef9074ea5 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -46,8 +46,17 @@ The agent receives only the task and archived broken tree. It does not receive tResults are written under `evals/coding-agent/results/` unless `--results-dir` is supplied. Use `--keep` only for debugging a failed trial.-Agent-process and harness failures are reported as `error` and excluded from+Agent-process, grader-process and harness failures are reported as `error` and excluded fromthe success denominator. They are never silently converted into model failures.+Grader timeouts and terminating signals cannot establish a failing baseline:+`--verify` requires a behavioral `fail` followed by an oracle `pass`.+Ordinary nonzero grader exits are still treated as behavioral failures, so+inspect grader logs to distinguish assertion failures from setup failures.++Run the synthetic runner controls with `bun test ./evals/coding-agent/run.test.ts`.+They cover baseline/reference timeouts and signals, a genuine failing baseline,+an already-passing baseline, and exclusion of process errors from scored trials.+The controls use temporary local Git fixtures and invoke no model or provider.The same runner can score another checkout and manifest with `--repo` and`--manifest`; the website corpus uses this so both repositories share exactlydiff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjsindex 9b144a259..22ad40ce8 100644--- a/evals/coding-agent/run.mjs+++ b/evals/coding-agent/run.mjs@@ -370,8 +370,9 @@ function runTrial(evalCase, mode, trial, options, runDir) {writeFileSync(join(trialDir, "grader.stdout.log"), grader.stdout);writeFileSync(join(trialDir, "grader.stderr.log"), grader.stderr);const agentOk = !["agent", "regrade"].includes(mode) || (agent.status === 0 && !agent.error);- const passed = agentOk && grader.status === 0;- const outcome = !agentOk ? "error" : passed ? "pass" : "fail";+ const graderErrored = Boolean(grader.error || grader.signal || grader.status === null);+ const passed = agentOk && !graderErrored && grader.status === 0;+ const outcome = !agentOk || graderErrored ? "error" : passed ? "pass" : "fail";const result = {case_id: evalCase.id,trial,@@ -382,6 +383,8 @@ function runTrial(evalCase, mode, trial, options, runDir) {agent_exit: agent.status,agent_usage: extractAgentUsage(agent.stdout),grader_exit: grader.status,+ grader_signal: grader.signal,+ grader_error: grader.error,changed_files: candidate.changedFiles,workspace: options.keep ? workspace : undefined,};@@ -487,7 +490,7 @@ function main() {for (const evalCase of cases) {const baseline = runTrial(evalCase, "baseline", 1, options, join(runDir, "baseline"));const oracle = runTrial(evalCase, "oracle", 1, options, join(runDir, "oracle"));- const valid = !baseline.passed && oracle.passed && !baseline.harness_error && !oracle.harness_error;+ const valid = baseline.outcome === "fail" && oracle.outcome === "pass";verification.push({ case_id: evalCase.id, valid, baseline, oracle });console.log(`${evalCase.id}: broken=${baseline.passed ? "PASS" : "FAIL"} oracle=${oracle.passed ? "PASS" : "FAIL"}`);}diff --git a/evals/coding-agent/run.test.ts b/evals/coding-agent/run.test.tsnew file mode 100644index 000000000..762af3f61--- /dev/null+++ b/evals/coding-agent/run.test.ts@@ -0,0 +1,90 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { expect, test } from "bun:test";+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { spawnSync } from "node:child_process";++test("process failures cannot prove a regression or count as scored failures", () => {+ const repo = mkdtempSync(join(tmpdir(), "eval-runner-control-"));+ const runner = new URL("./run.mjs", import.meta.url).pathname;+ const git = (...args: string[]) => {+ const result = spawnSync("git", args, { cwd: repo, encoding: "utf8", timeout: 10_000 });+ if (result.status !== 0) throw new Error(result.stderr || String(result.error));+ return result.stdout.trim();+ };+ try {+ git("init", "-q");+ git("config", "user.name", "Synthetic eval control");+ git("config", "user.email", "[email protected]");+ git("config", "commit.gpgsign", "false");+ writeFileSync(join(repo, "state.txt"), "broken");+ git("add", "state.txt");+ git("commit", "-qm", "Synthetic broken state");+ const base = git("rev-parse", "HEAD");+ writeFileSync(join(repo, "state.txt"), "fixed");+ git("commit", "-qam", "Synthetic reference state");+ const fix = git("rev-parse", "HEAD");+ const controls = [+ { id: "baseline-timeout", baseline: "error", oracle: "pass", valid: false },+ { id: "intended-failure", baseline: "fail", oracle: "pass", valid: true },+ { id: "both-pass", baseline: "pass", oracle: "pass", valid: false },+ { id: "reference-timeout", baseline: "pass", oracle: "error", valid: false },+ { id: "baseline-signal", baseline: "error", oracle: "pass", valid: false },+ { id: "reference-signal", baseline: "pass", oracle: "error", valid: false },+ ];+ writeFileSync(join(repo, "grade.mjs"), `+import { readFileSync } from "node:fs";+const broken = readFileSync("state.txt", "utf8") === "broken";+const id = process.env.SCREENPIPE_EVAL_CASE_ID;+const affected = id.startsWith("baseline-") ? broken : !broken;+if (id.endsWith("-timeout") && affected) setInterval(() => {}, 1000);+else if (id.endsWith("-signal") && affected) process.kill(process.pid, "SIGTERM");+else process.exit(id === "intended-failure" && broken ? 1 : 0);+`);+ const manifest = join(repo, "cases.json");+ writeFileSync(manifest, JSON.stringify({+ schema_version: 1, suite: "synthetic-runner-controls", dataset_version: "1",+ cases: controls.map(({ id }) => ({+ id, name: id, base_ref: base, oracle_ref: fix, oracle_paths: ["state.txt"],+ source: { kind: "git_regression", fix_commit: fix },+ prompt: "Synthetic harness control; no model is invoked.",+ grader: { timeout_seconds: 1, command: "exec node grade.mjs",+ fixtures: [{ local_path: "grade.mjs", destination_path: "grade.mjs" }] },+ })),+ }));+ const invoke = (directory: string, ...args: string[]) => spawnSync("node", [+ runner, "--repo", repo, "--manifest", manifest, "--results-dir", join(repo, directory), ...args,+ ], { cwd: repo, encoding: "utf8", timeout: 60_000 });+ const verified = invoke("verification", "--verify");+ expect(verified.error).toBeUndefined();+ expect(verified.status).toBe(1); // Invalid controls must make the CLI reject verification.+ const results = JSON.parse(readFileSync(join(repo, "verification/verification.json"), "utf8"));+ expect(results).toHaveLength(controls.length);+ for (const control of controls) {+ const result = results.find((item: { case_id: string }) => item.case_id === control.id);+ expect(result.baseline.harness_error).toBeUndefined();+ expect(result.oracle.harness_error).toBeUndefined();+ expect(result.baseline.outcome).toBe(control.baseline);+ expect(result.oracle.outcome).toBe(control.oracle);+ expect(result.valid).toBe(control.valid);+ const errored = control.baseline === "error" ? result.baseline :+ control.oracle === "error" ? result.oracle : null;+ if (errored) {+ expect(errored.passed).toBe(false);+ expect(errored.grader_exit).toBeNull();+ expect(errored.grader_signal).toBe("SIGTERM");+ if (control.id.endsWith("timeout")) expect(errored.grader_error).toContain("ETIMEDOUT");+ }+ }+ const scored = invoke("scoring", "--mode", "baseline", "--case", "baseline-signal");+ expect(scored.error).toBeUndefined();+ expect(scored.status).toBe(0);+ const summary = JSON.parse(readFileSync(join(repo, "scoring/summary.json"), "utf8"));+ expect(summary.cases[0]).toMatchObject({ errors: 1, scored_trials: 0, success_rate: null });+ } finally {+ rmSync(repo, { recursive: true, force: true });+ }+}, 90_000);