YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(evals): classify wrapped Vitest setup errors
louis04ccb672e7ab
diff --git a/evals/coding-agent/grader-outcome.mjs b/evals/coding-agent/grader-outcome.mjsindex d2095b2ba..e763ba291 100644--- a/evals/coding-agent/grader-outcome.mjs+++ b/evals/coding-agent/grader-outcome.mjs@@ -34,6 +34,11 @@ export function classifyGraderError(grader) {((/^Error: Failed to resolve import ['"][^\r\n]+['"] from ['"][^\r\n]+['"]\. Does the file exist\?\s*$/m.test(stderr) &&!/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) ||/^Error: Failed to load url /m.test(stderr) ||+ // Vitest wraps failures inside vi.mock factories. Require its wrapper,+ // the caused-by missing URL, zero executed tests, and no assertions.+ (/^Error: \[vitest\] There was an error when mocking a module\./m.test(stderr) &&+ /^Caused by: Error: Failed to load url [^\n]+Does the file exist\?\s*$/m.test(stderr) &&+ !/^AssertionError(?: \[[^\]]+\])?:/m.test(stderr)) ||/^Error: Cannot find module ['"][^\n]+['"] imported from ['"][^\n]+['"]\.\s*$/m.test(stderr))) return "vitest_collection_error";// A beforeAll build failure can register tests but skip every affected test.// Passing neighboring suites do not turn a compile failure into an assertion.diff --git a/evals/coding-agent/wrapped-setup.test.ts b/evals/coding-agent/wrapped-setup.test.tsnew file mode 100644index 000000000..cfe7d0ffe--- /dev/null+++ b/evals/coding-agent/wrapped-setup.test.ts@@ -0,0 +1,43 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import {expect,test} from "bun:test";+import {mkdtempSync,writeFileSync,readFileSync,rmSync} from "node:fs";+import {join} from "node:path";+import {tmpdir} from "node:os";+import {spawnSync} from "node:child_process";+import {classifyGraderError} from "./grader-outcome.mjs";++const collection={status:1,stdout:" RUN v1.6.1 /synthetic\n Test Files 1 failed (1)\n Tests no tests\n",+ stderr:"Failed Suites 1\nError: [vitest] There was an error when mocking a module. If you are using vi.mock factory, check imports.\nCaused by: Error: Failed to load url synthetic-provider (resolved id: synthetic-provider) in /synthetic/fixture.test.ts. Does the file exist?\n"};+test("wrapped missing dependencies need complete collection evidence",()=>{+ expect(classifyGraderError(collection)).toBe("vitest_collection_error");+ expect(classifyGraderError({...collection,stdout:"\x1b[31m"+collection.stdout+"\x1b[0m",stderr:"\x1b[31m"+collection.stderr+"\x1b[0m"})).toBe("vitest_collection_error");+ for(const stdout of ["",collection.stdout.replace(" Tests no tests\n",""),collection.stdout.replace("no tests","1 failed (1)"),collection.stdout+" Tests 1 failed (1)\n",collection.stdout.replace("Test Files 1 failed (1)","Test Files 1 passed (1)")])expect(classifyGraderError({...collection,stdout})).toBeNull();+ for(const stderr of [collection.stderr.replace("Error: [vitest] There was an error when mocking a module.","Error: unrecognized wrapper"),collection.stderr.replace("Caused by: Error: Failed to load url", "Caused by: TypeError: unexpected"),collection.stderr.replace(" Does the file exist?", ""),collection.stderr+"AssertionError: actual result differs\n"])expect(classifyGraderError({...collection,stderr})).toBeNull();+ expect(classifyGraderError({...collection,status:0})).toBeNull();+});+test("wrapped collection errors invalidate corpus verification and stay out of scored trials",()=>{+ const repo=mkdtempSync(join(tmpdir(),"wrapped-setup-control-"));+ const run=(program:string,args:string[])=>{const z=spawnSync(program,args,{cwd:repo,encoding:"utf8",timeout:30000});expect(z.error).toBeUndefined();return z;};+ const git=(...args:string[])=>{const z=run("git",args);expect(z.status).toBe(0);return z.stdout.trim();};+ try{+ git("init","-q");git("config","user.name","Synthetic fixture");git("config","user.email","[email protected]");git("config","commit.gpgsign","false");+ writeFileSync(join(repo,"state.txt"),"broken");git("add","state.txt");git("commit","-qm","broken");const base=git("rev-parse","HEAD");writeFileSync(join(repo,"state.txt"),"fixed");git("commit","-qam","fixed");const fixed=git("rev-parse","HEAD");+ writeFileSync(join(repo,"grade.mjs"),`import {readFileSync} from 'node:fs';+const broken=readFileSync('state.txt','utf8')==='broken';const id=process.env.SCREENPIPE_EVAL_CASE_ID;+const affected=id==='wrapped-reference'?!broken:broken;+if(affected){process.stdout.write(${JSON.stringify(collection.stdout)});process.stderr.write(${JSON.stringify(collection.stderr)});if(id==='real-assertion'){process.stdout.write('Tests 1 failed (1)\\n');process.stderr.write('AssertionError: expected behavior failed\\n');}process.exit(1);}+process.exit(id==='wrapped-reference'&&broken?1:0);`);+ const cases=["wrapped-baseline","wrapped-reference","real-assertion"].map(id=>({id,name:id,base_ref:base,oracle_ref:fixed,source:{kind:"git_regression",fix_commit:fixed},oracle_paths:["state.txt"],prompt:"Synthetic runner control; no agent trial.",grader:{command:"node grade.mjs",fixtures:[{local_path:"grade.mjs",destination_path:"grade.mjs"}]}}));+ writeFileSync(join(repo,"cases.json"),JSON.stringify({schema_version:1,suite:"wrapped-setup-controls",dataset_version:"1",cases}));+ const runner=new URL("./run.mjs",import.meta.url).pathname;+ const invoke=(dir:string,...args:string[])=>run("node",[runner,"--repo",repo,"--manifest",join(repo,"cases.json"),"--results-dir",join(repo,dir),...args]);+ expect(invoke("verify","--verify").status).toBe(1);+ const results=JSON.parse(readFileSync(join(repo,"verify/verification.json"),"utf8"));+ expect(results.map((r:any)=>[r.baseline.outcome,r.oracle.outcome,r.valid])).toEqual([["error","pass",false],["fail","error",false],["fail","pass",true]]);+ expect(invoke("scoring","--mode","baseline","--case","wrapped-baseline").status).toBe(0);+ const summary=JSON.parse(readFileSync(join(repo,"scoring/summary.json"),"utf8"));+ expect(summary.cases[0]).toMatchObject({errors:1,scored_trials:0,success_rate:null});+ }finally{rmSync(repo,{recursive:true,force:true});}+},45000);