YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: prevent MCP grader test-script bypass
Louis Beaumonta5306f0640ad
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 744fdf654..5258f869c 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -532,7 +532,7 @@ local API credentials are synthetic. Only the two production source changesare applied by the oracle; builds are created inside each grading workspace.Run `bun test evals/coding-agent/calibrate-mcp-team-reload.test.js` with the MCP-package dependencies installed. Eight controls retain the parent/reference+package dependencies installed. Ten controls retain the parent/referencecontrast, reject an unused-helper fix, cached settings, stale gateway pairingand blanket denial, accept equivalent naming, and distinguish a missing-sourcebuild failure. These are deterministic corpus/grader tests, not model trials.@@ -1736,3 +1736,9 @@ required. Run `bun test evals/coding-agent/calibrate-keyboard.test.js`. Hiddenfixtures and runtime links appear only at grading. This does not verify theseparate live-capture prepend index repair, full timeline rendering, nativeevent delivery, recording persistence, enforced isolation or model performance.++The MCP credential-reload and live-database-boundary graders invoke Vitest directly,+so changing the candidate package’s `test` script cannot skip hidden assertions.+Their calibration suites reject broken source with a no-op test script and still+accept correct source with the same script. This closes that command-dispatch+bypass; it does not establish an execution sandbox or tamper-proof dependencies.diff --git a/evals/coding-agent/calibrate-mcp-db-boundary.test.js b/evals/coding-agent/calibrate-mcp-db-boundary.test.jsindex a479f41bd..8a98ba802 100644--- a/evals/coding-agent/calibrate-mcp-db-boundary.test.js+++ b/evals/coding-agent/calibrate-mcp-db-boundary.test.js@@ -11,13 +11,14 @@ const show=ref=>execFileSync('git',['show',`${ref}:${index}`],{cwd:repo,encodingconst root=mkdtempSync(join(tmpdir(),'mcp-db-calibration-')),template=join(root,'template');mkdirSync(template);execFileSync('tar',['-xf','-','-C',template],{input:execFileSync('git',['archive',item.base_ref,pkg],{cwd:repo,maxBuffer:64*1024*1024})});afterAll(()=>rmSync(root,{recursive:true,force:true}));-function grade(name,source,extra){+function grade(name,source,extra,noOpTest=false){const cwd=join(root,name);cpSync(template,cwd,{recursive:true});+ if(noOpTest){const file=join(cwd,pkg,'package.json');const data=JSON.parse(readFileSync(file,'utf8'));data.scripts.test='true';writeFileSync(file,JSON.stringify(data));}if(source===null)rmSync(join(cwd,index));else writeFileSync(join(cwd,index),source);if(extra)writeFileSync(join(cwd,index+'.unused.ts'),extra);for(const f of item.grader.fixtures)writeFileSync(join(cwd,f.destination_path),readFileSync(join(import.meta.dir,f.local_path)));symlinkSync(join(repo,pkg,'node_modules'),join(cwd,pkg,'node_modules'),'dir');- return spawnSync(process.execPath,['run','test','--','src/eval-db-boundary.test.ts'],{cwd:join(cwd,pkg),encoding:'utf8',timeout:45000,env:{PATH:process.env.PATH,HOME:cwd,BUN_BIN:process.execPath,CI:'true',NO_COLOR:'1',SCREENPIPE_DISABLE_TELEMETRY:'1'}});+ return spawnSync('sh',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:45000,env:{PATH:process.env.PATH,HOME:cwd,BUN_BIN:process.execPath,CI:'true',NO_COLOR:'1',SCREENPIPE_DISABLE_TELEMETRY:'1'}});}function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stderr).not.toContain('Could not resolve');}function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}@@ -30,3 +31,6 @@ test('direct database file reads are rejected even when their exception is swalltest('removing supported CLI recovery is rejected',()=>fails(grade('no-cli',replace(fixed,'const home = os.homedir();','return ""; const home = os.homedir();'))));test('blanket keyless requests fail preserved access',()=>fails(grade('deny',replace(fixed,'function ensureApiKey(): Promise<string> {','function ensureApiKey(): Promise<string> { return Promise.resolve("");'))));test('missing entrypoint is a build failure, not an intended behavioral contrast',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Could not resolve|ModuleNotFound|not found/);expect(r.stderr).not.toContain('AssertionError');});++test('no-op package test script cannot pass unchanged broken behavior',()=>{const r=grade('no-op-broken',broken,undefined,true);fails(r);expect(r.stdout).toContain('failed');},60_000);+test('correct behavior still passes with a no-op package test script',()=>{const r=grade('no-op-correct',fixed,undefined,true);expect(r.status).toBe(0);expect(r.stdout).toContain('7 passed');},60_000);diff --git a/evals/coding-agent/calibrate-mcp-team-reload.test.js b/evals/coding-agent/calibrate-mcp-team-reload.test.jsindex 1861dbf6a..d6cceec27 100644--- a/evals/coding-agent/calibrate-mcp-team-reload.test.js+++ b/evals/coding-agent/calibrate-mcp-team-reload.test.js@@ -16,14 +16,15 @@ const template=join(root,'template');mkdirSync(template);const archive=execFileSync('git',['archive',item.base_ref,pkg],{cwd:repo,maxBuffer:64*1024*1024});execFileSync('tar',['-xf','-','-C',template],{input:archive});afterAll(()=>rmSync(root,{recursive:true,force:true}));-function grade(name,sources){+function grade(name,sources,noOpTest=false){const cwd=join(root,name);cpSync(template,cwd,{recursive:true});+ if(noOpTest){const file=join(cwd,pkg,'package.json');const data=JSON.parse(readFileSync(file,'utf8'));data.scripts.test='true';writeFileSync(file,JSON.stringify(data));}for(const [p,s] of Object.entries(sources)){if(s===null)rmSync(join(cwd,p));else writeFileSync(join(cwd,p),s);}writeFileSync(join(cwd,pkg,'src/eval-team-auth.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-team-reload.fixture.ts')));symlinkSync(join(repo,pkg,'node_modules'),join(cwd,pkg,'node_modules'),'dir');- const r=spawnSync(process.execPath,['run','test','--','src/eval-team-auth.test.ts'],{cwd:join(cwd,pkg),encoding:'utf8',timeout:45_000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true',NO_COLOR:'1',FORCE_COLOR:'0',SCREENPIPE_DISABLE_TELEMETRY:'1'}});+ const r=spawnSync('sh',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:45_000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true',NO_COLOR:'1',FORCE_COLOR:'0',SCREENPIPE_DISABLE_TELEMETRY:'1'}});for(const key of ['stdout','stderr'])r[key]=r[key]?.replace(/\x1b\[[0-9;]*m/g,'');return r;}@@ -47,3 +48,6 @@ test('new token paired with stale gateway is rejected',()=>{test('missing config source is a build error rather than behavior evidence',()=>{const r=grade('missing-source',{...fixed,[config]:null});expect(r.status).toBe(1);expect(r.stdout).toContain('skipped');expect(r.stderr).toContain('Could not resolve');expect(r.stderr).not.toContain('AssertionError');},60_000);++test('no-op package test script cannot pass unchanged broken behavior',()=>{const r=grade('no-op-broken',broken,true);fails(r);expect(r.stdout).toContain('failed');},60_000);+test('correct behavior still passes with a no-op package test script',()=>{const r=grade('no-op-correct',fixed,true);expect(r.status).toBe(0);expect(r.stdout).toContain('7 passed');},60_000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 383cc4103..e9ee56827 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -2007,7 +2007,7 @@"destination_path": "packages/screenpipe-mcp/src/eval-team-auth.test.ts"}],- "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-team-auth.test.ts"+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 node node_modules/vitest/vitest.mjs run src/eval-team-auth.test.ts"}},{@@ -2407,7 +2407,7 @@"destination_path": "packages/screenpipe-mcp/src/eval-db-boundary.preload.cjs"}],- "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-db-boundary.test.ts"+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 node node_modules/vitest/vitest.mjs run src/eval-db-boundary.test.ts"}},{