Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: prevent MCP grader test-script bypass

Louis Beaumonta5306f0640ad
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 744fdf654..5258f869c 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -532,7 +532,7 @@ local API credentials are synthetic. Only the two production source changes
are applied by the oracle; builds are created inside each grading workspace.
Run `bun test evals/coding-agent/calibrate-mcp-team-reload.test.js` with the MCP
-package dependencies installed. Eight controls retain the parent/reference
+package dependencies installed. Ten controls retain the parent/reference
contrast, reject an unused-helper fix, cached settings, stale gateway pairing
and blanket denial, accept equivalent naming, and distinguish a missing-source
build failure. These are deterministic corpus/grader tests, not model trials.
@@ -1736,3 +1736,9 @@ required. Run `bun test evals/coding-agent/calibrate-keyboard.test.js`. Hidden
fixtures and runtime links appear only at grading. This does not verify the
separate live-capture prepend index repair, full timeline rendering, native
event delivery, recording persistence, enforced isolation or model performance.
+
+The MCP credential-reload and live-database-boundary graders invoke Vitest directly,
+so changing the candidate package’s `test` script cannot skip hidden assertions.
+Their calibration suites reject broken source with a no-op test script and still
+accept correct source with the same script. This closes that command-dispatch
+bypass; it does not establish an execution sandbox or tamper-proof dependencies.
diff --git a/evals/coding-agent/calibrate-mcp-db-boundary.test.js b/evals/coding-agent/calibrate-mcp-db-boundary.test.js
index a479f41bd..8a98ba802 100644
--- a/evals/coding-agent/calibrate-mcp-db-boundary.test.js
+++ b/evals/coding-agent/calibrate-mcp-db-boundary.test.js
@@ -11,13 +11,14 @@ const show=ref=>execFileSync('git',['show',`${ref}:${index}`],{cwd:repo,encoding
const root=mkdtempSync(join(tmpdir(),'mcp-db-calibration-')),template=join(root,'template');mkdirSync(template);
execFileSync('tar',['-xf','-','-C',template],{input:execFileSync('git',['archive',item.base_ref,pkg],{cwd:repo,maxBuffer:64*1024*1024})});
afterAll(()=>rmSync(root,{recursive:true,force:true}));
-function grade(name,source,extra){
+function grade(name,source,extra,noOpTest=false){
const cwd=join(root,name);cpSync(template,cwd,{recursive:true});
+ if(noOpTest){const file=join(cwd,pkg,'package.json');const data=JSON.parse(readFileSync(file,'utf8'));data.scripts.test='true';writeFileSync(file,JSON.stringify(data));}
if(source===null)rmSync(join(cwd,index));else writeFileSync(join(cwd,index),source);
if(extra)writeFileSync(join(cwd,index+'.unused.ts'),extra);
for(const f of item.grader.fixtures)writeFileSync(join(cwd,f.destination_path),readFileSync(join(import.meta.dir,f.local_path)));
symlinkSync(join(repo,pkg,'node_modules'),join(cwd,pkg,'node_modules'),'dir');
- return spawnSync(process.execPath,['run','test','--','src/eval-db-boundary.test.ts'],{cwd:join(cwd,pkg),encoding:'utf8',timeout:45000,env:{PATH:process.env.PATH,HOME:cwd,BUN_BIN:process.execPath,CI:'true',NO_COLOR:'1',SCREENPIPE_DISABLE_TELEMETRY:'1'}});
+ return spawnSync('sh',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:45000,env:{PATH:process.env.PATH,HOME:cwd,BUN_BIN:process.execPath,CI:'true',NO_COLOR:'1',SCREENPIPE_DISABLE_TELEMETRY:'1'}});
}
function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stderr).not.toContain('Could not resolve');}
function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
@@ -30,3 +31,6 @@ test('direct database file reads are rejected even when their exception is swall
test('removing supported CLI recovery is rejected',()=>fails(grade('no-cli',replace(fixed,'const home = os.homedir();','return ""; const home = os.homedir();'))));
test('blanket keyless requests fail preserved access',()=>fails(grade('deny',replace(fixed,'function ensureApiKey(): Promise<string> {','function ensureApiKey(): Promise<string> { return Promise.resolve("");'))));
test('missing entrypoint is a build failure, not an intended behavioral contrast',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Could not resolve|ModuleNotFound|not found/);expect(r.stderr).not.toContain('AssertionError');});
+
+test('no-op package test script cannot pass unchanged broken behavior',()=>{const r=grade('no-op-broken',broken,undefined,true);fails(r);expect(r.stdout).toContain('failed');},60_000);
+test('correct behavior still passes with a no-op package test script',()=>{const r=grade('no-op-correct',fixed,undefined,true);expect(r.status).toBe(0);expect(r.stdout).toContain('7 passed');},60_000);
diff --git a/evals/coding-agent/calibrate-mcp-team-reload.test.js b/evals/coding-agent/calibrate-mcp-team-reload.test.js
index 1861dbf6a..d6cceec27 100644
--- a/evals/coding-agent/calibrate-mcp-team-reload.test.js
+++ b/evals/coding-agent/calibrate-mcp-team-reload.test.js
@@ -16,14 +16,15 @@ const template=join(root,'template');mkdirSync(template);
const archive=execFileSync('git',['archive',item.base_ref,pkg],{cwd:repo,maxBuffer:64*1024*1024});
execFileSync('tar',['-xf','-','-C',template],{input:archive});
afterAll(()=>rmSync(root,{recursive:true,force:true}));
-function grade(name,sources){
+function grade(name,sources,noOpTest=false){
const cwd=join(root,name);cpSync(template,cwd,{recursive:true});
+ if(noOpTest){const file=join(cwd,pkg,'package.json');const data=JSON.parse(readFileSync(file,'utf8'));data.scripts.test='true';writeFileSync(file,JSON.stringify(data));}
for(const [p,s] of Object.entries(sources)){
if(s===null)rmSync(join(cwd,p));else writeFileSync(join(cwd,p),s);
}
writeFileSync(join(cwd,pkg,'src/eval-team-auth.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-team-reload.fixture.ts')));
symlinkSync(join(repo,pkg,'node_modules'),join(cwd,pkg,'node_modules'),'dir');
- const r=spawnSync(process.execPath,['run','test','--','src/eval-team-auth.test.ts'],{cwd:join(cwd,pkg),encoding:'utf8',timeout:45_000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true',NO_COLOR:'1',FORCE_COLOR:'0',SCREENPIPE_DISABLE_TELEMETRY:'1'}});
+ const r=spawnSync('sh',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:45_000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true',NO_COLOR:'1',FORCE_COLOR:'0',SCREENPIPE_DISABLE_TELEMETRY:'1'}});
for(const key of ['stdout','stderr'])r[key]=r[key]?.replace(/\x1b\[[0-9;]*m/g,'');
return r;
}
@@ -47,3 +48,6 @@ test('new token paired with stale gateway is rejected',()=>{
test('missing config source is a build error rather than behavior evidence',()=>{
const r=grade('missing-source',{...fixed,[config]:null});expect(r.status).toBe(1);expect(r.stdout).toContain('skipped');expect(r.stderr).toContain('Could not resolve');expect(r.stderr).not.toContain('AssertionError');
},60_000);
+
+test('no-op package test script cannot pass unchanged broken behavior',()=>{const r=grade('no-op-broken',broken,true);fails(r);expect(r.stdout).toContain('failed');},60_000);
+test('correct behavior still passes with a no-op package test script',()=>{const r=grade('no-op-correct',fixed,true);expect(r.status).toBe(0);expect(r.stdout).toContain('7 passed');},60_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 383cc4103..e9ee56827 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -2007,7 +2007,7 @@
"destination_path": "packages/screenpipe-mcp/src/eval-team-auth.test.ts"
}
],
- "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-team-auth.test.ts"
+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 node node_modules/vitest/vitest.mjs run src/eval-team-auth.test.ts"
}
},
{
@@ -2407,7 +2407,7 @@
"destination_path": "packages/screenpipe-mcp/src/eval-db-boundary.preload.cjs"
}
],
- "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-db-boundary.test.ts"
+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 node node_modules/vitest/vitest.mjs run src/eval-db-boundary.test.ts"
}
},
{