Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover HTTP MCP search source evidence

Louis Beaumont01aac0d55e21
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 5a72933c9..a33ba65e4 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1805,3 +1805,9 @@ and current source pass eleven. Run
Fixtures and dependency links appear only at grading. This does not establish
image decoding/rendering, path-only attachment handling, real discovery, native
durability, enforced isolation or model performance.
+
+### HTTP MCP search evidence
+
+`app-mcp-search-source-evidence` exercises the actual HTTP MCP server and SDK client against a synthetic loopback backend. It checks frame/chunk references, input-event text, bounded and explicitly unlimited reads, pagination, missing IDs, empty results and upstream refusal. The broken parent fails five outcomes while preserving four; the historical reference passes nine.
+
+Run `bun test evals/coding-agent/calibrate-mcp-search-evidence.test.js` to check the historical contrast, current source, equivalent implementation and broken/bypass controls. Dependencies are linked only for grading. This does not establish stdio/native telemetry, disconnect cleanup, live authentication, model performance or enforced host isolation. The existing activity-summary case tests another public tool.
diff --git a/evals/coding-agent/calibrate-mcp-search-evidence.test.js b/evals/coding-agent/calibrate-mcp-search-evidence.test.js
new file mode 100644
index 000000000..91e11db20
--- /dev/null
+++ b/evals/coding-agent/calibrate-mcp-search-evidence.test.js
@@ -0,0 +1,33 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';
+import {resolve,join} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-mcp-search-source-evidence');
+const root=mkdtempSync(join(tmpdir(),'mcp-search-evidence-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',ref,'packages/screenpipe-mcp'],{cwd:repo,maxBuffer:24*1024*1024})});
+ const pkg=join(cwd,'packages/screenpipe-mcp');symlinkSync(join(repo,'packages/screenpipe-mcp/node_modules'),join(pkg,'node_modules'),'dir');
+ writeFileSync(join(pkg,'src/eval-evidence.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-search-evidence.fixture.ts.txt')));mutate(pkg);
+ const home=join(cwd,'home');mkdirSync(home);
+ const r=spawnSync(process.env.NODE_BIN||'node',['node_modules/vitest/vitest.mjs','run','src/eval-evidence.test.ts'],{cwd:pkg,encoding:'utf8',timeout:60_000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:home,CI:'true',TZ:'UTC',SCREENPIPE_MCP_TELEMETRY_DISABLED:'1',DO_NOT_TRACK:'1',SCREENPIPE_API_URL:'http://127.0.0.1:1'}});
+ if(process.env.MCP_EVIDENCE_CALIBRATION_RESULTS){mkdirSync(process.env.MCP_EVIDENCE_CALIBRATION_RESULTS,{recursive:true});writeFileSync(join(process.env.MCP_EVIDENCE_CALIBRATION_RESULTS,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null,stdout:r.stdout,stderr:r.stderr},null,2)+'\n');}
+ return r;
+}
+const clean=s=>s.replace(/\x1b\[[0-9;]*m/g,'');
+function pass(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status,r.stdout+r.stderr).toBe(0);expect(clean(r.stdout)).toContain('9 passed');}
+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stderr).not.toMatch(/Cannot find module|Failed to load url/);}
+function edit(pkg,file,from,to){const p=join(pkg,'src',file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}
+test('broken HTTP caller fails five outcomes and preserves four',()=>{const r=grade('parent',item.base_ref);fail(r);expect(clean(r.stdout)).toContain('5 failed');expect(clean(r.stdout)).toContain('4 passed');});
+test('historical reference passes all nine HTTP outcomes',()=>pass(grade('reference',item.oracle_ref)));
+test('current source preserves the public search contract',()=>pass(grade('current','HEAD')));
+test('unused correct helper does not repair the broken HTTP caller',()=>fail(grade('unused',item.base_ref,pkg=>writeFileSync(join(pkg,'src/unused-search-result.ts'),execFileSync('git',['show',`${item.oracle_ref}:packages/screenpipe-mcp/src/search-result.ts`],{cwd:repo})))));
+test('equivalent truncation expression is accepted',()=>pass(grade('equivalent',item.oracle_ref,pkg=>edit(pkg,'search-result.ts','const left = Math.floor(cap / 2);','const left = Math.trunc(cap / 2);'))));
+test('wrong source identifiers cannot pass',()=>fail(grade('wrong-id',item.oracle_ref,pkg=>edit(pkg,'search-result.ts','? value : undefined','? value + 1 : undefined'))));
+test('unbounded default text cannot pass',()=>fail(grade('unbounded',item.oracle_ref,pkg=>edit(pkg,'search-result.ts','if (cap === 0 || text.length <= cap) return text;','if (true) return text;'))));
+test('blanket empty response cannot pass',()=>fail(grade('blanket',item.oracle_ref,pkg=>edit(pkg,'http-server.ts','const results = data.data || [];','const results = [];'))));
+test('missing HTTP module is setup failure rather than intended regression',()=>{const r=grade('missing',item.oracle_ref,pkg=>rmSync(join(pkg,'src/http-server.ts')));expect(r.status).not.toBe(0);expect(r.stderr).toMatch(/Cannot find module|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index c8b93576f..ca39f34f5 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-10-08-codex-embedded-images",
+ "dataset_version": "2026-10-09-mcp-search-evidence",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -5932,6 +5932,49 @@
],
"command": "cargo test --offline --manifest-path .eval-downmix/Cargo.toml --test outcomes -- --nocapture"
}
+ },
+ {
+ "id": "app-mcp-search-source-evidence",
+ "name": "Preserve source evidence through HTTP MCP search",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "fd797a4607f7def147171ac23a6c061cf4920d9f"
+ },
+ "base_ref": "0d082079f354e42ef8b51baafe4bc258b30cded2",
+ "oracle_ref": "fd797a4607f7def147171ac23a6c061cf4920d9f",
+ "oracle_paths": [
+ "packages/screenpipe-mcp/src/http-server.ts",
+ "packages/screenpipe-mcp/src/search-result.ts",
+ "packages/screenpipe-mcp/src/call-observation.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/screenpipe-mcp/src/**"
+ ],
+ "tags": [
+ "regression",
+ "mcp",
+ "evidence",
+ "pagination"
+ ],
+ "prompt": "HTTP MCP search_content loses the source references needed to inspect returned evidence and can discard input-event text. Repair the existing public tool. Screen and input results with valid backend frame IDs must retain frame_id=<id> and screenpipe://frame/<id>; audio results must retain chunk_id=<id> and transcription. Preserve zero and other nonnegative safe integer IDs, and never invent references for missing, negative or nonnumeric IDs. Keep source text in the response. Default long screen text must be bounded while preserving its beginning, end and source reference: for one 3008-character result, the complete response must stay below 1500 characters. An explicitly scoped max_content_length:0 read must preserve the full text. Preserve next-page offset hints when more results remain, but do not suggest another offset on a final nonempty page. Keep empty-result messages and upstream HTTP403 failures distinct. Exercise the actual HTTP MCP server with its SDK client and synthetic loopback backend. No private helper name, extraction or formatting outside these public references is required. This task does not establish stdio behavior, native telemetry, real customer retrieval, live authentication or client-disconnection cleanup. Do not call live services.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/screenpipe-mcp/node_modules",
+ "destination_path": "packages/screenpipe-mcp/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/mcp-search-evidence.fixture.ts.txt",
+ "destination_path": "packages/screenpipe-mcp/src/eval-evidence.test.ts"
+ }
+ ],
+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 node node_modules/vitest/vitest.mjs run src/eval-evidence.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/mcp-search-evidence.fixture.ts.txt b/evals/coding-agent/graders/mcp-search-evidence.fixture.ts.txt
new file mode 100644
index 000000000..81cc26f0e
--- /dev/null
+++ b/evals/coding-agent/graders/mcp-search-evidence.fixture.ts.txt
@@ -0,0 +1,28 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {beforeEach,afterEach,it,expect,vi} from 'vitest';
+import {createServer} from 'node:http';
+import {Client} from '@modelcontextprotocol/sdk/client/index.js';
+import {StreamableHTTPClientTransport} from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import {buildHttpServer} from './http-server';
+let api:any,server:any,client:Client,data:any[],pagination:any,status:number;
+const long='HEAD'+ 'x'.repeat(3000)+'TAIL';
+beforeEach(async()=>{
+ data=[];pagination={total:1,offset:0};status=200;
+ api=createServer((req,res)=>{res.setHeader('content-type','application/json');if(req.url?.startsWith('/search')){res.statusCode=status;res.end(JSON.stringify(status===200?{data,pagination}:{error:'synthetic unavailable'}));}else res.end('{}');});
+ await new Promise<void>(r=>api.listen(0,'127.0.0.1',r));const port=api.address().port;
+ vi.stubEnv('SCREENPIPE_API_URL',`http://127.0.0.1:${port}`);vi.stubEnv('SCREENPIPE_MCP_TELEMETRY_DISABLED','1');vi.stubEnv('DO_NOT_TRACK','1');
+ server=buildHttpServer({mcpPort:0,screenpipePort:port,host:'127.0.0.1'});await new Promise<void>(r=>server.listen(0,'127.0.0.1',r));
+ client=new Client({name:'synthetic-evidence-check',version:'1.0.0'});await client.connect(new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${server.address().port}/mcp`)));
+});
+afterEach(async()=>{await client?.close();server?.closeAllConnections();api?.closeAllConnections();await Promise.all([server,api].filter(Boolean).map(s=>new Promise<void>(r=>s.close(()=>r()))));vi.unstubAllEnvs();});
+async function search(args:any={}){const r=await client.callTool({name:'search_content',arguments:{q:'synthetic',...args}});expect(r.isError).not.toBe(true);return (r.content as any[]).map(c=>c.text??'').join('\n');}
+it('retains the backend frame reference in ordinary screen results',async()=>{for(const id of [0,72,901]){data=[{type:'OCR',content:{frame_id:id,text:'synthetic screen '+id,text_source:'accessibility'}}];const t=await search();expect(t).toContain('frame_id='+id);expect(t).toContain('screenpipe://frame/'+id);expect(t).toContain('synthetic screen '+id);}});
+it('bounds default long content while retaining both ends and source',async()=>{data=[{type:'OCR',content:{frame_id:72,text:long}}];const t=await search();expect(t).toContain('HEAD');expect(t).toContain('TAIL');expect(t.length).toBeLessThan(1500);expect(t).toContain('screenpipe://frame/72');});
+it('explicit scoped unlimited read preserves complete text',async()=>{data=[{type:'OCR',content:{text:long}}];expect(await search({max_content_length:0})).toContain(long);});
+it('retains audio chunk evidence',async()=>{for(const id of [0,31,77]){data=[{type:'Audio',content:{chunk_id:id,timestamp:'2026-10-01T00:00:00Z',transcription:'synthetic speech '+id}}];const t=await search();expect(t).toContain('chunk_id='+id);expect(t).toContain('synthetic speech '+id);}});
+it('input events remain visible with their source',async()=>{data=[{type:'Input',content:{frame_id:72,text_content:'synthetic input'}}];const t=await search();expect(t).toContain('synthetic input');expect(t).toContain('screenpipe://frame/72');});
+it('final nonempty page does not suggest another offset',async()=>{data=[{type:'OCR',content:{text:'last page'}}];pagination={total:6,offset:5};expect(await search({offset:5,limit:1})).not.toContain('offset=');});
+it('nonfinal page retains useful next offset',async()=>{data=[{type:'OCR',content:{text:'next page'}}];pagination={total:6,offset:2};expect(await search({offset:2,limit:1})).toContain('offset=3');});
+it('missing or invalid identifiers do not acquire invented source references',async()=>{data=[{type:'OCR',content:{frame_id:-1,text:'no frame'}},{type:'Audio',content:{chunk_id:'invalid',transcription:'no chunk'}}];const t=await search();expect(t).not.toContain('frame_id=');expect(t).not.toContain('chunk_id=');expect(t).not.toContain('screenpipe://frame/');});
+it('empty results and upstream refusal remain distinct',async()=>{const t=await search();expect(t).toContain('No results');status=403;await expect(search()).rejects.toThrow('403');});