Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): cover MCP HTTP session deletion and accounting

Louis Beaumont1bc4bd01855c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 2fd26676a..a61679a67 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -103,7 +103,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 146 git-mined regressions. See
+The current app corpus contains 147 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1840,3 +1840,18 @@ Run `bun test evals/coding-agent/calibrate-mcp-search-evidence.test.js` to check
## Fixed-term manual grants
`app-manual-grant-payment-prompts` executes the actual trial and contextual-card eligibility functions with synthetic accounts and a frozen clock. The parent fails four grant outcomes and preserves eighteen; the source-only reference and current source pass all twenty-two. Signup trials, ordinary payers, enterprise accounts, expired/invalid evidence and eligibility refusals remain covered. Run `bun test evals/coding-agent/calibrate-manual-grant.test.js` for correct, broken, equivalent, disconnected-caller, blanket-exemption and missing-source controls. Fixtures and dependencies appear only at grading. This does not establish rendered UI, native recording, live billing, identity-provider verification, enforced agent isolation or model performance.
+
+## HTTP MCP session lifecycle
+
+`app-mcp-session-lifecycle` exercises actual HTTP initialization, repeated tool
+listing, independent sessions, explicit deletion, later reinitialization and
+health accounting. Six outcomes preserve unknown-session refusal, route and
+preflight behavior. The parent fails four outcomes and preserves two; the fix
+and current server pass all six. The existing search-evidence case catches
+missing registration but accepts an omitted cleanup callback; this case checks
+that distinct deletion/accounting gap. Run
+`bun test evals/coding-agent/calibrate-mcp-session-lifecycle.test.js`.
+The hidden fixture and runtime link appear only at grading. Synthetic loopback
+responses establish logical invalidation and active-session accounting, not heap
+reclamation, disconnect cleanup, LAN authentication, concurrency, agent isolation
+or model performance.
diff --git a/evals/coding-agent/calibrate-mcp-session-lifecycle.test.js b/evals/coding-agent/calibrate-mcp-session-lifecycle.test.js
new file mode 100644
index 000000000..4771036cb
--- /dev/null
+++ b/evals/coding-agent/calibrate-mcp-session-lifecycle.test.js
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';
+import {resolve,join} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-mcp-session-lifecycle');
+const root=mkdtempSync(join(tmpdir(),'mcp-session-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',ref,'packages/screenpipe-mcp'],{cwd:repo,maxBuffer:24*1024*1024})});
+ const pkg=join(cwd,'packages/screenpipe-mcp');symlinkSync(join(repo,'packages/screenpipe-mcp/node_modules'),join(pkg,'node_modules'),'dir');
+ writeFileSync(join(pkg,'src/eval-session.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-session-lifecycle.fixture.ts.txt')));mutate(pkg);
+ const r=spawnSync(process.env.NODE_BIN||'node',['node_modules/vitest/vitest.mjs','run','src/eval-session.test.ts'],{cwd:pkg,encoding:'utf8',timeout:60_000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,CI:'true',TZ:'UTC',SCREENPIPE_DISABLE_TELEMETRY:'1',SCREENPIPE_MCP_TELEMETRY_DISABLED:'1',DO_NOT_TRACK:'1',SCREENPIPE_API_URL:'http://127.0.0.1:1'}});
+ if(process.env.MCP_SESSION_CALIBRATION_RESULTS){mkdirSync(process.env.MCP_SESSION_CALIBRATION_RESULTS,{recursive:true});writeFileSync(join(process.env.MCP_SESSION_CALIBRATION_RESULTS,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null,stdout:r.stdout,stderr:r.stderr},null,2)+'\n');}return r;
+}
+const clean=s=>s.replace(/\x1b\[[0-9;]*m/g,'');
+function pass(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status,r.stdout+r.stderr).toBe(0);expect(clean(r.stdout)).toContain('6 passed');}
+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stderr).not.toMatch(/Cannot find module|Failed to load url/);}
+function edit(pkg,from,to){const p=join(pkg,'src/http-server.ts'),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}
+const noOp=pkg=>{const p=join(pkg,'package.json'),v=JSON.parse(readFileSync(p));v.scripts.test='exit 0';writeFileSync(p,JSON.stringify(v));};
+test('parent fails four outcomes and preserves two',()=>{const r=grade('parent',item.base_ref);fail(r);expect(clean(r.stdout)).toContain('4 failed');expect(clean(r.stdout)).toContain('2 passed');});
+test('reference passes six public outcomes',()=>pass(grade('reference',item.oracle_ref)));
+test('current source passes six public outcomes',()=>pass(grade('current','HEAD')));
+test('unused correct server cannot repair the caller',()=>fail(grade('unused',item.base_ref,pkg=>writeFileSync(join(pkg,'src/unused-server.ts'),execFileSync('git',['show',`${item.oracle_ref}:packages/screenpipe-mcp/src/http-server.ts`],{cwd:repo})))));
+test('post-request registration is a valid alternative',()=>pass(grade('equivalent',item.oracle_ref,pkg=>{edit(pkg,' sessions.set(newSessionId, { server, transport });',' // registration occurs after request handling');edit(pkg,' await session.transport.handleRequest(req, res);',' await session.transport.handleRequest(req, res);\n if (req.method === "POST" && session.transport.sessionId) sessions.set(session.transport.sessionId, session);');})));
+test('registration without deletion accounting fails',()=>fail(grade('no-cleanup',item.oracle_ref,pkg=>edit(pkg,' sessions.delete(closedSessionId);',' // omitted'))));
+test('deleting every session on one close fails preserved clients',()=>fail(grade('clear-all',item.oracle_ref,pkg=>edit(pkg,' sessions.delete(closedSessionId);',' sessions.clear();'))));
+test('constant health zero fails real active accounting',()=>fail(grade('fake-health',item.oracle_ref,pkg=>edit(pkg,'sessions: sessions.size','sessions: 0'))));
+test('empty tool listings cannot pass',()=>fail(grade('empty-tools',item.oracle_ref,pkg=>edit(pkg,'async () => ({ tools: TOOLS })','async () => ({ tools: [] })'))));
+test('no-op package script does not hide broken behavior',()=>fail(grade('noop-broken',item.base_ref,noOp)));
+test('no-op package script does not reject correct behavior',()=>pass(grade('noop-correct',item.oracle_ref,noOp)));
+test('missing server is setup failure',()=>{const r=grade('missing',item.oracle_ref,pkg=>rmSync(join(pkg,'src/http-server.ts')));expect(r.status).not.toBe(0);expect(r.stderr).toMatch(/Cannot find module|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 1a2b7777b..c42b93ebd 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6091,6 +6091,47 @@
],
"command": "bun --no-env-file test packages/ai-gateway/eval-vertex-tool-content.test.ts"
}
+ },
+ {
+ "id": "app-mcp-session-lifecycle",
+ "name": "Preserve HTTP MCP sessions and explicit deletion",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "5e0b1bdbe72f1628bd9b0a82acb63c3d79aba136"
+ },
+ "base_ref": "5aa887630cb04307c4734366b18b9d8ddccca45a",
+ "oracle_ref": "5e0b1bdbe72f1628bd9b0a82acb63c3d79aba136",
+ "oracle_paths": [
+ "packages/screenpipe-mcp/src/http-server.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/screenpipe-mcp/src/http-server.ts"
+ ],
+ "tags": [
+ "regression",
+ "mcp",
+ "sessions",
+ "lifecycle"
+ ],
+ "prompt": "HTTP MCP clients initialize successfully but cannot reliably reuse or delete their sessions. Repair the public HTTP session lifecycle. Initialization must return a nonempty session ID; repeated tools/list requests using it must succeed and include search_content. Independently initialized clients need distinct IDs and must remain usable independently. The unauthenticated health endpoint must report status ok and the accurate number of active initialized sessions, beginning at zero. An explicit DELETE of an initialized session succeeds, removes it from the active count and makes that ID unusable without disturbing other sessions. Closing every session returns the count to zero; a later initialization gets a fresh usable identity. Unknown or deleted session IDs must receive an HTTP error instead of tools, without entering the active count. Preserve unauthenticated loopback access, HTTP 404 for unknown routes and HTTP 204 for OPTIONS. Exercise synthetic loopback requests only; no actual screen recordings, credentials, live services or particular helper/storage implementation is required. This task checks public responses, active-session accounting and logical invalidation, not heap reclamation, dropped TCP connections, LAN authentication, stdio or concurrency guarantees.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/screenpipe-mcp/node_modules",
+ "destination_path": "packages/screenpipe-mcp/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/mcp-session-lifecycle.fixture.ts.txt",
+ "destination_path": "packages/screenpipe-mcp/src/eval-session.test.ts"
+ }
+ ],
+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 node node_modules/vitest/vitest.mjs run src/eval-session.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/mcp-session-lifecycle.fixture.ts.txt b/evals/coding-agent/graders/mcp-session-lifecycle.fixture.ts.txt
new file mode 100644
index 000000000..366c7176d
--- /dev/null
+++ b/evals/coding-agent/graders/mcp-session-lifecycle.fixture.ts.txt
@@ -0,0 +1,19 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {beforeEach,afterEach,it,expect} from 'vitest';
+import {buildHttpServer} from './http-server';
+let server:any,url:string;
+const headers={'content-type':'application/json',accept:'application/json, text/event-stream','mcp-protocol-version':'2024-11-05'};
+beforeEach(async()=>{server=buildHttpServer({mcpPort:0,screenpipePort:1,host:'127.0.0.1'});await new Promise<void>(r=>server.listen(0,'127.0.0.1',r));url=`http://127.0.0.1:${server.address().port}`;});
+afterEach(async()=>{server?.closeAllConnections();await new Promise<void>(r=>server.close(()=>r()));});
+async function rpc(method:string,session?:string){return fetch(url+'/mcp',{method:'POST',headers:{...headers,...(session?{'mcp-session-id':session}:{})},body:JSON.stringify({jsonrpc:'2.0',id:1,method,params:method==='initialize'?{protocolVersion:'2024-11-05',capabilities:{},clientInfo:{name:'synthetic-session',version:'1.0.0'}}:{}}),signal:AbortSignal.timeout(3000)});}
+async function init(){const r=await rpc('initialize');expect(r.status).toBe(200);const id=r.headers.get('mcp-session-id');await r.text();expect(id).toBeTruthy();return id!;}
+async function list(id:string){const r=await rpc('tools/list',id);expect(r.status).toBe(200);const text=await r.text();const payload=r.headers.get('content-type')?.includes('text/event-stream')?JSON.parse(text.split('\n').filter(line=>line.startsWith('data:')).map(line=>line.slice(5).trim()).join('\n')):JSON.parse(text);expect(payload.result.tools.some((tool:any)=>tool.name==='search_content')).toBe(true);}
+async function count(n:number){const r=await fetch(url+'/health',{signal:AbortSignal.timeout(3000)});expect(r.status).toBe(200);expect(await r.json()).toEqual(expect.objectContaining({status:'ok',sessions:n}));}
+async function close(id:string){const r=await fetch(url+'/mcp',{method:'DELETE',headers:{...headers,'mcp-session-id':id},signal:AbortSignal.timeout(3000)});await r.text();expect(r.status).toBe(200);}
+it('initial health, unknown routes and preflight remain available',async()=>{await count(0);const r=await fetch(url+'/missing',{signal:AbortSignal.timeout(3000)});expect(r.status).toBe(404);await r.text();const o=await fetch(url+'/mcp',{method:'OPTIONS',signal:AbortSignal.timeout(3000)});expect(o.status).toBe(204);});
+it('unknown session cannot list tools or enter health count',async()=>{const r=await rpc('tools/list','synthetic-missing');expect(r.status).toBeGreaterThanOrEqual(400);await r.text();await count(0);});
+it('initialized session remains usable across requests',async()=>{const id=await init();await list(id);await list(id);await count(1);});
+it('independent sessions remain individually usable',async()=>{const a=await init(),b=await init();expect(a).not.toBe(b);await list(a);await list(b);await count(2);});
+it('deleting one session removes only that session',async()=>{const a=await init(),b=await init();await close(a);await count(1);await list(b);const old=await rpc('tools/list',a);expect(old.status).toBeGreaterThanOrEqual(400);await old.text();await count(1);});
+it('all closed sessions leave the active count and later initialization recovers',async()=>{const a=await init();await close(a);await count(0);const b=await init();expect(b).not.toBe(a);await list(b);await count(1);await close(b);await count(0);});