YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover HTTP MCP browser authority boundaries
Louis Beaumontf505ad1a67c8
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 68e8d322f..7f7a58a4b 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -2002,3 +2002,18 @@ unresponsive startup stop collection as setup errors. Calibrate withThe Vertex upstream JSON case executes public JSON and streaming completion methods with synthetic OAuth and completion transport. Empty or malformed successful bodies must produce status 502 while valid content, reasoning fallback, client refusal and streaming remain intact. It does not prove gateway retry behavior, live provider reliability or agent isolation.`app-pipe-watch-switch-preservation` mounts the real conversation-switch hook and session store. It distinguishes equal-count stale pipe-watch text from normal panel ownership, preserves longer stored transcripts, and checks incoming chat identity. This is separate from saved terminal pipe transcripts and overlapping-load races. Run `bun test evals/coding-agent/calibrate-pipe-watch-switch.test.js` for calibration. Native delivery and disk durability are outside its scope.++### HTTP MCP browser authority++`app-mcp-browser-boundary` exercises the actual HTTP server and MCP transport+with synthetic loopback requests. Foreign Host/Origin inputs must not initialize,+list tools through, or delete a session without matching bearer credentials.+Literal local clients, explicit bearer authorization, session accounting, health,+preflight and route refusal remain usable. The parent fails five security+outcomes and preserves four; the source-only historical reference passes nine.+Run `bun test evals/coding-agent/calibrate-mcp-browser-boundary.test.js` for+broken/correct, alternative implementation, partial-fix, disconnected-caller,+blanket-denial, no-op-script and missing-source controls. Hidden fixtures and+third-party dependencies appear only when grading starts. This is synthetic+HTTP evidence, not a real browser/DNS exploit test, remote network validation,+all header-parser ambiguities, model performance or enforced host isolation.diff --git a/evals/coding-agent/calibrate-mcp-browser-boundary.test.js b/evals/coding-agent/calibrate-mcp-browser-boundary.test.jsnew file mode 100644index 000000000..077a13aae--- /dev/null+++ b/evals/coding-agent/calibrate-mcp-browser-boundary.test.js@@ -0,0 +1,37 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';+import {resolve,join} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-mcp-browser-boundary');+const root=mkdtempSync(join(tmpdir(),'mcp-browser-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',ref,'packages/screenpipe-mcp'],{cwd:repo,maxBuffer:24*1024*1024})});+ const pkg=join(cwd,'packages/screenpipe-mcp');symlinkSync(join(repo,'packages/screenpipe-mcp/node_modules'),join(pkg,'node_modules'),'dir');+ writeFileSync(join(pkg,'src/eval-browser-boundary.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-browser-boundary.fixture.ts.txt')));mutate(pkg);+ const r=spawnSync(process.env.NODE_BIN||'node',['node_modules/vitest/vitest.mjs','run','src/eval-browser-boundary.test.ts'],{cwd:pkg,encoding:'utf8',timeout:60_000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,CI:'true',TZ:'UTC',SCREENPIPE_DISABLE_TELEMETRY:'1',SCREENPIPE_MCP_TELEMETRY_DISABLED:'1',DO_NOT_TRACK:'1',SCREENPIPE_API_URL:'http://127.0.0.1:1'}});+ if(process.env.MCP_BROWSER_CALIBRATION_RESULTS){mkdirSync(process.env.MCP_BROWSER_CALIBRATION_RESULTS,{recursive:true});writeFileSync(join(process.env.MCP_BROWSER_CALIBRATION_RESULTS,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null,stdout:r.stdout,stderr:r.stderr},null,2)+'\n');}return r;+}+const clean=s=>s.replace(/\x1b\[[0-9;]*m/g,'');+function pass(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status,r.stdout+r.stderr).toBe(0);expect(clean(r.stdout)).toContain('9 passed');}+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stderr).not.toMatch(/Cannot find module|Failed to load url/);}+function edit(pkg,from,to){const p=join(pkg,'src/http-server.ts'),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+const noOp=pkg=>{const p=join(pkg,'package.json'),v=JSON.parse(readFileSync(p));v.scripts.test='exit 0';writeFileSync(p,JSON.stringify(v));};+test('parent fails five security outcomes and preserves four',()=>{const r=grade('parent',item.base_ref);fail(r);expect(clean(r.stdout)).toContain('5 failed');expect(clean(r.stdout)).toContain('4 passed');});+test('reference preserves all nine outcomes',()=>pass(grade('reference',item.oracle_ref)));+test('current source preserves all nine outcomes',()=>pass(grade('current','HEAD')));+test('unused correct source cannot fix caller',()=>fail(grade('unused',item.base_ref,pkg=>writeFileSync(join(pkg,'src/unused.ts'),execFileSync('git',['show',`${item.oracle_ref}:packages/screenpipe-mcp/src/http-server.ts`],{cwd:repo})))));+test('an authority allowlist is an equivalent implementation',()=>pass(grade('equivalent',item.oracle_ref,pkg=>{const p=join(pkg,'src/http-server.ts');const s=readFileSync(p,'utf8');const line=s.split('\n').find(x=>x.includes('return /^(?:localhost|'));expect(line).toBeDefined();writeFileSync(p,s.replace(line," return new Set(['localhost', '127.0.0.1', '[::1]']).has(host.toLowerCase().replace(/:[0-9]{1,5}$/, ''));").replaceAll('isTrustedLocalHost','safeAuthority').replaceAll('isTrustedLocalOrigin','safeOrigin'));})));+test('host-only gate is insufficient',()=>fail(grade('host-only',item.oracle_ref,pkg=>edit(pkg,'(req.headers.origin === undefined || isTrustedLocalOrigin(req.headers.origin))','true'))));+test('origin-only gate is insufficient',()=>fail(grade('origin-only',item.oracle_ref,pkg=>edit(pkg,'isTrustedLocalHost(req.headers.host)','true'))));+test('blanket refusal fails preserved clients',()=>fail(grade('refuse-all',item.oracle_ref,pkg=>edit(pkg,'if (!isAuthorized(req, config.apiKey))','if (true)'))));+test('auth bypass at caller fails despite correct helper',()=>fail(grade('caller-bypass',item.oracle_ref,pkg=>edit(pkg,'if (!isAuthorized(req, config.apiKey))','if (false)'))));+test('removing bearer fallback rejects authorized clients',()=>fail(grade('no-bearer',item.oracle_ref,pkg=>edit(pkg,'return constantTimeEq(got, expected);','return false;'))));+test('no-op package script cannot hide broken behavior',()=>fail(grade('noop-broken',item.base_ref,noOp)));+test('no-op package script preserves correct behavior',()=>pass(grade('noop-correct',item.oracle_ref,noOp)));+test('missing source is a setup error',()=>{const r=grade('missing',item.oracle_ref,pkg=>rmSync(join(pkg,'src/http-server.ts')));expect(r.status).not.toBe(0);expect(r.stderr).toMatch(/Cannot find module|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');});+test('unkeyed deployments cannot bypass the boundary',()=>fail(grade('unkeyed-bypass',item.oracle_ref,pkg=>edit(pkg,'if (!apiKey) return false;','if (!apiKey) return true;'))));diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 4c2a81eb5..4f5fec701 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6697,6 +6697,48 @@],"command": "node .eval-audio-drop/audio-drop.mjs && cargo test --locked -p screenpipe-audio --lib eval_audio_drop -- --test-threads=1"}+ },+ {+ "id": "app-mcp-browser-boundary",+ "name": "Reject foreign browser authority at the HTTP MCP boundary",+ "source": {+ "kind": "git_regression",+ "fix_commit": "52297f8edee9c85ada55581f6ae7287b0643df82"+ },+ "base_ref": "6bb753f85c6a01cc3a2fc26890ce68cef5cbad85",+ "oracle_ref": "52297f8edee9c85ada55581f6ae7287b0643df82",+ "oracle_paths": [+ "packages/screenpipe-mcp/src/http-server.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/screenpipe-mcp/src/http-server.ts"+ ],+ "tags": [+ "regression",+ "security",+ "mcp",+ "privacy",+ "preserved-behavior"+ ],+ "agent_timeout_seconds": 900,+ "prompt": "The HTTP MCP listener trusts every loopback TCP peer, allowing browser requests with attacker-controlled Host or Origin headers to initialize or use sessions. Repair the actual HTTP boundary. Without a matching configured bearer token, reject foreign authorities, misleading localhost/IP suffixes, userinfo-like authorities, numeric IP aliases, foreign origins and opaque null origins with HTTP 401 and no new session. Apply that protection to initialization, existing-session tools/list and DELETE, including /mcp query URLs; rejected deletion must leave the legitimate session usable. Preserve native clients using literal 127.0.0.1, localhost (case insensitive) or bracketed ::1 authorities with optional ports, and local HTTP/HTTPS origins using those literals. Explicit matching bearer credentials must still authorize otherwise untrusted authorities/origins; wrong or malformed bearer values must not. Successful initialization must return a real usable session and advertise tools, including search_content on listing. Preserve health status/session accounting, unauthenticated /health and OPTIONS, and 404 for unknown authorized routes. Use synthetic loopback traffic with telemetry disabled and no real tool/backend calls. No particular private helper or implementation path is required. This task does not test a real browser/DNS resolver, remote network reachability, all header-parser ambiguities, native recording, telemetry delivery or registry packaging.",+ "grader_dependency_links": [+ {+ "source_path": "packages/screenpipe-mcp/node_modules",+ "destination_path": "packages/screenpipe-mcp/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/mcp-browser-boundary.fixture.ts.txt",+ "destination_path": "packages/screenpipe-mcp/src/eval-browser-boundary.test.ts"+ }+ ],+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 SCREENPIPE_API_URL=http://127.0.0.1:1 node node_modules/vitest/vitest.mjs run src/eval-browser-boundary.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/mcp-browser-boundary.fixture.ts.txt b/evals/coding-agent/graders/mcp-browser-boundary.fixture.ts.txtnew file mode 100644index 000000000..22f87bc05--- /dev/null+++ b/evals/coding-agent/graders/mcp-browser-boundary.fixture.ts.txt@@ -0,0 +1,100 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, it, expect } from 'vitest';+import { request } from 'node:http';+import { buildHttpServer } from './http-server';+let server: any, port: number;+const token = 'synthetic-eval-token';+const originalFetch = globalThis.fetch;+beforeEach(async () => {+ globalThis.fetch = (() => { throw new Error('Unexpected upstream access'); }) as typeof fetch;+ server = buildHttpServer({ host: '127.0.0.1', mcpPort: 0, screenpipePort: 1, apiKey: token });+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));+ port = server.address().port;+});+afterEach(async () => {+ server?.closeAllConnections();+ await new Promise<void>(resolve => server.close(() => resolve()));+ globalThis.fetch = originalFetch;+});+function call(headers: Record<string, string> = {}, method = 'POST', path = '/mcp', rpc = 'initialize') {+ return new Promise<{ status: number; session?: string; body: string }>((resolve, reject) => {+ const req = request({ hostname: '127.0.0.1', port, path, method, headers: {+ host: `127.0.0.1:${port}`, 'content-type': 'application/json',+ accept: 'application/json, text/event-stream', ...headers,+ } }, res => {+ let body = ''; res.on('data', chunk => body += chunk);+ res.on('end', () => resolve({ status: res.statusCode!, session: res.headers['mcp-session-id'] as string | undefined, body }));+ });+ req.setTimeout(3000, () => req.destroy(new Error('Synthetic HTTP timeout')));+ req.on('error', reject);+ req.end(method === 'POST' ? JSON.stringify({ jsonrpc: '2.0', id: 1, method: rpc,+ params: rpc === 'initialize' ? { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'synthetic-boundary', version: '1' } } : {} }) : undefined);+ });+}+function payload(body: string) {+ return JSON.parse(body.startsWith('event:') || body.startsWith('data:') ? body.split('\n').filter(x => x.startsWith('data:')).map(x => x.slice(5).trim()).join('\n') : body);+}+async function count(expected: number) {+ const r = await call({}, 'GET', '/health'); expect(r.status).toBe(200);+ expect(JSON.parse(r.body)).toEqual(expect.objectContaining({ status: 'ok', sessions: expected }));+}+async function init(headers: Record<string, string> = {}) {+ const r = await call(headers); expect(r.status).toBe(200); expect(r.session).toBeTruthy();+ expect(payload(r.body).result.capabilities.tools).toBeDefined(); return r.session!;+}+async function denied(headers: Record<string, string>, method = 'POST', path = '/mcp', rpc = 'initialize') {+ const r = await call(headers, method, path, rpc); expect(r.status).toBe(401); expect(r.session).toBeUndefined();+}+it('refuses attacker authorities even when the TCP peer is loopback', async () => {+ for (const host of [`rebind.invalid:${port}`, `localhost.evil.invalid:${port}`, `127.0.0.1.evil.invalid:${port}`, `2130706433:${port}`, `127.1:${port}`, `[email protected]:${port}`]) {+ await denied({ host }); await count(0);+ }+});+it('refuses foreign, opaque and misleading origins before creating sessions', async () => {+ for (const origin of ['https://evil.invalid', 'null', 'http://[email protected]', 'http://localhost.evil.invalid', 'http://127.0.0.1.evil.invalid']) {+ await denied({ origin }); await count(0);+ }+});+it('checks reused sessions on tool listing and deletion, including query URLs', async () => {+ const id = await init();+ for (const headers of [{ host: 'rebind.invalid' }, { origin: 'https://evil.invalid' }]) {+ await denied({ ...headers, 'mcp-session-id': id }, 'POST', '/mcp?probe=1', 'tools/list');+ await denied({ ...headers, 'mcp-session-id': id }, 'DELETE'); await count(1);+ }+ const good = await call({ 'mcp-session-id': id }, 'POST', '/mcp', 'tools/list');+ expect(good.status).toBe(200); expect(payload(good.body).result.tools.some((t: any) => t.name === 'search_content')).toBe(true);+});+it('wrong bearer credentials cannot rescue an untrusted browser', async () => {+ for (const authorization of ['Bearer wrong', 'Basic synthetic', `Bearer ${token}x`]) {+ await denied({ origin: 'https://evil.invalid', authorization }); await count(0);+ }+});+it('preserves native clients using literal loopback authorities', async () => {+ for (const host of [`127.0.0.1:${port}`, `localhost:${port}`, `[::1]:${port}`, 'LOCALHOST']) await init({ host });+ await count(4);+});+it('preserves trusted local browser origins', async () => {+ for (const origin of [`http://localhost:${port}`, `https://127.0.0.1:${port}`, `http://[::1]:${port}`]) await init({ origin });+ await count(3);+});+it('explicit matching bearer credentials authorize otherwise untrusted clients', async () => {+ const id = await init({ host: 'remote.invalid', origin: 'https://remote.invalid', authorization: `Bearer ${token}` });+ const r = await call({ host: 'remote.invalid', origin: 'https://remote.invalid', authorization: `Bearer ${token}`, 'mcp-session-id': id }, 'POST', '/mcp', 'tools/list');+ expect(r.status).toBe(200); expect(payload(r.body).result.tools.length).toBeGreaterThan(0); await count(1);+});+it('preserves unauthenticated health and preflight and authorized route refusal', async () => {+ const r = await call({ host: 'untrusted.invalid', origin: 'https://evil.invalid' }, 'GET', '/health');+ expect(r.status).toBe(200); expect(JSON.parse(r.body)).toEqual({ status: 'ok', sessions: 0 });+ expect((await call({ origin: 'https://evil.invalid' }, 'OPTIONS')).status).toBe(204);+ expect((await call({}, 'GET', '/not-a-route')).status).toBe(404); await count(0);+});++it('protects the default deployment without an API key while retaining native access', async () => {+ server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve()));+ server = buildHttpServer({ host: '127.0.0.1', mcpPort: 0, screenpipePort: 1 });+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); port = server.address().port;+ await denied({ host: 'rebind.invalid' }); await denied({ origin: 'https://evil.invalid' });+ await denied({ origin: 'https://evil.invalid', authorization: `Bearer ${token}` });+ await count(0); await init(); await count(1);+});