YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): confine local API credentials to configured authority
louis4fd2330fdf56
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex cfbedb5dd..64241df57 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1139,3 +1139,16 @@ establish native environment parsing, actual capture delivery, unknown preferencor failed IPC policy, execution isolation or model performance.`app-chat-timeline-offset` renders the actual MarkdownBlock and Markdown parser with synthetic native/store ports. Twelve outcomes cover raw and encoded offsets, captured instants, timestamp precedence over start_time, invalid timestamps and preserved frame/artifact/web links. Run `bun test evals/coding-agent/calibrate-chat-timeline-offset.test.js` for grader controls. This is not native event delivery, OS deeplink handling, frame retrieval, sub-millisecond precision, execution isolation or a model trial. Graders and dependency links are installed only when grading begins.++`app-local-api-authority` runs the actual localFetch wrapper, global fetch+interceptor and shared notification action executor with synthetic native config+and transport. Twenty-four outcomes cover deceptive URL authorities, configured+ports, document-relative URLs, notification refusal and preserved local auth,+request bodies, explicit auth, disabled auth and key-refresh retry. The parent+fails fourteen and preserves ten; reference/current pass all twenty-four.+Run `bun test evals/coding-agent/calibrate-local-api-authority.test.js` for eight+controls, including equivalent helper names, inactive correct code, blanket auth+suppression, missing notification guards and missing-source setup classification.+Fixtures/dependencies are installed only at grading. This does not establish+native transport, redirects/cookies, toast component behavior, server authorization,+agent isolation or model performance.diff --git a/evals/coding-agent/calibrate-local-api-authority.test.js b/evals/coding-agent/calibrate-local-api-authority.test.jsnew file mode 100644index 000000000..01745ddef--- /dev/null+++ b/evals/coding-agent/calibrate-local-api-authority.test.js@@ -0,0 +1,46 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-local-api-authority');+const apiPath = `${app}/lib/api.ts`, actionPath = `${app}/lib/notifications/actions.ts`;+const root = mkdtempSync(join(tmpdir(), 'local-api-authority-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('24 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails intended credential and dispatch outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('14 failed | 10 passed');},60000);+test('historical reference passes twenty-four outcomes',()=>passes(grade('reference')),60000);+test('current source passes twenty-four outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent private authority helper naming passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{for(const path of [apiPath,actionPath]){const f=join(cwd,path);writeFileSync(f,readFileSync(f,'utf8').replaceAll('isLocalApiUrl','isConfiguredLoopbackTarget'));}})),60000);+test('inactive correct modules cannot hide broken active code',()=>fails(grade('unused',item.oracle_ref,cwd=>{for(const path of [apiPath,actionPath]){writeFileSync(join(cwd,path+'.unused.ts'),readFileSync(join(cwd,path)));writeFileSync(join(cwd,path),execFileSync('git',['show',`${item.base_ref}:${path}`],{cwd:repo}));}})),60000);+test('blanket key suppression fails local preservation',()=>fails(grade('no-auth',item.oracle_ref,cwd=>{const f=join(cwd,apiPath);writeFileSync(f,readFileSync(f,'utf8').replace('_authEnabled = config.auth_enabled;', '_authEnabled = false;'));})),60000);+test('safe fetch alone does not permit external notification dispatch',()=>fails(grade('no-action-guard',item.oracle_ref,cwd=>replace(cwd,actionPath,'if (!isLocalApiUrl(action.url)) {','if (false) {'))),60000);+test('missing API source is setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,apiPath)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 2ef92e299..2bb03384d 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3978,6 +3978,50 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/chat/eval-timeline-offset.test.tsx"}+ },+ {+ "id": "app-local-api-authority",+ "name": "Keep local API credentials within the configured authority",+ "source": {+ "kind": "git_regression",+ "fix_commit": "107d957aca3523cd309f56d93bd5ab20ce6519bf"+ },+ "base_ref": "ebeef3e7249ebcc2d1a9d1824c3b2fb68a1cfacb",+ "oracle_ref": "107d957aca3523cd309f56d93bd5ab20ce6519bf",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/api.ts",+ "apps/screenpipe-app-tauri/lib/notifications/actions.ts"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/api.ts",+ "apps/screenpipe-app-tauri/lib/notifications/actions.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "privacy",+ "credentials",+ "notifications"+ ],+ "prompt": "The local API fetch wrapper and global fetch interceptor can attach the local bearer key to an external URL, and the shared notification API action executor can dispatch to an external target. Repair these actual request paths. Only HTTP localhost or 127.0.0.1 at the configured API port may receive automatically supplied local authentication; do not mistake URL credentials, host suffixes or path/query text for authority. localFetch must continue passing absolute non-local URLs through without adding its key. Resolve localFetch relative paths against the configured local API, and global fetch relative paths against document.baseURI. Preserve authenticated local requests, disabled authentication, explicit caller Authorization, and a local401 key-refresh retry. Shared notification API actions must refuse non-local targets (including protocol-relative URLs) with a rejected operation and zero transport calls, while preserving local POST method/body. Check non-default configured ports and wrong-protocol/wrong-port targets. Use existing public entrypoints and contracts; no particular helper or parser is required. This task covers the actual JavaScript wrapper/interceptor/shared action executor with synthetic native configuration and transport, not the separate toast component, browser redirects/cookies, native network delivery or server authorization.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/local-api-authority.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/eval-local-api-authority.test.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts lib/eval-local-api-authority.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/local-api-authority.fixture.ts.txt b/evals/coding-agent/graders/local-api-authority.fixture.ts.txtnew file mode 100644index 000000000..002075e6d--- /dev/null+++ b/evals/coding-agent/graders/local-api-authority.fixture.ts.txt@@ -0,0 +1,84 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, beforeEach, expect, it, vi } from 'vitest';+const ports = vi.hoisted(() => ({ config: vi.fn() }));+vi.mock('@/lib/utils/tauri', () => ({ commands: { getLocalApiConfig: ports.config } }));+vi.mock('@tauri-apps/api/event', () => ({ emit: vi.fn(async () => {}) }));+vi.mock('@/lib/chat-utils', () => ({ showChatWithPrefill: vi.fn(async () => {}) }));+let api: typeof import('./api');+let actions: typeof import('./notifications/actions');+let transport: ReturnType<typeof vi.fn>;+let originalFetch: typeof window.fetch;+const key = 'synthetic-local-key';+function headers(index = 0) { return new Headers(transport.mock.calls[index]?.[1]?.headers); }+beforeEach(async () => {+ vi.resetModules();+ ports.config.mockReset();+ ports.config.mockResolvedValue({ key, port: 4317, auth_enabled: true });+ originalFetch = window.fetch;+ transport = vi.fn(async () => new Response(null, { status: 200 }));+ window.fetch = transport as typeof window.fetch;+ api = await import('./api');+ await api.ensureApiReady();+ actions = await import('./notifications/actions');+});+afterEach(() => { window.fetch = originalFetch; document.querySelectorAll('base[data-eval]').forEach(el => el.remove()); });+it.each([+ 'https://external.invalid/collect',+ 'http://localhost:[email protected]/collect',+ 'http://external.invalid/localhost:4317',+ 'http://localhost:4317.external.invalid/collect',+ 'http://localhost:4318/health',+ 'https://localhost:4317/health',+])('localFetch never supplies its key to %s', async url => {+ await api.localFetch(url);+ expect(transport).toHaveBeenCalledTimes(1);+ expect(transport.mock.calls[0][0]).toBe(url);+ expect(headers().has('Authorization')).toBe(false);+});+it.each(['/health', 'health', 'http://localhost:4317/health', 'http://127.0.0.1:4317/health'])('configured local target retains authentication: %s', async url => {+ await api.localFetch(url);+ expect(transport).toHaveBeenCalledTimes(1);+ expect(headers().get('Authorization')).toBe(`Bearer ${key}`);+ expect(transport.mock.calls[0][0]).toBe(url.startsWith('http') ? url : 'http://localhost:4317/health');+});+it.each(['https://external.invalid/?next=localhost:4317', 'http://localhost:[email protected]/collect', '/app-relative', 'https://external.invalid/plain'])('global fetch uses actual authority for %s', async url => {+ await window.fetch(url);+ expect(transport).toHaveBeenCalledTimes(1);+ expect(headers().has('Authorization')).toBe(false);+});+it('global fetch honors a local document base', async () => {+ const base = document.createElement('base'); base.dataset.eval = 'true'; base.href = 'http://localhost:4317/'; document.head.append(base);+ await window.fetch('/health');+ expect(headers().get('Authorization')).toBe(`Bearer ${key}`);+});+it.each(['https://external.invalid/collect', 'http://localhost:[email protected]/collect', 'http://localhost:4318/health', 'https://localhost:4317/health', '//external.invalid/collect'])('notification API executor refuses non-local target without transport: %s', async url => {+ let rejected = false;+ try { await actions.executeNotificationAction({ type: 'api', url }); } catch { rejected = true; }+ expect(rejected).toBe(true);+ expect(transport).not.toHaveBeenCalled();+});+it('notification local action preserves request and body', async () => {+ await actions.executeNotificationAction({ type: 'api', url: '/capture/hd/start', method: 'POST', body: { fps: 1 } });+ expect(transport).toHaveBeenCalledTimes(1);+ expect(transport.mock.calls[0][0]).toBe('http://localhost:4317/capture/hd/start');+ expect(transport.mock.calls[0][1].method).toBe('POST');+ expect(JSON.parse(transport.mock.calls[0][1].body)).toEqual({ fps: 1 });+ expect(headers().get('Authorization')).toBe(`Bearer ${key}`);+});+it('disabled authentication does not inject the key', async () => {+ api.configureApi({ authEnabled: false }); await api.localFetch('/health');+ expect(headers().has('Authorization')).toBe(false);+});+it('explicit caller authorization is preserved', async () => {+ await api.localFetch('/health', { headers: { Authorization: 'Bearer synthetic-explicit' } });+ expect(headers().get('Authorization')).toBe('Bearer synthetic-explicit');+});+it('local unauthorized response refreshes the key for one retry', async () => {+ transport.mockResolvedValueOnce(new Response(null, { status: 401 }));+ ports.config.mockResolvedValue({ key: 'synthetic-replacement', port: 4317, auth_enabled: true });+ await api.localFetch('/health');+ expect(transport).toHaveBeenCalledTimes(2);+ expect(headers(0).get('Authorization')).toBe(`Bearer ${key}`);+ expect(headers(1).get('Authorization')).toBe('Bearer synthetic-replacement');+});