YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): grade connection probe response compatibility
louis0ea711d853af
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex ad46454e2..b39edf76b 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 75 git-mined regressions. See+The current app corpus contains 76 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -830,3 +830,21 @@ blanket suppression and missing-source setup failure. Setdependency links are materialized only for grading. These are corpus checks,not model trials, full chat UI or native persistence evidence, or proof ofexecution isolation. No real files, recordings or provider accounts are used.++## Connection-probe response compatibility++`app-connection-probe-response` exercises the public connection-test service with+synthetic provider responses and real request/error helpers. Sixteen outcomes+cover streaming-default custom, OpenAI, Ollama and Anthropic gateways, token+parameter recovery, configured connection identity, JSON compatibility, empty+messages, error details, reply length and cancellation. The historical parent+fails five response assertions and preserves eleven outcomes; the source-only+reference and current service pass all sixteen.++Run `bun test evals/coding-agent/calibrate-connection-probe.test.js` with desktop+Vitest dependencies installed. Ten controls reject unused fixes, retry/provider+regressions, constant replies and lost credentials. A valid SSE parser passes+without requiring a particular request flag. Missing source remains a setup+error. Hidden fixtures and dependencies are materialized only for grading.+These are corpus checks, not model trials, settings persistence, native/live+provider acceptance or proof of execution isolation.diff --git a/evals/coding-agent/calibrate-connection-probe.test.js b/evals/coding-agent/calibrate-connection-probe.test.jsnew file mode 100644index 000000000..965508e67--- /dev/null+++ b/evals/coding-agent/calibrate-connection-probe.test.js@@ -0,0 +1,68 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-connection-probe-response');+const hook = `${app}/lib/utils/ai-preset-connection.ts`;+const body = `${app}/lib/utils/chat-test-body.ts`;+const root = mkdtempSync(join(tmpdir(), 'connection-probe-calibration-')), archives = new Map();+const paths = [`${app}/lib`];+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, { recursive: true, force: true }));+function change(cwd, from, to) {+ const path = join(cwd, hook), source = readFileSync(path, 'utf8');+ expect(source.split(from)).toHaveLength(2);+ writeFileSync(path, source.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['archive', ref, ...paths], { cwd: repo, maxBuffer: 8 * 1024 * 1024 }));+ execFileSync('tar', ['-x', '-C', cwd], { input: archives.get(ref) });+ for (const f of item.grader.fixtures) writeFileSync(join(cwd, f.destination_path), readFileSync(join(import.meta.dir, f.local_path)));+ for (const link of item.grader_dependency_links) symlinkSync(join(repo, link.source_path), join(cwd, link.destination_path), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024,+ env: { PATH: process.env.PATH, HOME: cwd, CI: 'true', TZ: 'UTC', NO_COLOR: '1' } });+ if (receipts) {+ mkdirSync(receipts, { recursive: true });+ writeFileSync(join(receipts, `${name}.stdout`), r.stdout || '');+ writeFileSync(join(receipts, `${name}.stderr`), r.stderr || '');+ writeFileSync(join(receipts, `${name}.json`), JSON.stringify({ ref, command: item.grader.command, exit_code: r.status, signal: r.signal, error: r.error?.message }, null, 2));+ }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull();+ return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('16 passed'); expect(r.stdout + r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(/AssertionError|wrong provider identity/); expect(r.stdout + r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails five response outcomes and preserves eleven', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('5 failed | 11 passed'); }, 60000);+test('reference passes all response outcomes', () => passes(grade('reference')), 60000);+test('current source passes all response outcomes', () => passes(grade('current', '4c638a9cf2a028133a67231393c9b40baf4211bc')), 60000);+test('unused correct source cannot hide the broken public service', () => fails(grade('unused', item.oracle_ref, cwd => {+ writeFileSync(join(cwd, `${app}/lib/utils/unused-correct.ts`), readFileSync(join(cwd, hook)));+ for (const path of [hook, body]) writeFileSync(join(cwd, path), execFileSync('git', ['show', `${item.base_ref}:${path}`], { cwd: repo }));+})), 60000);+test('an alternative SSE parser passes without requesting non-streaming', () => passes(grade('sse-alternative', item.base_ref, cwd => {+ change(cwd, 'const data = await response.json();', String.raw`const wire = await response.text();+ let data;+ if (response.headers.get('content-type')?.includes('text/event-stream')) {+ let text = '';+ for (const line of wire.split('\n')) {+ if (!line.startsWith('data: ') || line.slice(6).trim() === '[DONE]') continue;+ const event = JSON.parse(line.slice(6));+ text += isAnthropic ? (event.delta?.text || '') : (event.choices?.[0]?.delta?.content || '');+ }+ data = isAnthropic ? {content: [{text}]} : {choices: [{message: {content: text}}]};+ } else data = JSON.parse(wire);`);+})), 60000);+test('losing response compatibility on token retry fails', () => fails(grade('retry-loss', item.oracle_ref, cwd => change(cwd, ' response = await request(endpoint, {', ' (body as any).stream = true; response = await request(endpoint, {'))), 60000);+test('losing Anthropic response compatibility fails', () => fails(grade('anthropic-loss', item.oracle_ref, cwd => change(cwd, 'stream: false,', 'stream: true,'))), 60000);+test('constant successful replies fail', () => fails(grade('constant', item.oracle_ref, cwd => change(cwd, ' reply,', ' reply: "hi",'))), 60000);+test('dropping provider credentials fails', () => fails(grade('identity-loss', item.oracle_ref, cwd => change(cwd, 'return headers;', 'return {"Content-Type": "application/json"};'))), 60000);+test('missing public service stays a setup error', () => {+ const r = grade('missing', item.oracle_ref, cwd => rmSync(join(cwd, hook)));+ expect(r.status).toBe(1); expect(r.stdout).toContain('no tests'); expect(r.stdout + r.stderr).toMatch(/Failed to resolve import|Failed to load url/);+}, 60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 7dd9a9f8d..d098e636f 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3091,6 +3091,56 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/voice-transcription-budget.test.js"}+ },+ {+ "id": "app-connection-probe-response",+ "name": "Return valid connection-probe replies from streaming-default gateways",+ "source": {+ "kind": "git_regression",+ "fix_commit": "9a4950abb4bbcddec07942609297b8d57de5e437"+ },+ "base_ref": "22d2053908e55e86d7b84365e3ba508896e63060",+ "oracle_ref": "9a4950abb4bbcddec07942609297b8d57de5e437",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/utils/ai-preset-connection.ts",+ "apps/screenpipe-app-tauri/lib/utils/chat-test-body.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "connection-test",+ "provider-compatibility",+ "recovery"+ ],+ "prompt": "The public testAiPresetConnection service reports a connection failure for working OpenAI-compatible gateways that default to server-sent events. Restore the actual provider reply for custom, OpenAI, native Ollama and Anthropic connections, including gateways that require max_completion_tokens instead of max_tokens. Preserve configured model, endpoint and provider credentials, a nonnegative finite latency, ordinary JSON replies, existing empty-text message compatibility and the 100-character displayed reply limit. Missing messages, provider authorization errors, array-shaped provider errors and transport failures must remain failures with their meaningful details; caller cancellation must reach the provider request. Any implementation that achieves these outcomes is acceptable, including successful SSE handling. Preserve the public service interface. Use synthetic provider ports only. This covers the connection service and its internal request/error helpers, not settings UI persistence, native networking or live provider acceptance.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/connection-probe.fixture.ts",+ "destination_path": "apps/screenpipe-app-tauri/eval-connection-probe.test.ts"+ },+ {+ "local_path": "graders/connection-probe.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-connection-probe.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-connection-probe.config.mjs"+ },+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/utils/ai-preset-connection.ts",+ "apps/screenpipe-app-tauri/lib/utils/chat-test-body.ts",+ "apps/screenpipe-app-tauri/lib/utils/ai-endpoint-url.ts",+ "apps/screenpipe-app-tauri/lib/utils/validation.ts"+ ]}]}diff --git a/evals/coding-agent/graders/connection-probe.config.mjs b/evals/coding-agent/graders/connection-probe.config.mjsnew file mode 100644index 000000000..4ed0248ce--- /dev/null+++ b/evals/coding-agent/graders/connection-probe.config.mjs@@ -0,0 +1,11 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({+ esbuild: { jsx: 'automatic' },+ test: { include: ['eval-connection-probe.test.ts'], environment: 'node',+ pool: 'forks', poolOptions: { forks: { singleFork: true } } },+ resolve: { alias: { '@': path.resolve('.') } },+ css: { postcss: { plugins: [] } },+});diff --git a/evals/coding-agent/graders/connection-probe.fixture.ts b/evals/coding-agent/graders/connection-probe.fixture.tsnew file mode 100644index 000000000..1d68b5f75--- /dev/null+++ b/evals/coding-agent/graders/connection-probe.fixture.ts@@ -0,0 +1,84 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, expect, test, vi } from 'vitest';+import { testAiPresetConnection } from './lib/utils/ai-preset-connection';+vi.mock('@/lib/http/tauri-fetch', () => ({ tauriFetchWithDeadline: () => { throw new Error('unexpected native network'); } }));+afterEach(() => vi.unstubAllGlobals());+const presets = [+ { provider: 'custom', url: 'https://gateway.example.test/v1', model: 'custom-model', apiKey: 'synthetic-key' },+ { provider: 'openai', model: 'openai-model', apiKey: 'synthetic-key' },+ { provider: 'native-ollama', url: 'http://localhost:11434/v1', model: 'local-model' },+ { provider: 'anthropic', model: 'anthropic-model', apiKey: 'synthetic-key' },+] as const;+function json(data: unknown, status = 200) { return new Response(JSON.stringify(data), { status, headers: { 'content-type': 'application/json' } }); }+function reply(anthropic: boolean, text: string) { return anthropic ? { content: [{ type: 'text', text }] } : { choices: [{ message: { role: 'assistant', content: text } }] }; }+function gateway(preset: typeof presets[number], text: string, streamingDefault = true) {+ return vi.fn(async (url: string, init?: RequestInit) => {+ const body = JSON.parse(String(init?.body)), headers = new Headers(init?.headers);+ const anthropic = preset.provider === 'anthropic';+ const endpoint = preset.provider === 'openai' ? 'https://api.openai.com/v1/chat/completions'+ : anthropic ? 'https://api.anthropic.com/v1/messages' : `${preset.url}/chat/completions`;+ // A synthetic provider rejects misrouted or unauthenticated requests, rather+ // than accepting a canned result unrelated to the configured connection.+ if (url !== endpoint || init?.method !== 'POST' || body.model !== preset.model || !body.messages?.some((m: any) => m.role === 'user' && m.content)) return json({ error: { message: 'wrong connection request' } }, 400);+ if ('apiKey' in preset && (anthropic ? headers.get('x-api-key') !== preset.apiKey : headers.get('authorization') !== `Bearer ${preset.apiKey}`)) return json({ error: { message: 'wrong provider identity' } }, 401);+ if (!streamingDefault || body.stream === false) return json(reply(anthropic, text));+ const records = [text.slice(0, 4), text.slice(4)].map(part => anthropic+ ? `event: content_block_delta\ndata: ${JSON.stringify({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: part } })}\n\n`+ : `data: ${JSON.stringify({ choices: [{ delta: { content: part } }] })}\n\n`);+ return new Response(records.join('') + (anthropic ? 'event: message_stop\ndata: {"type":"message_stop"}\n\n' : 'data: [DONE]\n\n'), { headers: { 'content-type': 'text/event-stream' } });+ });+}+for (const preset of presets) test(`returns the actual reply from a streaming-default ${preset.provider} gateway`, async () => {+ vi.stubGlobal('fetch', () => { throw new Error('live network forbidden'); });+ const expected = `${preset.provider}-reply-37`, request = gateway(preset, expected);+ const pending = testAiPresetConnection(preset, { fetch: request });+ await expect(pending).resolves.toMatchObject({ reply: expected });+ const result = await pending;+ expect(result.reply).toBe(expected); expect(request.mock.calls.length).toBeGreaterThan(0);+ expect(Number.isFinite(result.latencyMs)).toBe(true); expect(result.latencyMs).toBeGreaterThanOrEqual(0);+});+test('token-parameter recovery returns a streaming-default gateway reply', async () => {+ const preset = presets[0], good = gateway(preset, 'retry-reply-92');+ const request = vi.fn(async (url: string, init?: RequestInit) => {+ const body = JSON.parse(String(init?.body));+ if (body.max_tokens !== undefined || body.max_completion_tokens === undefined) return json({ error: { message: 'Unsupported parameter: use max_completion_tokens' } }, 400);+ return good(url, init);+ });+ await expect(testAiPresetConnection(preset, { fetch: request })).resolves.toMatchObject({ reply: 'retry-reply-92' });+ expect(good).toHaveBeenCalled();+});+for (const preset of [presets[0], presets[3]]) {+ test(`preserves ordinary JSON replies for ${preset.provider}`, async () => {+ expect((await testAiPresetConnection(preset, { fetch: gateway(preset, 'plain-reply-53', false) })).reply).toBe('plain-reply-53');+ });+ test(`rejects a missing message for ${preset.provider}`, async () => {+ await expect(testAiPresetConnection(preset, { fetch: async () => json({}) })).rejects.toThrow();+ });+ test(`preserves valid empty-text message compatibility for ${preset.provider}`, async () => {+ const result = await testAiPresetConnection(preset, { fetch: async () => json(reply(preset.provider === 'anthropic', '')) });+ expect(typeof result.reply).toBe('string'); expect(result.reply.trim().length).toBeGreaterThan(0);+ });+}+test('surfaces array-shaped provider error details', async () => {+ await expect(testAiPresetConnection(presets[0], { fetch: async () => json([{ error: { message: 'synthetic project disabled', status: 'PERMISSION_DENIED' } }], 403) })).rejects.toThrow(/synthetic project disabled/);+});+test('does not turn an authorization refusal into success', async () => {+ const request = vi.fn(async () => json({ error: { message: 'synthetic credential rejected' } }, 401));+ await expect(testAiPresetConnection(presets[3], { fetch: request })).rejects.toThrow(/401.*synthetic credential rejected/);+});+test('preserves transport failure', async () => {+ await expect(testAiPresetConnection(presets[0], { fetch: async () => { throw new Error('synthetic transport refused'); } })).rejects.toThrow('synthetic transport refused');+});+test('bounds the displayed reply to 100 characters', async () => {+ const text = '0123456789'.repeat(14);+ expect((await testAiPresetConnection(presets[0], { fetch: gateway(presets[0], text, false) })).reply).toBe(text.slice(0, 100));+});+test('forwards caller cancellation to the provider port', async () => {+ const controller = new AbortController(); controller.abort();+ const request = vi.fn(async (_url: string, init?: RequestInit) => {+ if (init?.signal?.aborted) throw new Error('synthetic request aborted');+ return json(reply(false, 'must not succeed'));+ });+ await expect(testAiPresetConnection(presets[0], { fetch: request, signal: controller.signal })).rejects.toThrow('synthetic request aborted');+});