YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover registered pipe-run outputs through public hook
louisfa0165d4f046
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 701809b7a..de4aad429 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 72 git-mined regressions. See+The current app corpus contains 73 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -786,3 +786,21 @@ The twelve outcomes also pass against current persistence source using thehistorical synthetic test setup. This is not the complete current app suite,independent native webview scheduling, real filesystem durability, proof ofworkspace isolation or a model trial.++## Registered pipe-run outputs++`app-pipe-run-registered-outputs` executes the public chat-inspector hook with+real artifact and citation helpers against a synthetic local API. Six outcomes+cover active-run visibility without tool calls, other-run exclusion, explicit+output deduplication, ordinary chat, leaving the pipe context and recovery after+a temporary API refusal. The parent fails three intended outcomes and preserves+three; the hook-only reference passes all six.++Run `bun test evals/coding-agent/calibrate-pipe-run-outputs.test.js` with desktop+React/Vitest dependencies installed. Seven controls cover parent/reference, an+unused corrected hook, equivalent helper naming, removed source filtering,+blanket suppression and missing-source setup failure. Set+`SCREENPIPE_EVAL_CALIBRATION_RECEIPTS` to retain per-control logs. Fixtures and+dependency links are materialized only for grading. These are corpus checks,+not model trials, full chat UI or native persistence evidence, or proof of+execution isolation. No real files, recordings or provider accounts are used.diff --git a/evals/coding-agent/calibrate-pipe-run-outputs.test.js b/evals/coding-agent/calibrate-pipe-run-outputs.test.jsnew file mode 100644index 000000000..2734d1d2e--- /dev/null+++ b/evals/coding-agent/calibrate-pipe-run-outputs.test.js@@ -0,0 +1,54 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-pipe-run-registered-outputs');+const hook = `${app}/lib/hooks/use-chat-inspector.ts`;+const root = mkdtempSync(join(tmpdir(), 'pipe-outputs-calibration-')), archives = new Map();+const paths = [hook, `${app}/lib/hooks/use-unified-artifacts.ts`, `${app}/lib/source-citations.ts`, `${app}/lib/api.ts`];+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, { recursive: true, force: true }));+function change(cwd, from, to) {+ const path = join(cwd, hook), source = readFileSync(path, 'utf8');+ expect(source.split(from)).toHaveLength(2);+ writeFileSync(path, source.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['archive', ref, ...paths], { cwd: repo, maxBuffer: 8 * 1024 * 1024 }));+ execFileSync('tar', ['-x', '-C', cwd], { input: archives.get(ref) });+ for (const f of item.grader.fixtures) writeFileSync(join(cwd, f.destination_path), readFileSync(join(import.meta.dir, f.local_path)));+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024,+ env: { PATH: process.env.PATH, HOME: cwd, CI: 'true', TZ: 'UTC', NO_COLOR: '1' } });+ if (receipts) {+ mkdirSync(receipts, { recursive: true });+ writeFileSync(join(receipts, `${name}.stdout`), r.stdout || '');+ writeFileSync(join(receipts, `${name}.stderr`), r.stderr || '');+ writeFileSync(join(receipts, `${name}.json`), JSON.stringify({ ref, command: item.grader.command, exit_code: r.status, signal: r.signal, error: r.error?.message }, null, 2));+ }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull();+ return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('6 passed'); expect(r.stdout + r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout + r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails three intended outcomes and preserves three', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('3 failed | 3 passed'); }, 60000);+test('reference passes all public hook outcomes', () => passes(grade('reference')), 60000);+test('equivalent private helper naming passes', () => passes(grade('equivalent', item.oracle_ref, cwd => {+ const p = join(cwd, hook); writeFileSync(p, readFileSync(p, 'utf8').replaceAll('mergePipeRunArtifactOutputs', 'combineRegisteredOutputs'));+})), 60000);+test('unused correct hook cannot hide the broken entry point', () => fails(grade('unused', item.oracle_ref, cwd => {+ writeFileSync(join(cwd, `${app}/lib/hooks/unused-correct.ts`), readFileSync(join(cwd, hook)));+ writeFileSync(join(cwd, hook), execFileSync('git', ['show', `${item.base_ref}:${hook}`], { cwd: repo }));+})), 60000);+test('cross-run output acceptance fails', () => fails(grade('source-bypass', item.oracle_ref, cwd => change(cwd, 'if (artifact.source !== pipeRunArtifactSource) continue;', '// calibration: source filter removed'))), 60000);+test('blanket registered-output suppression fails', () => fails(grade('blanket', item.oracle_ref, cwd => change(cwd, 'const outputs = [...toolOutputs];', 'const outputs = [...toolOutputs]; return outputs;'))), 60000);+test('missing public hook is a setup error', () => {+ const r = grade('missing', item.oracle_ref, cwd => rmSync(join(cwd, hook)));+ expect(r.status).toBe(1); expect(r.stdout).toContain('no tests'); expect(r.stdout + r.stderr).toMatch(/Failed to resolve import|Failed to load url/);+}, 60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 05d4a9a6c..ab87f262c 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-22.8",+ "dataset_version": "2026-09-23.2","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -2953,6 +2953,54 @@],"command": "bun --no-env-file test evals/coding-agent/graders/daily-summary-errors.test.js"}+ },+ {+ "id": "app-pipe-run-registered-outputs",+ "name": "Expose registered outputs for the active pipe run",+ "source": {+ "kind": "git_regression",+ "fix_commit": "d3569717c055679ca232b822491cd72ac58ca06c"+ },+ "base_ref": "67f8cc7ceb74e5772ff44970419a664e7827fe74",+ "oracle_ref": "d3569717c055679ca232b822491cd72ac58ca06c",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-chat-inspector.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-chat-inspector.ts",+ "apps/screenpipe-app-tauri/lib/hooks/use-unified-artifacts.ts",+ "apps/screenpipe-app-tauri/lib/source-citations.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "pipe-outputs",+ "execution-scope",+ "recovery"+ ],+ "prompt": "The public useChatInspector hook only exposes explicit artifact tool results, so a pipe run whose output was registered separately appears to have no output. Restore outputs for the supplied active pipe-run source even when the transcript has no artifact tool call. Query the artifact API with that source and exclude rows belonging to other runs. Preserve explicit tool outputs and avoid duplicate registered outputs when their original path already identifies an explicit output. Ordinary chat without a pipe-run source must preserve explicit outputs without reading the artifact API. Removing the pipe context must clear its registered outputs. An initial artifact API refusal must leave registered outputs empty and a later transcript update must recover the registered output after the API recovers. Preserve the public hook interface; evaluate observable outputs and requests rather than private helper names or UI wording. Use synthetic local API responses only. This task covers the actual hook and internal artifact/citation helpers, not full chat UI wiring, native registration, database persistence, or live services.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/pipe-run-outputs.fixture.ts",+ "destination_path": "apps/screenpipe-app-tauri/eval-pipe-run-outputs.test.ts"+ },+ {+ "local_path": "graders/pipe-run-outputs.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-pipe-run-outputs.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-pipe-run-outputs.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/pipe-run-outputs.config.mjs b/evals/coding-agent/graders/pipe-run-outputs.config.mjsnew file mode 100644index 000000000..de72baa4a--- /dev/null+++ b/evals/coding-agent/graders/pipe-run-outputs.config.mjs@@ -0,0 +1,11 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({+ esbuild: { jsx: 'automatic' },+ test: { include: ['eval-pipe-run-outputs.test.ts'], environment: 'node',+ pool: 'forks', poolOptions: { forks: { singleFork: true } } },+ resolve: { alias: { '@': path.resolve('.') } },+ css: { postcss: { plugins: [] } },+});diff --git a/evals/coding-agent/graders/pipe-run-outputs.fixture.ts b/evals/coding-agent/graders/pipe-run-outputs.fixture.tsnew file mode 100644index 000000000..da3a3c22c--- /dev/null+++ b/evals/coding-agent/graders/pipe-run-outputs.fixture.ts@@ -0,0 +1,24 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+// @vitest-environment jsdom+import React from 'react';+import {createRoot} from 'react-dom/client';+import {act} from 'react-dom/test-utils';+import {beforeEach,afterEach,it,expect,vi} from 'vitest';+import {useChatInspector} from '@/lib/hooks/use-chat-inspector';+const port=vi.hoisted(()=>({fetch:vi.fn()}));+vi.mock('@/lib/api',()=>({localFetch:(...args:any[])=>port.fetch(...args)}));+let root:any,host:any,value:any,rows:any[];+const source='pipe:synthetic:7';const output='/synthetic/run7/report.md';+const row=(s=source,path=output)=>({registered:true,id:7,source:s,source_type:'pipe-run',title:'Synthetic report',kind:'markdown',path,original_path:'/synthetic/declared/report.md',size_bytes:100,preview:null,saf_kind:null,artifact_id:null,saf_version:null,modified_at:'2026-08-17T00:00:00Z',created_at:'2026-08-17T00:00:00Z'});+const messages=[{contentBlocks:[{type:'tool',toolCall:{toolName:'save_artifact',args:{title:'Synthetic explicit report'},result:'Saved "report" to Artifacts (/synthetic/declared/report.md)',isRunning:false}}]}];+function Probe({s,m}:any){value=(useChatInspector as any)(m,s);return null;}+async function render(s:any=source,m:any=[]){await act(async()=>root.render(React.createElement(Probe,{s,m})));}+beforeEach(()=>{(globalThis as any).IS_REACT_ACT_ENVIRONMENT=true;vi.useFakeTimers();host=document.createElement('div');document.body.append(host);root=createRoot(host);rows=[row()];port.fetch.mockReset().mockImplementation(async()=>({ok:true,text:async()=>JSON.stringify({data:rows,pagination:{total:rows.length},sources:[source]})}));});+afterEach(async()=>{await act(async()=>root.unmount());host.remove();vi.clearAllTimers();vi.useRealTimers();});+it('registered active-run output appears without an explicit tool call',async()=>{await render();expect(value.outputs.map((x:any)=>x.path)).toEqual([output]);expect(port.fetch.mock.calls.length).toBeGreaterThan(0);for(const [url] of port.fetch.mock.calls)expect(new URL(url,'http://synthetic.invalid').searchParams.get('source')).toBe(source);});+it('another run is excluded while active output remains',async()=>{rows=[row('pipe:synthetic:8','/synthetic/other/report.md'),row()];await render();expect(value.outputs.map((x:any)=>x.path)).toEqual([output]);});+it('explicit output is preserved and registered original-path duplicate suppressed',async()=>{await render(source,messages);expect(value.outputs.map((x:any)=>x.path)).toEqual(['/synthetic/declared/report.md']);});+it('ordinary chat preserves explicit outputs without artifact API reads',async()=>{await render(null,messages);expect(value.outputs.map((x:any)=>x.path)).toEqual(['/synthetic/declared/report.md']);expect(port.fetch).not.toHaveBeenCalled();});+it('switching away from pipe context clears registered outputs',async()=>{await render();await render(null);expect(value.outputs).toEqual([]);});+it('transient artifact failure can recover when transcript grows',async()=>{port.fetch.mockResolvedValue({ok:false,status:503});await render();expect(value.outputs).toEqual([]);port.fetch.mockImplementation(async()=>({ok:true,text:async()=>JSON.stringify({data:[row()],pagination:{total:1},sources:[source]})}));await render(source,[{contentBlocks:[]}]);expect(value.outputs.map((x:any)=>x.path)).toEqual([output]);});