Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: cover shared running-pipe tracker lifecycle

louis1d679bd35c24
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 29ddfdb10..33f635ef1 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,16 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-shared-running-pipes-lifecycle` mounts the actual React hook and Zustand
+store with synthetic HTTP/event ports and timers. Ten outcomes cover overlapping
+consumers, final unmount, remount, delayed fetch and event-bus completion, pipe
+events, filtering and poll recovery. The parent fails six lifecycle outcomes and
+preserves four; the reference and current source pass ten. Run
+`bun test evals/coding-agent/calibrate-running-pipes.test.js` for calibration.
+The fixture and runtime links appear only at grading. This does not establish
+native event delivery, real-server behavior, memory usage, execution isolation or
+model improvement.
+
`app-capture-cadence-durable-before-restart` drives the actual recording-settings
change/apply callbacks, local write queue and apply bar with synthetic hook state,
settings storage and native ports. Four outcomes cover a delayed save, a later
diff --git a/evals/coding-agent/calibrate-running-pipes.test.js b/evals/coding-agent/calibrate-running-pipes.test.js
new file mode 100644
index 000000000..547c24f7f
--- /dev/null
+++ b/evals/coding-agent/calibrate-running-pipes.test.js
@@ -0,0 +1,37 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-shared-running-pipes-lifecycle');
+const hook=`${app}/lib/hooks/use-running-pipes.ts`, root=mkdtempSync(join(tmpdir(),'running-pipes-calibration-'));
+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(cwd,from,to){const p=join(cwd,hook),text=readFileSync(p,'utf8');expect(text.split(from)).toHaveLength(2);writeFileSync(p,text.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);
+ // Exact product modules only; no agent, network, native process or user config.
+ for(const file of [hook,`${app}/lib/events/types.ts`,`${app}/lib/api.ts`,`${app}/lib/events/bus.ts`]){const dest=join(cwd,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,execFileSync('git',['show',`${ref}:${file}`],{cwd:repo}));}
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,classification:classifyGraderError(r)},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('10 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/);}
+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(classifyGraderError(r)).toBeNull();expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/);}
+test('parent fails lifecycle outcomes and preserves nearby behavior',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('6 failed | 4 passed')},60000);
+test('reference passes',()=>passes(grade('reference')),60000);
+test('current passes',()=>passes(grade('current','HEAD')),60000);
+test('declared oracle path repairs parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>{const patch=execFileSync('git',['diff',item.base_ref,item.oracle_ref,'--',...item.oracle_paths],{cwd:repo});execFileSync('git',['apply','--whitespace=nowarn','-'],{cwd,input:patch})})),60000);
+test('unused correct hook cannot hide active broken hook',()=>fails(grade('unused',item.oracle_ref,cwd=>{writeFileSync(join(cwd,app,'unused-correct-hook.ts'),readFileSync(join(cwd,hook)));writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}))})),60000);
+test('equivalent private names preserve outcomes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,hook);writeFileSync(p,readFileSync(p,'utf8').replaceAll('trackerGeneration','lifecycleEpoch').replaceAll('consumerCount','subscribers'))})),60000);
+test('premature shared teardown is rejected',()=>fails(grade('premature',item.oracle_ref,cwd=>replace(cwd,'if (--consumerCount > 0) return;','--consumerCount;'))),60000);
+test('lost stale-generation check is rejected',()=>fails(grade('stale',item.oracle_ref,cwd=>replace(cwd,'mounted && consumerCount > 0 && generation === trackerGeneration','mounted && consumerCount > 0'))),60000);
+test('dangling poll is rejected',()=>fails(grade('poll',item.oracle_ref,cwd=>replace(cwd,'clearInterval(pollHandle);','/* keep polling */'))),60000);
+test('blanket event suppression is rejected',()=>fails(grade('events',item.oracle_ref,cwd=>replace(cwd,'if (envelope.source !== "pipe") return;','return;'))),60000);
+test('missing hook is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(classifyGraderError(r)).toBe('vitest_collection_error')},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 5fa86e0cc..86ea3e864 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-30-capture-cadence",
+ "dataset_version": "2026-09-30-running-pipes-lifecycle",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -4455,6 +4455,53 @@
],
"command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" test ../../evals/coding-agent/graders/capture-cadence.test.js"
}
+ },
+ {
+ "id": "app-shared-running-pipes-lifecycle",
+ "name": "Stop abandoned running-pipe trackers without disrupting active consumers",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "94b670bc0cc3fd09a7b3fa02e1ae7fe93586599a",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5116"
+ },
+ "base_ref": "9df282bf7daa7efb2e4e23759b7752eee445cefd",
+ "oracle_ref": "94b670bc0cc3fd09a7b3fa02e1ae7fe93586599a",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-running-pipes.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-running-pipes.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "lifecycle",
+ "shared-state",
+ "stale-response"
+ ],
+ "prompt": "The running-pipes sidebar tracker keeps polling and listening after its final consumer unmounts. Fix the existing useRunningPipes hook so active consumers share one tracker, removing one consumer preserves the others, and removing the last stops polling and event subscriptions. A later mount must refresh its state. Fetches and event-bus setup that finish after their original mount was abandoned must not start background work, overwrite a newer mount, or cancel the newer tracker. Preserve the existing 30-second recovery poll, recovery after temporary HTTP failures, running-only filtering, start-time ordering, pipe event updates and terminal removal; unrelated or malformed session events must remain ignored. Keep the existing hook interface. This task concerns React lifecycle with controlled HTTP and event ports, not native event delivery.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/running-pipes-lifecycle.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-running-pipes.test.tsx"
+ },
+ {
+ "local_path": "graders/running-pipes-lifecycle.config.mjs",
+ "destination_path": "apps/screenpipe-app-tauri/eval-running-pipes.config.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-running-pipes.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/running-pipes-lifecycle.config.mjs b/evals/coding-agent/graders/running-pipes-lifecycle.config.mjs
new file mode 100644
index 000000000..7cc131803
--- /dev/null
+++ b/evals/coding-agent/graders/running-pipes-lifecycle.config.mjs
@@ -0,0 +1,5 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { defineConfig } from 'vitest/config';
+import path from 'node:path';
+export default defineConfig({esbuild:{jsx:'automatic'},resolve:{alias:{'@':path.resolve('.')}},test:{include:['eval-running-pipes.test.tsx'],environment:'jsdom',pool:'forks',poolOptions:{forks:{singleFork:true}}},css:{postcss:{plugins:[]}}});
diff --git a/evals/coding-agent/graders/running-pipes-lifecycle.fixture.tsx.txt b/evals/coding-agent/graders/running-pipes-lifecycle.fixture.tsx.txt
new file mode 100644
index 000000000..86eb61bf2
--- /dev/null
+++ b/evals/coding-agent/graders/running-pipes-lifecycle.fixture.tsx.txt
@@ -0,0 +1,25 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterEach, it, expect, vi } from 'vitest';
+import { renderHook, act, cleanup } from '@testing-library/react';
+import './lib/hooks/use-running-pipes';
+const f=vi.hoisted(()=>({fetch:vi.fn(),mount:vi.fn(),listeners:new Set<any>()}));
+vi.mock('@/lib/api',()=>({localFetch:f.fetch}));
+vi.mock('@/lib/events/bus',()=>{const register=(fn:any)=>{f.listeners.add(fn);return ()=>f.listeners.delete(fn)};return {mountAgentEventBus:f.mount,registerDefault:register,registerObserver:register}});
+const tick=async()=>{await act(async()=>{for(let i=0;i<8;i++)await Promise.resolve()})};
+const row=(name:string,id=1,started='2026-01-01T01:00:00Z')=>({is_running:true,config:{name,config:{title:name}},recent_executions:[{id,status:'running',started_at:started}]});
+const response=(data:any[])=>({ok:true,json:async()=>({data})});
+const deferred=()=>{let resolve!:(v:any)=>void;const promise=new Promise<any>(r=>resolve=r);return {promise,resolve}};
+let useRunningPipes:any;
+beforeEach(async()=>{vi.resetModules();vi.useFakeTimers();vi.setSystemTime(new Date('2026-01-01T02:00:00Z'));f.listeners.clear();f.fetch.mockReset().mockResolvedValue(response([row('one')]));f.mount.mockReset().mockResolvedValue(undefined);globalThis.fetch=vi.fn(()=>{throw Error('network forbidden')}) as any;useRunningPipes=(await import('./lib/hooks/use-running-pipes')).useRunningPipes});
+afterEach(()=>{cleanup();vi.clearAllTimers();vi.useRealTimers();vi.unstubAllGlobals()});
+it('stops polling and event listening after the final consumer leaves',async()=>{const h=renderHook(()=>useRunningPipes());await tick();expect(h.result.current.map((x:any)=>x.pipeName)).toEqual(['one']);h.unmount();await tick();const before=f.fetch.mock.calls.length;await act(async()=>{await vi.advanceTimersByTimeAsync(90000)});expect(f.fetch).toHaveBeenCalledTimes(before);expect(f.listeners.size).toBe(0)});
+it('preserves a shared tracker while another consumer remains',async()=>{const a=renderHook(()=>useRunningPipes());const b=renderHook(()=>useRunningPipes());await tick();expect(f.fetch).toHaveBeenCalledTimes(1);expect(f.listeners.size).toBe(1);a.unmount();f.fetch.mockResolvedValue(response([row('two')]));await act(async()=>{await vi.advanceTimersByTimeAsync(30000)});expect(b.result.current.map((x:any)=>x.pipeName)).toEqual(['two']);expect(f.listeners.size).toBe(1);b.unmount()});
+it('does not start work when the initial fetch resolves after unmount',async()=>{const d=deferred();f.fetch.mockReturnValueOnce(d.promise);const h=renderHook(()=>useRunningPipes());h.unmount();d.resolve(response([row('late')]));await tick();await act(async()=>{await vi.advanceTimersByTimeAsync(90000)});expect(f.fetch).toHaveBeenCalledTimes(1);expect(f.listeners.size).toBe(0)});
+it('starts a fresh fetch after all consumers unmount and remount',async()=>{const a=renderHook(()=>useRunningPipes());await tick();a.unmount();f.fetch.mockResolvedValue(response([row('new')]));const b=renderHook(()=>useRunningPipes());await tick();expect(b.result.current.map((x:any)=>x.pipeName)).toEqual(['new']);expect(f.listeners.size).toBe(1);b.unmount()});
+it('ignores an old initial fetch after a new consumer has mounted',async()=>{const d=deferred();f.fetch.mockReturnValueOnce(d.promise);const a=renderHook(()=>useRunningPipes());a.unmount();f.fetch.mockResolvedValue(response([row('fresh')]));const b=renderHook(()=>useRunningPipes());await tick();d.resolve(response([row('stale')]));await tick();expect(b.result.current.map((x:any)=>x.pipeName)).toEqual(['fresh']);expect(f.listeners.size).toBe(1);b.unmount()});
+it('late event-bus setup cannot install an abandoned subscription',async()=>{const d=deferred();f.mount.mockReturnValueOnce(d.promise);const a=renderHook(()=>useRunningPipes());await tick();a.unmount();d.resolve(undefined);await tick();expect(f.listeners.size).toBe(0);const before=f.fetch.mock.calls.length;await act(async()=>{await vi.advanceTimersByTimeAsync(60000)});expect(f.fetch).toHaveBeenCalledTimes(before)});
+it('old event-bus completion preserves a newer active tracker',async()=>{const d=deferred();f.mount.mockReturnValueOnce(d.promise);const a=renderHook(()=>useRunningPipes());await tick();a.unmount();const b=renderHook(()=>useRunningPipes());await tick();d.resolve(undefined);await tick();expect(f.listeners.size).toBe(1);f.fetch.mockResolvedValue(response([row('newer')]));await act(async()=>{await vi.advanceTimersByTimeAsync(30000)});expect(b.result.current.map((x:any)=>x.pipeName)).toEqual(['newer']);b.unmount();expect(f.listeners.size).toBe(0)});
+it('preserves pipe events and ignores unrelated events',async()=>{f.fetch.mockResolvedValue(response([]));const a=renderHook(()=>useRunningPipes());await tick();const send=(source:string,id:string,type:string)=>act(()=>{for(const fn of f.listeners)fn({source,sessionId:id,event:{type}})});send('pi','pipe:unrelated:8','message');expect(a.result.current).toEqual([]);send('pipe','not-a-pipe','message');expect(a.result.current).toEqual([]);send('pipe','pipe:live:9','message');expect(a.result.current.map((x:any)=>[x.pipeName,x.executionId])).toEqual([['live',9]]);send('pipe','pipe:live:9','pipe_done');expect(a.result.current).toEqual([]);a.unmount()});
+it('preserves running-only filtering and stable start-time order',async()=>{f.fetch.mockResolvedValue(response([row('z',2,'2026-01-01T00:00:00Z'),row('a',1),{...row('stopped'),is_running:false}]));const a=renderHook(()=>useRunningPipes());await tick();expect(a.result.current.map((x:any)=>x.pipeName)).toEqual(['z','a']);a.unmount()});
+it('recovers polling after a temporary HTTP failure',async()=>{f.fetch.mockResolvedValueOnce({ok:false});const a=renderHook(()=>useRunningPipes());await tick();expect(a.result.current).toEqual([]);await act(async()=>{await vi.advanceTimersByTimeAsync(30000)});expect(a.result.current.map((x:any)=>x.pipeName)).toEqual(['one']);a.unmount()});