YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover active recording volume recovery
Louis Beaumont8b75c0048763
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex c5f00452e..e244c687a 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -33,7 +33,7 @@ images, blanket paste refusal and missing-source classification. Fixtures anddependency links appear only when grading begins. This does not execute nativeclipboard delivery, image encoding, disk autosave, agent isolation or model trials.-The current app corpus contains 80 git-mined regressions. See+The current app corpus contains 121 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1486,3 +1486,17 @@ refusal, lost reports, ignored hashes, lost pending state and missing source.Fixtures appear only at grading. This checks the extension and its local receipts;real skill-store durability, native authorization, context privacy filtering,execution isolation and model performance remain outside this evidence.++## Active recording storage volume++`app-active-recording-storage-volume` runs the actual React storage hook with+synthetic native and settings ports. Ten outcomes compare low and healthy active+volumes against conflicting saved paths, then cover refresh, native reserve+boundaries, probe failures, disabled checks and late results. The parent fails+four volume outcomes and preserves six; the historical fix and current hook+pass all ten. Run `bun test evals/coding-agent/calibrate-active-storage-volume.test.js`+for twelve controls, including unused correct code, a wrong-volume bypass,+blanket warning suppression, an equivalent boundary and missing-source errors.+Fixtures and runtime links appear only when grading begins. This does not test+real disk space, IPC delivery, native recording or migration policy, execution+isolation or model performance.diff --git a/evals/coding-agent/calibrate-active-storage-volume.test.js b/evals/coding-agent/calibrate-active-storage-volume.test.jsnew file mode 100644index 000000000..cf38eddde--- /dev/null+++ b/evals/coding-agent/calibrate-active-storage-volume.test.js@@ -0,0 +1,39 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-active-recording-storage-volume');+const hook=`${app}/lib/hooks/use-recording-storage.ts`, root=mkdtempSync(join(tmpdir(),'active-storage-volume-calibration-'));+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,from,to){const p=join(cwd,hook),text=readFileSync(p,'utf8');expect(text.split(from)).toHaveLength(2);writeFileSync(p,text.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ // Exact product modules only; no agent, network, native process or user config.+ for(const file of [hook]){const dest=join(cwd,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,execFileSync('git',['show',`${ref}:${file}`],{cwd:repo}));}+ for(const file of ['lib/hooks/use-settings.ts','lib/utils/tauri.ts']){const dest=join(cwd,app,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,'throw new Error("unmocked external port");');}+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,classification:classifyGraderError(r)},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('10 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/);}+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(classifyGraderError(r)).toBeNull();expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/);}+test('parent fails four volume outcomes and preserves six nearby outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 6 passed')},60000);+test('reference passes',()=>passes(grade('reference')),60000);+test('current passes',()=>passes(grade('current','HEAD')),60000);+test('declared oracle path repairs parent',()=>passes(grade('oracle-only',item.base_ref,cwd=>{const patch=execFileSync('git',['diff',item.base_ref,item.oracle_ref,'--',...item.oracle_paths],{cwd:repo});execFileSync('git',['apply','--whitespace=nowarn','-'],{cwd,input:patch})})),60000);+test('unused correct code cannot hide the active broken hook',()=>fails(grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,app,'unused-correct-hook.ts'),execFileSync('git',['show',`${item.oracle_ref}:${hook}`],{cwd:repo})))),60000);+test('equivalent inclusive boundary passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,'availableBytes <= thresholdBytes','!(availableBytes > thresholdBytes)'))),60000);+test('blanket warning suppression is rejected',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,'? { availableBytes, thresholdBytes } : null','? null : null'))),60000);+test('wrong-volume probe despite an active-directory lookup is rejected',()=>fails(grade('wrong-volume',item.oracle_ref,cwd=>replace(cwd,'commands.getDiskUsage(true, directory)','commands.getDiskUsage(true, "/synthetic/saved-volume")'))),60000);+test('exclusive threshold boundary is rejected',()=>fails(grade('boundary',item.oracle_ref,cwd=>replace(cwd,'availableBytes <= thresholdBytes','availableBytes < thresholdBytes'))),60000);+test('probing while capture is active is rejected',()=>fails(grade('active-scan',item.oracle_ref,cwd=>replace(cwd,'const enabled = paused &&','const enabled ='))),60000);+test('late old-directory result overwrite is rejected',()=>fails(grade('stale',item.oracle_ref,cwd=>{const f=join(cwd,hook);writeFileSync(f,readFileSync(f,'utf8').replaceAll('if (!cancelled)', 'if (true)'))})),60000);+test('missing hook is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(classifyGraderError(r)).toBe('vitest_collection_error')},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex c3b8d018b..a93c5734b 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4963,6 +4963,53 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/learning-save-readback.test.ts"}+ },+ {+ "id": "app-active-recording-storage-volume",+ "name": "Check low storage on the active recording volume",+ "source": {+ "kind": "git_regression",+ "fix_commit": "8e51453598d2587ac57828e1238d3cec0a92f05c",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7460"+ },+ "base_ref": "b93f07684b845a3baaedafc288d18d79b642fbf0",+ "oracle_ref": "8e51453598d2587ac57828e1238d3cec0a92f05c",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-recording-storage.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-recording-storage.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "recording",+ "storage",+ "volume"+ ],+ "prompt": "Paused recording can show a low-storage warning for the saved settings directory while the running engine uses another volume. Fix the existing useRecordingStorage hook so its warning reflects the currently active recording directory, including after manual refresh. A healthy saved volume must not hide low space on the active volume, and a low saved volume must not block a healthy active one. If the active directory or disk probe is unavailable, report an error with no warning instead of treating failure as zero free space or using a different volume. Preserve the hook interface, native reserve and inclusive low-space boundary, invalid-native-value errors, manual recovery, and checks only while paused with the low-disk guard enabled. Keep checking state accurate and prevent late old-directory results or resumed-recording results from restoring stale warnings. This task concerns React behavior with controlled native ports, not filesystem operations or native recording policy.",+ "agent_timeout_seconds": 1200,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/active-storage-volume.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-active-storage-volume.test.tsx"+ },+ {+ "local_path": "graders/active-storage-volume.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-active-storage-volume.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-active-storage-volume.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/active-storage-volume.config.mjs b/evals/coding-agent/graders/active-storage-volume.config.mjsnew file mode 100644index 000000000..c10056b65--- /dev/null+++ b/evals/coding-agent/graders/active-storage-volume.config.mjs@@ -0,0 +1,5 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({esbuild:{jsx:'automatic'},resolve:{alias:{'@':path.resolve('.')}},test:{include:['eval-active-storage-volume.test.tsx'],environment:'jsdom',pool:'forks',poolOptions:{forks:{singleFork:true}}},css:{postcss:{plugins:[]}}});diff --git a/evals/coding-agent/graders/active-storage-volume.fixture.tsx.txt b/evals/coding-agent/graders/active-storage-volume.fixture.tsx.txtnew file mode 100644index 000000000..e808386c7--- /dev/null+++ b/evals/coding-agent/graders/active-storage-volume.fixture.tsx.txt@@ -0,0 +1,79 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { act, cleanup, renderHook } from '@testing-library/react';+import { afterEach, beforeEach, expect, test, vi } from 'vitest';+const ports = vi.hoisted(() => ({+ settings: { dataDir: 'default', stopRecordingOnLowDisk: true },+ getDataDir: vi.fn(), getActiveDataDir: vi.fn(), getDiskUsage: vi.fn(), getLowDiskGuardConfig: vi.fn(),+}));+vi.mock('@/lib/hooks/use-settings', () => ({useSettings: () => ports}));+vi.mock('@/lib/utils/tauri', () => ({commands: ports}));+import { useRecordingStorage } from './lib/hooks/use-recording-storage';+const active = '/synthetic/recording-volume', saved = '/synthetic/saved-volume';+const usage = (n:number) => ({status:'ok',data:{available_space_bytes:n}});+const settle = async () => {await act(async()=>{});};+beforeEach(()=>{+ vi.resetAllMocks(); ports.settings.dataDir='default'; ports.settings.stopRecordingOnLowDisk=true;+ ports.getDataDir.mockResolvedValue(saved);+ ports.getActiveDataDir.mockResolvedValue({status:'ok',data:active});+ ports.getLowDiskGuardConfig.mockResolvedValue({thresholdBytes:500});+ ports.getDiskUsage.mockImplementation(async (_quick:boolean,path:string)=>usage(path===active?300:900));+});+afterEach(cleanup);+test('warns when the active volume is low even though the saved volume is healthy',async()=>{+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ expect(r.result.current).toMatchObject({warning:{availableBytes:300,thresholdBytes:500},checking:false,error:false});+});+test('does not show a saved-volume warning when the active volume is healthy',async()=>{+ ports.getDiskUsage.mockImplementation(async (_quick:boolean,path:string)=>usage(path===active?900:300));+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ expect(r.result.current).toMatchObject({warning:null,checking:false,error:false});+});+test('preserves an ordinary matching directory and the native threshold boundary',async()=>{+ ports.getDataDir.mockResolvedValue(active);ports.getDiskUsage.mockResolvedValue(usage(500));+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ expect(r.result.current.warning).toEqual({availableBytes:500,thresholdBytes:500});+});+test('a manual refresh checks the newly active volume and clears the warning',async()=>{+ ports.getDataDir.mockResolvedValue(active);+ const r=renderHook(()=>useRecordingStorage(true));await settle();expect(r.result.current.warning).not.toBeNull();+ ports.getActiveDataDir.mockResolvedValue({status:'ok',data:saved});+ act(()=>r.result.current.refresh());await settle();+ expect(r.result.current).toMatchObject({warning:null,checking:false,error:false});+});+test('does not inspect storage during recording or with the guard disabled',async()=>{+ const r=renderHook(({paused})=>useRecordingStorage(paused),{initialProps:{paused:false}});await settle();+ expect(ports.getDiskUsage).not.toHaveBeenCalled();expect(r.result.current.warning).toBeNull();+ ports.settings.stopRecordingOnLowDisk=false;r.rerender({paused:true});await settle();+ expect(ports.getDiskUsage).not.toHaveBeenCalled();expect(r.result.current).toMatchObject({warning:null,checking:false,error:false});+});+test('active-directory lookup failure reports unavailable rather than falling back to a different volume',async()=>{+ ports.getActiveDataDir.mockResolvedValue({status:'error',error:'synthetic unavailable'});+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ expect(r.result.current).toMatchObject({warning:null,checking:false,error:true});+});+test('failed disk probes are not zero free bytes and can recover manually',async()=>{+ ports.getDataDir.mockResolvedValue(active);ports.getDiskUsage.mockRejectedValueOnce(new Error('synthetic unavailable'));+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ expect(r.result.current).toMatchObject({warning:null,checking:false,error:true});+ act(()=>r.result.current.refresh());await settle();expect(r.result.current.error).toBe(false);expect(r.result.current.warning?.availableBytes).toBe(300);+});+test('invalid native values report unavailable',async()=>{+ ports.getDiskUsage.mockResolvedValue(usage(-1));+ const r=renderHook(()=>useRecordingStorage(true));await settle();expect(r.result.current).toMatchObject({warning:null,error:true});+ ports.getDiskUsage.mockResolvedValue(usage(300));ports.getLowDiskGuardConfig.mockResolvedValue({thresholdBytes:NaN});+ act(()=>r.result.current.refresh());await settle();expect(r.result.current).toMatchObject({warning:null,error:true});+});+test('late probes cannot restore warnings after recording resumes',async()=>{+ let finish!:(v:any)=>void;ports.getDiskUsage.mockReturnValueOnce(new Promise(r=>{finish=r}));+ const r=renderHook(({paused})=>useRecordingStorage(paused),{initialProps:{paused:true}});await settle();expect(r.result.current.checking).toBe(true);+ r.rerender({paused:false});await act(async()=>finish(usage(0)));+ expect(r.result.current).toMatchObject({warning:null,checking:false,error:false});+});+test('late old-directory probes cannot overwrite a new healthy result',async()=>{+ let finish!:(v:any)=>void;ports.getDiskUsage.mockReturnValueOnce(new Promise(r=>{finish=r}));+ const r=renderHook(()=>useRecordingStorage(true));await settle();+ ports.settings.dataDir=saved;ports.getDataDir.mockResolvedValue(saved);ports.getActiveDataDir.mockResolvedValue({status:'ok',data:saved});+ r.rerender();expect(r.result.current.warning).toBeNull();await settle();+ await act(async()=>finish(usage(0)));expect(r.result.current).toMatchObject({warning:null,checking:false,error:false});+});