Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify learned skill save identity and content

Louis Beaumontf43a075d0e18
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 44214bbec..c5f00452e 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1473,3 +1473,16 @@ must remain infrastructure errors, not evidence of the historical defect.
`app-enterprise-local-policy-waits` runs the actual enterprise authentication hook with synthetic HTTP, settings and native-command boundaries. Fourteen outcomes cover six stalled local operations, late install metadata, rejected/expired credentials, credential choice, account-only access, recording pause, native denial and healthy policy persistence. The task allows ten seconds for one stalled operation; it does not require a particular timer helper or warning message.
Run `bun test evals/coding-agent/calibrate-enterprise-policy-waits.test.js` to check the broken parent, historical reference, current hook, equivalent helper names, disconnected correct source, omitted persistence, excessive wait, native/pause bypasses and missing-source infrastructure failure. These checks do not establish native recording enforcement, real disk durability, host isolation or model performance.
+
+`app-learning-save-readback` runs the actual opted-in learning extension with
+synthetic service replies and temporary local state. Five outcomes reject a
+mismatched saved identity, origin or content despite matching response hashes.
+Seven nearby outcomes cover hash failures, uncertain writes, restart protection,
+successful creation, whitespace normalization and owned updates. The parent fails
+five and preserves seven; the reference and current source pass all twelve.
+Run `bun test evals/coding-agent/calibrate-learning-save-readback.test.js` for ten
+controls, including equivalent helper naming, unused correct code, blanket save
+refusal, lost reports, ignored hashes, lost pending state and missing source.
+Fixtures appear only at grading. This checks the extension and its local receipts;
+real skill-store durability, native authorization, context privacy filtering,
+execution isolation and model performance remain outside this evidence.
diff --git a/evals/coding-agent/calibrate-learning-save-readback.test.js b/evals/coding-agent/calibrate-learning-save-readback.test.js
new file mode 100644
index 000000000..e336f73a1
--- /dev/null
+++ b/evals/coding-agent/calibrate-learning-save-readback.test.js
@@ -0,0 +1,33 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..'), source='crates/screenpipe-core/assets/extensions/skill-learning.ts';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-learning-save-readback');
+const root=mkdtempSync(join(tmpdir(),'calibrate-learning-readback-')),receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(cwd,from,to){const p=join(cwd,source),s=readFileSync(p,'utf8');expect(s.includes(from)).toBe(true);writeFileSync(p,s.replaceAll(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(dirname(join(cwd,source)),{recursive:true});
+ writeFileSync(join(cwd,source),execFileSync('git',['show',`${ref}:${source}`],{cwd:repo}));
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');expect(r.stderr).toContain('0 fail');}
+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('expect(');expect(r.stderr).not.toMatch(/Cannot find module|error: Unexpected|SyntaxError/);}
+test('broken parent exposes five false successes and preserves seven outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('7 pass');});
+test('historical reference passes',()=>passes(grade('reference')));
+test('current source passes',()=>passes(grade('current','HEAD')));
+test('equivalent helper naming passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,'saveState','persistLearningReceipt'))));
+test('unused correct code cannot hide the broken caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{writeFileSync(join(cwd,'unused-correct.ts'),readFileSync(join(cwd,source)));writeFileSync(join(cwd,source),execFileSync('git',['show',`${item.base_ref}:${source}`],{cwd:repo}));})));
+test('blanket save refusal fails preserved successful writes',()=>fails(grade('refuse-all',item.oracle_ref,cwd=>replace(cwd,' const current = inventory.get(input.name);',' throw new Error("All writes refused");\n const current = inventory.get(input.name);'))));
+test('lost review report fails successful preservation',()=>fails(grade('no-report',item.oracle_ref,cwd=>replace(cwd,' const output = join(root, "output");',' if (name === "latest-change.md") return;\n const output = join(root, "output");'))));
+test('ignoring returned hashes fails uncertain-write checks',()=>fails(grade('ignore-hash',item.oracle_ref,cwd=>replace(cwd,'if (!skill?.sha256 || saved?.sha256 !== skill.sha256','if (false'))));
+test('clearing pending evidence fails durable uncertainty',()=>fails(grade('lose-pending',item.oracle_ref,cwd=>replace(cwd,'async function saveState() { await localFile("learning-state.json", JSON.stringify(state)); }','async function saveState() { const {pending, ...rest} = state; await localFile("learning-state.json", JSON.stringify(rest)); }'))));
+test('missing active source remains an infrastructure failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,source)));expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index fafd079a1..c3b8d018b 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4930,6 +4930,39 @@
],
"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts lib/hooks/__tests__/eval-enterprise-policy-waits.test.tsx"
}
+ },
+ {
+ "id": "app-learning-save-readback",
+ "name": "Verify the saved skill before reporting learning success",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "ebfae02d4464e87983c1516bb1f4e95a40197397",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7434"
+ },
+ "base_ref": "fffff2a42ef5584969e7dd680b54bafde59803c4",
+ "oracle_ref": "ebfae02d4464e87983c1516bb1f4e95a40197397",
+ "oracle_paths": [
+ "crates/screenpipe-core/assets/extensions/skill-learning.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "local-skills",
+ "write-verification",
+ "preserved-behavior"
+ ],
+ "prompt": "The opted-in learning task can report a successful skill save even when the service stored a different method or identity. Repair the active learning extension so a save succeeds only when read-back has matching nonempty write/read hashes and the requested key, name, agent origin, description and instructions. Accept the storage service trimming surrounding whitespace. A mismatched result or failed write must report an error, keep a durable pending receipt, leave consumed evidence, ownership and change count unchanged, and avoid a success report or another write in that run. A pending receipt must also block a save after restart. Preserve successful creation and updating of an owned skill, the existing optimistic concurrency hash, evidence consumption, ownership, one-change limit and the review report including the previous method for updates. Use synthetic activity and service replies; no real recordings or private data are needed.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/learning-save-readback.test.ts",
+ "destination_path": "evals/coding-agent/graders/learning-save-readback.test.ts"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/learning-save-readback.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/learning-save-readback.test.ts b/evals/coding-agent/graders/learning-save-readback.test.ts
new file mode 100644
index 000000000..c09bfb793
--- /dev/null
+++ b/evals/coding-agent/graders/learning-save-readback.test.ts
@@ -0,0 +1,61 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, expect, test } from 'bun:test';
+import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import learning from '../../../crates/screenpipe-core/assets/extensions/skill-learning';
+const proposal = { name:'screenpipe-learned-brief-check', description:'Check a brief before saving', instructions:'For a recurring brief, verify the source and unresolved action. Stop when the source is ambiguous. Confirm each retained action has support.', checks:['Intended use: verify a supported action.', 'Counterexample: leave an unrelated task alone.', 'Privacy: keep personal identities out of the method.'] };
+const originalFetch = globalThis.fetch, roots: string[] = [];
+afterEach(async()=>{globalThis.fetch=originalFetch;for(const root of roots.splice(0))await rm(root,{recursive:true,force:true});});
+async function harness(options: any = {}) {
+ const root=await mkdtemp(join(tmpdir(),'eval-learning-readback-'));roots.push(root);
+ await writeFile(join(root,'.screenpipe-learning-config.json'),JSON.stringify({port:3130}));
+ await writeFile(join(root,'.screenpipe-permissions.json'),JSON.stringify({pipe_token:'synthetic-eval-token'}));
+ const previous=options.patch?{...proposal,key:proposal.name,origin:'agent',sha256:'old-sha',instructions:'Previous short method.'}:null;
+ const initial=options.state??{used:[],owned:previous?{[proposal.name]:'old-sha'}:{},changes:0};
+ await mkdir(join(root,'output'));await writeFile(join(root,'output/learning-state.json'),JSON.stringify(initial));
+ const hooks=new Map(),tools=new Map(),writes:any[]=[];let stored:any=previous;
+ globalThis.fetch=(async(url:any,init:any)=>{
+ expect(new URL(String(url)).origin).toBe('http://127.0.0.1:3130');
+ const path=new URL(String(url)).pathname, body=init.body?JSON.parse(init.body):{};
+ if(path==='/search')return Response.json({data:[{content:{text:'A synthetic brief omitted a supported action on one occasion.',timestamp:'2026-01-01T00:00:00Z'}},{content:{text:'Another synthetic brief needed its source checked on a separate occasion.',timestamp:'2026-01-02T00:00:00Z'}}]});
+ expect(path).toBe('/agent/skills/manage');
+ if(body.action==='list')return Response.json({skills:previous?[previous]:[]});
+ if(body.action==='read')return Response.json({skill:stored});
+ expect(['create','patch']).toContain(body.action);writes.push(body);
+ if(options.writeFailure)return Response.json({error:'synthetic unavailable'},{status:503});
+ const saved={...body,name:body.name,key:body.name,origin:'agent',description:body.description.trim(),instructions:body.instructions.trim(),sha256:'new-sha',...options.saved};
+ stored={...saved,...options.readback};return Response.json({skill:saved});
+ }) as any;
+ learning({on:(n:string,fn:any)=>hooks.set(n,fn),registerTool:(t:any)=>tools.set(t.name,t),setActiveTools:()=>{}} as any);
+ await hooks.get('session_start')({}, {cwd:root});
+ const call=async(n:string,input:any={})=>{const r=await tools.get(n).execute('synthetic-call',input);return {...JSON.parse(r.content[0].text),isError:r.isError};};
+ const prepare=async()=>{await call('learning_inventory');if(previous)await call('learning_inventory',{name:proposal.name});return (await call('learning_context',{source:'activity'})).items.map((x:any)=>x.ref);};
+ const state=async()=>JSON.parse(await readFile(join(root,'output/learning-state.json'),'utf8'));
+ return {root,writes,call,prepare,state,initial};
+}
+async function uncertain(options:any){
+ const h=await harness(options),evidence=await h.prepare(),input={...proposal,evidence};
+ const r=await h.call('learning_save',input);expect(r.isError).toBe(true);expect(r.saved).toBeUndefined();
+ const state=await h.state();expect(state.pending.name).toBe(proposal.name);expect(state.used).toEqual(h.initial.used);expect(state.owned).toEqual(h.initial.owned);expect(state.changes).toBe(0);
+ expect(await readFile(join(h.root,'output/latest-change.md'),'utf8').catch(()=>null)).toBeNull();
+ expect((await h.call('learning_save',input)).isError).toBe(true);expect(h.writes).toHaveLength(1);
+}
+for(const [field,value] of [['key','screenpipe-learned-other'],['name','screenpipe-learned-other'],['origin','user'],['description','Different trigger'],['instructions','Different method']])test(`matching hashes cannot verify a mismatched ${field}`,()=>uncertain({saved:{[field]:value}}));
+test('changed readback hash stays uncertain',()=>uncertain({readback:{sha256:'other-sha'}}));
+test('empty hashes cannot establish a saved method',()=>uncertain({saved:{sha256:''}}));
+test('provider write failure preserves a pending receipt without retry',()=>uncertain({writeFailure:true}));
+for(const mode of ['create','trim','patch'])test(`verified ${mode} preserves evidence, ownership and report`,async()=>{
+ const h=await harness({patch:mode==='patch'}),evidence=await h.prepare();
+ const input={...proposal,evidence,...(mode==='trim'?{description:` ${proposal.description} `,instructions:`\n${proposal.instructions}\n`}:{})};
+ const result=await h.call('learning_save',input);expect(result.isError).not.toBe(true);expect(result.saved).toBe(proposal.name);
+ expect(h.writes).toHaveLength(1);expect(h.writes[0].action).toBe(mode==='patch'?'patch':'create');if(mode==='patch')expect(h.writes[0].expected_sha256).toBe('old-sha');
+ const state=await h.state();expect(state.pending).toBeUndefined();expect(state.owned[proposal.name]).toBe('new-sha');expect(state.used).toEqual(evidence);expect(state.changes).toBe(1);
+ const report=await readFile(join(h.root,result.report),'utf8');expect(report).toContain(proposal.instructions);if(mode==='patch')expect(report).toContain('Previous short method.');
+ expect((await h.call('learning_save',input)).isError).toBe(true);expect(h.writes).toHaveLength(1);
+});
+test('persisted uncertain write blocks another save after restart',async()=>{
+ const initial={used:[],owned:{},changes:0,pending:{name:proposal.name,evidence:['one','two']}};
+ const h=await harness({state:initial});expect((await h.call('learning_save',{...proposal,evidence:['one','two']})).isError).toBe(true);expect(h.writes).toEqual([]);expect(await h.state()).toEqual(initial);
+});