YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover workflow save receipts and context transport
louis5eba88efc07e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 94a66675a..08fcb440b 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 59 git-mined regressions. See+The current app corpus contains 60 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -496,3 +496,21 @@ predicate logic and distinguishing missing source from behavior failure.Only the error module is the oracle. These synthetic function outcomes do notprove rendered UI, actual retry dispatch, TLS recovery or provider availability.No network calls, model trials or native builds run.++## Workflow save receipts and readable context++`app-workflow-save-receipt-boundary` drives the registered workflow tool against+a synthetic loopback recorder. Fourteen outcomes cover rejected operations,+capability and abort boundaries, exact literal payloads, compact context indexes,+owned draft selectors, private snapshots readable through the real transport+normalizer, and successful save receipts that survive a failed follow-up read.+The parent fails eleven intended outcomes and preserves three; reference and+selected current source pass all fourteen.++Run `bun test evals/coding-agent/calibrate-workflow-receipt.test.js`. Eleven+controls reject unused fixes, swallowed errors, oversized inline context,+world-readable snapshots, discarded remaining state and repeated writes, while+accepting equivalent internal naming. Missing imports remain setup failures.+Only the extension is the historical oracle; the existing public transport+normalizer runs unchanged. These outcomes do not prove native persistence,+real agent repair choices, model performance or enforced process isolation.diff --git a/evals/coding-agent/calibrate-workflow-receipt.test.js b/evals/coding-agent/calibrate-workflow-receipt.test.jsnew file mode 100644index 000000000..0d055c96a--- /dev/null+++ b/evals/coding-agent/calibrate-workflow-receipt.test.js@@ -0,0 +1,32 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'workflow-receipt-calibration-'));+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-workflow-save-receipt-boundary');+const source='crates/screenpipe-core/assets/extensions/workflow-workspace.ts';+const show=ref=>execFileSync('git',['show',`${ref}:${source}`],{cwd:repo,encoding:'utf8'}),fixed=show(item.oracle_ref),broken=show(item.base_ref);+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,body,unused){+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',item.base_ref,source,'crates/screenpipe-core/assets/extensions/lib/glm-protocol.ts'],{cwd:repo,maxBuffer:32*1024*1024})});+ if(body===null)rmSync(join(cwd,source));else writeFileSync(join(cwd,source),body);+ if(unused)writeFileSync(join(cwd,source+'.unused.ts'),unused);+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ return spawnSync(process.execPath,['test',item.grader.fixtures[0].destination_path],{cwd,encoding:'utf8',timeout:15000,env:{PATH:process.env.PATH,HOME:cwd,NO_COLOR:'1'}});+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toMatch(/Cannot find|ModuleNotFound|Export named .* not found/);expect(r.stderr).toContain('Ran 14 tests across 1 file');expect(r.stderr).not.toMatch(/\n \d+ errors?\n/);}+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}+test('parent fails eleven workflow outcomes and preserves three',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('11 fail');expect(r.stderr).toContain('3 pass');});+test('reference passes fourteen outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');});+test('unused extension cannot repair the registered tool',()=>fails(grade('unused',broken,fixed)));+test('equivalent internal request function name passes',()=>expect(grade('equivalent',fixed.replaceAll(/\bcall\b/g,'requestWorkspace')).status).toBe(0));+test('returning error-shaped success instead of rejecting is caught',()=>fails(grade('swallow',replace(fixed,'throw new Error(`Workflow operation failed: ${error.message}`);','return {isError:true,content:[{type:"text",text:error.message}],details:{saved:false}};'))));+test('old large inline threshold loses medium transport context',()=>fails(grade('threshold',replace(fixed,'text.length > 8_000','text.length > 24_000'))));+test('world-readable snapshots are rejected',()=>fails(grade('permissions',replace(fixed,'{mode:0o600}','{mode:0o644}'))));+test('discarding remaining-work state is rejected',()=>fails(grade('remaining',fixed.replaceAll('remaining:', 'discarded:'))));+test('successful save must survive a failed follow-up read',()=>fails(grade('follow-up',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','throw new Error("follow-up read failed"); result = {...result, remaining: {unavailable:true, next:'))));+test('retrying successful mutation after read failure is rejected',()=>fails(grade('repeat-save',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','await call("/workflows/workspace", {...input,task}); result = {...result, remaining: {unavailable:true, next:'))));+test('missing extension remains an import failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex d19b05177..94c363ca4 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.10",+ "dataset_version": "2026-09-20.11","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1948,6 +1948,38 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/certificate-error.test.ts"}+ },+ {+ "id": "app-workflow-save-receipt-boundary",+ "name": "Preserve workflow save receipts and expose actual operation failure",+ "source": {+ "kind": "git_regression",+ "fix_commit": "1f8ecb0858b63e82dc324050338953e7e4b2eb17"+ },+ "base_ref": "52706da629809407bf151c09294b677b65a32a78",+ "oracle_ref": "1f8ecb0858b63e82dc324050338953e7e4b2eb17",+ "oracle_paths": [+ "crates/screenpipe-core/assets/extensions/workflow-workspace.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "workflow",+ "retry-safety",+ "context-integrity"+ ],+ "prompt": "The workflow_workspace extension can report failed operations as successful tool results, omit remaining work after saves and lose medium-size context through the private transport. Execute the actual registered tool. Conflicts, missing capability, aborted operations and missing draft selectors must reject; missing drafts should report exact open ids and titles owned by the current task without substituting another draft or retrying a mutation. Preserve structured literal payloads and the current task. A successful save must retain its receipt and expose freshly read remaining work/current revisions in remaining (revision, catalogRevision, cycleStatus, canFinish, openDrafts); if that follow-up read fails, return the successful receipt with remaining.unavailable true and do not repeat the save. Keep context indexes bounded, explicit selectors complete and oversized context in a private readable snapshot with exact data and revision fields, including context between the private transport limit and the old larger inline threshold. Preserve compact index behavior and full draft/catalog reads. Use synthetic loopback recorder responses and temporary files; no real recorder, workflow publication, agent dispatch, model or native build. These are extension-boundary outcomes, not backend durability or full stop/resume behavior.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/workflow-receipt.fixture.ts",+ "destination_path": "crates/screenpipe-core/assets/extensions/workflow-workspace.eval.test.ts"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test crates/screenpipe-core/assets/extensions/workflow-workspace.eval.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/workflow-receipt.fixture.ts b/evals/coding-agent/graders/workflow-receipt.fixture.tsnew file mode 100644index 000000000..0ddbe5d23--- /dev/null+++ b/evals/coding-agent/graders/workflow-receipt.fixture.ts@@ -0,0 +1,128 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, beforeEach, expect, test } from "bun:test";+import { mkdtempSync, writeFileSync, readFileSync, statSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import extension from "./workflow-workspace";+import { normalizeGlmRequest } from "./lib/glm-protocol";+const compactGlmToolResultText = (content: string): string => normalizeGlmRequest({model:"synthetic", tools:[{type:"function",function:{name:"subagent"}}], messages:[{role:"tool",content}]}).messages[0].content;+async function checked(promise: Promise<any>) { await expect(promise).resolves.toBeDefined(); return promise; }+let cwd:string, dir:string, originalTask:string|undefined, tool:any, server:any, requests:any[];+beforeEach(()=>{+ cwd=process.cwd();originalTask=process.env.SCREENPIPE_PIPE_NAME;+ dir=mkdtempSync(join(tmpdir(),"workflow-tool-"));process.chdir(dir);+ process.env.SCREENPIPE_PIPE_NAME="workflow-review";requests=[];+ server=Bun.serve({port:0,hostname:"127.0.0.1",fetch:async(req)=>{+ expect(req.headers.get("authorization")).toBe("Bearer fixture-capability");+ const body=req.method==="POST"?await req.json():null;requests.push(body);+ return Response.json({saved:true,revision:2});+ }});+ writeFileSync(join(dir,".screenpipe-permissions.json"),JSON.stringify({api_base:`http://127.0.0.1:${server.port}`,pipe_token:"fixture-capability"}));+ extension({registerTool:(t:any)=>{tool=t}} as any);+});+afterEach(()=>{server.stop(true);process.chdir(cwd);rmSync(dir,{recursive:true});if(originalTask===undefined)delete process.env.SCREENPIPE_PIPE_NAME;else process.env.SCREENPIPE_PIPE_NAME=originalTask;});+test("structured saves serialize quotes and JSON-like source content without shell construction",async()=>{+ const payload={title:'Literal ]} and "quote"',sources:[{quote:'$() `command` \\ \n trailing ]}'}]};+ const result=await checked(tool.execute("id",{action:"handoff",expected_revision:1,draft_id:"a",assignee:"workflow-review",note:"checked",payload},new AbortController().signal));+ expect(result.isError).not.toBe(true);expect(requests[0].payload).toEqual(payload);expect(requests[0].task).toBe("workflow-review");+});+test("server conflicts are errors, not success text",async()=>{+ server.reload({fetch:()=>Response.json({error:"Workspace changed"},{status:409})});+ await expect(tool.execute("id",{action:"publish",draft_id:"a"},new AbortController().signal)).rejects.toThrow("Workspace changed");+});+test("missing capability never falls back to a broad owner token",async()=>{+ writeFileSync(join(dir,".screenpipe-permissions.json"),"{}");+ await expect(tool.execute("id",{action:"context"},new AbortController().signal)).rejects.toThrow("Local recorder capability unavailable");+ expect(requests).toHaveLength(0);+});+test("an already-aborted task cannot save",async()=>{+ const abort=new AbortController();abort.abort();+ await expect(tool.execute("id",{action:"publish",draft_id:"a"},abort.signal)).rejects.toThrow();+ expect(requests).toHaveLength(0);+});++function contextFixture(payload:any = {title:"Invoice review",evidence:[{quote:"Full source"}]}) {+ const draft={id:"draft-a",status:"open",assignee:"workflow-review",payload,history:[{note:"Check the source"}]};+ const workflow={id:"wf-a",title:"Existing invoice review",stages:[{procedure:[{text:"Existing action"}]}]};+ server.reload({fetch:(req:Request)=>Response.json(new URL(req.url).pathname==="/workflows/workspace"+ ? {workspace:{revision:7,cycle:{id:"cycle",notes:{large:"not in index"}},drafts:{"draft-a":draft}},catalogRevision:4,ready:true}+ : {workflows:[workflow],profile:{},outputContract:"Exact output contract"})});+ return {draft,workflow};+}+test("context index avoids hiding full drafts behind shared harness truncation",async()=>{+ contextFixture({title:"Invoice review",evidence:"x".repeat(40000)});+ const result=await checked(tool.execute("id",{action:"context"},new AbortController().signal));+ const index=JSON.parse(result.content[0].text);+ expect(index.revision).toBe(7);expect(index.catalogRevision).toBe(4);+ expect(index.drafts[0].id).toBe("draft-a");expect(index.drafts[0].payload).toBeUndefined();+ expect(index.workflows[0].stages).toBeUndefined();expect(index.cycle.notes).toBeUndefined();+ expect(result.content[0].text.length).toBeLessThan(24000);+});+test("explicit context selectors preserve the exact draft and catalog record",async()=>{+ const {draft,workflow}=contextFixture();+ const call=async(input:any)=>JSON.parse((await tool.execute("id",{action:"context",...input},new AbortController().signal)).content[0].text);+ expect((await call({draft_id:"draft-a"})).draft).toEqual(draft);+ expect((await call({workflow_id:"wf-a"})).workflow).toEqual(workflow);+ expect((await call({draft_id:"draft-a"})).outputContract).toBe("Exact output contract");+ await expect(tool.execute("id",{action:"context",draft_id:"missing"},new AbortController().signal)).rejects.toThrow("not found");+});+test("oversized selected context is preserved in a private readable snapshot",async()=>{+ const {draft}=contextFixture({title:"Invoice review",evidence:"exact source ".repeat(4000)});+ const result=await checked(tool.execute("id",{action:"context",draft_id:"draft-a"},new AbortController().signal));+ const pointer=JSON.parse(result.content[0].text);+ expect(result.content[0].text.length).toBeLessThan(24000);+ expect(JSON.parse(readFileSync(pointer.path,"utf8")).draft).toEqual(draft);+ expect(statSync(pointer.path).mode & 0o777).toBe(0o600);+ expect(pointer.revision).toBe(7);expect(pointer.catalogRevision).toBe(4);+});+test("medium context survives the Private transport with an exact snapshot",async()=>{+ const {draft}=contextFixture({title:"Invoice review",evidence:"source fragment ".repeat(800)});+ const result=await checked(tool.execute("id",{action:"context",draft_id:"draft-a"},new AbortController().signal));+ // The native failure lived between the old 24K inline limit and Private's+ // 8K tool-text limit: its JSON became unparseable, with no snapshot to read.+ const delivered=compactGlmToolResultText(result.content[0].text);+ expect(()=>JSON.parse(delivered)).not.toThrow();+ const pointer=JSON.parse(delivered);+ expect(delivered).toBe(result.content[0].text);+ expect(JSON.parse(readFileSync(pointer.path,"utf8")).draft).toEqual(draft);+ expect(pointer.revision).toBe(7);+ expect(pointer.catalogRevision).toBe(4);+});++test("mistyped draft selectors return exact owned ids without substituting or writing",async()=>{+ contextFixture();+ const error=await tool.execute("id",{action:"context",draft_id:"draft-b"},new AbortController().signal).catch((e:Error)=>e);+ expect(error).toBeInstanceOf(Error);expect(error.message).toContain("draft-a");expect(error.message).toContain("Invoice review");+});+test.each(["handoff", "publish", "reject"])("unknown draft %s returns exact candidates but never retries the mutation",async(action)=>{+ let writes=0;+ server.reload({fetch:(req:Request)=>{+ if(req.method==="POST"){writes++;return Response.json({error:"Draft not found."},{status:409});}+ return Response.json({workspace:{drafts:{a:{id:"exact-id",status:"open",assignee:"workflow-review",payload:{title:"Review invoice"}},b:{id:"other-owner",status:"open",assignee:"workflow-deepen"}}}});+ }});+ const error=await tool.execute("id",{action,draft_id:"typo",note:"repair"},new AbortController().signal).catch((e:Error)=>e);+ expect(error).toBeInstanceOf(Error);expect(writes).toBe(1);+ expect(error.message).toContain("exact-id");+ expect(error.message).not.toContain("other-owner");+});+++test("publication returns remaining work with current revisions, distinct from its save receipt",async()=>{+ server.reload({fetch:(req:Request)=>Response.json(req.method==="POST"+ ? {revision:8,changes:{created:1},checkedThrough:"previous-window"}+ : {workspace:{revision:12,cycle:{status:"running"},drafts:{done:{status:"published"}}},catalogRevision:8,canFinish:true})});+ const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));+ expect(result.isError).not.toBe(true);+ expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{revision:12,catalogRevision:8,cycleStatus:"running",canFinish:true,openDrafts:[]}});+});+test("a failed state read after saving preserves the successful receipt",async()=>{+ let writes=0;+ server.reload({fetch:(req:Request)=>{+ if(req.method==="POST"){writes++;return Response.json({revision:8,changes:{created:1}});}+ return Response.json({error:"Recorder unavailable"},{status:503});+ }});+ const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));+ expect(result.isError).not.toBe(true);expect(writes).toBe(1);+ expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{unavailable:true}});+});