YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: preserve maintained MCP workflow retrieval
louis0932e5b4ebe2
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 4f21f55a7..2fd5ead3f 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 63 git-mined regressions. See+The current app corpus contains 64 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -587,3 +587,25 @@ establish native migration/recovery durability, disk or NAS behavior, recordingcontinuity, native authorization, deployment reach or enforced trial isolation.The native rollout-policy finding remains open. No native build or productionoperation is performed; runtime links and fixtures appear only at grading time.++## Maintained workflow retrieval++`mcp-maintained-workflow-retrieval` preserves the escaped #7161 failure: listing+maintained workflows succeeded while retrieval rejected their UUID-based IDs.+The actual workflow tool function runs with a synthetic API port. Seventeen+outcomes cover maintained and legacy IDs, automation-detail defaults and explicit+options, list-to-detail continuity, complete JSON, safe query encoding, pagination,+malformed IDs and injection, refusal before requests, provider failures and a+later explicit retry. The parent fails seven intended outcomes and preserves ten;+the workflow-tools-only reference and current source pass all seventeen.++Run `bun test evals/coding-agent/calibrate-workflow-ids.test.js`. Fourteen controls+reject unused fixes, broad ID acceptance, UUID-only compatibility loss, wrong+identity, option/validation bypasses, empty success and blanket refusal; equivalent+private constant naming passes. Missing source remains a setup failure.++No installed packages or dependency links are required. The hidden fixture is+materialized only for grading. These are corpus and grader checks, not model+trials, native catalog persistence, stdio dispatch, authentication or bounded+stalled-response evidence. No real recordings, credentials or provider calls+are used; execution isolation and model improvement remain unproven.diff --git a/evals/coding-agent/calibrate-workflow-ids.test.js b/evals/coding-agent/calibrate-workflow-ids.test.jsnew file mode 100644index 000000000..e67ac6fb8--- /dev/null+++ b/evals/coding-agent/calibrate-workflow-ids.test.js@@ -0,0 +1,29 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),path='packages/screenpipe-mcp/src/workflow-tools.ts';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='mcp-maintained-workflow-retrieval');+const source=ref=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});+const broken=source(item.base_ref),fixed=source(item.oracle_ref),fixture='evals/coding-agent/graders/mcp-workflow-ids.test.js';+const grader=readFileSync(join(repo,fixture)),root=mkdtempSync(join(tmpdir(),'workflow-ids-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,code,unused){const cwd=join(root,name);mkdirSync(dirname(join(cwd,path)),{recursive:true});if(code!==null)writeFileSync(join(cwd,path),code);if(unused)writeFileSync(join(cwd,dirname(path),'unused.ts'),unused);mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),grader);return spawnSync(process.execPath,['test','./'+fixture],{cwd,encoding:'utf8',timeout:10_000,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toMatch(/[1-9][0-9]* fail/);expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function change(old,next){expect(fixed.split(old)).toHaveLength(2);return fixed.replace(old,next);}+test('broken parent fails seven intended outcomes, preserving ten',()=>{const r=grade('broken',broken);fails(r);expect(r.stderr).toContain('7 fail');expect(r.stderr).toContain('10 pass');});+test('reference passes seventeen outcomes',()=>{const r=grade('fixed',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('17 pass');});+test('equivalent private constant name passes',()=>expect(grade('equivalent',fixed.replaceAll('WORKFLOW_ID','VALID_CATALOG_ID')).status).toBe(0));+test('unused correct module cannot repair retrieval',()=>fails(grade('unused',broken,fixed)));+test('accepting every ID fails malformed request boundaries',()=>fails(grade('no-id-guard',change('typeof args.id !== "string" || args.id.trim() !== args.id || !WORKFLOW_ID.test(args.id)','false'))));+test('UUID-only fix fails preserved legacy retrieval',()=>fails(grade('uuid-only',change('[0-9a-f]{64}|',''))));+test('wrong identity fails response and request outcomes',()=>fails(grade('wrong-id',change('endpoint = `/workflows/${args.id}?','endpoint = `/workflows/wf-${"b".repeat(64)}?'))));+test('ignoring explicit false fails automation boundary',()=>fails(grade('auto-true',change('args.include_automation !== false','true'))));+test('nonboolean options must not reach backend',()=>fails(grade('nonboolean',change('if (args.include_automation !== undefined && typeof args.include_automation !== "boolean")','if (false)'))));+test('removing pagination validation fails preserved refusal',()=>fails(grade('pagination',change('throw new Error(`Invalid ${key}`);\n params.set','{}\n params.set'))));+test('reporting provider HTTP failure as success fails',()=>fails(grade('http-success',change('if (!response.ok)','if (false)'))));+test('response-only empty catalog fails successful retrieval',()=>fails(grade('empty-success',change('JSON.stringify(data)','JSON.stringify({ data: [] })'))));+test('blanket refusal fails valid reads',()=>fails(grade('deny',change('let endpoint: string;','throw new Error("denied"); let endpoint: string;'))));+test('missing module is setup failure, never behavioral evidence',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 6db7a7214..5bf334e84 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.14",+ "dataset_version": "2026-09-21.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -2080,6 +2080,38 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts components/storage-migration-prompt.test.tsx --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "mcp-maintained-workflow-retrieval",+ "name": "Retrieve maintained workflows without weakening ID request boundaries",+ "source": {+ "kind": "git_regression",+ "fix_commit": "9c38b194f5e324b1fcf78fbc3ece1adf4af29274"+ },+ "base_ref": "ea0a6852e1bbb89b87535a1618a44d0bc752640b",+ "oracle_ref": "9c38b194f5e324b1fcf78fbc3ece1adf4af29274",+ "oracle_paths": [+ "packages/screenpipe-mcp/src/workflow-tools.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "mcp",+ "workflow",+ "input-validation"+ ],+ "prompt": "The MCP workflow tools can list maintained workflows but reject their returned IDs when retrieving details. Make maintained wf-prefixed lowercase hexadecimal UUID IDs retrievable while preserving legacy wf-prefixed 64-character lowercase hexadecimal IDs. Preserve the exact workflow identity and complete returned JSON, including unknown nested fields and evidence. The automation-detail option defaults to true and respects explicit true or false; nonboolean options must fail before a backend call. Reject malformed IDs, wrong lengths, nonhex content, surrounding whitespace and path/query/fragment or encoded-path injection before requesting anything. Preserve list search encoding, empty catalogs and safe integer pagination (limit 1 through 100, offset zero or greater), rejecting invalid inputs and unknown tools without requests. Each valid invocation makes one backend read. Non-success HTTP responses, transport failures and malformed JSON must surface failure instead of an empty success; a later explicit retry can recover. Exercise the actual workflow tool function using a synthetic API port. Do not access real recordings, credentials or providers. Native catalog persistence, stdio transport, authorization and stalled-response timeouts are outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/mcp-workflow-ids.test.js",+ "destination_path": "evals/coding-agent/graders/mcp-workflow-ids.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/mcp-workflow-ids.test.js"+ }}]}diff --git a/evals/coding-agent/graders/mcp-workflow-ids.test.js b/evals/coding-agent/graders/mcp-workflow-ids.test.jsnew file mode 100644index 000000000..141bb0a7e--- /dev/null+++ b/evals/coding-agent/graders/mcp-workflow-ids.test.js@@ -0,0 +1,42 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { readWorkflowTool } from '../../../packages/screenpipe-mcp/src/workflow-tools.ts';+const ids = ['wf-12345678-1234-4234-8234-123456789abc', 'wf-abcdef01-2345-6789-abcd-ef0123456789', `wf-${'a'.repeat(64)}`];+for (const id of ids) for (const include of [undefined, true, false]) {+ test(`retrieves ${id} with automation ${include}`, async () => {+ const seen=[];const payload={id,workflow:{stages:[{name:'Synthetic review',unknownField:{keep:true}}]},automationEvidence:[{status:'no_captured_frame'}]};+ const result=await readWorkflowTool('get-workflow',{id,...(include===undefined?{}:{include_automation:include})},async endpoint=>{seen.push(endpoint);return Response.json(payload);});+ expect(seen).toHaveLength(1);const url=new URL(seen[0],'http://fixture.invalid');+ expect(url.pathname).toBe(`/workflows/${id}`);expect(url.searchParams.get('include_automation')).toBe(String(include!==false));expect([...url.searchParams.keys()]).toEqual(['include_automation']);+ expect(result.content).toHaveLength(1);expect(result.content[0].type).toBe('text');expect(JSON.parse(result.content[0].text)).toEqual(payload);+ });+}+test('lists workflows and retrieves the maintained ID returned by that list',async()=>{+ const requests=[];const id=ids[0];const call=async endpoint=>{requests.push(endpoint);return Response.json(endpoint.startsWith('/workflows?')?{data:[{id,title:'Synthetic workflow'}]}:{id,workflow:{stages:[]}});};+ const list=await readWorkflowTool('list-workflows',{},call);const chosen=JSON.parse(list.content[0].text).data[0].id;+ const detail=await readWorkflowTool('get-workflow',{id:chosen},call);expect(JSON.parse(detail.content[0].text).id).toBe(id);expect(requests).toHaveLength(2);+});+test('preserves empty catalogs and safely encodes search and pagination',async()=>{+ for(const args of [{},{q:'a&offset=999 / ? # +',limit:100,offset:0},{limit:1,offset:900}]){+ const seen=[];const result=await readWorkflowTool('list-workflows',args,async endpoint=>{seen.push(endpoint);return Response.json({data:[],pagination:{total:0}});});+ expect(seen).toHaveLength(1);const url=new URL(seen[0],'http://fixture.invalid');expect(url.pathname).toBe('/workflows');+ expect(Object.fromEntries(url.searchParams)).toEqual(Object.fromEntries(Object.entries(args).map(([k,v])=>[k,String(v)])));expect(JSON.parse(result.content[0].text)).toEqual({data:[],pagination:{total:0}});+ }+});+test('rejects malformed IDs and path/query injection before any backend request',async()=>{+ const invalid=[undefined,null,7,'wf-','wf-1234',`wf-${'a'.repeat(63)}`,`wf-${'a'.repeat(65)}`,'wf-12345678-1234-4234-8234-123456789abz','wf-12345678123442348234123456789abc',`${ids[0]}/../catalog`,`${ids[0]}?include_automation=false`,`${ids[0]}#fragment`,`${ids[0]}%2f..`,`${ids[0]}\n`,`${ids[0]} `,` ${ids[0]}`];+ for(const id of invalid){let calls=0;await expect(readWorkflowTool('get-workflow',{id},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}+});+test('rejects nonboolean automation options before any backend request',async()=>{+ for(const include_automation of ['false',0,1,null,{}]){let calls=0;await expect(readWorkflowTool('get-workflow',{id:ids[2],include_automation},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}+});+test('rejects invalid list inputs and unknown tools before requests',async()=>{+ for(const args of [{q:7},{limit:0},{limit:101},{limit:1.5},{limit:'1'},{offset:-1},{offset:1.5},{offset:Number.MAX_SAFE_INTEGER+1}]){let calls=0;await expect(readWorkflowTool('list-workflows',args,async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}+ let calls=0;await expect(readWorkflowTool('execute-workflow',{id:ids[2]},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);+});+for(const failure of ['http','transport','json'])test(`provider ${failure} failure is not an empty success; explicit retry can recover`,async()=>{+ let calls=0;const call=async()=>{calls++;if(calls===1){if(failure==='transport')throw Error('synthetic offline');if(failure==='http')return new Response('{}',{status:503});return new Response('{',{status:200});}return Response.json({id:ids[2],workflow:{stages:[]}});};+ await expect(readWorkflowTool('get-workflow',{id:ids[2]},call)).rejects.toThrow();expect(calls).toBe(1);+ const result=await readWorkflowTool('get-workflow',{id:ids[2]},call);expect(calls).toBe(2);expect(JSON.parse(result.content[0].text).id).toBe(ids[2]);+});