Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve maintained MCP workflow retrieval

louis0932e5b4ebe2
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 4f21f55a7..2fd5ead3f 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 63 git-mined regressions. See
+The current app corpus contains 64 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -587,3 +587,25 @@ establish native migration/recovery durability, disk or NAS behavior, recording
continuity, native authorization, deployment reach or enforced trial isolation.
The native rollout-policy finding remains open. No native build or production
operation is performed; runtime links and fixtures appear only at grading time.
+
+## Maintained workflow retrieval
+
+`mcp-maintained-workflow-retrieval` preserves the escaped #7161 failure: listing
+maintained workflows succeeded while retrieval rejected their UUID-based IDs.
+The actual workflow tool function runs with a synthetic API port. Seventeen
+outcomes cover maintained and legacy IDs, automation-detail defaults and explicit
+options, list-to-detail continuity, complete JSON, safe query encoding, pagination,
+malformed IDs and injection, refusal before requests, provider failures and a
+later explicit retry. The parent fails seven intended outcomes and preserves ten;
+the workflow-tools-only reference and current source pass all seventeen.
+
+Run `bun test evals/coding-agent/calibrate-workflow-ids.test.js`. Fourteen controls
+reject unused fixes, broad ID acceptance, UUID-only compatibility loss, wrong
+identity, option/validation bypasses, empty success and blanket refusal; equivalent
+private constant naming passes. Missing source remains a setup failure.
+
+No installed packages or dependency links are required. The hidden fixture is
+materialized only for grading. These are corpus and grader checks, not model
+trials, native catalog persistence, stdio dispatch, authentication or bounded
+stalled-response evidence. No real recordings, credentials or provider calls
+are used; execution isolation and model improvement remain unproven.
diff --git a/evals/coding-agent/calibrate-workflow-ids.test.js b/evals/coding-agent/calibrate-workflow-ids.test.js
new file mode 100644
index 000000000..e67ac6fb8
--- /dev/null
+++ b/evals/coding-agent/calibrate-workflow-ids.test.js
@@ -0,0 +1,29 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),path='packages/screenpipe-mcp/src/workflow-tools.ts';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='mcp-maintained-workflow-retrieval');
+const source=ref=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+const broken=source(item.base_ref),fixed=source(item.oracle_ref),fixture='evals/coding-agent/graders/mcp-workflow-ids.test.js';
+const grader=readFileSync(join(repo,fixture)),root=mkdtempSync(join(tmpdir(),'workflow-ids-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,code,unused){const cwd=join(root,name);mkdirSync(dirname(join(cwd,path)),{recursive:true});if(code!==null)writeFileSync(join(cwd,path),code);if(unused)writeFileSync(join(cwd,dirname(path),'unused.ts'),unused);mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),grader);return spawnSync(process.execPath,['test','./'+fixture],{cwd,encoding:'utf8',timeout:10_000,env:{PATH:dirname(process.execPath)}});}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toMatch(/[1-9][0-9]* fail/);expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}
+function change(old,next){expect(fixed.split(old)).toHaveLength(2);return fixed.replace(old,next);}
+test('broken parent fails seven intended outcomes, preserving ten',()=>{const r=grade('broken',broken);fails(r);expect(r.stderr).toContain('7 fail');expect(r.stderr).toContain('10 pass');});
+test('reference passes seventeen outcomes',()=>{const r=grade('fixed',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('17 pass');});
+test('equivalent private constant name passes',()=>expect(grade('equivalent',fixed.replaceAll('WORKFLOW_ID','VALID_CATALOG_ID')).status).toBe(0));
+test('unused correct module cannot repair retrieval',()=>fails(grade('unused',broken,fixed)));
+test('accepting every ID fails malformed request boundaries',()=>fails(grade('no-id-guard',change('typeof args.id !== "string" || args.id.trim() !== args.id || !WORKFLOW_ID.test(args.id)','false'))));
+test('UUID-only fix fails preserved legacy retrieval',()=>fails(grade('uuid-only',change('[0-9a-f]{64}|',''))));
+test('wrong identity fails response and request outcomes',()=>fails(grade('wrong-id',change('endpoint = `/workflows/${args.id}?','endpoint = `/workflows/wf-${"b".repeat(64)}?'))));
+test('ignoring explicit false fails automation boundary',()=>fails(grade('auto-true',change('args.include_automation !== false','true'))));
+test('nonboolean options must not reach backend',()=>fails(grade('nonboolean',change('if (args.include_automation !== undefined && typeof args.include_automation !== "boolean")','if (false)'))));
+test('removing pagination validation fails preserved refusal',()=>fails(grade('pagination',change('throw new Error(`Invalid ${key}`);\n params.set','{}\n params.set'))));
+test('reporting provider HTTP failure as success fails',()=>fails(grade('http-success',change('if (!response.ok)','if (false)'))));
+test('response-only empty catalog fails successful retrieval',()=>fails(grade('empty-success',change('JSON.stringify(data)','JSON.stringify({ data: [] })'))));
+test('blanket refusal fails valid reads',()=>fails(grade('deny',change('let endpoint: string;','throw new Error("denied"); let endpoint: string;'))));
+test('missing module is setup failure, never behavioral evidence',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 6db7a7214..5bf334e84 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-20.14",
+ "dataset_version": "2026-09-21.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -2080,6 +2080,38 @@
],
"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts components/storage-migration-prompt.test.tsx --maxWorkers=1 --minWorkers=1"
}
+ },
+ {
+ "id": "mcp-maintained-workflow-retrieval",
+ "name": "Retrieve maintained workflows without weakening ID request boundaries",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "9c38b194f5e324b1fcf78fbc3ece1adf4af29274"
+ },
+ "base_ref": "ea0a6852e1bbb89b87535a1618a44d0bc752640b",
+ "oracle_ref": "9c38b194f5e324b1fcf78fbc3ece1adf4af29274",
+ "oracle_paths": [
+ "packages/screenpipe-mcp/src/workflow-tools.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "mcp",
+ "workflow",
+ "input-validation"
+ ],
+ "prompt": "The MCP workflow tools can list maintained workflows but reject their returned IDs when retrieving details. Make maintained wf-prefixed lowercase hexadecimal UUID IDs retrievable while preserving legacy wf-prefixed 64-character lowercase hexadecimal IDs. Preserve the exact workflow identity and complete returned JSON, including unknown nested fields and evidence. The automation-detail option defaults to true and respects explicit true or false; nonboolean options must fail before a backend call. Reject malformed IDs, wrong lengths, nonhex content, surrounding whitespace and path/query/fragment or encoded-path injection before requesting anything. Preserve list search encoding, empty catalogs and safe integer pagination (limit 1 through 100, offset zero or greater), rejecting invalid inputs and unknown tools without requests. Each valid invocation makes one backend read. Non-success HTTP responses, transport failures and malformed JSON must surface failure instead of an empty success; a later explicit retry can recover. Exercise the actual workflow tool function using a synthetic API port. Do not access real recordings, credentials or providers. Native catalog persistence, stdio transport, authorization and stalled-response timeouts are outside this task.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/mcp-workflow-ids.test.js",
+ "destination_path": "evals/coding-agent/graders/mcp-workflow-ids.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/mcp-workflow-ids.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/mcp-workflow-ids.test.js b/evals/coding-agent/graders/mcp-workflow-ids.test.js
new file mode 100644
index 000000000..141bb0a7e
--- /dev/null
+++ b/evals/coding-agent/graders/mcp-workflow-ids.test.js
@@ -0,0 +1,42 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { readWorkflowTool } from '../../../packages/screenpipe-mcp/src/workflow-tools.ts';
+const ids = ['wf-12345678-1234-4234-8234-123456789abc', 'wf-abcdef01-2345-6789-abcd-ef0123456789', `wf-${'a'.repeat(64)}`];
+for (const id of ids) for (const include of [undefined, true, false]) {
+ test(`retrieves ${id} with automation ${include}`, async () => {
+ const seen=[];const payload={id,workflow:{stages:[{name:'Synthetic review',unknownField:{keep:true}}]},automationEvidence:[{status:'no_captured_frame'}]};
+ const result=await readWorkflowTool('get-workflow',{id,...(include===undefined?{}:{include_automation:include})},async endpoint=>{seen.push(endpoint);return Response.json(payload);});
+ expect(seen).toHaveLength(1);const url=new URL(seen[0],'http://fixture.invalid');
+ expect(url.pathname).toBe(`/workflows/${id}`);expect(url.searchParams.get('include_automation')).toBe(String(include!==false));expect([...url.searchParams.keys()]).toEqual(['include_automation']);
+ expect(result.content).toHaveLength(1);expect(result.content[0].type).toBe('text');expect(JSON.parse(result.content[0].text)).toEqual(payload);
+ });
+}
+test('lists workflows and retrieves the maintained ID returned by that list',async()=>{
+ const requests=[];const id=ids[0];const call=async endpoint=>{requests.push(endpoint);return Response.json(endpoint.startsWith('/workflows?')?{data:[{id,title:'Synthetic workflow'}]}:{id,workflow:{stages:[]}});};
+ const list=await readWorkflowTool('list-workflows',{},call);const chosen=JSON.parse(list.content[0].text).data[0].id;
+ const detail=await readWorkflowTool('get-workflow',{id:chosen},call);expect(JSON.parse(detail.content[0].text).id).toBe(id);expect(requests).toHaveLength(2);
+});
+test('preserves empty catalogs and safely encodes search and pagination',async()=>{
+ for(const args of [{},{q:'a&offset=999 / ? # +',limit:100,offset:0},{limit:1,offset:900}]){
+ const seen=[];const result=await readWorkflowTool('list-workflows',args,async endpoint=>{seen.push(endpoint);return Response.json({data:[],pagination:{total:0}});});
+ expect(seen).toHaveLength(1);const url=new URL(seen[0],'http://fixture.invalid');expect(url.pathname).toBe('/workflows');
+ expect(Object.fromEntries(url.searchParams)).toEqual(Object.fromEntries(Object.entries(args).map(([k,v])=>[k,String(v)])));expect(JSON.parse(result.content[0].text)).toEqual({data:[],pagination:{total:0}});
+ }
+});
+test('rejects malformed IDs and path/query injection before any backend request',async()=>{
+ const invalid=[undefined,null,7,'wf-','wf-1234',`wf-${'a'.repeat(63)}`,`wf-${'a'.repeat(65)}`,'wf-12345678-1234-4234-8234-123456789abz','wf-12345678123442348234123456789abc',`${ids[0]}/../catalog`,`${ids[0]}?include_automation=false`,`${ids[0]}#fragment`,`${ids[0]}%2f..`,`${ids[0]}\n`,`${ids[0]} `,` ${ids[0]}`];
+ for(const id of invalid){let calls=0;await expect(readWorkflowTool('get-workflow',{id},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}
+});
+test('rejects nonboolean automation options before any backend request',async()=>{
+ for(const include_automation of ['false',0,1,null,{}]){let calls=0;await expect(readWorkflowTool('get-workflow',{id:ids[2],include_automation},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}
+});
+test('rejects invalid list inputs and unknown tools before requests',async()=>{
+ for(const args of [{q:7},{limit:0},{limit:101},{limit:1.5},{limit:'1'},{offset:-1},{offset:1.5},{offset:Number.MAX_SAFE_INTEGER+1}]){let calls=0;await expect(readWorkflowTool('list-workflows',args,async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);}
+ let calls=0;await expect(readWorkflowTool('execute-workflow',{id:ids[2]},async()=>{calls++;return Response.json({});})).rejects.toThrow();expect(calls).toBe(0);
+});
+for(const failure of ['http','transport','json'])test(`provider ${failure} failure is not an empty success; explicit retry can recover`,async()=>{
+ let calls=0;const call=async()=>{calls++;if(calls===1){if(failure==='transport')throw Error('synthetic offline');if(failure==='http')return new Response('{}',{status:503});return new Response('{',{status:200});}return Response.json({id:ids[2],workflow:{stages:[]}});};
+ await expect(readWorkflowTool('get-workflow',{id:ids[2]},call)).rejects.toThrow();expect(calls).toBe(1);
+ const result=await readWorkflowTool('get-workflow',{id:ids[2]},call);expect(calls).toBe(2);expect(JSON.parse(result.content[0].text).id).toBe(ids[2]);
+});