Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: supply valid revisions in workflow receipt fixtures

louise3c0728a0271
diff --git a/evals/coding-agent/calibrate-workflow-receipt.test.js b/evals/coding-agent/calibrate-workflow-receipt.test.js
index 0d055c96a..d3d8f80b0 100644
--- a/evals/coding-agent/calibrate-workflow-receipt.test.js
+++ b/evals/coding-agent/calibrate-workflow-receipt.test.js
@@ -30,3 +30,13 @@ test('discarding remaining-work state is rejected',()=>fails(grade('remaining',f
test('successful save must survive a failed follow-up read',()=>fails(grade('follow-up',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','throw new Error("follow-up read failed"); result = {...result, remaining: {unavailable:true, next:'))));
test('retrying successful mutation after read failure is rejected',()=>fails(grade('repeat-save',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','await call("/workflows/workspace", {...input,task}); result = {...result, remaining: {unavailable:true, next:'))));
test('missing extension remains an import failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});
+
+// A correct implementation may validate optimistic-concurrency inputs before I/O.
+test('valid revision guards do not turn fixture omissions into product failures',()=>{
+ const guarded=replace(fixed,'const permissions =',`
+ if (input.action !== "context" && input.action !== "start" && (!Number.isSafeInteger(input.expected_revision) || input.expected_revision < 0)) throw new Error("expected revision required");
+ if (input.action === "publish" && (!Number.isSafeInteger(input.catalog_revision) || input.catalog_revision < 0)) throw new Error("catalog revision required");
+ const permissions =`);
+ const r=grade('revision-guard',guarded);
+ expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');
+});
diff --git a/evals/coding-agent/graders/workflow-receipt.fixture.ts b/evals/coding-agent/graders/workflow-receipt.fixture.ts
index 0ddbe5d23..22255117e 100644
--- a/evals/coding-agent/graders/workflow-receipt.fixture.ts
+++ b/evals/coding-agent/graders/workflow-receipt.fixture.ts
@@ -29,7 +29,7 @@ test("structured saves serialize quotes and JSON-like source content without she
});
test("server conflicts are errors, not success text",async()=>{
server.reload({fetch:()=>Response.json({error:"Workspace changed"},{status:409})});
- await expect(tool.execute("id",{action:"publish",draft_id:"a"},new AbortController().signal)).rejects.toThrow("Workspace changed");
+ await expect(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"a"},new AbortController().signal)).rejects.toThrow("Workspace changed");
});
test("missing capability never falls back to a broad owner token",async()=>{
writeFileSync(join(dir,".screenpipe-permissions.json"),"{}");
@@ -38,7 +38,7 @@ test("missing capability never falls back to a broad owner token",async()=>{
});
test("an already-aborted task cannot save",async()=>{
const abort=new AbortController();abort.abort();
- await expect(tool.execute("id",{action:"publish",draft_id:"a"},abort.signal)).rejects.toThrow();
+ await expect(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"a"},abort.signal)).rejects.toThrow();
expect(requests).toHaveLength(0);
});
@@ -101,7 +101,7 @@ test.each(["handoff", "publish", "reject"])("unknown draft %s returns exact cand
if(req.method==="POST"){writes++;return Response.json({error:"Draft not found."},{status:409});}
return Response.json({workspace:{drafts:{a:{id:"exact-id",status:"open",assignee:"workflow-review",payload:{title:"Review invoice"}},b:{id:"other-owner",status:"open",assignee:"workflow-deepen"}}}});
}});
- const error=await tool.execute("id",{action,draft_id:"typo",note:"repair"},new AbortController().signal).catch((e:Error)=>e);
+ const error=await tool.execute("id",{action,expected_revision:1,catalog_revision:1,draft_id:"typo",note:"repair"},new AbortController().signal).catch((e:Error)=>e);
expect(error).toBeInstanceOf(Error);expect(writes).toBe(1);
expect(error.message).toContain("exact-id");
expect(error.message).not.toContain("other-owner");
@@ -112,7 +112,7 @@ test("publication returns remaining work with current revisions, distinct from i
server.reload({fetch:(req:Request)=>Response.json(req.method==="POST"
? {revision:8,changes:{created:1},checkedThrough:"previous-window"}
: {workspace:{revision:12,cycle:{status:"running"},drafts:{done:{status:"published"}}},catalogRevision:8,canFinish:true})});
- const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));
+ const result=await checked(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"done"},new AbortController().signal));
expect(result.isError).not.toBe(true);
expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{revision:12,catalogRevision:8,cycleStatus:"running",canFinish:true,openDrafts:[]}});
});
@@ -122,7 +122,7 @@ test("a failed state read after saving preserves the successful receipt",async()
if(req.method==="POST"){writes++;return Response.json({revision:8,changes:{created:1}});}
return Response.json({error:"Recorder unavailable"},{status:503});
}});
- const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));
+ const result=await checked(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"done"},new AbortController().signal));
expect(result.isError).not.toBe(true);expect(writes).toBe(1);
expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{unavailable:true}});
});