YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: supply valid revisions in workflow receipt fixtures
louise3c0728a0271
diff --git a/evals/coding-agent/calibrate-workflow-receipt.test.js b/evals/coding-agent/calibrate-workflow-receipt.test.jsindex 0d055c96a..d3d8f80b0 100644--- a/evals/coding-agent/calibrate-workflow-receipt.test.js+++ b/evals/coding-agent/calibrate-workflow-receipt.test.js@@ -30,3 +30,13 @@ test('discarding remaining-work state is rejected',()=>fails(grade('remaining',ftest('successful save must survive a failed follow-up read',()=>fails(grade('follow-up',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','throw new Error("follow-up read failed"); result = {...result, remaining: {unavailable:true, next:'))));test('retrying successful mutation after read failure is rejected',()=>fails(grade('repeat-save',replace(fixed,'result = {...result, remaining: {unavailable:true, next:','await call("/workflows/workspace", {...input,task}); result = {...result, remaining: {unavailable:true, next:'))));test('missing extension remains an import failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});++// A correct implementation may validate optimistic-concurrency inputs before I/O.+test('valid revision guards do not turn fixture omissions into product failures',()=>{+ const guarded=replace(fixed,'const permissions =',`+ if (input.action !== "context" && input.action !== "start" && (!Number.isSafeInteger(input.expected_revision) || input.expected_revision < 0)) throw new Error("expected revision required");+ if (input.action === "publish" && (!Number.isSafeInteger(input.catalog_revision) || input.catalog_revision < 0)) throw new Error("catalog revision required");+ const permissions =`);+ const r=grade('revision-guard',guarded);+ expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');+});diff --git a/evals/coding-agent/graders/workflow-receipt.fixture.ts b/evals/coding-agent/graders/workflow-receipt.fixture.tsindex 0ddbe5d23..22255117e 100644--- a/evals/coding-agent/graders/workflow-receipt.fixture.ts+++ b/evals/coding-agent/graders/workflow-receipt.fixture.ts@@ -29,7 +29,7 @@ test("structured saves serialize quotes and JSON-like source content without she});test("server conflicts are errors, not success text",async()=>{server.reload({fetch:()=>Response.json({error:"Workspace changed"},{status:409})});- await expect(tool.execute("id",{action:"publish",draft_id:"a"},new AbortController().signal)).rejects.toThrow("Workspace changed");+ await expect(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"a"},new AbortController().signal)).rejects.toThrow("Workspace changed");});test("missing capability never falls back to a broad owner token",async()=>{writeFileSync(join(dir,".screenpipe-permissions.json"),"{}");@@ -38,7 +38,7 @@ test("missing capability never falls back to a broad owner token",async()=>{});test("an already-aborted task cannot save",async()=>{const abort=new AbortController();abort.abort();- await expect(tool.execute("id",{action:"publish",draft_id:"a"},abort.signal)).rejects.toThrow();+ await expect(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"a"},abort.signal)).rejects.toThrow();expect(requests).toHaveLength(0);});@@ -101,7 +101,7 @@ test.each(["handoff", "publish", "reject"])("unknown draft %s returns exact candif(req.method==="POST"){writes++;return Response.json({error:"Draft not found."},{status:409});}return Response.json({workspace:{drafts:{a:{id:"exact-id",status:"open",assignee:"workflow-review",payload:{title:"Review invoice"}},b:{id:"other-owner",status:"open",assignee:"workflow-deepen"}}}});}});- const error=await tool.execute("id",{action,draft_id:"typo",note:"repair"},new AbortController().signal).catch((e:Error)=>e);+ const error=await tool.execute("id",{action,expected_revision:1,catalog_revision:1,draft_id:"typo",note:"repair"},new AbortController().signal).catch((e:Error)=>e);expect(error).toBeInstanceOf(Error);expect(writes).toBe(1);expect(error.message).toContain("exact-id");expect(error.message).not.toContain("other-owner");@@ -112,7 +112,7 @@ test("publication returns remaining work with current revisions, distinct from iserver.reload({fetch:(req:Request)=>Response.json(req.method==="POST"? {revision:8,changes:{created:1},checkedThrough:"previous-window"}: {workspace:{revision:12,cycle:{status:"running"},drafts:{done:{status:"published"}}},catalogRevision:8,canFinish:true})});- const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));+ const result=await checked(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"done"},new AbortController().signal));expect(result.isError).not.toBe(true);expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{revision:12,catalogRevision:8,cycleStatus:"running",canFinish:true,openDrafts:[]}});});@@ -122,7 +122,7 @@ test("a failed state read after saving preserves the successful receipt",async()if(req.method==="POST"){writes++;return Response.json({revision:8,changes:{created:1}});}return Response.json({error:"Recorder unavailable"},{status:503});}});- const result=await checked(tool.execute("id",{action:"publish",draft_id:"done"},new AbortController().signal));+ const result=await checked(tool.execute("id",{action:"publish",expected_revision:1,catalog_revision:1,draft_id:"done"},new AbortController().signal));expect(result.isError).not.toBe(true);expect(writes).toBe(1);expect(JSON.parse(result.content[0].text)).toMatchObject({revision:8,remaining:{unavailable:true}});});