YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify WIF token retries through the public provider
Louis Beaumontcb6db4451c6e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex e244c687a..9a64f74b5 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -33,7 +33,7 @@ images, blanket paste refusal and missing-source classification. Fixtures anddependency links appear only when grading begins. This does not execute nativeclipboard delivery, image encoding, disk autosave, agent isolation or model trials.-The current app corpus contains 121 git-mined regressions. See+The current app corpus contains 122 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1500,3 +1500,17 @@ blanket warning suppression, an equivalent boundary and missing-source errors.Fixtures and runtime links appear only when grading begins. This does not testreal disk space, IPC delivery, native recording or migration policy, executionisolation or model performance.+++`ai-gateway-wif-token-retry` exercises the public Vertex token method with real+ephemeral RSA signing and synthetic transport. Each exchange recovers from+transient HTTP and network failures, bounds retries, and refuses permanent+errors. Signed claims, exchange credentials, valid-token caching and legacy+authentication remain intact. The parent fails eight outcomes and preserves six;+the historical fix and current source pass fourteen. Run+`bun test evals/coding-agent/calibrate-wif-token-retry.test.js` for fourteen+controls, including equivalent backoff, an unused repair, disconnected exchanges,+wrong credentials and lost caching. Fixtures appear only at grading and require+no dependency links. Missing provider source is a setup failure. This does not+establish real provider availability, gateway HTTP admission, concurrent or+cross-account cache isolation, execution isolation or model performance.diff --git a/evals/coding-agent/calibrate-wif-token-retry.test.js b/evals/coding-agent/calibrate-wif-token-retry.test.jsnew file mode 100644index 000000000..63981a60d--- /dev/null+++ b/evals/coding-agent/calibrate-wif-token-retry.test.js@@ -0,0 +1,42 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from "bun:test";+import { execFileSync, spawnSync } from "node:child_process";+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { tmpdir } from "node:os";+const repo=resolve(import.meta.dir,"../..");+const item=JSON.parse(readFileSync(join(import.meta.dir,"cases.json"),"utf8")).cases.find(x=>x.id==="ai-gateway-wif-token-retry");+const path="packages/ai-gateway/src/providers/vertex.ts";+const show=(ref,p)=>execFileSync("git",["show",`${ref}:${p}`],{cwd:repo,encoding:"utf8"});+const broken=show(item.base_ref,path),fixed=show(item.oracle_ref,path);+const root=mkdtempSync(join(tmpdir(),"wif-retry-calibration-"));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,source,extra) {+ const workspace=join(root,name);+ const write=(p,data)=>{mkdirSync(dirname(join(workspace,p)),{recursive:true});writeFileSync(join(workspace,p),data);};+ if(source!==null)write(path,source);+ for(const p of ["packages/ai-gateway/src/providers/base.ts","packages/ai-gateway/src/types.ts"])write(p,show(item.base_ref,p));+ if(extra)write("packages/ai-gateway/src/providers/unused-repair.ts",extra);+ const fixture=item.grader.fixtures[0];mkdirSync(dirname(join(workspace,fixture.destination_path)),{recursive:true});copyFileSync(join(import.meta.dir,fixture.local_path),join(workspace,fixture.destination_path));+ const r=spawnSync(process.execPath,["--no-env-file","test",fixture.destination_path],{cwd:workspace,encoding:"utf8",timeout:30000,env:{PATH:dirname(process.execPath)}});+ if(process.env.WIF_CALIBRATION_RESULTS){const out=resolve(process.env.WIF_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});writeFileSync(join(out,name+".stdout"),r.stdout??"");writeFileSync(join(out,name+".stderr"),r.stderr??"");writeFileSync(join(out,name+".json"),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null}));}+ return r;+}+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain("expect(received)");expect(r.stderr).not.toContain("Cannot find module");}+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain("14 pass");}+function patch(anchor,replacement){expect(fixed.split(anchor)).toHaveLength(2);return fixed.replace(anchor,replacement);}+test("parent fails eight outcomes and preserves six",()=>{const r=grade("parent",broken);fail(r);expect(r.stderr).toContain("8 fail");expect(r.stderr).toContain("6 pass");},40000);+test("historical reference passes public token outcomes",()=>pass(grade("reference",fixed)),40000);+test("current source preserves the same authentication contract",()=>pass(grade("current",show("HEAD",path))),40000);+test("equivalent retry naming and backoff are accepted",()=>pass(grade("equivalent",fixed.replaceAll("wifFetchWithRetry","exchangeWithRecovery").replace("200 * (attempt + 1) + Math.random() * 150","25 * (attempt + 1)"))),40000);+test("unused correct retry helper cannot hide broken caller",()=>fail(grade("unused",broken,fixed)),40000);+test("only STS retry cannot hide broken impersonation",()=>fail(grade("sts-only",patch("const impResp = await wifFetchWithRetry(","const impResp = await fetch("))),40000);+test("only impersonation retry cannot hide broken STS",()=>fail(grade("iam-only",patch("const stsResp = await wifFetchWithRetry(","const stsResp = await fetch("))),40000);+test("429 cannot bypass transient recovery",()=>fail(grade("no429",patch("resp.status < 500 && resp.status !== 429","resp.status < 500"))),40000);+test("permanent refusal must not retry",()=>fail(grade("retry400",patch("resp.status < 500 && resp.status !== 429","resp.status < 400"))),40000);+test("retry budget stays bounded",()=>fail(grade("overbudget",patch("const MAX = 3;","const MAX = 4;"))),40000);+test("valid cached WIF token avoids new requests",()=>fail(grade("nocache",patch("if (tokenCache && tokenCache.expiresAt > Date.now() + 60000)","if (false && tokenCache && tokenCache.expiresAt > Date.now() + 60000)"))),40000);+test("retry preserves the STS token at impersonation",()=>fail(grade("credential",patch("Authorization: `Bearer ${sts.access_token}`","Authorization: `Bearer synthetic-wrong`"))),40000);+test("blanket refusal cannot replace recovery",()=>fail(grade("deny",patch("public async getAccessToken(): Promise<string> {","public async getAccessToken(): Promise<string> { throw new Error('refused');"))),40000);+test("missing provider is an infrastructure failure",()=>{const r=grade("missing",null);expect(r.status).not.toBe(0);expect(r.stderr).toContain("Cannot find module");expect(r.stderr).not.toContain("expect(received)");},40000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex a93c5734b..eaf59d365 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5010,6 +5010,41 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-active-storage-volume.config.mjs"}+ },+ {+ "id": "ai-gateway-wif-token-retry",+ "name": "Recover transient WIF authentication failures",+ "source": {+ "kind": "git_regression",+ "fix_commit": "a82653675fcdaca022c88b7c3599bac3e3542d89"+ },+ "base_ref": "b6e2cb2249d75d60a92c85d1f02938fe840e52cf",+ "oracle_ref": "a82653675fcdaca022c88b7c3599bac3e3542d89",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/vertex.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/providers/vertex.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "authentication",+ "retry"+ ],+ "prompt": "The public VertexAIProvider.getAccessToken method fails an entire request when either Workload Identity Federation token exchange encounters a temporary outage. Repair the actual STS exchange and service-account impersonation paths so 522/other 5xx, 429 and thrown transport errors can recover on the third attempt after two transient failures. Bound each exchange to three attempts; persistent failures must reject instead of returning or caching a token. Permanent 400/403 responses must reject immediately without retry, and failed STS must not reach impersonation. Preserve the original POST exchange data, signed JWT issuer/audience/subject, selected service-account URL and STS bearer credential during retry. Successful WIF tokens must be returned and reused from the existing valid-token cache without new requests. Preserve successful legacy service-account authentication and its cached token. Keep the existing public class, constructor and getAccessToken contract. No particular helper or retry implementation is required; exact delay values are not prescribed. Use synthetic credentials and intercepted transport only. Real provider availability, gateway HTTP admission, concurrent or cross-account cache isolation, cancellation, expiry-boundary policy and live model calls are outside this historical task.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/wif-token-retry.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/eval-wif-token-retry.test.ts"+ }+ ],+ "command": "bun --no-env-file test packages/ai-gateway/eval-wif-token-retry.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/wif-token-retry.fixture.ts.txt b/evals/coding-agent/graders/wif-token-retry.fixture.ts.txtnew file mode 100644index 000000000..f637c5d75--- /dev/null+++ b/evals/coding-agent/graders/wif-token-retry.fixture.ts.txt@@ -0,0 +1,76 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, beforeAll, expect, test } from "bun:test";+import { VertexAIProvider, resetTokenCache } from "./src/providers/vertex";+const realFetch = globalThis.fetch;+let pem: string;+beforeAll(async () => {+ const pair = await crypto.subtle.generateKey({name:"RSASSA-PKCS1-v1_5",modulusLength:2048,publicExponent:new Uint8Array([1,0,1]),hash:"SHA-256"},true,["sign","verify"]);+ const bytes = new Uint8Array(await crypto.subtle.exportKey("pkcs8",pair.privateKey));+ pem = "-----BEGIN PRIVATE KEY-----\n" + btoa(String.fromCharCode(...bytes)) + "\n-----END PRIVATE KEY-----";+});+afterEach(() => { globalThis.fetch=realFetch; resetTokenCache(); });+const stsURL="https://sts.googleapis.com/v1/token";+const iamURL="https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken";+const legacyURL="https://oauth2.googleapis.com/token";+const jwtClaims=(jwt:string) => JSON.parse(atob(jwt.split(".")[1].replace(/-/g,"+").replace(/_/g,"/")));+function harness(stage="none",fault:number|string=522,failures=0,wif=true) {+ resetTokenCache();+ const calls:Array<{url:string,init:RequestInit}>=[];+ let n=0;+ globalThis.fetch=(async (input:any,init:RequestInit={}) => {+ const url=String(input);calls.push({url,init});+ if (![stsURL,iamURL,legacyURL].includes(url)) throw new Error("Unexpected synthetic transport target");+ if(url===(stage==="sts"?stsURL:stage==="iam"?iamURL:"none") && ++n<=failures) {+ if(fault==="network")throw new Error("synthetic network failure");+ return new Response("synthetic refusal",{status:Number(fault)});+ }+ return Response.json(url===stsURL?{access_token:"synthetic-federated"}:url===iamURL?{accessToken:"synthetic-minted",expireTime:new Date(Date.now()+3600000).toISOString()}:{access_token:"synthetic-legacy",expires_in:3600});+ }) as typeof fetch;+ const provider=new VertexAIProvider(JSON.stringify({project_id:"synthetic-project",client_email:"[email protected]",private_key:pem}),"synthetic-project","global",wif?{signingKey:pem,kid:"synthetic-kid",issuer:"https://issuer.eval.invalid",jwtAudience:"synthetic-jwt-audience",subject:"synthetic-subject",stsAudience:"synthetic-sts-audience",saEmail:"[email protected]",projectId:"synthetic-project"}:undefined);+ return {provider,calls};+}+function assertRequests(calls:Array<{url:string,init:RequestInit}>) {+ for(const {url,init} of calls) {+ expect(init.method).toBe("POST");+ if(url===stsURL) {+ const body=new URLSearchParams(String(init.body));+ expect(body.get("audience")).toBe("synthetic-sts-audience");+ expect(body.get("grant_type")).toBe("urn:ietf:params:oauth:grant-type:token-exchange");+ expect(jwtClaims(body.get("subject_token")!)).toMatchObject({iss:"https://issuer.eval.invalid",aud:"synthetic-jwt-audience",sub:"synthetic-subject"});+ } else if(url===iamURL) {+ expect(new Headers(init.headers).get("authorization")).toBe("Bearer synthetic-federated");+ expect(JSON.parse(String(init.body)).scope).toEqual(["https://www.googleapis.com/auth/cloud-platform"]);+ }+ }+}+for(const stage of ["sts","iam"]) {+ for(const fault of [522,429,"network"])test(`${stage} recovers after two ${fault} failures`,async()=>{+ const h=harness(stage,fault,2);let token,error;+ try{token=await h.provider.getAccessToken();}catch(e){error=e;}+ expect(error).toBeUndefined();expect(token).toBe("synthetic-minted");+ expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(3);+ assertRequests(h.calls);+ const count=h.calls.length;expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(h.calls).toHaveLength(count);+ },10000);+ test(`${stage} persistent503 stops and rejects`,async()=>{+ const h=harness(stage,503,99);let error;+ try{await h.provider.getAccessToken();}catch(e){error=e;}+ expect(error).toBeDefined();expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(3);+ if(stage==="sts")expect(h.calls.some(x=>x.url===iamURL)).toBe(false);+ assertRequests(h.calls);+ },10000);+ for(const fault of [400,403])test(`${stage} permanent${fault} refuses without retry`,async()=>{+ const h=harness(stage,fault,99);let error;+ try{await h.provider.getAccessToken();}catch(e){error=e;}+ expect(error).toBeDefined();expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(1);+ if(stage==="sts")expect(h.calls.some(x=>x.url===iamURL)).toBe(false);+ },10000);+}+test("WIF ordinary success and valid cache are preserved",async()=>{+ const h=harness();expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(h.calls.map(x=>x.url)).toEqual([stsURL,iamURL]);assertRequests(h.calls);+});+test("legacy successful authentication and cache are preserved",async()=>{+ const h=harness("none",522,0,false);expect(await h.provider.getAccessToken()).toBe("synthetic-legacy");expect(await h.provider.getAccessToken()).toBe("synthetic-legacy");expect(h.calls.map(x=>x.url)).toEqual([legacyURL]);+ const body=new URLSearchParams(String(h.calls[0].init.body));expect(body.get("grant_type")).toBe("urn:ietf:params:oauth:grant-type:jwt-bearer");expect(jwtClaims(body.get("assertion")!)).toMatchObject({iss:"[email protected]",aud:legacyURL});+});