Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify WIF token retries through the public provider

Louis Beaumontcb6db4451c6e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index e244c687a..9a64f74b5 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -33,7 +33,7 @@ images, blanket paste refusal and missing-source classification. Fixtures and
dependency links appear only when grading begins. This does not execute native
clipboard delivery, image encoding, disk autosave, agent isolation or model trials.
-The current app corpus contains 121 git-mined regressions. See
+The current app corpus contains 122 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1500,3 +1500,17 @@ blanket warning suppression, an equivalent boundary and missing-source errors.
Fixtures and runtime links appear only when grading begins. This does not test
real disk space, IPC delivery, native recording or migration policy, execution
isolation or model performance.
+
+
+`ai-gateway-wif-token-retry` exercises the public Vertex token method with real
+ephemeral RSA signing and synthetic transport. Each exchange recovers from
+transient HTTP and network failures, bounds retries, and refuses permanent
+errors. Signed claims, exchange credentials, valid-token caching and legacy
+authentication remain intact. The parent fails eight outcomes and preserves six;
+the historical fix and current source pass fourteen. Run
+`bun test evals/coding-agent/calibrate-wif-token-retry.test.js` for fourteen
+controls, including equivalent backoff, an unused repair, disconnected exchanges,
+wrong credentials and lost caching. Fixtures appear only at grading and require
+no dependency links. Missing provider source is a setup failure. This does not
+establish real provider availability, gateway HTTP admission, concurrent or
+cross-account cache isolation, execution isolation or model performance.
diff --git a/evals/coding-agent/calibrate-wif-token-retry.test.js b/evals/coding-agent/calibrate-wif-token-retry.test.js
new file mode 100644
index 000000000..63981a60d
--- /dev/null
+++ b/evals/coding-agent/calibrate-wif-token-retry.test.js
@@ -0,0 +1,42 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+const repo=resolve(import.meta.dir,"../..");
+const item=JSON.parse(readFileSync(join(import.meta.dir,"cases.json"),"utf8")).cases.find(x=>x.id==="ai-gateway-wif-token-retry");
+const path="packages/ai-gateway/src/providers/vertex.ts";
+const show=(ref,p)=>execFileSync("git",["show",`${ref}:${p}`],{cwd:repo,encoding:"utf8"});
+const broken=show(item.base_ref,path),fixed=show(item.oracle_ref,path);
+const root=mkdtempSync(join(tmpdir(),"wif-retry-calibration-"));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,source,extra) {
+ const workspace=join(root,name);
+ const write=(p,data)=>{mkdirSync(dirname(join(workspace,p)),{recursive:true});writeFileSync(join(workspace,p),data);};
+ if(source!==null)write(path,source);
+ for(const p of ["packages/ai-gateway/src/providers/base.ts","packages/ai-gateway/src/types.ts"])write(p,show(item.base_ref,p));
+ if(extra)write("packages/ai-gateway/src/providers/unused-repair.ts",extra);
+ const fixture=item.grader.fixtures[0];mkdirSync(dirname(join(workspace,fixture.destination_path)),{recursive:true});copyFileSync(join(import.meta.dir,fixture.local_path),join(workspace,fixture.destination_path));
+ const r=spawnSync(process.execPath,["--no-env-file","test",fixture.destination_path],{cwd:workspace,encoding:"utf8",timeout:30000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.WIF_CALIBRATION_RESULTS){const out=resolve(process.env.WIF_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});writeFileSync(join(out,name+".stdout"),r.stdout??"");writeFileSync(join(out,name+".stderr"),r.stderr??"");writeFileSync(join(out,name+".json"),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message??null}));}
+ return r;
+}
+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain("expect(received)");expect(r.stderr).not.toContain("Cannot find module");}
+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain("14 pass");}
+function patch(anchor,replacement){expect(fixed.split(anchor)).toHaveLength(2);return fixed.replace(anchor,replacement);}
+test("parent fails eight outcomes and preserves six",()=>{const r=grade("parent",broken);fail(r);expect(r.stderr).toContain("8 fail");expect(r.stderr).toContain("6 pass");},40000);
+test("historical reference passes public token outcomes",()=>pass(grade("reference",fixed)),40000);
+test("current source preserves the same authentication contract",()=>pass(grade("current",show("HEAD",path))),40000);
+test("equivalent retry naming and backoff are accepted",()=>pass(grade("equivalent",fixed.replaceAll("wifFetchWithRetry","exchangeWithRecovery").replace("200 * (attempt + 1) + Math.random() * 150","25 * (attempt + 1)"))),40000);
+test("unused correct retry helper cannot hide broken caller",()=>fail(grade("unused",broken,fixed)),40000);
+test("only STS retry cannot hide broken impersonation",()=>fail(grade("sts-only",patch("const impResp = await wifFetchWithRetry(","const impResp = await fetch("))),40000);
+test("only impersonation retry cannot hide broken STS",()=>fail(grade("iam-only",patch("const stsResp = await wifFetchWithRetry(","const stsResp = await fetch("))),40000);
+test("429 cannot bypass transient recovery",()=>fail(grade("no429",patch("resp.status < 500 && resp.status !== 429","resp.status < 500"))),40000);
+test("permanent refusal must not retry",()=>fail(grade("retry400",patch("resp.status < 500 && resp.status !== 429","resp.status < 400"))),40000);
+test("retry budget stays bounded",()=>fail(grade("overbudget",patch("const MAX = 3;","const MAX = 4;"))),40000);
+test("valid cached WIF token avoids new requests",()=>fail(grade("nocache",patch("if (tokenCache && tokenCache.expiresAt > Date.now() + 60000)","if (false && tokenCache && tokenCache.expiresAt > Date.now() + 60000)"))),40000);
+test("retry preserves the STS token at impersonation",()=>fail(grade("credential",patch("Authorization: `Bearer ${sts.access_token}`","Authorization: `Bearer synthetic-wrong`"))),40000);
+test("blanket refusal cannot replace recovery",()=>fail(grade("deny",patch("public async getAccessToken(): Promise<string> {","public async getAccessToken(): Promise<string> { throw new Error('refused');"))),40000);
+test("missing provider is an infrastructure failure",()=>{const r=grade("missing",null);expect(r.status).not.toBe(0);expect(r.stderr).toContain("Cannot find module");expect(r.stderr).not.toContain("expect(received)");},40000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index a93c5734b..eaf59d365 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5010,6 +5010,41 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-active-storage-volume.config.mjs"
}
+ },
+ {
+ "id": "ai-gateway-wif-token-retry",
+ "name": "Recover transient WIF authentication failures",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "a82653675fcdaca022c88b7c3599bac3e3542d89"
+ },
+ "base_ref": "b6e2cb2249d75d60a92c85d1f02938fe840e52cf",
+ "oracle_ref": "a82653675fcdaca022c88b7c3599bac3e3542d89",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/vertex.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/vertex.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "authentication",
+ "retry"
+ ],
+ "prompt": "The public VertexAIProvider.getAccessToken method fails an entire request when either Workload Identity Federation token exchange encounters a temporary outage. Repair the actual STS exchange and service-account impersonation paths so 522/other 5xx, 429 and thrown transport errors can recover on the third attempt after two transient failures. Bound each exchange to three attempts; persistent failures must reject instead of returning or caching a token. Permanent 400/403 responses must reject immediately without retry, and failed STS must not reach impersonation. Preserve the original POST exchange data, signed JWT issuer/audience/subject, selected service-account URL and STS bearer credential during retry. Successful WIF tokens must be returned and reused from the existing valid-token cache without new requests. Preserve successful legacy service-account authentication and its cached token. Keep the existing public class, constructor and getAccessToken contract. No particular helper or retry implementation is required; exact delay values are not prescribed. Use synthetic credentials and intercepted transport only. Real provider availability, gateway HTTP admission, concurrent or cross-account cache isolation, cancellation, expiry-boundary policy and live model calls are outside this historical task.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/wif-token-retry.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/eval-wif-token-retry.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test packages/ai-gateway/eval-wif-token-retry.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/wif-token-retry.fixture.ts.txt b/evals/coding-agent/graders/wif-token-retry.fixture.ts.txt
new file mode 100644
index 000000000..f637c5d75
--- /dev/null
+++ b/evals/coding-agent/graders/wif-token-retry.fixture.ts.txt
@@ -0,0 +1,76 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, beforeAll, expect, test } from "bun:test";
+import { VertexAIProvider, resetTokenCache } from "./src/providers/vertex";
+const realFetch = globalThis.fetch;
+let pem: string;
+beforeAll(async () => {
+ const pair = await crypto.subtle.generateKey({name:"RSASSA-PKCS1-v1_5",modulusLength:2048,publicExponent:new Uint8Array([1,0,1]),hash:"SHA-256"},true,["sign","verify"]);
+ const bytes = new Uint8Array(await crypto.subtle.exportKey("pkcs8",pair.privateKey));
+ pem = "-----BEGIN PRIVATE KEY-----\n" + btoa(String.fromCharCode(...bytes)) + "\n-----END PRIVATE KEY-----";
+});
+afterEach(() => { globalThis.fetch=realFetch; resetTokenCache(); });
+const stsURL="https://sts.googleapis.com/v1/token";
+const iamURL="https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken";
+const legacyURL="https://oauth2.googleapis.com/token";
+const jwtClaims=(jwt:string) => JSON.parse(atob(jwt.split(".")[1].replace(/-/g,"+").replace(/_/g,"/")));
+function harness(stage="none",fault:number|string=522,failures=0,wif=true) {
+ resetTokenCache();
+ const calls:Array<{url:string,init:RequestInit}>=[];
+ let n=0;
+ globalThis.fetch=(async (input:any,init:RequestInit={}) => {
+ const url=String(input);calls.push({url,init});
+ if (![stsURL,iamURL,legacyURL].includes(url)) throw new Error("Unexpected synthetic transport target");
+ if(url===(stage==="sts"?stsURL:stage==="iam"?iamURL:"none") && ++n<=failures) {
+ if(fault==="network")throw new Error("synthetic network failure");
+ return new Response("synthetic refusal",{status:Number(fault)});
+ }
+ return Response.json(url===stsURL?{access_token:"synthetic-federated"}:url===iamURL?{accessToken:"synthetic-minted",expireTime:new Date(Date.now()+3600000).toISOString()}:{access_token:"synthetic-legacy",expires_in:3600});
+ }) as typeof fetch;
+ const provider=new VertexAIProvider(JSON.stringify({project_id:"synthetic-project",client_email:"[email protected]",private_key:pem}),"synthetic-project","global",wif?{signingKey:pem,kid:"synthetic-kid",issuer:"https://issuer.eval.invalid",jwtAudience:"synthetic-jwt-audience",subject:"synthetic-subject",stsAudience:"synthetic-sts-audience",saEmail:"[email protected]",projectId:"synthetic-project"}:undefined);
+ return {provider,calls};
+}
+function assertRequests(calls:Array<{url:string,init:RequestInit}>) {
+ for(const {url,init} of calls) {
+ expect(init.method).toBe("POST");
+ if(url===stsURL) {
+ const body=new URLSearchParams(String(init.body));
+ expect(body.get("audience")).toBe("synthetic-sts-audience");
+ expect(body.get("grant_type")).toBe("urn:ietf:params:oauth:grant-type:token-exchange");
+ expect(jwtClaims(body.get("subject_token")!)).toMatchObject({iss:"https://issuer.eval.invalid",aud:"synthetic-jwt-audience",sub:"synthetic-subject"});
+ } else if(url===iamURL) {
+ expect(new Headers(init.headers).get("authorization")).toBe("Bearer synthetic-federated");
+ expect(JSON.parse(String(init.body)).scope).toEqual(["https://www.googleapis.com/auth/cloud-platform"]);
+ }
+ }
+}
+for(const stage of ["sts","iam"]) {
+ for(const fault of [522,429,"network"])test(`${stage} recovers after two ${fault} failures`,async()=>{
+ const h=harness(stage,fault,2);let token,error;
+ try{token=await h.provider.getAccessToken();}catch(e){error=e;}
+ expect(error).toBeUndefined();expect(token).toBe("synthetic-minted");
+ expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(3);
+ assertRequests(h.calls);
+ const count=h.calls.length;expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(h.calls).toHaveLength(count);
+ },10000);
+ test(`${stage} persistent503 stops and rejects`,async()=>{
+ const h=harness(stage,503,99);let error;
+ try{await h.provider.getAccessToken();}catch(e){error=e;}
+ expect(error).toBeDefined();expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(3);
+ if(stage==="sts")expect(h.calls.some(x=>x.url===iamURL)).toBe(false);
+ assertRequests(h.calls);
+ },10000);
+ for(const fault of [400,403])test(`${stage} permanent${fault} refuses without retry`,async()=>{
+ const h=harness(stage,fault,99);let error;
+ try{await h.provider.getAccessToken();}catch(e){error=e;}
+ expect(error).toBeDefined();expect(h.calls.filter(x=>x.url===(stage==="sts"?stsURL:iamURL))).toHaveLength(1);
+ if(stage==="sts")expect(h.calls.some(x=>x.url===iamURL)).toBe(false);
+ },10000);
+}
+test("WIF ordinary success and valid cache are preserved",async()=>{
+ const h=harness();expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(await h.provider.getAccessToken()).toBe("synthetic-minted");expect(h.calls.map(x=>x.url)).toEqual([stsURL,iamURL]);assertRequests(h.calls);
+});
+test("legacy successful authentication and cache are preserved",async()=>{
+ const h=harness("none",522,0,false);expect(await h.provider.getAccessToken()).toBe("synthetic-legacy");expect(await h.provider.getAccessToken()).toBe("synthetic-legacy");expect(h.calls.map(x=>x.url)).toEqual([legacyURL]);
+ const body=new URLSearchParams(String(h.calls[0].init.body));expect(body.get("grant_type")).toBe("urn:ietf:params:oauth:grant-type:jwt-bearer");expect(jwtClaims(body.get("assertion")!)).toMatchObject({iss:"[email protected]",aud:legacyURL});
+});