Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): honour a GFS 429 and surface its retry hint to the model
Alfredo Lopez80ccbbfb5fae
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 5a509c804..77d0f6c62 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -131,7 +131,7 @@ jobs:- name: Testworking-directory: ${{ matrix.service }}env:- PINNED_SUITES: ${{ matrix.service == 'gfs-controller' && 'serve.agentRateLimit.test.ts' || '' }}+ PINNED_SUITES: ${{ matrix.service == 'gfs-controller' && 'serve.agentRateLimit.test.ts' || matrix.service == 'mcp-host' && 'gfsClient.retry429.test.ts' || '' }}run: |set -euo pipefailif [ -z "$PINNED_SUITES" ]; thendiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex cd947ff9e..31e677e92 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.368",+ "version": "0.1.369","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.368",+ "version": "0.1.369","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 3dc925d7b..826596dc8 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.368",+ "version": "0.1.369","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/core/tools/nativeToolRegistry.ts b/mcp-host/src/core/tools/nativeToolRegistry.tsindex 14f2b047b..cde20945c 100644--- a/mcp-host/src/core/tools/nativeToolRegistry.ts+++ b/mcp-host/src/core/tools/nativeToolRegistry.ts@@ -210,7 +210,7 @@ export class NativeToolRegistry implements ToolRegistry {}const gfsScopes = getGfsToolScopes(gfsEnv)if (gfsScopes && gfsScopes.size > 0) {- const gfsClient = createGfscClient(gfsEnv)+ const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: config.toolTimeout })const gfsTools = [...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),diff --git a/mcp-host/src/internalTools/gfs.test.ts b/mcp-host/src/internalTools/gfs.test.tsindex ed2b8d154..d743f94bc 100644--- a/mcp-host/src/internalTools/gfs.test.ts+++ b/mcp-host/src/internalTools/gfs.test.ts@@ -6,6 +6,7 @@ import {buildGfsReadTools,buildGfsWriteTools,} from './gfs'+import { GfscHttpError } from './gfsClient'/*** P3-S04 — agent gfs READ tools. Four read-only tools (list/read/stat/resolve)@@ -84,6 +85,37 @@ describe('buildGfsReadTools', () => {expect(r.error).not.toContain('blob/key')})+ it('passes a 429 retry hint to the model and still redacts the body', async () => {+ const c = client({+ read: vi.fn(async () => {+ throw new GfscHttpError(429, 'rate limited on /internal/server/path blob/key', 30)+ }),+ })+ const r = await toolMap(c)+ .get('clerum__gfs_read')!+ .execute({ drive: 'main', resourceId: 'abc' }, '')+ expect(r.success).toBe(false)+ expect(r.error).toBe('GFS read failed (gfsc 429: rate_limited, retry after 30s)')+ expect(r.error).not.toContain('/internal/server/path')+ expect(r.error).not.toContain('blob/key')+ })++ it.each([+ [429, 'rate_limited'],+ [400, 'invalid_request'],+ [503, 'unavailable'],+ ])('maps gfsc %s without a hint to %s', async (status, category) => {+ const c = client({+ stat: vi.fn(async () => {+ throw new GfscHttpError(status, 'private path=/data/gfs/secret')+ }),+ })+ const r = await toolMap(c)+ .get('clerum__gfs_stat')!+ .execute({ drive: 'main', resourceId: 'abc' }, '')+ expect(r).toEqual({ success: false, error: `GFS read failed (gfsc ${status}: ${category})` })+ })+it('strips a non-gfsc-shaped read error down to the generic label', async () => {const c = client({read: vi.fn(async () => {diff --git a/mcp-host/src/internalTools/gfs.ts b/mcp-host/src/internalTools/gfs.tsindex 3b4fd4680..b797c32e8 100644--- a/mcp-host/src/internalTools/gfs.ts+++ b/mcp-host/src/internalTools/gfs.ts@@ -1,4 +1,5 @@import type { InternalToolDefinition, InternalToolResult } from '../workflow/types'+import { GfscHttpError } from './gfsClient'/*** Agent gfs READ tools (spec community.md §Operator surfaces, plan P3-S04).@@ -183,10 +184,18 @@ function redactedFail(label: string, error: unknown): InternalToolResult {? 'precondition_failed': status === 413? 'limit_exceeded'- : status >= 500- ? 'unavailable'- : 'failed'- return { success: false, error: `${label} (gfsc ${status}: ${category})` }+ : status === 429+ ? 'rate_limited'+ : status >= 500+ ? 'unavailable'+ : 'failed'+ // The retry hint comes from the typed error's parsed Retry-After header, an+ // integer, never from the response body.+ const hint =+ error instanceof GfscHttpError && error.status === 429 && error.retryAfterSeconds !== undefined+ ? `, retry after ${error.retryAfterSeconds}s`+ : ''+ return { success: false, error: `${label} (gfsc ${status}: ${category}${hint})` }}function mutationFail(error: unknown): InternalToolResult {diff --git a/mcp-host/src/internalTools/gfsClient.retry429.test.ts b/mcp-host/src/internalTools/gfsClient.retry429.test.tsnew file mode 100644index 000000000..6d95f7b04--- /dev/null+++ b/mcp-host/src/internalTools/gfsClient.retry429.test.ts@@ -0,0 +1,169 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { buildGfsReadTools } from './gfs'+import { GfscHttpError, createGfscClient } from './gfsClient'++/**+ * #699 stress: N agent pods rate limited by gfsc at once. One mcp-host process+ * holds one subject token, so N clients with distinct tokens stand for N+ * agents. The fetch stub is keyed by the bearer token and timers are faked.+ */++const N = 50++type Call = { token: string; at: number }++function rateLimited(retryAfter?: string): Response {+ const headers: Record<string, string> = { 'content-type': 'application/json' }+ if (retryAfter !== undefined) headers['retry-after'] = retryAfter+ return new Response(JSON.stringify({ ok: false, error: { code: 'rate_limited' } }), {+ status: 429,+ headers,+ })+}++function ok(token: string): Response {+ return new Response(JSON.stringify({ token }), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })+}++function tokenOf(init?: RequestInit): string {+ const auth = (init?.headers as Record<string, string> | undefined)?.authorization+ if (!auth?.startsWith('Bearer ')) throw new Error('request carried no bearer token')+ return auth.slice('Bearer '.length)+}++// answer(token, attempt) decides the response for the attempt-th call (1-based)+// made with that token.+function fleet(answer: (token: string, attempt: number) => Response, maxRetryWaitMs: number) {+ const calls: Call[] = []+ const fetchFn = vi.fn(async (_input: string, init?: RequestInit) => {+ const token = tokenOf(init)+ calls.push({ token, at: Date.now() })+ return answer(token, calls.filter(c => c.token === token).length)+ })+ const clients = Array.from({ length: N }, (_, i) => ({+ token: `agent-${i}`,+ client: createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? `agent-${i}` : undefined),+ fetch: fetchFn,+ },+ { maxRetryWaitMs }+ ),+ }))+ const perToken = (token: string) => calls.filter(c => c.token === token)+ return { calls, clients, perToken }+}++describe('gfsc client under a concurrent 429 burst', () => {+ beforeEach(() => {+ vi.useFakeTimers()+ })+ afterEach(() => {+ vi.useRealTimers()+ })++ it('M1: every agent retries once after Retry-After and succeeds', async () => {+ const { calls, clients, perToken } = fleet(+ (token, attempt) => (attempt === 1 ? rateLimited('2') : ok(token)),+ 60_000+ )++ const results = Promise.all(+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))+ )+ await vi.advanceTimersByTimeAsync(0)+ expect(calls).toHaveLength(N)+ await vi.advanceTimersByTimeAsync(1999)+ expect(calls).toHaveLength(N)+ await vi.advanceTimersByTimeAsync(1)+ expect(calls).toHaveLength(2 * N)++ await expect(results).resolves.toEqual(clients.map(({ token }) => ({ token })))+ for (const { token } of clients) {+ const [first, retry] = perToken(token)+ expect(perToken(token)).toHaveLength(2)+ expect(retry!.at - first!.at).toBeGreaterThanOrEqual(2000)+ }+ })++ it('M2: a 429 without Retry-After fails every agent at once', async () => {+ const { calls, clients } = fleet(() => rateLimited(), 60_000)++ const settled = await Promise.allSettled(+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))+ )+ expect(calls).toHaveLength(N)+ expect(settled.every(s => s.status === 'rejected')).toBe(true)+ for (const s of settled) {+ const reason = (s as PromiseRejectedResult).reason+ expect(reason).toBeInstanceOf(GfscHttpError)+ expect(reason).toMatchObject({ status: 429, retryAfterSeconds: undefined })+ }+ expect(vi.getTimerCount()).toBe(0)+ })++ it('M3: a Retry-After above the tool budget fails at once with the hint', async () => {+ const { calls, clients } = fleet(() => rateLimited('2'), 1000)++ const settled = await Promise.allSettled(+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))+ )+ expect(calls).toHaveLength(N)+ for (const s of settled) {+ expect(s.status).toBe('rejected')+ const reason = (s as PromiseRejectedResult).reason as GfscHttpError+ expect(reason).toMatchObject({ status: 429, retryAfterSeconds: 2 })+ expect(reason.message).toContain('(retry after 2s)')+ }+ expect(vi.getTimerCount()).toBe(0)+ })++ it('M4: an agent still denied after its retry stops at two calls', async () => {+ const { calls, clients, perToken } = fleet(() => rateLimited('2'), 60_000)++ const settled = Promise.allSettled(+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))+ )+ await vi.advanceTimersByTimeAsync(2000)+ expect(calls).toHaveLength(2 * N)+ await vi.advanceTimersByTimeAsync(120_000)+ expect(calls).toHaveLength(2 * N)++ for (const s of await settled) {+ expect(s.status).toBe('rejected')+ expect((s as PromiseRejectedResult).reason).toMatchObject({+ status: 429,+ retryAfterSeconds: 2,+ })+ }+ for (const { token } of clients) expect(perToken(token)).toHaveLength(2)+ expect(vi.getTimerCount()).toBe(0)+ })++ it('M5: the read tool returns rate_limited with the hint and never throws', async () => {+ const { calls, clients } = fleet(() => rateLimited('2'), 60_000)+ const statTools = clients.map(({ client }) => {+ const tool = buildGfsReadTools(client).find(t => t.name === 'clerum__gfs_stat')+ if (!tool) throw new Error('clerum__gfs_stat is not registered')+ return tool+ })++ const results = Promise.all(+ statTools.map(tool => tool.execute({ drive: 'main', resourceId: 'rid' }, ''))+ )+ await vi.advanceTimersByTimeAsync(2000)+ expect(calls).toHaveLength(2 * N)++ const settled = await results+ expect(settled).toHaveLength(N)+ for (const result of settled) {+ expect(result).toEqual({+ success: false,+ error: 'GFS read failed (gfsc 429: rate_limited, retry after 2s)',+ })+ }+ })+})diff --git a/mcp-host/src/internalTools/gfsClient.test.ts b/mcp-host/src/internalTools/gfsClient.test.tsindex a575f6355..d8492e369 100644--- a/mcp-host/src/internalTools/gfsClient.test.ts+++ b/mcp-host/src/internalTools/gfsClient.test.ts@@ -1,6 +1,7 @@-import { describe, expect, it, vi } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'import {DEFAULT_GFS_ACCESS_FILE,+ GfscHttpError,createGfscClient,getGfsToolScopes,hasGfsRuntimeAccess,@@ -19,6 +20,9 @@ function jsonResponse(body: unknown, status = 200): Response {})}+// Every test client gets a budget far above any Retry-After used here.+const clientOptions = { maxRetryWaitMs: 60_000 }+function textResponse(body: string, status: number, statusText = ''): Response {return new Response(body, { status, statusText })}@@ -54,10 +58,13 @@ describe('gfs runtime gfsc client', () => {const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>jsonResponse({ ok: true, data: { items: [] } }))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await client.accessible({ drive: 'main' })expect(fetchFn.mock.calls[0]?.[0]).toBe('http://gfsc.gfs.svc.cluster.local:8087/v1/accessible?drive=main'@@ -71,10 +78,13 @@ describe('gfs runtime gfsc client', () => {const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>jsonResponse({ ok: true, data: { version: 3 } }))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await client.write({ drive: 'main', resourceId: 'rid', content: 'new', ifMatch: 2 })expect(fetchFn.mock.calls[0]?.[0]).toBe('http://gfsc-writer.gfs.svc.cluster.local:8087/v1/resources/rid/content?drive=main'@@ -89,10 +99,13 @@ describe('gfs runtime gfsc client', () => {const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>jsonResponse({ ok: true, data: {} }))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await client.createFile({drive: 'main',parentResourceId: 'parent/id',@@ -136,11 +149,14 @@ describe('gfs runtime gfsc client', () => {})it('fails loud when the mounted runtime token file is empty', async () => {- const client = createGfscClient({- get: () => undefined,- readFile: async () => ' \n',- fetch: vi.fn(async () => jsonResponse({ ok: true })),- })+ const client = createGfscClient(+ {+ get: () => undefined,+ readFile: async () => ' \n',+ fetch: vi.fn(async () => jsonResponse({ ok: true })),+ },+ clientOptions+ )await expect(client.accessible({ drive: 'main' })).rejects.toThrow(/MCP_HOST_GFS_TOKEN_FILE is empty/@@ -149,10 +165,13 @@ describe('gfs runtime gfsc client', () => {it('surfaces gfsc authorization denials with the response body', async () => {const fetchFn = vi.fn(async () => textResponse('not authorized to write this resource', 403))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await expect(client.write({ drive: 'main', resourceId: 'rid', content: 'new', ifMatch: 2 })@@ -161,10 +180,13 @@ describe('gfs runtime gfsc client', () => {it('surfaces gfsc read denials with the response body', async () => {const fetchFn = vi.fn(async () => textResponse('not authorized to read this resource', 403))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await expect(client.read({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(/gfsc 403: not authorized to read this resource/@@ -185,10 +207,13 @@ describe('gfs runtime gfsc client', () => {},})const fetchFn = vi.fn(async () => textResponse(envelope, 503))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await expect(client.resolve({ uri: 'gfs://main/docs/report.md' })).rejects.toThrow(`gfsc 503: ${envelope}`@@ -197,10 +222,13 @@ describe('gfs runtime gfsc client', () => {it('surfaces gfsc transient failures with the status text when the body is empty', async () => {const fetchFn = vi.fn(async () => textResponse('', 503, 'Service Unavailable'))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await expect(client.stat({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(/gfsc 503: Service Unavailable/@@ -209,13 +237,174 @@ describe('gfs runtime gfsc client', () => {it('surfaces gfsc list transient failures with the status text when the body is empty', async () => {const fetchFn = vi.fn(async () => textResponse('', 503, 'Service Unavailable'))- const client = createGfscClient({- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),- fetch: fetchFn,- })+ const client = createGfscClient(+ {+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ },+ clientOptions+ )await expect(client.list({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(/gfsc 503: Service Unavailable/)})})++function rateLimited(retryAfter?: string): Response {+ const headers: Record<string, string> = { 'content-type': 'application/json' }+ if (retryAfter !== undefined) headers['retry-after'] = retryAfter+ // gfsc's envelope also carries retryAfterSeconds; the client reads only the+ // header, so the no-header cases below prove the body is never consulted.+ return new Response(+ JSON.stringify({ ok: false, error: { code: 'rate_limited', retryAfterSeconds: 5 } }),+ { status: 429, headers }+ )+}++function tokenEnv(fetchFn: (input: string, init?: RequestInit) => Promise<Response>) {+ return {+ get: (key: string) => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),+ fetch: fetchFn,+ }+}++describe('gfsc client on a 429', () => {+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it('retries once after the stated delay and resolves', async () => {+ vi.useFakeTimers()+ const callTimes: number[] = []+ const fetchFn = vi.fn(async () => {+ callTimes.push(Date.now())+ return callTimes.length === 1 ? rateLimited('2') : jsonResponse({ ok: true, version: 4 })+ })+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)++ const pending = client.stat({ drive: 'main', resourceId: 'rid' })+ await vi.advanceTimersByTimeAsync(0)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(2000)+ expect(fetchFn).toHaveBeenCalledTimes(2)+ await expect(pending).resolves.toEqual({ ok: true, version: 4 })+ expect(callTimes[1]! - callTimes[0]!).toBe(2000)+ })++ it('waits exactly the advertised Retry-After, not a constant', async () => {+ vi.useFakeTimers()+ const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout')+ let calls = 0+ const fetchFn = vi.fn(async () => (++calls === 1 ? rateLimited('7') : jsonResponse({})))+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)++ const pending = client.read({ drive: 'main', resourceId: 'rid' })+ await vi.advanceTimersByTimeAsync(0)+ expect(setTimeoutSpy).toHaveBeenCalledTimes(1)+ expect(setTimeoutSpy.mock.calls[0]?.[1]).toBe(7000)+ await vi.advanceTimersByTimeAsync(6999)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(1)+ expect(fetchFn).toHaveBeenCalledTimes(2)+ await expect(pending).resolves.toEqual({})+ })++ it('gives up after one retry and keeps the hint on the thrown error', async () => {+ vi.useFakeTimers()+ const fetchFn = vi.fn(async () => rateLimited('2'))+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)++ const rejection = expect(+ client.write({ drive: 'main', resourceId: 'rid', content: 'x', ifMatch: 1 })+ ).rejects.toMatchObject({+ name: 'GfscHttpError',+ status: 429,+ retryAfterSeconds: 2,+ message: expect.stringMatching(/^gfsc 429: .* \(retry after 2s\)$/),+ })+ await vi.advanceTimersByTimeAsync(2000)+ await rejection+ await vi.advanceTimersByTimeAsync(60_000)+ expect(fetchFn).toHaveBeenCalledTimes(2)+ })++ it.each([+ ['absent', undefined],+ ['an HTTP-date', 'Wed, 23 Sep 2026 10:00:00 GMT'],+ ['zero', '0'],+ ['fractional', '1.5'],+ ])('throws at once without retrying when Retry-After is %s', async (_label, retryAfter) => {+ vi.useFakeTimers()+ const answers = [rateLimited(retryAfter), rateLimited('1'), jsonResponse({ ok: true })]+ const fetchFn = vi.fn(async () => answers.shift()!)+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)++ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)+ expect(error).toBeInstanceOf(GfscHttpError)+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: undefined })+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(vi.getTimerCount()).toBe(0)++ // Control on the same client: a 429 with a usable hint is retried, so+ // the single call above is the hint check at work, not a missing retry.+ const control = client.stat({ drive: 'main', resourceId: 'rid' })+ await vi.advanceTimersByTimeAsync(1000)+ await expect(control).resolves.toEqual({ ok: true })+ expect(fetchFn).toHaveBeenCalledTimes(3)+ })++ it('throws at once with the hint when Retry-After exceeds the tool budget', async () => {+ vi.useFakeTimers()+ const answers = [rateLimited('30'), rateLimited('10'), jsonResponse({ ok: true })]+ const fetchFn = vi.fn(async () => answers.shift()!)+ const client = createGfscClient(tokenEnv(fetchFn), { maxRetryWaitMs: 10_000 })++ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: 30 })+ expect((error as Error).message).toContain('(retry after 30s)')+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(vi.getTimerCount()).toBe(0)++ // Control at the boundary: a delay equal to the budget is still waited out.+ const control = client.stat({ drive: 'main', resourceId: 'rid' })+ await vi.advanceTimersByTimeAsync(10_000)+ await expect(control).resolves.toEqual({ ok: true })+ expect(fetchFn).toHaveBeenCalledTimes(3)+ })++ it('does not retry any other status, even with a Retry-After header', async () => {+ vi.useFakeTimers()+ const answers = [+ new Response('unavailable', { status: 503, headers: { 'retry-after': '2' } }),+ rateLimited('2'),+ jsonResponse({ ok: true }),+ ]+ const fetchFn = vi.fn(async () => answers.shift()!)+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)++ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)+ expect(error).toMatchObject({ status: 503, message: 'gfsc 503: unavailable' })+ expect(error).toMatchObject({ retryAfterSeconds: undefined })+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(vi.getTimerCount()).toBe(0)++ // Control: the same header on a 429 is honoured.+ const control = client.stat({ drive: 'main', resourceId: 'rid' })+ await vi.advanceTimersByTimeAsync(2000)+ await expect(control).resolves.toEqual({ ok: true })+ expect(fetchFn).toHaveBeenCalledTimes(3)+ })++ it('rejects a construction without a usable retry budget', () => {+ const env = tokenEnv(vi.fn(async () => jsonResponse({})))+ expect(() => createGfscClient(env, { maxRetryWaitMs: Number.NaN })).toThrow(+ /maxRetryWaitMs must be a non-negative number/+ )+ expect(() => createGfscClient(env, { maxRetryWaitMs: -1 })).toThrow(+ /maxRetryWaitMs must be a non-negative number/+ )+ expect(createGfscClient(env, { maxRetryWaitMs: 0 })).toHaveProperty('stat')+ })+})diff --git a/mcp-host/src/internalTools/gfsClient.ts b/mcp-host/src/internalTools/gfsClient.tsindex 115cc542f..939f40155 100644--- a/mcp-host/src/internalTools/gfsClient.ts+++ b/mcp-host/src/internalTools/gfsClient.ts@@ -61,7 +61,48 @@ export function hasGfsRuntimeAccess(env: GfsRuntimeEnv): boolean {return (env.fileExists ?? existsSync)(filePath)}-export function createGfscClient(env: GfsRuntimeEnv): GfscWriteClient {+/**+ * A non-2xx answer from gfsc. The message keeps the `gfsc <status>: …` prefix+ * that `redactedFail` parses. `retryAfterSeconds` is set only on a 429, from+ * gfsc's `Retry-After` header when it is a positive integer of seconds.+ */+export class GfscHttpError extends Error {+ constructor(+ readonly status: number,+ detail: string,+ readonly retryAfterSeconds?: number+ ) {+ super(+ `gfsc ${status}: ${detail}` ++ (retryAfterSeconds === undefined ? '' : ` (retry after ${retryAfterSeconds}s)`)+ )+ this.name = 'GfscHttpError'+ }+}++export interface GfscClientOptions {+ /**+ * The longest `Retry-After` the client will sleep through before its single+ * retry: the calling tool's timeout. A longer advertised delay fails at once+ * instead of sleeping into that timeout. Time already spent inside the same+ * tool call is not subtracted; against gfsc's 60 s window and the default+ * tool budget that difference is not reachable.+ */+ maxRetryWaitMs: number+}++function retryAfterHeader(res: Response): number | undefined {+ const raw = res.headers.get('retry-after')?.trim()+ return raw !== undefined && /^[1-9][0-9]*$/.test(raw) ? Number(raw) : undefined+}++export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions): GfscWriteClient {+ const { maxRetryWaitMs } = options+ if (!Number.isFinite(maxRetryWaitMs) || maxRetryWaitMs < 0) {+ throw new Error(+ `gfsc client maxRetryWaitMs must be a non-negative number, got ${maxRetryWaitMs}`+ )+ }const fetchFn = env.fetch ?? fetchconst readerBase = envValue(env, 'MCP_HOST_GFSC_BASE_URL', DEFAULT_GFSC_BASE_URL).replace(/\/+$/,@@ -83,23 +124,46 @@ export function createGfscClient(env: GfsRuntimeEnv): GfscWriteClient {return value}- async function request(baseUrl: string, path: string, init: RequestInit = {}): Promise<unknown> {- const res = await fetchFn(`${baseUrl}${path}`, {+ async function send(baseUrl: string, path: string, init: RequestInit): Promise<Response> {+ return fetchFn(`${baseUrl}${path}`, {...init,headers: {[AUTH_HEADER]: `${AUTH_SCHEME} ${await accessValue()}`,...(init.headers ?? {}),},})- if (!res.ok) {- const body = await res.text().catch(() => '')- throw new Error(`gfsc ${res.status}: ${body || res.statusText}`)- }+ }++ async function httpError(res: Response): Promise<GfscHttpError> {+ const body = await res.text().catch(() => '')+ const retryAfter = res.status === 429 ? retryAfterHeader(res) : undefined+ return new GfscHttpError(res.status, body || res.statusText, retryAfter)+ }++ async function decode(res: Response): Promise<unknown> {const contentType = res.headers.get('content-type') ?? ''if (contentType.includes('application/json')) return res.json()return res.text()}+ // A 429 that states a delay is retried once, after that delay. gfsc's agent+ // limiter answers before the permission store and the executor run, so the+ // denied attempt had no effect and repeating a write is safe. A 429 without a+ // delay, a delay beyond maxRetryWaitMs, or a second denial fails at once.+ async function request(baseUrl: string, path: string, init: RequestInit = {}): Promise<unknown> {+ const first = await send(baseUrl, path, init)+ if (first.ok) return decode(first)+ const denied = await httpError(first)+ const waitSeconds = denied.retryAfterSeconds+ if (denied.status !== 429 || waitSeconds === undefined || waitSeconds * 1000 > maxRetryWaitMs) {+ throw denied+ }+ await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000))+ const second = await send(baseUrl, path, init)+ if (second.ok) return decode(second)+ throw await httpError(second)+ }+return {accessible: ({ drive, cursor }) => {const q = new URLSearchParams({ drive })diff --git a/mcp-host/src/workflow/workflowService.ts b/mcp-host/src/workflow/workflowService.tsindex a3bc15882..adfd50867 100644--- a/mcp-host/src/workflow/workflowService.ts+++ b/mcp-host/src/workflow/workflowService.ts@@ -932,7 +932,7 @@ export class WorkflowService {const validGfsScopes = rawGfsScopes.every(scope => knownGfsScopes.has(scope))const gfsScopes = new Set(validGfsScopes ? rawGfsScopes : [])if (requestedGfsToolNames.size > 0 && gfsScopes.size > 0 && hasGfsRuntimeAccess(gfsEnv)) {- const gfsClient = createGfscClient(gfsEnv)+ const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: timeoutMs })const scopedGfsTools = [...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),