Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): honour a GFS 429 and surface its retry hint to the model

Alfredo Lopez80ccbbfb5fae
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 5a509c804..77d0f6c62 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -131,7 +131,7 @@ jobs:
- name: Test
working-directory: ${{ matrix.service }}
env:
- PINNED_SUITES: ${{ matrix.service == 'gfs-controller' && 'serve.agentRateLimit.test.ts' || '' }}
+ PINNED_SUITES: ${{ matrix.service == 'gfs-controller' && 'serve.agentRateLimit.test.ts' || matrix.service == 'mcp-host' && 'gfsClient.retry429.test.ts' || '' }}
run: |
set -euo pipefail
if [ -z "$PINNED_SUITES" ]; then
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index cd947ff9e..31e677e92 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 3dc925d7b..826596dc8 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/tools/nativeToolRegistry.ts b/mcp-host/src/core/tools/nativeToolRegistry.ts
index 14f2b047b..cde20945c 100644
--- a/mcp-host/src/core/tools/nativeToolRegistry.ts
+++ b/mcp-host/src/core/tools/nativeToolRegistry.ts
@@ -210,7 +210,7 @@ export class NativeToolRegistry implements ToolRegistry {
}
const gfsScopes = getGfsToolScopes(gfsEnv)
if (gfsScopes && gfsScopes.size > 0) {
- const gfsClient = createGfscClient(gfsEnv)
+ const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: config.toolTimeout })
const gfsTools = [
...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),
...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),
diff --git a/mcp-host/src/internalTools/gfs.test.ts b/mcp-host/src/internalTools/gfs.test.ts
index ed2b8d154..d743f94bc 100644
--- a/mcp-host/src/internalTools/gfs.test.ts
+++ b/mcp-host/src/internalTools/gfs.test.ts
@@ -6,6 +6,7 @@ import {
buildGfsReadTools,
buildGfsWriteTools,
} from './gfs'
+import { GfscHttpError } from './gfsClient'
/**
* P3-S04 — agent gfs READ tools. Four read-only tools (list/read/stat/resolve)
@@ -84,6 +85,37 @@ describe('buildGfsReadTools', () => {
expect(r.error).not.toContain('blob/key')
})
+ it('passes a 429 retry hint to the model and still redacts the body', async () => {
+ const c = client({
+ read: vi.fn(async () => {
+ throw new GfscHttpError(429, 'rate limited on /internal/server/path blob/key', 30)
+ }),
+ })
+ const r = await toolMap(c)
+ .get('clerum__gfs_read')!
+ .execute({ drive: 'main', resourceId: 'abc' }, '')
+ expect(r.success).toBe(false)
+ expect(r.error).toBe('GFS read failed (gfsc 429: rate_limited, retry after 30s)')
+ expect(r.error).not.toContain('/internal/server/path')
+ expect(r.error).not.toContain('blob/key')
+ })
+
+ it.each([
+ [429, 'rate_limited'],
+ [400, 'invalid_request'],
+ [503, 'unavailable'],
+ ])('maps gfsc %s without a hint to %s', async (status, category) => {
+ const c = client({
+ stat: vi.fn(async () => {
+ throw new GfscHttpError(status, 'private path=/data/gfs/secret')
+ }),
+ })
+ const r = await toolMap(c)
+ .get('clerum__gfs_stat')!
+ .execute({ drive: 'main', resourceId: 'abc' }, '')
+ expect(r).toEqual({ success: false, error: `GFS read failed (gfsc ${status}: ${category})` })
+ })
+
it('strips a non-gfsc-shaped read error down to the generic label', async () => {
const c = client({
read: vi.fn(async () => {
diff --git a/mcp-host/src/internalTools/gfs.ts b/mcp-host/src/internalTools/gfs.ts
index 3b4fd4680..b797c32e8 100644
--- a/mcp-host/src/internalTools/gfs.ts
+++ b/mcp-host/src/internalTools/gfs.ts
@@ -1,4 +1,5 @@
import type { InternalToolDefinition, InternalToolResult } from '../workflow/types'
+import { GfscHttpError } from './gfsClient'
/**
* Agent gfs READ tools (spec community.md §Operator surfaces, plan P3-S04).
@@ -183,10 +184,18 @@ function redactedFail(label: string, error: unknown): InternalToolResult {
? 'precondition_failed'
: status === 413
? 'limit_exceeded'
- : status >= 500
- ? 'unavailable'
- : 'failed'
- return { success: false, error: `${label} (gfsc ${status}: ${category})` }
+ : status === 429
+ ? 'rate_limited'
+ : status >= 500
+ ? 'unavailable'
+ : 'failed'
+ // The retry hint comes from the typed error's parsed Retry-After header, an
+ // integer, never from the response body.
+ const hint =
+ error instanceof GfscHttpError && error.status === 429 && error.retryAfterSeconds !== undefined
+ ? `, retry after ${error.retryAfterSeconds}s`
+ : ''
+ return { success: false, error: `${label} (gfsc ${status}: ${category}${hint})` }
}
function mutationFail(error: unknown): InternalToolResult {
diff --git a/mcp-host/src/internalTools/gfsClient.retry429.test.ts b/mcp-host/src/internalTools/gfsClient.retry429.test.ts
new file mode 100644
index 000000000..6d95f7b04
--- /dev/null
+++ b/mcp-host/src/internalTools/gfsClient.retry429.test.ts
@@ -0,0 +1,169 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildGfsReadTools } from './gfs'
+import { GfscHttpError, createGfscClient } from './gfsClient'
+
+/**
+ * #699 stress: N agent pods rate limited by gfsc at once. One mcp-host process
+ * holds one subject token, so N clients with distinct tokens stand for N
+ * agents. The fetch stub is keyed by the bearer token and timers are faked.
+ */
+
+const N = 50
+
+type Call = { token: string; at: number }
+
+function rateLimited(retryAfter?: string): Response {
+ const headers: Record<string, string> = { 'content-type': 'application/json' }
+ if (retryAfter !== undefined) headers['retry-after'] = retryAfter
+ return new Response(JSON.stringify({ ok: false, error: { code: 'rate_limited' } }), {
+ status: 429,
+ headers,
+ })
+}
+
+function ok(token: string): Response {
+ return new Response(JSON.stringify({ token }), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ })
+}
+
+function tokenOf(init?: RequestInit): string {
+ const auth = (init?.headers as Record<string, string> | undefined)?.authorization
+ if (!auth?.startsWith('Bearer ')) throw new Error('request carried no bearer token')
+ return auth.slice('Bearer '.length)
+}
+
+// answer(token, attempt) decides the response for the attempt-th call (1-based)
+// made with that token.
+function fleet(answer: (token: string, attempt: number) => Response, maxRetryWaitMs: number) {
+ const calls: Call[] = []
+ const fetchFn = vi.fn(async (_input: string, init?: RequestInit) => {
+ const token = tokenOf(init)
+ calls.push({ token, at: Date.now() })
+ return answer(token, calls.filter(c => c.token === token).length)
+ })
+ const clients = Array.from({ length: N }, (_, i) => ({
+ token: `agent-${i}`,
+ client: createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? `agent-${i}` : undefined),
+ fetch: fetchFn,
+ },
+ { maxRetryWaitMs }
+ ),
+ }))
+ const perToken = (token: string) => calls.filter(c => c.token === token)
+ return { calls, clients, perToken }
+}
+
+describe('gfsc client under a concurrent 429 burst', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('M1: every agent retries once after Retry-After and succeeds', async () => {
+ const { calls, clients, perToken } = fleet(
+ (token, attempt) => (attempt === 1 ? rateLimited('2') : ok(token)),
+ 60_000
+ )
+
+ const results = Promise.all(
+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ expect(calls).toHaveLength(N)
+ await vi.advanceTimersByTimeAsync(1999)
+ expect(calls).toHaveLength(N)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(calls).toHaveLength(2 * N)
+
+ await expect(results).resolves.toEqual(clients.map(({ token }) => ({ token })))
+ for (const { token } of clients) {
+ const [first, retry] = perToken(token)
+ expect(perToken(token)).toHaveLength(2)
+ expect(retry!.at - first!.at).toBeGreaterThanOrEqual(2000)
+ }
+ })
+
+ it('M2: a 429 without Retry-After fails every agent at once', async () => {
+ const { calls, clients } = fleet(() => rateLimited(), 60_000)
+
+ const settled = await Promise.allSettled(
+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))
+ )
+ expect(calls).toHaveLength(N)
+ expect(settled.every(s => s.status === 'rejected')).toBe(true)
+ for (const s of settled) {
+ const reason = (s as PromiseRejectedResult).reason
+ expect(reason).toBeInstanceOf(GfscHttpError)
+ expect(reason).toMatchObject({ status: 429, retryAfterSeconds: undefined })
+ }
+ expect(vi.getTimerCount()).toBe(0)
+ })
+
+ it('M3: a Retry-After above the tool budget fails at once with the hint', async () => {
+ const { calls, clients } = fleet(() => rateLimited('2'), 1000)
+
+ const settled = await Promise.allSettled(
+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))
+ )
+ expect(calls).toHaveLength(N)
+ for (const s of settled) {
+ expect(s.status).toBe('rejected')
+ const reason = (s as PromiseRejectedResult).reason as GfscHttpError
+ expect(reason).toMatchObject({ status: 429, retryAfterSeconds: 2 })
+ expect(reason.message).toContain('(retry after 2s)')
+ }
+ expect(vi.getTimerCount()).toBe(0)
+ })
+
+ it('M4: an agent still denied after its retry stops at two calls', async () => {
+ const { calls, clients, perToken } = fleet(() => rateLimited('2'), 60_000)
+
+ const settled = Promise.allSettled(
+ clients.map(({ client }) => client.stat({ drive: 'main', resourceId: 'rid' }))
+ )
+ await vi.advanceTimersByTimeAsync(2000)
+ expect(calls).toHaveLength(2 * N)
+ await vi.advanceTimersByTimeAsync(120_000)
+ expect(calls).toHaveLength(2 * N)
+
+ for (const s of await settled) {
+ expect(s.status).toBe('rejected')
+ expect((s as PromiseRejectedResult).reason).toMatchObject({
+ status: 429,
+ retryAfterSeconds: 2,
+ })
+ }
+ for (const { token } of clients) expect(perToken(token)).toHaveLength(2)
+ expect(vi.getTimerCount()).toBe(0)
+ })
+
+ it('M5: the read tool returns rate_limited with the hint and never throws', async () => {
+ const { calls, clients } = fleet(() => rateLimited('2'), 60_000)
+ const statTools = clients.map(({ client }) => {
+ const tool = buildGfsReadTools(client).find(t => t.name === 'clerum__gfs_stat')
+ if (!tool) throw new Error('clerum__gfs_stat is not registered')
+ return tool
+ })
+
+ const results = Promise.all(
+ statTools.map(tool => tool.execute({ drive: 'main', resourceId: 'rid' }, ''))
+ )
+ await vi.advanceTimersByTimeAsync(2000)
+ expect(calls).toHaveLength(2 * N)
+
+ const settled = await results
+ expect(settled).toHaveLength(N)
+ for (const result of settled) {
+ expect(result).toEqual({
+ success: false,
+ error: 'GFS read failed (gfsc 429: rate_limited, retry after 2s)',
+ })
+ }
+ })
+})
diff --git a/mcp-host/src/internalTools/gfsClient.test.ts b/mcp-host/src/internalTools/gfsClient.test.ts
index a575f6355..d8492e369 100644
--- a/mcp-host/src/internalTools/gfsClient.test.ts
+++ b/mcp-host/src/internalTools/gfsClient.test.ts
@@ -1,6 +1,7 @@
-import { describe, expect, it, vi } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import {
DEFAULT_GFS_ACCESS_FILE,
+ GfscHttpError,
createGfscClient,
getGfsToolScopes,
hasGfsRuntimeAccess,
@@ -19,6 +20,9 @@ function jsonResponse(body: unknown, status = 200): Response {
})
}
+// Every test client gets a budget far above any Retry-After used here.
+const clientOptions = { maxRetryWaitMs: 60_000 }
+
function textResponse(body: string, status: number, statusText = ''): Response {
return new Response(body, { status, statusText })
}
@@ -54,10 +58,13 @@ describe('gfs runtime gfsc client', () => {
const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>
jsonResponse({ ok: true, data: { items: [] } })
)
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await client.accessible({ drive: 'main' })
expect(fetchFn.mock.calls[0]?.[0]).toBe(
'http://gfsc.gfs.svc.cluster.local:8087/v1/accessible?drive=main'
@@ -71,10 +78,13 @@ describe('gfs runtime gfsc client', () => {
const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>
jsonResponse({ ok: true, data: { version: 3 } })
)
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await client.write({ drive: 'main', resourceId: 'rid', content: 'new', ifMatch: 2 })
expect(fetchFn.mock.calls[0]?.[0]).toBe(
'http://gfsc-writer.gfs.svc.cluster.local:8087/v1/resources/rid/content?drive=main'
@@ -89,10 +99,13 @@ describe('gfs runtime gfsc client', () => {
const fetchFn = vi.fn(async (_input: string, _init?: RequestInit) =>
jsonResponse({ ok: true, data: {} })
)
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await client.createFile({
drive: 'main',
parentResourceId: 'parent/id',
@@ -136,11 +149,14 @@ describe('gfs runtime gfsc client', () => {
})
it('fails loud when the mounted runtime token file is empty', async () => {
- const client = createGfscClient({
- get: () => undefined,
- readFile: async () => ' \n',
- fetch: vi.fn(async () => jsonResponse({ ok: true })),
- })
+ const client = createGfscClient(
+ {
+ get: () => undefined,
+ readFile: async () => ' \n',
+ fetch: vi.fn(async () => jsonResponse({ ok: true })),
+ },
+ clientOptions
+ )
await expect(client.accessible({ drive: 'main' })).rejects.toThrow(
/MCP_HOST_GFS_TOKEN_FILE is empty/
@@ -149,10 +165,13 @@ describe('gfs runtime gfsc client', () => {
it('surfaces gfsc authorization denials with the response body', async () => {
const fetchFn = vi.fn(async () => textResponse('not authorized to write this resource', 403))
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await expect(
client.write({ drive: 'main', resourceId: 'rid', content: 'new', ifMatch: 2 })
@@ -161,10 +180,13 @@ describe('gfs runtime gfsc client', () => {
it('surfaces gfsc read denials with the response body', async () => {
const fetchFn = vi.fn(async () => textResponse('not authorized to read this resource', 403))
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await expect(client.read({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(
/gfsc 403: not authorized to read this resource/
@@ -185,10 +207,13 @@ describe('gfs runtime gfsc client', () => {
},
})
const fetchFn = vi.fn(async () => textResponse(envelope, 503))
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await expect(client.resolve({ uri: 'gfs://main/docs/report.md' })).rejects.toThrow(
`gfsc 503: ${envelope}`
@@ -197,10 +222,13 @@ describe('gfs runtime gfsc client', () => {
it('surfaces gfsc transient failures with the status text when the body is empty', async () => {
const fetchFn = vi.fn(async () => textResponse('', 503, 'Service Unavailable'))
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await expect(client.stat({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(
/gfsc 503: Service Unavailable/
@@ -209,13 +237,174 @@ describe('gfs runtime gfsc client', () => {
it('surfaces gfsc list transient failures with the status text when the body is empty', async () => {
const fetchFn = vi.fn(async () => textResponse('', 503, 'Service Unavailable'))
- const client = createGfscClient({
- get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
- fetch: fetchFn,
- })
+ const client = createGfscClient(
+ {
+ get: key => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ },
+ clientOptions
+ )
await expect(client.list({ drive: 'main', resourceId: 'rid' })).rejects.toThrow(
/gfsc 503: Service Unavailable/
)
})
})
+
+function rateLimited(retryAfter?: string): Response {
+ const headers: Record<string, string> = { 'content-type': 'application/json' }
+ if (retryAfter !== undefined) headers['retry-after'] = retryAfter
+ // gfsc's envelope also carries retryAfterSeconds; the client reads only the
+ // header, so the no-header cases below prove the body is never consulted.
+ return new Response(
+ JSON.stringify({ ok: false, error: { code: 'rate_limited', retryAfterSeconds: 5 } }),
+ { status: 429, headers }
+ )
+}
+
+function tokenEnv(fetchFn: (input: string, init?: RequestInit) => Promise<Response>) {
+ return {
+ get: (key: string) => (key === 'MCP_HOST_GFS_TOKEN' ? 'gfs-access' : undefined),
+ fetch: fetchFn,
+ }
+}
+
+describe('gfsc client on a 429', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('retries once after the stated delay and resolves', async () => {
+ vi.useFakeTimers()
+ const callTimes: number[] = []
+ const fetchFn = vi.fn(async () => {
+ callTimes.push(Date.now())
+ return callTimes.length === 1 ? rateLimited('2') : jsonResponse({ ok: true, version: 4 })
+ })
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const pending = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(0)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(2000)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ await expect(pending).resolves.toEqual({ ok: true, version: 4 })
+ expect(callTimes[1]! - callTimes[0]!).toBe(2000)
+ })
+
+ it('waits exactly the advertised Retry-After, not a constant', async () => {
+ vi.useFakeTimers()
+ const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout')
+ let calls = 0
+ const fetchFn = vi.fn(async () => (++calls === 1 ? rateLimited('7') : jsonResponse({})))
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const pending = client.read({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(0)
+ expect(setTimeoutSpy).toHaveBeenCalledTimes(1)
+ expect(setTimeoutSpy.mock.calls[0]?.[1]).toBe(7000)
+ await vi.advanceTimersByTimeAsync(6999)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ await expect(pending).resolves.toEqual({})
+ })
+
+ it('gives up after one retry and keeps the hint on the thrown error', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi.fn(async () => rateLimited('2'))
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const rejection = expect(
+ client.write({ drive: 'main', resourceId: 'rid', content: 'x', ifMatch: 1 })
+ ).rejects.toMatchObject({
+ name: 'GfscHttpError',
+ status: 429,
+ retryAfterSeconds: 2,
+ message: expect.stringMatching(/^gfsc 429: .* \(retry after 2s\)$/),
+ })
+ await vi.advanceTimersByTimeAsync(2000)
+ await rejection
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ })
+
+ it.each([
+ ['absent', undefined],
+ ['an HTTP-date', 'Wed, 23 Sep 2026 10:00:00 GMT'],
+ ['zero', '0'],
+ ['fractional', '1.5'],
+ ])('throws at once without retrying when Retry-After is %s', async (_label, retryAfter) => {
+ vi.useFakeTimers()
+ const answers = [rateLimited(retryAfter), rateLimited('1'), jsonResponse({ ok: true })]
+ const fetchFn = vi.fn(async () => answers.shift()!)
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)
+ expect(error).toBeInstanceOf(GfscHttpError)
+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: undefined })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+
+ // Control on the same client: a 429 with a usable hint is retried, so
+ // the single call above is the hint check at work, not a missing retry.
+ const control = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(1000)
+ await expect(control).resolves.toEqual({ ok: true })
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ })
+
+ it('throws at once with the hint when Retry-After exceeds the tool budget', async () => {
+ vi.useFakeTimers()
+ const answers = [rateLimited('30'), rateLimited('10'), jsonResponse({ ok: true })]
+ const fetchFn = vi.fn(async () => answers.shift()!)
+ const client = createGfscClient(tokenEnv(fetchFn), { maxRetryWaitMs: 10_000 })
+
+ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)
+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: 30 })
+ expect((error as Error).message).toContain('(retry after 30s)')
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+
+ // Control at the boundary: a delay equal to the budget is still waited out.
+ const control = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(10_000)
+ await expect(control).resolves.toEqual({ ok: true })
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ })
+
+ it('does not retry any other status, even with a Retry-After header', async () => {
+ vi.useFakeTimers()
+ const answers = [
+ new Response('unavailable', { status: 503, headers: { 'retry-after': '2' } }),
+ rateLimited('2'),
+ jsonResponse({ ok: true }),
+ ]
+ const fetchFn = vi.fn(async () => answers.shift()!)
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)
+ expect(error).toMatchObject({ status: 503, message: 'gfsc 503: unavailable' })
+ expect(error).toMatchObject({ retryAfterSeconds: undefined })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+
+ // Control: the same header on a 429 is honoured.
+ const control = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(2000)
+ await expect(control).resolves.toEqual({ ok: true })
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ })
+
+ it('rejects a construction without a usable retry budget', () => {
+ const env = tokenEnv(vi.fn(async () => jsonResponse({})))
+ expect(() => createGfscClient(env, { maxRetryWaitMs: Number.NaN })).toThrow(
+ /maxRetryWaitMs must be a non-negative number/
+ )
+ expect(() => createGfscClient(env, { maxRetryWaitMs: -1 })).toThrow(
+ /maxRetryWaitMs must be a non-negative number/
+ )
+ expect(createGfscClient(env, { maxRetryWaitMs: 0 })).toHaveProperty('stat')
+ })
+})
diff --git a/mcp-host/src/internalTools/gfsClient.ts b/mcp-host/src/internalTools/gfsClient.ts
index 115cc542f..939f40155 100644
--- a/mcp-host/src/internalTools/gfsClient.ts
+++ b/mcp-host/src/internalTools/gfsClient.ts
@@ -61,7 +61,48 @@ export function hasGfsRuntimeAccess(env: GfsRuntimeEnv): boolean {
return (env.fileExists ?? existsSync)(filePath)
}
-export function createGfscClient(env: GfsRuntimeEnv): GfscWriteClient {
+/**
+ * A non-2xx answer from gfsc. The message keeps the `gfsc <status>: …` prefix
+ * that `redactedFail` parses. `retryAfterSeconds` is set only on a 429, from
+ * gfsc's `Retry-After` header when it is a positive integer of seconds.
+ */
+export class GfscHttpError extends Error {
+ constructor(
+ readonly status: number,
+ detail: string,
+ readonly retryAfterSeconds?: number
+ ) {
+ super(
+ `gfsc ${status}: ${detail}` +
+ (retryAfterSeconds === undefined ? '' : ` (retry after ${retryAfterSeconds}s)`)
+ )
+ this.name = 'GfscHttpError'
+ }
+}
+
+export interface GfscClientOptions {
+ /**
+ * The longest `Retry-After` the client will sleep through before its single
+ * retry: the calling tool's timeout. A longer advertised delay fails at once
+ * instead of sleeping into that timeout. Time already spent inside the same
+ * tool call is not subtracted; against gfsc's 60 s window and the default
+ * tool budget that difference is not reachable.
+ */
+ maxRetryWaitMs: number
+}
+
+function retryAfterHeader(res: Response): number | undefined {
+ const raw = res.headers.get('retry-after')?.trim()
+ return raw !== undefined && /^[1-9][0-9]*$/.test(raw) ? Number(raw) : undefined
+}
+
+export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions): GfscWriteClient {
+ const { maxRetryWaitMs } = options
+ if (!Number.isFinite(maxRetryWaitMs) || maxRetryWaitMs < 0) {
+ throw new Error(
+ `gfsc client maxRetryWaitMs must be a non-negative number, got ${maxRetryWaitMs}`
+ )
+ }
const fetchFn = env.fetch ?? fetch
const readerBase = envValue(env, 'MCP_HOST_GFSC_BASE_URL', DEFAULT_GFSC_BASE_URL).replace(
/\/+$/,
@@ -83,23 +124,46 @@ export function createGfscClient(env: GfsRuntimeEnv): GfscWriteClient {
return value
}
- async function request(baseUrl: string, path: string, init: RequestInit = {}): Promise<unknown> {
- const res = await fetchFn(`${baseUrl}${path}`, {
+ async function send(baseUrl: string, path: string, init: RequestInit): Promise<Response> {
+ return fetchFn(`${baseUrl}${path}`, {
...init,
headers: {
[AUTH_HEADER]: `${AUTH_SCHEME} ${await accessValue()}`,
...(init.headers ?? {}),
},
})
- if (!res.ok) {
- const body = await res.text().catch(() => '')
- throw new Error(`gfsc ${res.status}: ${body || res.statusText}`)
- }
+ }
+
+ async function httpError(res: Response): Promise<GfscHttpError> {
+ const body = await res.text().catch(() => '')
+ const retryAfter = res.status === 429 ? retryAfterHeader(res) : undefined
+ return new GfscHttpError(res.status, body || res.statusText, retryAfter)
+ }
+
+ async function decode(res: Response): Promise<unknown> {
const contentType = res.headers.get('content-type') ?? ''
if (contentType.includes('application/json')) return res.json()
return res.text()
}
+ // A 429 that states a delay is retried once, after that delay. gfsc's agent
+ // limiter answers before the permission store and the executor run, so the
+ // denied attempt had no effect and repeating a write is safe. A 429 without a
+ // delay, a delay beyond maxRetryWaitMs, or a second denial fails at once.
+ async function request(baseUrl: string, path: string, init: RequestInit = {}): Promise<unknown> {
+ const first = await send(baseUrl, path, init)
+ if (first.ok) return decode(first)
+ const denied = await httpError(first)
+ const waitSeconds = denied.retryAfterSeconds
+ if (denied.status !== 429 || waitSeconds === undefined || waitSeconds * 1000 > maxRetryWaitMs) {
+ throw denied
+ }
+ await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000))
+ const second = await send(baseUrl, path, init)
+ if (second.ok) return decode(second)
+ throw await httpError(second)
+ }
+
return {
accessible: ({ drive, cursor }) => {
const q = new URLSearchParams({ drive })
diff --git a/mcp-host/src/workflow/workflowService.ts b/mcp-host/src/workflow/workflowService.ts
index a3bc15882..adfd50867 100644
--- a/mcp-host/src/workflow/workflowService.ts
+++ b/mcp-host/src/workflow/workflowService.ts
@@ -932,7 +932,7 @@ export class WorkflowService {
const validGfsScopes = rawGfsScopes.every(scope => knownGfsScopes.has(scope))
const gfsScopes = new Set(validGfsScopes ? rawGfsScopes : [])
if (requestedGfsToolNames.size > 0 && gfsScopes.size > 0 && hasGfsRuntimeAccess(gfsEnv)) {
- const gfsClient = createGfscClient(gfsEnv)
+ const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: timeoutMs })
const scopedGfsTools = [
...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),
...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),