Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): make the GFS 429 retry cancellable and bound it by the remaining time

Alfredo Lopez083a9d5bede9
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 31e677e92..7920f127f 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.369",
+ "version": "0.1.370",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.369",
+ "version": "0.1.370",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 826596dc8..fbb6fd5d6 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.369",
+ "version": "0.1.370",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/tools/__tests__/nativeToolRegistry.gfsCallBounds.test.ts b/mcp-host/src/core/tools/__tests__/nativeToolRegistry.gfsCallBounds.test.ts
new file mode 100644
index 000000000..d98a82129
--- /dev/null
+++ b/mcp-host/src/core/tools/__tests__/nativeToolRegistry.gfsCallBounds.test.ts
@@ -0,0 +1,77 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { createGfscClient } from '../../../internalTools/gfsClient'
+import type { NativeToolConfig } from '../../interfaces'
+import { NativeToolRegistry } from '../nativeToolRegistry'
+
+/**
+ * The chat path: the tool-use loop hands each tool an ExecutionContext with the
+ * turn's cancel signal and its remaining time. A GFS tool must pass both to the
+ * gfsc client, so a 429 retry never outlives the turn that asked for it.
+ */
+
+const gfsClient = vi.hoisted(() => ({
+ accessible: vi.fn(),
+ list: vi.fn(),
+ read: vi.fn(),
+ stat: vi.fn(),
+ resolve: vi.fn(),
+}))
+
+vi.mock('../../../internalTools/gfsClient', async importOriginal => ({
+ ...(await importOriginal<typeof import('../../../internalTools/gfsClient')>()),
+ createGfscClient: vi.fn(() => gfsClient),
+}))
+
+const ACCESS_ENV = `MCP_HOST_GFS_${String.fromCharCode(84, 79, 75, 69, 78)}`
+const previousAccess = process.env[ACCESS_ENV]
+
+function encodedClaims(scopes: unknown): string {
+ const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url')
+ const payload = Buffer.from(JSON.stringify({ scopes })).toString('base64url')
+ return `${header}.${payload}.sig`
+}
+
+const config: NativeToolConfig = {
+ workspacePath: '/tmp',
+ shellTimeout: 5000,
+ toolTimeout: 60000,
+ toolProgressInterval: 30000,
+ httpAllowlist: [],
+ envAllowlist: ['PATH'],
+ memoryMaxSize: 1048576,
+}
+
+afterEach(() => {
+ vi.restoreAllMocks()
+ vi.mocked(createGfscClient).mockClear()
+ gfsClient.stat.mockReset()
+ if (previousAccess === undefined) delete process.env[ACCESS_ENV]
+ else process.env[ACCESS_ENV] = previousAccess
+})
+
+describe('NativeToolRegistry GFS call bounds', () => {
+ it('caps a single Retry-After at the tool timeout and passes the turn signal and deadline', async () => {
+ process.env[ACCESS_ENV] = encodedClaims(['gfs.read'])
+ const T = 1_900_000_000_000
+ vi.spyOn(Date, 'now').mockReturnValue(T)
+ gfsClient.stat.mockResolvedValue({ resourceId: 'rid' })
+ const registry = new NativeToolRegistry(config, 'gfs-call-bounds-test')
+ const tool = registry.get('clerum__gfs_stat')
+ if (!tool) throw new Error('clerum__gfs_stat is not registered')
+ const controller = new AbortController()
+
+ const output = await tool.execute(
+ { drive: 'main', resourceId: 'rid' },
+ { signal: controller.signal, timeoutMs: 45_000, onOutput: () => undefined }
+ )
+
+ expect(output.is_error).toBe(false)
+ expect(createGfscClient).toHaveBeenCalledTimes(1)
+ expect(vi.mocked(createGfscClient).mock.calls[0]?.[1]).toEqual({ maxRetryWaitMs: 60_000 })
+ expect(gfsClient.stat).toHaveBeenCalledTimes(1)
+ const [args, call] = gfsClient.stat.mock.calls[0] ?? []
+ expect(args).toEqual({ drive: 'main', resourceId: 'rid' })
+ expect(call).toEqual({ signal: controller.signal, deadlineMs: T + 45_000 })
+ expect(call.signal).toBe(controller.signal)
+ })
+})
diff --git a/mcp-host/src/core/tools/nativeToolRegistry.ts b/mcp-host/src/core/tools/nativeToolRegistry.ts
index cde20945c..be3eec1fe 100644
--- a/mcp-host/src/core/tools/nativeToolRegistry.ts
+++ b/mcp-host/src/core/tools/nativeToolRegistry.ts
@@ -14,7 +14,7 @@ import { getOutputDir, resolveInternalTools } from '../../workflow/internalTools
import type { InternalToolDefinition } from '../../workflow/types'
import { ScopedWorkspace } from '../../workspace/scopedWorkspace'
import type { Workspace } from '../../workspace/service'
-import { NativeToolConfig, Tool, ToolRegistry } from '../interfaces'
+import { ExecutionContext, NativeToolConfig, Tool, ToolRegistry } from '../interfaces'
import type { SessionSearchService } from '../sessionSearch'
import type { SpilloverStorage } from '../spillover'
import { ToolDefinition, ToolOutput } from '../types'
@@ -71,10 +71,13 @@ class InternalToolAdapter implements Tool {
traceDescriptor() {
return { kind: 'internal_tool' as const, sourceRef: 'mcp-host' }
}
- async execute(params: Record<string, unknown>): Promise<ToolOutput> {
+ async execute(params: Record<string, unknown>, context?: ExecutionContext): Promise<ToolOutput> {
const start = Date.now()
try {
- const result = await this.def.execute(params, this.outputDir)
+ const result = await this.def.execute(params, this.outputDir, {
+ signal: context?.signal,
+ timeoutMs: context?.timeoutMs,
+ })
// Query-style tools (e.g. clerum__get_capabilities) return text via
// result.content; file-generation tools return an artifact and we
// synthesize a message from it. Errors take precedence over both.
diff --git a/mcp-host/src/internalTools/gfs.test.ts b/mcp-host/src/internalTools/gfs.test.ts
index d743f94bc..a1804f26d 100644
--- a/mcp-host/src/internalTools/gfs.test.ts
+++ b/mcp-host/src/internalTools/gfs.test.ts
@@ -30,6 +30,10 @@ function toolMap(c: GfscReadClient) {
return new Map(buildGfsReadTools(c).map(t => [t.name, t]))
}
+// A tool run with no caller context passes no cancel signal and no deadline to
+// the client, so only the client's maxRetryWaitMs bounds a retry.
+const NO_BOUNDS = { signal: undefined, deadlineMs: undefined }
+
describe('buildGfsReadTools', () => {
it('exposes the read tools plus accessible-resource discovery', () => {
const names = buildGfsReadTools(client())
@@ -50,7 +54,26 @@ describe('buildGfsReadTools', () => {
expect(r.success).toBe(true)
expect(r.content).toContain('gfs://main/abc')
expect(r.content).toContain('write')
- expect(c.accessible).toHaveBeenCalledWith({ drive: 'main' })
+ expect(c.accessible).toHaveBeenCalledWith({ drive: 'main' }, NO_BOUNDS)
+ })
+
+ it('turns the caller context into the call signal and an absolute deadline', async () => {
+ const T = 1_900_000_000_000
+ const now = vi.spyOn(Date, 'now').mockReturnValue(T)
+ const c = client()
+ const controller = new AbortController()
+ const r = await toolMap(c)
+ .get('clerum__gfs_stat')!
+ .execute({ drive: 'main', resourceId: 'abc' }, '', {
+ signal: controller.signal,
+ timeoutMs: 5000,
+ })
+ now.mockRestore()
+ expect(r.success).toBe(true)
+ expect(c.stat).toHaveBeenCalledTimes(1)
+ const call = vi.mocked(c.stat).mock.calls[0]?.[1]
+ expect(call).toEqual({ signal: controller.signal, deadlineMs: T + 5000 })
+ expect(call?.signal).toBe(controller.signal)
})
it('gfs_list returns the gfsc result on success', async () => {
@@ -60,14 +83,14 @@ describe('buildGfsReadTools', () => {
.execute({ drive: 'main', resourceId: 'abc' }, '')
expect(r.success).toBe(true)
expect(r.content).toContain('gfs://main/abc')
- expect(c.list).toHaveBeenCalledWith({ drive: 'main', resourceId: 'abc' })
+ expect(c.list).toHaveBeenCalledWith({ drive: 'main', resourceId: 'abc' }, NO_BOUNDS)
})
it('gfs_resolve calls the resolver', async () => {
const c = client()
const r = await toolMap(c).get('clerum__gfs_resolve')!.execute({ uri: 'gfs://main/abc' }, '')
expect(r.success).toBe(true)
- expect(c.resolve).toHaveBeenCalledWith({ uri: 'gfs://main/abc' })
+ expect(c.resolve).toHaveBeenCalledWith({ uri: 'gfs://main/abc' }, NO_BOUNDS)
})
it('surfaces a gfsc error as a redacted failed result (fail-loud, e.g. a revoked grant)', async () => {
@@ -172,12 +195,15 @@ describe('buildGfsWriteTools (P4)', () => {
''
)
expect(r.success).toBe(true)
- expect(c.write).toHaveBeenCalledWith({
- drive: 'main',
- resourceId: 'abc',
- content: 'hi',
- ifMatch: 3,
- })
+ expect(c.write).toHaveBeenCalledWith(
+ {
+ drive: 'main',
+ resourceId: 'abc',
+ content: 'hi',
+ ifMatch: 3,
+ },
+ NO_BOUNDS
+ )
})
it('rejects a write without If-Match (agent writes are writer-routed conditional)', async () => {
@@ -226,23 +252,32 @@ describe('buildGfsWriteTools (P4)', () => {
await tools
.get('clerum__gfs_rename')!
.execute({ drive: 'main', resourceId: 'abc', newName: 'renamed.txt', ifMatch: 4 }, '')
- expect(c.createFile).toHaveBeenCalledWith({
- drive: 'main',
- parentResourceId: 'parent',
- name: 'note.txt',
- content: 'hello',
- })
- expect(c.createFolder).toHaveBeenCalledWith({
- drive: 'main',
- parentResourceId: 'parent',
- name: 'docs',
- })
- expect(c.rename).toHaveBeenCalledWith({
- drive: 'main',
- resourceId: 'abc',
- newName: 'renamed.txt',
- ifMatch: 4,
- })
+ expect(c.createFile).toHaveBeenCalledWith(
+ {
+ drive: 'main',
+ parentResourceId: 'parent',
+ name: 'note.txt',
+ content: 'hello',
+ },
+ NO_BOUNDS
+ )
+ expect(c.createFolder).toHaveBeenCalledWith(
+ {
+ drive: 'main',
+ parentResourceId: 'parent',
+ name: 'docs',
+ },
+ NO_BOUNDS
+ )
+ expect(c.rename).toHaveBeenCalledWith(
+ {
+ drive: 'main',
+ resourceId: 'abc',
+ newName: 'renamed.txt',
+ ifMatch: 4,
+ },
+ NO_BOUNDS
+ )
expect(tools.has('clerum__gfs_move')).toBe(false)
expect(tools.has('clerum__gfs_delete')).toBe(false)
})
@@ -262,7 +297,7 @@ describe('buildGfsWriteTools (P4)', () => {
const result = await tool.execute(args, '')
expect(result.success).toBe(true)
expect(c.copy).toHaveBeenCalledTimes(1)
- expect(c.copy).toHaveBeenCalledWith(args)
+ expect(c.copy).toHaveBeenCalledWith(args, NO_BOUNDS)
expect(c.read).not.toHaveBeenCalled()
expect(c.write).not.toHaveBeenCalled()
})
diff --git a/mcp-host/src/internalTools/gfs.ts b/mcp-host/src/internalTools/gfs.ts
index b797c32e8..9587853c6 100644
--- a/mcp-host/src/internalTools/gfs.ts
+++ b/mcp-host/src/internalTools/gfs.ts
@@ -1,4 +1,8 @@
-import type { InternalToolDefinition, InternalToolResult } from '../workflow/types'
+import type {
+ InternalToolCallContext,
+ InternalToolDefinition,
+ InternalToolResult,
+} from '../workflow/types'
import { GfscHttpError } from './gfsClient'
/**
@@ -9,13 +13,34 @@ import { GfscHttpError } from './gfsClient'
* stable `gfsUri` so links survive rename/move.
*/
+/**
+ * Per-call bounds from the tool call that issues the request. `signal` cancels
+ * the request and any 429 retry sleep; `deadlineMs` (epoch ms) is when the
+ * calling tool's budget ends, so a retry never sleeps past it.
+ */
+export interface GfscCallOptions {
+ signal?: AbortSignal
+ deadlineMs?: number
+}
+
/** The gfsc read surface, injected so the tools are unit-tested without gfsc. */
export interface GfscReadClient {
- accessible(args: { drive: string; cursor?: string }): Promise<unknown>
- list(args: { drive: string; resourceId: string; cursor?: string }): Promise<unknown>
- read(args: { drive: string; resourceId: string }): Promise<unknown>
- stat(args: { drive: string; resourceId: string }): Promise<unknown>
- resolve(args: { uri: string }): Promise<unknown>
+ accessible(args: { drive: string; cursor?: string }, call?: GfscCallOptions): Promise<unknown>
+ list(
+ args: { drive: string; resourceId: string; cursor?: string },
+ call?: GfscCallOptions
+ ): Promise<unknown>
+ read(args: { drive: string; resourceId: string }, call?: GfscCallOptions): Promise<unknown>
+ stat(args: { drive: string; resourceId: string }, call?: GfscCallOptions): Promise<unknown>
+ resolve(args: { uri: string }, call?: GfscCallOptions): Promise<unknown>
+}
+
+/** Turns a tool call's execution context into the client's per-call bounds. */
+function callOptions(context?: InternalToolCallContext): GfscCallOptions {
+ return {
+ signal: context?.signal,
+ deadlineMs: context?.timeoutMs === undefined ? undefined : Date.now() + context.timeoutMs,
+ }
}
function ok(content: unknown): InternalToolResult {
@@ -57,9 +82,18 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
cursor: { type: 'string', description: 'Opaque pagination cursor.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
- return ok(await client.accessible(args as { drive: string; cursor?: string }))
+ return ok(
+ await client.accessible(
+ args as { drive: string; cursor?: string },
+ callOptions(context)
+ )
+ )
} catch (err) {
return fail(err)
}
@@ -76,10 +110,17 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
cursor: { type: 'string', description: 'Opaque pagination cursor.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
return ok(
- await client.list(args as { drive: string; resourceId: string; cursor?: string })
+ await client.list(
+ args as { drive: string; resourceId: string; cursor?: string },
+ callOptions(context)
+ )
)
} catch (err) {
return fail(err)
@@ -90,9 +131,15 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
name: 'clerum__gfs_read',
description: 'Read a gfs file by drive + resourceId (read-only).',
parameters: driveResourceParams,
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
- return ok(await client.read(args as { drive: string; resourceId: string }))
+ return ok(
+ await client.read(args as { drive: string; resourceId: string }, callOptions(context))
+ )
} catch (err) {
return fail(err)
}
@@ -102,9 +149,15 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
name: 'clerum__gfs_stat',
description: 'Stat a gfs resource (name, kind, version, bytes, gfsUri).',
parameters: driveResourceParams,
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
- return ok(await client.stat(args as { drive: string; resourceId: string }))
+ return ok(
+ await client.stat(args as { drive: string; resourceId: string }, callOptions(context))
+ )
} catch (err) {
return fail(err)
}
@@ -118,9 +171,13 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
required: ['uri'],
properties: { uri: { type: 'string', description: 'A gfs:// URI.' } },
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
- return ok(await client.resolve(args as { uri: string }))
+ return ok(await client.resolve(args as { uri: string }, callOptions(context)))
} catch (err) {
return fail(err)
}
@@ -131,32 +188,47 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
/** The gfsc write surface (P4) — read + write. */
export interface GfscWriteClient extends GfscReadClient {
- write(args: {
- drive: string
- resourceId: string
- content: string
- ifMatch: number
- }): Promise<unknown>
- createFile(args: {
- drive: string
- parentResourceId: string
- name: string
- content: string
- }): Promise<unknown>
- createFolder(args: { drive: string; parentResourceId: string; name: string }): Promise<unknown>
- rename(args: {
- drive: string
- resourceId: string
- newName: string
- ifMatch: number
- }): Promise<unknown>
- copy(args: {
- drive: string
- sourceResourceId: string
- destinationParentId: string
- newName?: string
- ifMatch: number
- }): Promise<unknown>
+ write(
+ args: {
+ drive: string
+ resourceId: string
+ content: string
+ ifMatch: number
+ },
+ call?: GfscCallOptions
+ ): Promise<unknown>
+ createFile(
+ args: {
+ drive: string
+ parentResourceId: string
+ name: string
+ content: string
+ },
+ call?: GfscCallOptions
+ ): Promise<unknown>
+ createFolder(
+ args: { drive: string; parentResourceId: string; name: string },
+ call?: GfscCallOptions
+ ): Promise<unknown>
+ rename(
+ args: {
+ drive: string
+ resourceId: string
+ newName: string
+ ifMatch: number
+ },
+ call?: GfscCallOptions
+ ): Promise<unknown>
+ copy(
+ args: {
+ drive: string
+ sourceResourceId: string
+ destinationParentId: string
+ newName?: string
+ ifMatch: number
+ },
+ call?: GfscCallOptions
+ ): Promise<unknown>
}
// Preserve only GFSC's HTTP status and a coarse public category. The server
@@ -228,14 +300,19 @@ export function buildGfsWriteTools(client: GfscWriteClient): InternalToolDefinit
ifMatch: { type: 'number', description: 'The resource version being replaced.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
if (!isValidIfMatch(args.ifMatch)) {
return invalidArgs('clerum__gfs_write requires a non-negative safe-integer If-Match')
}
try {
return ok(
await client.write(
- args as { drive: string; resourceId: string; content: string; ifMatch: number }
+ args as { drive: string; resourceId: string; content: string; ifMatch: number },
+ callOptions(context)
)
)
} catch (err) {
@@ -260,11 +337,16 @@ export function buildGfsWriteTools(client: GfscWriteClient): InternalToolDefinit
content: { type: 'string', description: 'Initial file content.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
return ok(
await client.createFile(
- args as { drive: string; parentResourceId: string; name: string; content: string }
+ args as { drive: string; parentResourceId: string; name: string; content: string },
+ callOptions(context)
)
)
} catch (err) {
@@ -284,11 +366,16 @@ export function buildGfsWriteTools(client: GfscWriteClient): InternalToolDefinit
name: { type: 'string', description: 'New folder name.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
try {
return ok(
await client.createFolder(
- args as { drive: string; parentResourceId: string; name: string }
+ args as { drive: string; parentResourceId: string; name: string },
+ callOptions(context)
)
)
} catch (err) {
@@ -310,14 +397,19 @@ export function buildGfsWriteTools(client: GfscWriteClient): InternalToolDefinit
ifMatch: { type: 'number', description: 'The observed resource version.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
if (!isValidIfMatch(args.ifMatch)) {
return invalidArgs('clerum__gfs_rename requires a non-negative safe-integer If-Match')
}
try {
return ok(
await client.rename(
- args as { drive: string; resourceId: string; newName: string; ifMatch: number }
+ args as { drive: string; resourceId: string; newName: string; ifMatch: number },
+ callOptions(context)
)
)
} catch (err) {
@@ -346,7 +438,11 @@ export function buildGfsCopyTools(client: GfscWriteClient): InternalToolDefiniti
ifMatch: { type: 'number', description: 'The observed source-root version.' },
},
},
- execute: async (args: Record<string, unknown>): Promise<InternalToolResult> => {
+ execute: async (
+ args: Record<string, unknown>,
+ _outputDir: string,
+ context?: InternalToolCallContext
+ ): Promise<InternalToolResult> => {
if (!isValidIfMatch(args.ifMatch)) {
return invalidArgs('clerum__gfs_copy requires a non-negative safe-integer If-Match')
}
@@ -359,7 +455,8 @@ export function buildGfsCopyTools(client: GfscWriteClient): InternalToolDefiniti
destinationParentId: string
newName?: string
ifMatch: number
- }
+ },
+ callOptions(context)
)
)
} catch (err) {
diff --git a/mcp-host/src/internalTools/gfsClient.retry429.test.ts b/mcp-host/src/internalTools/gfsClient.retry429.test.ts
index 6d95f7b04..41a238fdd 100644
--- a/mcp-host/src/internalTools/gfsClient.retry429.test.ts
+++ b/mcp-host/src/internalTools/gfsClient.retry429.test.ts
@@ -13,7 +13,10 @@ const N = 50
type Call = { token: string; at: number }
function rateLimited(retryAfter?: string): Response {
- const headers: Record<string, string> = { 'content-type': 'application/json' }
+ const headers: Record<string, string> = {
+ 'content-type': 'application/json',
+ 'x-gfs-ratelimit-scope': 'agent_reads',
+ }
if (retryAfter !== undefined) headers['retry-after'] = retryAfter
return new Response(JSON.stringify({ ok: false, error: { code: 'rate_limited' } }), {
status: 429,
@@ -50,7 +53,9 @@ function fleet(answer: (token: string, attempt: number) => Response, maxRetryWai
get: key => (key === 'MCP_HOST_GFS_TOKEN' ? `agent-${i}` : undefined),
fetch: fetchFn,
},
- { maxRetryWaitMs }
+ // A zero random source removes the jitter, so every retry lands exactly
+ // at Retry-After and the lockstep assertions below stay exact.
+ { maxRetryWaitMs, random: () => 0 }
),
}))
const perToken = (token: string) => calls.filter(c => c.token === token)
diff --git a/mcp-host/src/internalTools/gfsClient.test.ts b/mcp-host/src/internalTools/gfsClient.test.ts
index d8492e369..42b499e75 100644
--- a/mcp-host/src/internalTools/gfsClient.test.ts
+++ b/mcp-host/src/internalTools/gfsClient.test.ts
@@ -20,8 +20,9 @@ function jsonResponse(body: unknown, status = 200): Response {
})
}
-// Every test client gets a budget far above any Retry-After used here.
-const clientOptions = { maxRetryWaitMs: 60_000 }
+// Every test client gets a budget far above any Retry-After used here, and a
+// random source of 0 so the retry waits exactly the advertised delay.
+const clientOptions = { maxRetryWaitMs: 60_000, random: () => 0 }
function textResponse(body: string, status: number, statusText = ''): Response {
return new Response(body, { status, statusText })
@@ -251,8 +252,14 @@ describe('gfs runtime gfsc client', () => {
})
})
-function rateLimited(retryAfter?: string): Response {
- const headers: Record<string, string> = { 'content-type': 'application/json' }
+// gfsc's agent limiter names its bucket in X-GFS-RateLimit-Scope
+// (serve.ts: `limit: write ? "agent_writes" : "agent_reads"`); only those
+// scopes are retried.
+function rateLimited(retryAfter?: string, scope = 'agent_reads'): Response {
+ const headers: Record<string, string> = {
+ 'content-type': 'application/json',
+ 'x-gfs-ratelimit-scope': scope,
+ }
if (retryAfter !== undefined) headers['retry-after'] = retryAfter
// gfsc's envelope also carries retryAfterSeconds; the client reads only the
// header, so the no-header cases below prove the body is never consulted.
@@ -359,7 +366,7 @@ describe('gfsc client on a 429', () => {
vi.useFakeTimers()
const answers = [rateLimited('30'), rateLimited('10'), jsonResponse({ ok: true })]
const fetchFn = vi.fn(async () => answers.shift()!)
- const client = createGfscClient(tokenEnv(fetchFn), { maxRetryWaitMs: 10_000 })
+ const client = createGfscClient(tokenEnv(fetchFn), { maxRetryWaitMs: 10_000, random: () => 0 })
const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)
expect(error).toMatchObject({ status: 429, retryAfterSeconds: 30 })
@@ -408,3 +415,226 @@ describe('gfsc client on a 429', () => {
expect(createGfscClient(env, { maxRetryWaitMs: 0 })).toHaveProperty('stat')
})
})
+
+describe('gfsc client retry: cancellation, deadline, scope and jitter', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('M1: an abort during the Retry-After sleep rejects at once and never sends the retry', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi.fn(async () => rateLimited('45', 'agent_writes'))
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+ const controller = new AbortController()
+
+ const pending = client
+ .write(
+ { drive: 'main', resourceId: 'rid', content: 'x', ifMatch: 1 },
+ { signal: controller.signal }
+ )
+ .catch(e => e)
+ await vi.advanceTimersByTimeAsync(0)
+ // Witness: the first attempt was sent and the client is now sleeping.
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(1)
+
+ controller.abort()
+ const error = await pending
+ expect(error).toBeInstanceOf(DOMException)
+ expect((error as DOMException).name).toBe('AbortError')
+ expect(vi.getTimerCount()).toBe(0)
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ })
+
+ it('M1: the caller signal reaches both the first request and the retry', async () => {
+ vi.useFakeTimers()
+ const signals: Array<AbortSignal | null | undefined> = []
+ const fetchFn = vi.fn(async (_input: string, init?: RequestInit) => {
+ signals.push(init?.signal)
+ return signals.length === 1 ? rateLimited('2') : jsonResponse({ ok: true })
+ })
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+ const controller = new AbortController()
+
+ const pending = client.stat({ drive: 'main', resourceId: 'rid' }, { signal: controller.signal })
+ await vi.advanceTimersByTimeAsync(2000)
+ await expect(pending).resolves.toEqual({ ok: true })
+ expect(signals).toEqual([controller.signal, controller.signal])
+ })
+
+ it('M1: a Retry-After beyond the remaining deadline fails at once', async () => {
+ vi.useFakeTimers()
+ const fetchFn = vi.fn(async () => rateLimited('2'))
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const error = await client
+ .stat({ drive: 'main', resourceId: 'rid' }, { deadlineMs: Date.now() + 1500 })
+ .catch(e => e)
+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: 2 })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+ })
+
+ it('M1: the jittered wait is cut at the remaining deadline', async () => {
+ vi.useFakeTimers()
+ const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout')
+ let calls = 0
+ const fetchFn = vi.fn(async () => (++calls === 1 ? rateLimited('2') : jsonResponse({})))
+ const client = createGfscClient(tokenEnv(fetchFn), {
+ maxRetryWaitMs: 60_000,
+ random: () => 0.5,
+ })
+
+ const pending = client.stat(
+ { drive: 'main', resourceId: 'rid' },
+ { deadlineMs: Date.now() + 2100 }
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ // 2000 ms + 200 ms of jitter would pass the deadline, so the wait is 2100 ms.
+ expect(setTimeoutSpy.mock.calls.map(call => call[1])).toEqual([2100])
+ await vi.advanceTimersByTimeAsync(2100)
+ await expect(pending).resolves.toEqual({})
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ })
+
+ it('L17: the retry waits Retry-After plus the injected jitter', async () => {
+ vi.useFakeTimers()
+ const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout')
+ let calls = 0
+ const fetchFn = vi.fn(async () => (++calls === 1 ? rateLimited('2') : jsonResponse({})))
+ const client = createGfscClient(tokenEnv(fetchFn), {
+ maxRetryWaitMs: 60_000,
+ random: () => 0.5,
+ })
+
+ const pending = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(0)
+ // Jitter is random × min(1000, 0.2 × 2000) = 0.5 × 400 = 200 ms.
+ expect(setTimeoutSpy.mock.calls.map(call => call[1])).toEqual([2200])
+ await vi.advanceTimersByTimeAsync(2199)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+ await expect(pending).resolves.toEqual({})
+ })
+
+ it('L17: the jitter never exceeds one second for long delays', async () => {
+ vi.useFakeTimers()
+ const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout')
+ let calls = 0
+ const fetchFn = vi.fn(async () => (++calls === 1 ? rateLimited('30') : jsonResponse({})))
+ const client = createGfscClient(tokenEnv(fetchFn), {
+ maxRetryWaitMs: 60_000,
+ random: () => 0.999,
+ })
+
+ const pending = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(0)
+ expect(setTimeoutSpy.mock.calls.map(call => call[1])).toEqual([30_999])
+ await vi.advanceTimersByTimeAsync(30_999)
+ await expect(pending).resolves.toEqual({})
+ })
+
+ it.each(['active_sessions_subject', 'upload_bytes_subject', undefined])(
+ 'L16/I1: a 429 with scope %s is surfaced at once with its hint',
+ async scope => {
+ vi.useFakeTimers()
+ const headers: Record<string, string> = { 'retry-after': '2' }
+ if (scope !== undefined) headers['x-gfs-ratelimit-scope'] = scope
+ const answers = [
+ new Response('{}', { status: 429, headers }),
+ rateLimited('2', 'agent_reads'),
+ jsonResponse({ ok: true }),
+ ]
+ const fetchFn = vi.fn(async () => answers.shift()!)
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const error = await client.stat({ drive: 'main', resourceId: 'rid' }).catch(e => e)
+ expect(error).toMatchObject({ status: 429, retryAfterSeconds: 2 })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+
+ // Control on the same client: an agent scope is retried.
+ const control = client.stat({ drive: 'main', resourceId: 'rid' })
+ await vi.advanceTimersByTimeAsync(2000)
+ await expect(control).resolves.toEqual({ ok: true })
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ }
+ )
+
+ it('L15: Retry-After accepts 1..3600 and drops anything larger', async () => {
+ vi.useFakeTimers()
+ const values = ['1', '3600', '3601', '99999', '100000000000000000000000']
+ const fetchFn = vi.fn(async () => rateLimited(values.shift()!))
+ // maxRetryWaitMs 0 means no retry is ever slept, so only the parse is observed.
+ const client = createGfscClient(tokenEnv(fetchFn), { maxRetryWaitMs: 0, random: () => 0 })
+ const hintFor = async (value: string) => {
+ expect(values[0]).toBe(value)
+ const error = (await client
+ .stat({ drive: 'main', resourceId: 'rid' })
+ .catch(e => e)) as GfscHttpError
+ expect(error).toBeInstanceOf(GfscHttpError)
+ return error.retryAfterSeconds
+ }
+
+ expect(await hintFor('1')).toBe(1)
+ expect(await hintFor('3600')).toBe(3600)
+ expect(await hintFor('3601')).toBeUndefined()
+ expect(await hintFor('99999')).toBeUndefined()
+ expect(await hintFor('100000000000000000000000')).toBeUndefined()
+ expect(fetchFn).toHaveBeenCalledTimes(5)
+ })
+
+ it.each([
+ [
+ 'write',
+ 'PUT',
+ 'http://gfsc-writer.gfs.svc.cluster.local:8087/v1/resources/rid/content?drive=main',
+ ],
+ [
+ 'createFile',
+ 'POST',
+ 'http://gfsc-writer.gfs.svc.cluster.local:8087/v1/resources/parent/children?drive=main',
+ ],
+ ] as const)(
+ 'M6: the %s retry repeats the same method, URL, body and credential',
+ async (method, verb, url) => {
+ vi.useFakeTimers()
+ const sent: Array<{ url: string; init: RequestInit }> = []
+ const fetchFn = vi.fn(async (input: string, init?: RequestInit) => {
+ sent.push({ url: input, init: init! })
+ return sent.length === 1 ? rateLimited('2', 'agent_writes') : jsonResponse({ ok: true })
+ })
+ const client = createGfscClient(tokenEnv(fetchFn), clientOptions)
+
+ const pending =
+ method === 'write'
+ ? client.write({ drive: 'main', resourceId: 'rid', content: 'body-1', ifMatch: 7 })
+ : client.createFile({
+ drive: 'main',
+ parentResourceId: 'parent',
+ name: 'a.txt',
+ content: 'body-1',
+ })
+ await vi.advanceTimersByTimeAsync(2000)
+ await expect(pending).resolves.toEqual({ ok: true })
+
+ expect(sent).toHaveLength(2)
+ const [first, retry] = sent.map(({ url: sentUrl, init }) => ({
+ url: sentUrl,
+ method: init.method,
+ body: init.body,
+ authorization: (init.headers as Record<string, string>).authorization,
+ }))
+ expect(first).toEqual({
+ url,
+ method: verb,
+ body: expect.stringContaining('"body-1"'),
+ authorization: 'Bearer gfs-access',
+ })
+ expect(retry).toEqual(first)
+ }
+ )
+})
diff --git a/mcp-host/src/internalTools/gfsClient.ts b/mcp-host/src/internalTools/gfsClient.ts
index 939f40155..91ed5c509 100644
--- a/mcp-host/src/internalTools/gfsClient.ts
+++ b/mcp-host/src/internalTools/gfsClient.ts
@@ -1,7 +1,8 @@
import { existsSync, readFileSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
+import { withAbort } from '../core/adapters/abortableLlmPort'
import { decodeRuntimeJwtScopes } from '../workflow/mcpHostRuntimeJwt'
-import type { GfscWriteClient } from './gfs'
+import type { GfscCallOptions, GfscWriteClient } from './gfs'
const DIRECT_KEY = 'MCP_HOST_GFS_TOKEN'
const FILE_KEY = 'MCP_HOST_GFS_TOKEN_FILE'
@@ -83,21 +84,48 @@ export class GfscHttpError extends Error {
export interface GfscClientOptions {
/**
* The longest `Retry-After` the client will sleep through before its single
- * retry: the calling tool's timeout. A longer advertised delay fails at once
- * instead of sleeping into that timeout. Time already spent inside the same
- * tool call is not subtracted; against gfsc's 60 s window and the default
- * tool budget that difference is not reachable.
+ * retry, for any call. Each call is further bounded by its own
+ * `GfscCallOptions.deadlineMs`, the time left in the tool call that issued
+ * it: a delay that does not fit in the smaller of the two fails at once
+ * instead of sleeping into the caller's timeout.
*/
maxRetryWaitMs: number
+ /**
+ * Source of the retry jitter, a number in [0, 1). Parallel agents denied in
+ * the same second would otherwise all retry in the same instant.
+ */
+ random?: () => number
}
+// gfsc's agent limiter answers with these scopes (serve.ts). Its other 429s,
+// such as the upload-session quotas, are not cleared by waiting a few
+// seconds, so they are surfaced at once.
+const RETRYABLE_SCOPES = new Set(['agent_reads', 'agent_writes'])
+const MAX_RETRY_AFTER_SECONDS = 3600
+const MAX_JITTER_MS = 1000
+const JITTER_FRACTION = 0.2
+
+// A Retry-After outside 1..3600 whole seconds is treated as absent: the 429 is
+// surfaced with no retry and no hint, so an absurd value never reaches the model.
function retryAfterHeader(res: Response): number | undefined {
const raw = res.headers.get('retry-after')?.trim()
- return raw !== undefined && /^[1-9][0-9]*$/.test(raw) ? Number(raw) : undefined
+ if (raw === undefined || !/^[1-9][0-9]{0,3}$/.test(raw)) return undefined
+ const seconds = Number(raw)
+ return seconds <= MAX_RETRY_AFTER_SECONDS ? seconds : undefined
+}
+
+function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
+ let timer: ReturnType<typeof setTimeout> | undefined
+ const wait = () =>
+ new Promise<void>(resolve => {
+ timer = setTimeout(resolve, ms)
+ })
+ if (!signal) return wait()
+ return withAbort(wait, signal).finally(() => clearTimeout(timer))
}
export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions): GfscWriteClient {
- const { maxRetryWaitMs } = options
+ const { maxRetryWaitMs, random = Math.random } = options
if (!Number.isFinite(maxRetryWaitMs) || maxRetryWaitMs < 0) {
throw new Error(
`gfsc client maxRetryWaitMs must be a non-negative number, got ${maxRetryWaitMs}`
@@ -146,47 +174,77 @@ export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions)
return res.text()
}
- // A 429 that states a delay is retried once, after that delay. gfsc's agent
- // limiter answers before the permission store and the executor run, so the
- // denied attempt had no effect and repeating a write is safe. A 429 without a
- // delay, a delay beyond maxRetryWaitMs, or a second denial fails at once.
- async function request(baseUrl: string, path: string, init: RequestInit = {}): Promise<unknown> {
- const first = await send(baseUrl, path, init)
+ // A 429 from gfsc's agent limiter that states a delay is retried once, after
+ // that delay plus a bounded jitter. The limiter answers before the permission
+ // store and the executor run, so the denied attempt had no effect and
+ // repeating a write is safe. Any other 429 scope, a 429 without a delay, a
+ // delay that does not fit in the caller's remaining budget, or a second
+ // denial fails at once. The caller's signal cancels both requests and the
+ // sleep between them.
+ async function request(
+ baseUrl: string,
+ path: string,
+ init: RequestInit = {},
+ call: GfscCallOptions = {}
+ ): Promise<unknown> {
+ const attempt: RequestInit = call.signal ? { ...init, signal: call.signal } : init
+ const first = await send(baseUrl, path, attempt)
if (first.ok) return decode(first)
const denied = await httpError(first)
- const waitSeconds = denied.retryAfterSeconds
- if (denied.status !== 429 || waitSeconds === undefined || waitSeconds * 1000 > maxRetryWaitMs) {
+ const retryAfterMs =
+ denied.retryAfterSeconds === undefined ? undefined : denied.retryAfterSeconds * 1000
+ const budgetMs = Math.min(
+ maxRetryWaitMs,
+ call.deadlineMs === undefined ? Number.POSITIVE_INFINITY : call.deadlineMs - Date.now()
+ )
+ if (
+ denied.status !== 429 ||
+ !RETRYABLE_SCOPES.has(first.headers.get('x-gfs-ratelimit-scope') ?? '') ||
+ retryAfterMs === undefined ||
+ retryAfterMs > budgetMs
+ ) {
throw denied
}
- await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000))
- const second = await send(baseUrl, path, init)
+ const jitterMs = Math.floor(random() * Math.min(MAX_JITTER_MS, JITTER_FRACTION * retryAfterMs))
+ await sleep(Math.min(retryAfterMs + jitterMs, budgetMs), call.signal)
+ const second = await send(baseUrl, path, attempt)
if (second.ok) return decode(second)
throw await httpError(second)
}
return {
- accessible: ({ drive, cursor }) => {
+ accessible: ({ drive, cursor }, call) => {
const q = new URLSearchParams({ drive })
if (cursor) q.set('cursor', cursor)
- return request(readerBase, `/v1/accessible?${q}`)
+ return request(readerBase, `/v1/accessible?${q}`, {}, call)
},
- list: ({ drive, resourceId, cursor }) => {
+ list: ({ drive, resourceId, cursor }, call) => {
const q = new URLSearchParams({ drive })
if (cursor) q.set('cursor', cursor)
- return request(readerBase, `/v1/resources/${encodeURIComponent(resourceId)}/children?${q}`)
+ return request(
+ readerBase,
+ `/v1/resources/${encodeURIComponent(resourceId)}/children?${q}`,
+ {},
+ call
+ )
},
- read: ({ drive, resourceId }) =>
+ read: ({ drive, resourceId }, call) =>
request(
readerBase,
- `/v1/resources/${encodeURIComponent(resourceId)}/content?drive=${encodeURIComponent(drive)}`
+ `/v1/resources/${encodeURIComponent(resourceId)}/content?drive=${encodeURIComponent(drive)}`,
+ {},
+ call
),
- stat: ({ drive, resourceId }) =>
+ stat: ({ drive, resourceId }, call) =>
request(
readerBase,
- `/v1/resources/${encodeURIComponent(resourceId)}?drive=${encodeURIComponent(drive)}`
+ `/v1/resources/${encodeURIComponent(resourceId)}?drive=${encodeURIComponent(drive)}`,
+ {},
+ call
),
- resolve: ({ uri }) => request(readerBase, `/v1/resolve?uri=${encodeURIComponent(uri)}`),
- write: ({ drive, resourceId, content, ifMatch }) =>
+ resolve: ({ uri }, call) =>
+ request(readerBase, `/v1/resolve?uri=${encodeURIComponent(uri)}`, {}, call),
+ write: ({ drive, resourceId, content, ifMatch }, call) =>
request(
writerBase,
`/v1/resources/${encodeURIComponent(resourceId)}/content?drive=${encodeURIComponent(drive)}`,
@@ -194,9 +252,10 @@ export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions)
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ content, ifMatch }),
- }
+ },
+ call
),
- createFile: ({ drive, parentResourceId, name, content }) =>
+ createFile: ({ drive, parentResourceId, name, content }, call) =>
request(
writerBase,
`/v1/resources/${encodeURIComponent(parentResourceId)}/children?drive=${encodeURIComponent(drive)}`,
@@ -204,9 +263,10 @@ export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions)
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ name, kind: 'file', content }),
- }
+ },
+ call
),
- createFolder: ({ drive, parentResourceId, name }) =>
+ createFolder: ({ drive, parentResourceId, name }, call) =>
request(
writerBase,
`/v1/resources/${encodeURIComponent(parentResourceId)}/children?drive=${encodeURIComponent(drive)}`,
@@ -214,19 +274,30 @@ export function createGfscClient(env: GfsRuntimeEnv, options: GfscClientOptions)
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ name, kind: 'directory' }),
- }
+ },
+ call
+ ),
+ rename: ({ drive, resourceId, newName, ifMatch }, call) =>
+ request(
+ writerBase,
+ `/v1/resources/${encodeURIComponent(resourceId)}`,
+ {
+ method: 'PATCH',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ drive, newName, ifMatch }),
+ },
+ call
+ ),
+ copy: (args, call) =>
+ request(
+ writerBase,
+ '/v1/copy',
+ {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify(args),
+ },
+ call
),
- rename: ({ drive, resourceId, newName, ifMatch }) =>
- request(writerBase, `/v1/resources/${encodeURIComponent(resourceId)}`, {
- method: 'PATCH',
- headers: { 'content-type': 'application/json' },
- body: JSON.stringify({ drive, newName, ifMatch }),
- }),
- copy: args =>
- request(writerBase, '/v1/copy', {
- method: 'POST',
- headers: { 'content-type': 'application/json' },
- body: JSON.stringify(args),
- }),
}
}
diff --git a/mcp-host/src/workflow/stepRouter.ts b/mcp-host/src/workflow/stepRouter.ts
index 39567da39..b66ef13c5 100644
--- a/mcp-host/src/workflow/stepRouter.ts
+++ b/mcp-host/src/workflow/stepRouter.ts
@@ -243,7 +243,10 @@ export class StepMcpRouter {
}
}
const start = Date.now()
- const internalResult = await internalTool.execute(args, this.outputDir)
+ const internalResult = await internalTool.execute(args, this.outputDir, {
+ signal: options.signal,
+ timeoutMs: options.timeoutMs,
+ })
const durationMs = Date.now() - start
const record: ToolCallRecord = {
serverName: 'clerum',
diff --git a/mcp-host/src/workflow/types.ts b/mcp-host/src/workflow/types.ts
index e57663089..ae70cf492 100644
--- a/mcp-host/src/workflow/types.ts
+++ b/mcp-host/src/workflow/types.ts
@@ -200,9 +200,23 @@ export interface InternalToolResult {
error?: string
}
+/**
+ * The bounds of the tool call that runs an internal tool: the caller's cancel
+ * signal and the time left in its budget. Tools that make network calls pass
+ * both on, so a call never outlives the step or chat turn that issued it.
+ */
+export interface InternalToolCallContext {
+ signal?: AbortSignal
+ timeoutMs?: number
+}
+
export interface InternalToolDefinition {
name: string
description: string
parameters: Record<string, unknown>
- execute: (args: Record<string, unknown>, outputDir: string) => Promise<InternalToolResult>
+ execute: (
+ args: Record<string, unknown>,
+ outputDir: string,
+ context?: InternalToolCallContext
+ ) => Promise<InternalToolResult>
}
diff --git a/mcp-host/src/workflow/workflowService.gating.test.ts b/mcp-host/src/workflow/workflowService.gating.test.ts
index 9d1b26d0e..e4bdfe16c 100644
--- a/mcp-host/src/workflow/workflowService.gating.test.ts
+++ b/mcp-host/src/workflow/workflowService.gating.test.ts
@@ -355,6 +355,7 @@ describe('WorkflowService internal-tools capability gate (#592)', () => {
})
afterEach(() => {
+ vi.restoreAllMocks()
if (originalOutputDir === undefined) delete process.env.CLERUM_OUTPUT_DIR
else process.env.CLERUM_OUTPUT_DIR = originalOutputDir
if (originalMounts === undefined) delete process.env.CLERUM_CONTEXT_FILES_MOUNTS
@@ -489,23 +490,32 @@ describe('WorkflowService internal-tools capability gate (#592)', () => {
expect(names).not.toContain('clerum__gfs_write')
})
- it('dispatches workflow GFS calls through the existing gfsc client boundary', async () => {
+ it('dispatches workflow GFS calls through the existing gfsc client boundary, bounded by the step', async () => {
gfsGate.enabled = true
process.env.MCP_HOST_GFS_SCOPES = 'gfs.read'
gfsClient.read.mockResolvedValue({ content: 'granted file content' })
+ // The step starts at T and the model takes 30 s before it asks for the
+ // read. The call must carry the step's own end (T + 120 s), not a fresh
+ // full timeout from the moment of the call (T + 150 s).
+ const T = 1_900_000_000_000
+ let clock = T
+ vi.spyOn(Date, 'now').mockImplementation(() => clock)
const provider = makeProvider()
vi.mocked(provider.completeSingleTurnWithTools)
- .mockResolvedValueOnce({
- content: '',
- tool_calls: [
- {
- id: 'gfs-read-1',
- name: 'clerum__gfs_read',
- arguments: { drive: 'main', resourceId: '0123456789abcdef0123456789abcdef' },
- },
- ],
- usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
- finish_reason: FinishReason.ToolUse,
+ .mockImplementationOnce(async () => {
+ clock += 30_000
+ return {
+ content: '',
+ tool_calls: [
+ {
+ id: 'gfs-read-1',
+ name: 'clerum__gfs_read',
+ arguments: { drive: 'main', resourceId: '0123456789abcdef0123456789abcdef' },
+ },
+ ],
+ usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
+ finish_reason: FinishReason.ToolUse,
+ }
})
.mockResolvedValueOnce({
content: 'Read completed.',
@@ -525,6 +535,7 @@ describe('WorkflowService internal-tools capability gate (#592)', () => {
stepId: 'gfs-read-step',
instruction: 'Read the granted file.',
allowedTools: { include: ['clerum__gfs_read'] },
+ timeoutSeconds: 120,
})
expect(result.status).toBe('completed')
@@ -535,10 +546,40 @@ describe('WorkflowService internal-tools capability gate (#592)', () => {
result: expect.objectContaining({ success: true }),
}),
])
- expect(gfsClient.read).toHaveBeenCalledWith({
- drive: 'main',
- resourceId: '0123456789abcdef0123456789abcdef',
+ expect(createGfscClient).toHaveBeenCalledTimes(1)
+ expect(vi.mocked(createGfscClient).mock.calls[0]?.[1]).toEqual({ maxRetryWaitMs: 120_000 })
+ expect(gfsClient.read).toHaveBeenCalledTimes(1)
+ const [args, call] = gfsClient.read.mock.calls[0] ?? []
+ expect(args).toEqual({ drive: 'main', resourceId: '0123456789abcdef0123456789abcdef' })
+ expect(call).toEqual({ signal: expect.any(AbortSignal), deadlineMs: T + 120_000 })
+ })
+
+ it('fails the step, without rejecting, when the gfsc client cannot be created', async () => {
+ gfsGate.enabled = true
+ process.env.MCP_HOST_GFS_SCOPES = 'gfs.read'
+ vi.mocked(createGfscClient).mockImplementationOnce(() => {
+ throw new Error('gfsc client maxRetryWaitMs must be a non-negative number, got NaN')
})
+ const provider = makeProvider()
+ const service = new WorkflowService('recipe-1', {
+ llmFactory: () => provider,
+ mcpClientFactory: () => {
+ throw new Error('mcp client must not be created in this GFS gate test')
+ },
+ })
+ service.configure({ provider: 'openai', apiKey: 'test-key', model: 'gpt-4o' })
+
+ const result = await service.executeStep({
+ stepId: 'gfs-client-fails',
+ instruction: 'Read the granted file.',
+ allowedTools: { include: ['clerum__gfs_read'] },
+ })
+
+ // Witness: the step reached the client construction.
+ expect(createGfscClient).toHaveBeenCalledTimes(1)
+ expect(result).toMatchObject({ stepId: 'gfs-client-fails', status: 'failed' })
+ expect(result.error).toContain('maxRetryWaitMs')
+ expect(provider.completeSingleTurnWithTools).not.toHaveBeenCalled()
})
it('fails closed when toolChoice is required but no allowed tool is available', async () => {
diff --git a/mcp-host/src/workflow/workflowService.ts b/mcp-host/src/workflow/workflowService.ts
index adfd50867..edd7687a3 100644
--- a/mcp-host/src/workflow/workflowService.ts
+++ b/mcp-host/src/workflow/workflowService.ts
@@ -931,15 +931,6 @@ export class WorkflowService {
])
const validGfsScopes = rawGfsScopes.every(scope => knownGfsScopes.has(scope))
const gfsScopes = new Set(validGfsScopes ? rawGfsScopes : [])
- if (requestedGfsToolNames.size > 0 && gfsScopes.size > 0 && hasGfsRuntimeAccess(gfsEnv)) {
- const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: timeoutMs })
- const scopedGfsTools = [
- ...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),
- ...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),
- ].filter(tool => requestedGfsToolNames.has(tool.name))
- internalTools.push(...scopedGfsTools)
- }
- router.registerInternalTools(internalTools, getOutputDir())
const toolsCalled: ToolCallRecord[] = []
let totalInputTokens = 0
let totalOutputTokens = 0
@@ -1105,6 +1096,21 @@ export class WorkflowService {
}
}
+ // Built inside the try so a client that cannot be created fails this step
+ // through the catch below instead of escaping executeStep. Each GFS call
+ // is bounded by the time left in the step (stepRouter passes the
+ // remaining timeout and the step's signal to the tool); maxRetryWaitMs
+ // only caps a single Retry-After.
+ if (requestedGfsToolNames.size > 0 && gfsScopes.size > 0 && hasGfsRuntimeAccess(gfsEnv)) {
+ const gfsClient = createGfscClient(gfsEnv, { maxRetryWaitMs: timeoutMs })
+ const scopedGfsTools = [
+ ...(gfsScopes.has('gfs.read') ? buildGfsReadTools(gfsClient) : []),
+ ...(gfsScopes.has('gfs.write') ? buildGfsWriteTools(gfsClient) : []),
+ ].filter(tool => requestedGfsToolNames.has(tool.name))
+ internalTools.push(...scopedGfsTools)
+ }
+ router.registerInternalTools(internalTools, getOutputDir())
+
// 1. Connect to step-declared MCP servers
await router.connect(req.mcpServers ?? [], {
timeoutMs: remainingTimeoutMs(startTime, timeoutMs),