Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Merge branch 'dev' into codex/issue-833-mcp-host-idle-heap-fix

Alfredo Lopezd34485e86818
diff --cc mcp-host/src/__tests__/mcp/mcpClient.timeout.test.ts
index 20d3ab8ef,b96dfcdda..b3bc45dd7
--- a/mcp-host/src/__tests__/mcp/mcpClient.timeout.test.ts
+++ b/mcp-host/src/__tests__/mcp/mcpClient.timeout.test.ts
@@@ -9,8 -10,8 +10,10 @@@ const sdkState = vi.hoisted(() => (
callToolQueue: [] as Array<() => unknown>,
listToolsCalls: [] as unknown[][],
listToolsQueue: [] as Array<() => unknown>,
+ requestCalls: [] as unknown[][],
+ requestQueue: [] as Array<() => unknown>,
+ transportCloseCalls: [] as unknown[][],
+ transportCloseQueue: [] as Array<() => unknown>,
}))
vi.mock('@modelcontextprotocol/sdk/client/index.js', () => ({
@@@ -79,8 -103,8 +112,10 @@@ describe('McpClient SDK request timeout
sdkState.callToolQueue.length = 0
sdkState.listToolsCalls.length = 0
sdkState.listToolsQueue.length = 0
+ sdkState.requestCalls.length = 0
+ sdkState.requestQueue.length = 0
+ sdkState.transportCloseCalls.length = 0
+ sdkState.transportCloseQueue.length = 0
})
afterEach(() => {
@@@ -238,6 -481,243 +494,245 @@@
expect(c.isConnected).toBe(false)
expect(sdkState.connectCalls).toHaveLength(1)
expect(sdkState.listToolsCalls).toHaveLength(0)
- expect(sdkState.closeCalls).toHaveLength(1)
+ expect(sdkState.transportCloseCalls).toHaveLength(1)
+ })
+
+ it('detaches immediately and closes the owned transport when SDK protocol close hangs', async () => {
+ const releaseClose = deferred()
+ sdkState.transportCloseQueue.push(() => releaseClose.promise)
+ const c = client()
+ await c.connect()
+
+ const disconnecting = c.disconnect()
+ await Promise.resolve()
+
+ expect(c.isConnected).toBe(false)
+ expect(c.availableTools).toEqual([])
+ expect(sdkState.closeCalls).toHaveLength(0)
+ expect(sdkState.transportCloseCalls).toHaveLength(1)
+
+ releaseClose.resolve()
+ await expect(disconnecting).resolves.toBeUndefined()
+ })
+
+ it('does not reconnect or replay a tool after permanent close during the retry delay', async () => {
+ vi.useFakeTimers()
+ sdkState.callToolQueue.push(
+ () => {
+ throw Object.assign(new Error('session not found'), { code: -32003 })
+ },
+ () => ({ content: [{ type: 'text', text: 'must-not-run' }] })
+ )
+ const c = client()
+ await c.connect()
+
+ const pending = c.callTool('write', { value: 'side-effect' }, { timeoutMs: 30_000 })
+ const assertion = expect(pending).rejects.toThrow(/closed/)
+ await vi.advanceTimersByTimeAsync(0)
+ await retirePermanently(c)
+ await vi.advanceTimersByTimeAsync(1000)
+
+ await assertion
+ expect(sdkState.connectCalls).toHaveLength(1)
+ expect(sdkState.callToolCalls).toHaveLength(1)
+ expect(c.isConnected).toBe(false)
+ })
+
+ it('discards a successful tool result that arrives after permanent close', async () => {
+ let resolveCall!: (value: unknown) => void
+ const callResult = new Promise<unknown>(resolve => {
+ resolveCall = resolve
+ })
+ sdkState.callToolQueue.push(() => callResult)
+ const c = client()
+ await c.connect()
+
+ const pending = c.callTool('write', { value: 'side-effect' })
+ const assertion = expect(pending).rejects.toThrow(/closed/)
+ await vi.waitFor(() => expect(sdkState.callToolCalls).toHaveLength(1))
+ await retirePermanently(c)
+ resolveCall({ content: [{ type: 'text', text: 'late-success' }] })
+
+ await assertion
+ expect(sdkState.connectCalls).toHaveLength(1)
+ expect(sdkState.callToolCalls).toHaveLength(1)
+ expect(c.isConnected).toBe(false)
+ })
+
+ it('starts transport close synchronously when permanently retired', async () => {
+ const c = client()
+ await c.connect()
+
+ const cleanup = c.retire()
+
+ expect(c.isConnected).toBe(false)
+ expect(sdkState.transportCloseCalls).toHaveLength(1)
+ await cleanup()
+ expect(sdkState.transportCloseCalls).toHaveLength(1)
+ })
+
+ it('discards a retry result that arrives after permanent close', async () => {
+ vi.useFakeTimers()
+ const retryResult = deferred<unknown>()
+ sdkState.callToolQueue.push(
+ () => {
+ throw Object.assign(new Error('session not found'), { code: -32003 })
+ },
+ () => retryResult.promise
+ )
+ const c = client()
+ await c.connect()
+
+ const pending = c.callTool('write', { value: 'side-effect' }, { timeoutMs: 30_000 })
+ const assertion = expect(pending).rejects.toThrow(/closed/)
+ await vi.advanceTimersByTimeAsync(1000)
+ await vi.waitFor(() => expect(sdkState.callToolCalls).toHaveLength(2))
+
+ await retirePermanently(c)
+ retryResult.resolve({ content: [{ type: 'text', text: 'late-retry-success' }] })
+
+ await assertion
+ expect(sdkState.callToolCalls).toHaveLength(2)
+ expect(c.isConnected).toBe(false)
+ })
+
+ it('invalidates a reconnect handshake when permanent close arrives in flight', async () => {
+ vi.useFakeTimers()
+ const reconnectHandshake = deferred()
+ sdkState.callToolQueue.push(
+ () => {
+ throw Object.assign(new Error('session not found'), { code: -32003 })
+ },
+ () => ({ content: [{ type: 'text', text: 'must-not-run' }] })
+ )
+ const c = client()
+ await c.connect()
+ sdkState.connectQueue.push(() => reconnectHandshake.promise)
+
+ const pending = c.callTool('write', { value: 'side-effect' }, { timeoutMs: 30_000 })
+ const assertion = expect(pending).rejects.toThrow(/closed|superseded/)
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(sdkState.connectCalls).toHaveLength(2)
+
+ const closing = retirePermanently(c)
+ reconnectHandshake.resolve()
+ await closing
+ await vi.advanceTimersByTimeAsync(0)
+
+ await assertion
+ expect(sdkState.callToolCalls).toHaveLength(1)
+ expect(c.isConnected).toBe(false)
+ expect(sdkState.transportCloseCalls).toHaveLength(2)
+ expect(new Set(sdkState.transportCloseCalls.map(([transport]) => transport)).size).toBe(2)
+ })
+
+ it('rejects all recovery waiters promptly when retired while reconnect never settles', async () => {
+ vi.useFakeTimers()
+ const reconnectHandshake = deferred()
+ sdkState.callToolQueue.push(
+ () => {
+ throw Object.assign(new Error('session not found'), { code: -32003 })
+ },
+ () => {
+ throw Object.assign(new Error('session not found'), { code: -32003 })
+ }
+ )
+ const c = client()
+ await c.connect()
+ sdkState.connectQueue.push(() => reconnectHandshake.promise)
+
+ const settled = Promise.allSettled([
+ c.callTool('write', { id: 1 }, { timeoutMs: 30_000 }),
+ c.callTool('write', { id: 2 }, { timeoutMs: 30_000 }),
+ ])
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(sdkState.connectCalls).toHaveLength(2)
+
+ await retirePermanently(c)
+ await vi.advanceTimersByTimeAsync(0)
+
+ await expect(settled).resolves.toEqual([
+ {
+ status: 'rejected',
+ reason: expect.objectContaining({ message: expect.stringMatching(/closed/) }),
+ },
+ {
+ status: 'rejected',
+ reason: expect.objectContaining({ message: expect.stringMatching(/closed/) }),
+ },
+ ])
+ expect(sdkState.callToolCalls).toHaveLength(2)
+ expect(c.isConnected).toBe(false)
+ })
+
+ it('does not publish tools discovered after permanent close', async () => {
+ const discovery = deferred<{
+ tools: Array<{ name: string; description: string; inputSchema: Record<string, unknown> }>
+ }>()
+ sdkState.listToolsQueue.push(() => discovery.promise)
+ const c = client()
+
+ const pending = c.connect()
+ await vi.waitFor(() => expect(sdkState.listToolsCalls).toHaveLength(1))
+ const closing = retirePermanently(c)
+ discovery.resolve({
+ tools: [{ name: 'stale-tool', description: 'stale', inputSchema: {} }],
+ })
+
+ await closing
+ await expect(pending).rejects.toThrow(/closed/)
+ expect(c.availableTools).toEqual([])
+ expect(c.isConnected).toBe(false)
+ expect(sdkState.transportCloseCalls).toHaveLength(1)
+ })
+
+ it('does not let an old tools/list failure erase a reconnected tool cache', async () => {
+ const c = client()
+ await c.connect()
+
+ const staleDiscovery = deferred<{
+ tools: Array<{ name: string; description: string; inputSchema: Record<string, unknown> }>
+ }>()
+ sdkState.listToolsQueue.push(
+ () => staleDiscovery.promise,
+ () => ({
+ tools: [{ name: 'fresh-tool', description: 'fresh', inputSchema: {} }],
+ })
+ )
+ const staleRefresh = c.listTools()
+ await vi.waitFor(() => expect(sdkState.listToolsCalls).toHaveLength(2))
+
+ await c.reconnect()
+ expect(c.availableTools.map(tool => tool.name)).toEqual(['fresh-tool'])
+
+ staleDiscovery.reject(new Error('old tools/list failed'))
+ await expect(staleRefresh).rejects.toThrow(/superseded/)
+ expect(c.availableTools.map(tool => tool.name)).toEqual(['fresh-tool'])
+ expect(c.isConnected).toBe(true)
+ })
+
+ it('does not report a successful probe from a permanently closed connection', async () => {
+ const c = client()
+ await c.connect()
+ const staleProbe = deferred<{
+ tools: Array<{ name: string; description: string; inputSchema: Record<string, unknown> }>
+ }>()
- sdkState.listToolsQueue.push(() => staleProbe.promise)
++ // probeTools() issues a validated raw `request(tools/list)` (not listTools),
++ // so drive the in-flight probe through the request queue.
++ sdkState.requestQueue.push(() => staleProbe.promise)
+
+ const probing = c.probeTools()
- await vi.waitFor(() => expect(sdkState.listToolsCalls).toHaveLength(2))
++ await vi.waitFor(() => expect(sdkState.requestCalls).toHaveLength(1))
+ const closing = retirePermanently(c)
+ staleProbe.resolve({
+ tools: [{ name: 'stale-tool', description: 'stale', inputSchema: {} }],
+ })
+
+ await closing
+ await expect(probing).resolves.toEqual({
+ ok: false,
+ error: expect.objectContaining({ message: expect.stringMatching(/closed/) }),
+ stale: true,
+ })
})
})
diff --cc mcp-host/src/main.ts
index fc7b8c0ae,da9e3a5ed..92fefb858
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@@ -65,9 -66,21 +66,22 @@@ import { clerumPromptCacheInvalidations
import { ALL_PROVIDERS, type LlmProvider, descriptorFor, isLlmProvider } from './llm/registryCore'
import './logger'
import { McpManager } from './mcp'
+import { McpStatusHeartbeat } from './mcp/statusHeartbeat'
+ import {
+ AuthoritativeMcpFleetCoordinator,
+ DEFAULT_MCP_FLEET_RECONCILE_MAX_CONCURRENCY,
+ pollAuthoritativeMcpSnapshotIfCurrent,
+ reconcileAuthoritativeMcpSnapshot,
+ replaceAuthoritativeMcpFleet,
+ runAuthoritativeMcpInitialization,
+ } from './mcp/authoritativeFleet'
import { startMcpInitializationInBackground } from './mcpBackgroundInit'
import { IncomingMessageHandler, PendingTaskEntry } from './messageHandler'
+ import {
+ configurePluginWorkloadSdkBootstrapIdentity,
+ resolvePluginWorkloadSdkBootstrapCapabilityFamily,
+ } from './pluginWorkloadSdk/bootstrapIdentity'
+ import { PluginWorkloadSdkBootstrapServer } from './pluginWorkloadSdk/bootstrapServer'
import { maybeCreatePluginWorkloadSdkServer } from './pluginWorkloadSdk/server'
import type { PluginWorkloadSdkServer } from './pluginWorkloadSdk/server/sdkServer'
import { getDisplayName, sanitizeError } from './progress/intentExtraction'
@@@ -155,7 -165,8 +166,8 @@@ let failoverEngine: FailoverEngine | nu
let bootFallbackEntry: FallbackEntry | null = null
let hostWatcher: HostWatcher | null = null
let contextMapperPollTimer: ReturnType<typeof setInterval> | null = null
+ let contextMapperPollRunner: { trigger(): void; stop(): void } | null = null
-let mcpStatusHeartbeatTimer: ReturnType<typeof setInterval> | null = null
+let mcpStatusHeartbeat: McpStatusHeartbeat | null = null
let lastServerState: Map<string, string> = new Map()
let rpcServer: RPCServer | null = null
let mcpManager: McpManager | null = null
diff --cc mcp-host/src/mcp/__tests__/managerStatus.test.ts
index 3c6fa6de9,b4e2a4386..3a010b188
--- a/mcp-host/src/mcp/__tests__/managerStatus.test.ts
+++ b/mcp-host/src/mcp/__tests__/managerStatus.test.ts
@@@ -150,17 -540,74 +547,80 @@@ describe('McpManager.refreshAllServerSt
expect(s.reason).toBe('upstream_5xx')
})
+ it('does not let a probe from a superseded epoch degrade the same connected client', async () => {
+ const { McpClient } = await import('../client')
+ vi.spyOn(McpClient.prototype, 'connect').mockImplementation(async function (this: any) {
+ this.connected = true
+ this.tools = [
+ { name: 'tool-1', inputSchema: {}, serverName: 'svc' },
+ { name: 'tool-2', inputSchema: {}, serverName: 'svc' },
+ ]
+ })
+ vi.spyOn(McpClient.prototype, 'probeTools').mockResolvedValue({
+ ok: false,
+ error: new Error('MCP client svc is superseded'),
+ stale: true,
+ })
+ const m = new McpManager()
+ await m.addServer(serverInfo())
+
+ await m.refreshAllServerStatus()
+
+ expect(m.status.get('svc')).toMatchObject({
+ state: 'connected',
+ toolCount: 2,
+ reason: null,
+ })
+ })
+
- it('returns 0 and is a no-op when no servers are connected', async () => {
+ it('returns an empty summary and is a no-op when no servers are connected', async () => {
const m = new McpManager()
- const n = await m.refreshAllServerStatus()
- expect(n).toBe(0)
+ await expect(m.refreshAllServerStatus()).resolves.toEqual({
+ serverCount: 0,
+ succeeded: 0,
+ failed: 0,
+ toolCount: 0,
+ outputSchemaCount: 0,
+ aborted: false,
+ })
})
+
+ it('does not let a stale probe overwrite a replacement connection status', async () => {
+ const { McpClient } = await import('../client')
+ vi.spyOn(McpClient.prototype, 'connect')
+ .mockImplementationOnce(async function (this: any) {
+ this.connected = true
+ this.tools = [{ name: 'old-tool', inputSchema: {}, serverName: 'svc' }]
+ })
+ .mockImplementationOnce(async function (this: any) {
+ this.connected = true
+ this.tools = [
+ { name: 'new-tool-1', inputSchema: {}, serverName: 'svc' },
+ { name: 'new-tool-2', inputSchema: {}, serverName: 'svc' },
+ ]
+ })
+ const probeStarted = deferred()
- const releaseProbe = deferred<{ ok: true; toolCount: number }>()
++ const releaseProbe = deferred<{ ok: true; toolCount: number; outputSchemaCount: number }>()
+ vi.spyOn(McpClient.prototype, 'probeTools').mockImplementation(async () => {
+ probeStarted.resolve()
+ return releaseProbe.promise
+ })
+ const m = new McpManager()
+ await m.addServer(serverInfo())
+ const refresh = m.refreshAllServerStatus()
+ await probeStarted.promise
+
+ await m.replaceServer(
+ serverInfo({ transport: { type: 'streamableHttp', url: 'http://replacement/mcp' } }),
+ undefined,
+ { scheduleCleanup: vi.fn() }
+ )
+ expect(m.status.get('svc')).toMatchObject({ state: 'connected', toolCount: 2 })
+
- releaseProbe.resolve({ ok: true, toolCount: 99 })
++ releaseProbe.resolve({ ok: true, toolCount: 99, outputSchemaCount: 0 })
+ await refresh
+ expect(m.status.get('svc')).toMatchObject({ state: 'connected', toolCount: 2 })
+ })
})
describe('McpManager.disconnectAll resets the tracker', () => {
diff --cc mcp-host/src/mcp/client.ts
index b481d2522,febe8b8e4..e64add63a
--- a/mcp-host/src/mcp/client.ts
+++ b/mcp-host/src/mcp/client.ts
@@@ -5,23 -5,142 +5,32 @@@
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'
- import { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'
+import { ListToolsResultSchema } from '@modelcontextprotocol/sdk/types.js'
import { McpServerInfo, McpTool } from '../types'
+import {
+ type McpToolCallOptions,
+ ensureNotAborted,
+ remainingBudgetMs,
+ requestOptions,
+ resolveMcpRequestTimeoutMs,
+ withRequestTimeout,
+} from './requestOptions'
-const DEFAULT_MCP_TOOL_TIMEOUT_MS = 3_600_000
-const DEFAULT_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS = 3_600_000
-const MCP_TOOL_TIMEOUT_ENV = 'CLERUM_MCP_TOOL_TIMEOUT_MS'
-const MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV = 'CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS'
+export type { McpToolCallOptions } from './requestOptions'
+ type SupportedMcpTransport = SSEClientTransport | StreamableHTTPClientTransport
+
-export interface McpToolCallOptions {
- timeoutMs?: number
- signal?: AbortSignal
-}
-
-interface McpSdkRequestOptions {
- timeout: number
- maxTotalTimeout: number
- signal?: AbortSignal
-}
-
+ interface McpCallRecovery {
+ sourceEpoch: number
+ sourceClient: Client
+ targetEpoch?: number
+ targetClient?: Client
+ promise: Promise<void>
+ }
+
-type McpProbeResult =
- | { ok: true; toolCount: number }
- | { ok: false; error: unknown; stale: boolean }
-
-function readPositiveSafeIntegerEnv(name: string, defaultValue: number): number {
- const raw = process.env[name]
- if (raw === undefined || raw === '') return defaultValue
- if (!/^\d+$/.test(raw)) throw new Error(`${name} must be a positive safe integer`)
- const value = Number.parseInt(raw, 10)
- if (!Number.isSafeInteger(value) || value < 1) {
- throw new Error(`${name} must be a positive safe integer`)
- }
- return value
-}
-
-function resolveMcpRequestTimeoutMs(callerTimeoutMs?: number): number {
- const configuredTimeoutMs = readPositiveSafeIntegerEnv(
- MCP_TOOL_TIMEOUT_ENV,
- DEFAULT_MCP_TOOL_TIMEOUT_MS
- )
- const configuredMaxTotalTimeoutMs = readPositiveSafeIntegerEnv(
- MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV,
- DEFAULT_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS
- )
- if (
- configuredMaxTotalTimeoutMs < configuredTimeoutMs &&
- (callerTimeoutMs === undefined || callerTimeoutMs > configuredMaxTotalTimeoutMs)
- ) {
- throw new Error(
- `${MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV} must be greater than or equal to ${MCP_TOOL_TIMEOUT_ENV}`
- )
- }
- if (callerTimeoutMs !== undefined) {
- if (!Number.isSafeInteger(callerTimeoutMs) || callerTimeoutMs < 1) {
- throw new Error('caller MCP timeout must be a positive safe integer')
- }
- return Math.min(configuredTimeoutMs, configuredMaxTotalTimeoutMs, callerTimeoutMs)
- }
- return Math.min(configuredTimeoutMs, configuredMaxTotalTimeoutMs)
-}
-
-function ensureNotAborted(signal: AbortSignal | undefined): void {
- if (!signal?.aborted) return
- throw abortError(signal, 'aborted')
-}
-
-function abortError(signal: AbortSignal | undefined, fallback: string): Error {
- if (signal?.reason instanceof Error) return signal.reason
- const reason =
- typeof signal?.reason === 'string' && signal.reason.trim() ? signal.reason : fallback
- return new Error(reason)
-}
-
-function withRequestTimeout<T>(
- operation: Promise<T>,
- options: McpToolCallOptions,
- timeoutMessage: string
-): Promise<T> {
- ensureNotAborted(options.signal)
- const timeoutMs = resolveMcpRequestTimeoutMs(options.timeoutMs)
- return new Promise((resolve, reject) => {
- let settled = false
- const cleanup = () => {
- clearTimeout(timer)
- options.signal?.removeEventListener('abort', onAbort)
- }
- const resolveOnce = (value: T) => {
- if (settled) return
- settled = true
- cleanup()
- resolve(value)
- }
- const rejectOnce = (error: unknown) => {
- if (settled) return
- settled = true
- cleanup()
- reject(error)
- }
- const onAbort = () => rejectOnce(abortError(options.signal, 'aborted'))
- const timer = setTimeout(
- () =>
- rejectOnce(new Error(options.timeoutMs !== undefined ? 'step-timeout' : timeoutMessage)),
- timeoutMs
- )
- options.signal?.addEventListener('abort', onAbort, { once: true })
- operation.then(resolveOnce, rejectOnce)
- })
-}
-
-function remainingBudgetMs(deadlineMs: number | undefined): number | undefined {
- if (deadlineMs === undefined) return undefined
- const remainingMs = deadlineMs - Date.now()
- if (remainingMs <= 0) throw new Error('step-timeout')
- return remainingMs
-}
-
-function requestOptions(
- timeoutMs: number | undefined,
- signal: AbortSignal | undefined
-): McpSdkRequestOptions {
- ensureNotAborted(signal)
- const timeout = resolveMcpRequestTimeoutMs(timeoutMs)
- return { timeout, maxTotalTimeout: timeout, ...(signal ? { signal } : {}) }
-}
-
export class McpClient {
private client: Client | null = null
- private transport: Transport | null = null
+ private transport: SupportedMcpTransport | null = null
private serverConfig: McpServerInfo
private authToken?: string
private proxyUrl?: string
@@@ -208,30 -419,33 +309,47 @@@
* transient probe error during a heartbeat doesn't clobber the last known
* tool list.
*/
- async probeTools(options: McpToolCallOptions = {}): Promise<McpProbeResult> {
+ async probeTools(
+ options: McpToolCallOptions = {}
+ ): Promise<
- { ok: true; toolCount: number; outputSchemaCount: number } | { ok: false; error: unknown }
++ | { ok: true; toolCount: number; outputSchemaCount: number }
++ | { ok: false; error: unknown; stale: boolean }
+ > {
if (!this.client || !this.connected) {
- return { ok: false, error: new Error(`${this.name} is not connected`) }
+ return {
+ ok: false,
+ error: new Error(`${this.name} is not connected`),
+ stale: false,
+ }
}
+ const probeClient = this.client
+ const probeEpoch = this.connectionEpoch
try {
- const response = await probeClient.listTools(
- undefined,
+ // Deliberately use the validated raw request instead of Client.listTools().
+ // The SDK convenience method compiles output schemas and mutates task/output
+ // metadata caches, which must remain owned by connection and explicit refresh.
- const response = await this.client.request(
++ const response = await probeClient.request(
+ { method: 'tools/list', params: undefined },
+ ListToolsResultSchema,
requestOptions(options.timeoutMs, options.signal)
)
++ // Preserve connection-epoch staleness: a probe that resolved against a
++ // superseded/retired connection must not report counts for it.
+ this.ensureCallCurrent(probeEpoch, probeClient)
- const toolCount = (response.tools || []).length
- return { ok: true, toolCount }
+ return {
+ ok: true,
+ toolCount: response.tools.length,
+ outputSchemaCount: response.tools.filter(tool => tool.outputSchema !== undefined).length,
+ }
} catch (error) {
- return { ok: false, error }
+ if (!this.isCallCurrent(probeEpoch, probeClient)) {
+ return {
+ ok: false,
+ error: this.lifecycleError(this.retired ? 'closed' : 'superseded'),
+ stale: true,
+ }
+ }
+ return { ok: false, error, stale: false }
}
}
diff --cc mcp-host/src/mcp/manager.ts
index d50d89596,ab295411d..3e5c05c15
--- a/mcp-host/src/mcp/manager.ts
+++ b/mcp-host/src/mcp/manager.ts
@@@ -5,22 -5,24 +5,40 @@@ import { McpServerInfo, McpTool, ToolCa
import { McpClient, type McpToolCallOptions } from './client'
import { ServerStatusTracker } from './serverStatus'
+export interface McpStatusRefreshSummary {
+ serverCount: number
+ succeeded: number
+ failed: number
+ toolCount: number
+ outputSchemaCount: number
+ aborted: boolean
+}
+
+function deriveProbeSignal(roundSignal: AbortSignal | undefined): AbortSignal {
+ const probeController = new AbortController()
+ return roundSignal
+ ? AbortSignal.any([roundSignal, probeController.signal])
+ : probeController.signal
+}
+
+ export type McpAdmissionOutcome = 'applied' | 'stale'
+ export interface McpAdmissionControl {
+ isCurrent?: () => boolean
+ onCommit?: () => void
+ scheduleCleanup?: (cleanup: McpDetachedServerCleanup) => void
+ }
+ export type McpDetachedServerCleanup = () => Promise<void>
+
+ interface PendingAdmission {
+ attempt: symbol
+ client: McpClient
+ }
+
+ interface ClientRetirement {
+ cleanup: McpDetachedServerCleanup
+ claimed: boolean
+ }
+
export class McpManager {
private clients: Map<string, McpClient> = new Map()
private serverInfos: Map<string, McpServerInfo> = new Map()
@@@ -235,62 -451,23 +467,68 @@@
* Called by the background heartbeat to keep `observedAt` fresh and to
* classify refresh failures (spec §4.5 — stay connected, attach reason).
*
- * Returns the number of servers probed.
+ * Probes a stable client snapshot. State is written only after every probe
+ * settles; an aborted round never mutates the last known server status.
*/
- async refreshAllServerStatus(): Promise<number> {
+ async refreshAllServerStatus(options: McpToolCallOptions = {}): Promise<McpStatusRefreshSummary> {
const entries = [...this.clients.entries()]
- await Promise.all(
+ if (options.signal?.aborted) {
+ return {
+ serverCount: entries.length,
+ succeeded: 0,
+ failed: 0,
+ toolCount: 0,
+ outputSchemaCount: 0,
+ aborted: true,
+ }
+ }
+
+ const results = await Promise.all(
entries.map(async ([name, client]) => {
- const result = await client.probeTools()
- if (this.clients.get(name) !== client) return
- if (!result.ok && result.stale) return
- if (result.ok) {
- this.statusTracker.updateToolCount(name, result.toolCount)
- } else {
- this.statusTracker.updateToolCount(name, 0, { refreshError: result.error })
+ try {
+ return {
+ name,
++ client,
+ result: await client.probeTools({
+ timeoutMs: options.timeoutMs,
+ signal: deriveProbeSignal(options.signal),
+ }),
+ }
+ } catch (error) {
- return { name, result: { ok: false as const, error } }
++ return { name, client, result: { ok: false as const, error, stale: false } }
}
})
)
- return entries.length
+ const succeeded = results.filter(({ result }) => result.ok).length
+ const summary: McpStatusRefreshSummary = {
+ serverCount: entries.length,
+ succeeded,
+ failed: entries.length - succeeded,
+ toolCount: results.reduce(
+ (total, { result }) => total + (result.ok ? result.toolCount : 0),
+ 0
+ ),
+ outputSchemaCount: results.reduce(
+ (total, { result }) => total + (result.ok ? result.outputSchemaCount : 0),
+ 0
+ ),
+ aborted: options.signal?.aborted === true,
+ }
+ if (summary.aborted) return summary
+
- for (const { name, result } of results) {
++ for (const { name, client, result } of results) {
++ // Connection-epoch staleness (dev hardening): never write status for a
++ // client that was swapped out mid-round, and skip stale-error probes that
++ // raced a reconnect — they carry no authoritative signal for this client.
++ if (this.clients.get(name) !== client) continue
++ if (!result.ok && result.stale) continue
+ if (result.ok) {
+ this.statusTracker.updateToolCount(name, result.toolCount)
+ } else {
+ this.statusTracker.updateToolCount(name, 0, { refreshError: result.error })
+ }
+ }
+ return summary
}
/**
diff --cc mcp-host/src/mcp/requestOptions.ts
index 7790cd295,000000000..66baa8430
mode 100644,000000..100644
--- a/mcp-host/src/mcp/requestOptions.ts
+++ b/mcp-host/src/mcp/requestOptions.ts
@@@ -1,117 -1,0 +1,119 @@@
+const DEFAULT_MCP_TOOL_TIMEOUT_MS = 3_600_000
+const DEFAULT_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS = 3_600_000
+const MCP_TOOL_TIMEOUT_ENV = 'CLERUM_MCP_TOOL_TIMEOUT_MS'
+const MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV = 'CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS'
+
+export interface McpToolCallOptions {
+ timeoutMs?: number
+ signal?: AbortSignal
+}
+
+export interface McpSdkRequestOptions {
+ timeout: number
+ maxTotalTimeout: number
+ signal?: AbortSignal
+}
+
+function readPositiveSafeIntegerEnv(name: string, defaultValue: number): number {
+ const raw = process.env[name]
+ if (raw === undefined || raw === '') return defaultValue
+ if (!/^\d+$/.test(raw)) throw new Error(`${name} must be a positive safe integer`)
+ const value = Number.parseInt(raw, 10)
+ if (!Number.isSafeInteger(value) || value < 1) {
+ throw new Error(`${name} must be a positive safe integer`)
+ }
+ return value
+}
+
+export function resolveMcpRequestTimeoutMs(callerTimeoutMs?: number): number {
+ const configuredTimeoutMs = readPositiveSafeIntegerEnv(
+ MCP_TOOL_TIMEOUT_ENV,
+ DEFAULT_MCP_TOOL_TIMEOUT_MS
+ )
+ const configuredMaxTotalTimeoutMs = readPositiveSafeIntegerEnv(
+ MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV,
+ DEFAULT_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS
+ )
+ if (
+ configuredMaxTotalTimeoutMs < configuredTimeoutMs &&
+ (callerTimeoutMs === undefined || callerTimeoutMs > configuredMaxTotalTimeoutMs)
+ ) {
+ throw new Error(
+ `${MCP_TOOL_MAX_TOTAL_TIMEOUT_ENV} must be greater than or equal to ${MCP_TOOL_TIMEOUT_ENV}`
+ )
+ }
+ if (callerTimeoutMs !== undefined) {
+ if (!Number.isSafeInteger(callerTimeoutMs) || callerTimeoutMs < 1) {
+ throw new Error('caller MCP timeout must be a positive safe integer')
+ }
+ return Math.min(configuredTimeoutMs, configuredMaxTotalTimeoutMs, callerTimeoutMs)
+ }
+ return Math.min(configuredTimeoutMs, configuredMaxTotalTimeoutMs)
+}
+
+export function ensureNotAborted(signal: AbortSignal | undefined): void {
+ if (!signal?.aborted) return
- const reason =
- typeof signal.reason === 'string' && signal.reason.trim() ? signal.reason : 'aborted'
- throw new Error(reason)
++ throw abortError(signal, 'aborted')
+}
+
+function abortError(signal: AbortSignal | undefined, fallback: string): Error {
++ // Preserve an Error abort reason verbatim (e.g. the lifecycle 'closed' /
++ // 'superseded' error propagated through retirementController.abort()), so a
++ // retired/superseded connection surfaces its real cause instead of 'aborted'.
++ if (signal?.reason instanceof Error) return signal.reason
+ const reason =
+ typeof signal?.reason === 'string' && signal.reason.trim() ? signal.reason : fallback
+ return new Error(reason)
+}
+
+export function withRequestTimeout<T>(
+ operation: Promise<T>,
+ options: McpToolCallOptions,
+ timeoutMessage: string
+): Promise<T> {
+ ensureNotAborted(options.signal)
+ const timeoutMs = resolveMcpRequestTimeoutMs(options.timeoutMs)
+ return new Promise((resolve, reject) => {
+ let settled = false
+ const cleanup = () => {
+ clearTimeout(timer)
+ options.signal?.removeEventListener('abort', onAbort)
+ }
+ const resolveOnce = (value: T) => {
+ if (settled) return
+ settled = true
+ cleanup()
+ resolve(value)
+ }
+ const rejectOnce = (error: unknown) => {
+ if (settled) return
+ settled = true
+ cleanup()
+ reject(error)
+ }
+ const onAbort = () => rejectOnce(abortError(options.signal, 'aborted'))
+ const timer = setTimeout(
+ () =>
+ rejectOnce(new Error(options.timeoutMs !== undefined ? 'step-timeout' : timeoutMessage)),
+ timeoutMs
+ )
+ options.signal?.addEventListener('abort', onAbort, { once: true })
+ operation.then(resolveOnce, rejectOnce)
+ })
+}
+
+export function remainingBudgetMs(deadlineMs: number | undefined): number | undefined {
+ if (deadlineMs === undefined) return undefined
+ const remainingMs = deadlineMs - Date.now()
+ if (remainingMs <= 0) throw new Error('step-timeout')
+ return remainingMs
+}
+
+export function requestOptions(
+ timeoutMs: number | undefined,
+ signal: AbortSignal | undefined
+): McpSdkRequestOptions {
+ ensureNotAborted(signal)
+ const timeout = resolveMcpRequestTimeoutMs(timeoutMs)
+ return { timeout, maxTotalTimeout: timeout, ...(signal ? { signal } : {}) }
+}