Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* MCP Manager - manages connections to multiple MCP servers.
*
* Live connections are keyed by ClientKey = (serverName, principal) where
* principal ∈ { SHARED, userId }. Static/oauth-context servers use a single
* SHARED partition; oauth-user servers keep one token-less SHARED representative
* (populates the catalog) plus N lazily-admitted per-user partitions. Desired
* state / catalog / fencing still reason by serverName — see `byServer`.
*/
import { logger } from '../logger'
import { McpServerInfo, McpTool, ToolCallResult } from '../types'
import { catalogBootstrapMetrics } from './catalogBootstrapMetrics'
import {
McpAuthError,
McpClient,
McpTokenProvider,
type McpToolCallOptions,
staticTokenProvider,
} from './client'
import type { GrantExistsQuery, GrantExistsResult } from './grantExistenceClient'
import { grantCoordKey } from './grantExistenceClient'
import {
type GrantExistenceChecker,
type GrantProbe,
type McpCatalogBootstrapConfig,
createGrantProbe,
} from './grantProbe'
import { ServerStatusTracker, extractStructuredHttpStatus, isAuthError } from './serverStatus'
export interface McpStatusRefreshSummary {
serverCount: number
succeeded: number
failed: number
toolCount: number
outputSchemaCount: number
aborted: boolean
}
export type McpAdmissionOutcome = 'applied' | 'stale'
export interface McpAdmissionControl {
isCurrent?: () => boolean
onCommit?: () => void
scheduleCleanup?: (cleanup: McpDetachedServerCleanup) => void
}
export type McpDetachedServerCleanup = () => Promise<void>
/**
* Optional dependencies for eager catalog bootstrap and the SHARED oauth-context
* grant gate. Omitting `grantExistence` (or its policy config) keeps the manager
* on today's lazy admission — no bootstrap, no gate — so every existing 3-arg
* `new McpManager(...)` construction behaves identically.
*/
export interface McpManagerOptions {
/** Batch grant-existence checker (control-api `grants/exists`); no checker → no probe. */
grantExistence?: GrantExistenceChecker
/** LRU cap on live per-user partitions (main.ts OAUTH_USER_PARTITION_MAX). */
userPartitionMax?: number
/** Probe-policy knobs, injected so grantProbe never imports config.ts. */
catalogBootstrap?: McpCatalogBootstrapConfig
/** Injectable clock for tests. */
now?: () => number
}
/** Per-turn options for the eager catalog bootstrap (§6.1). */
export interface McpCatalogBootstrapOptions {
/**
* Ends this turn's WAIT for its admissions early — the call returns with
* whatever landed and the rest counted as `pending`, exactly as when the wait
* budget expires. It does NOT abort the admissions: they are owned by the
* manager and coalesced (per-user via `ensureInFlight`, SHARED cross-user), so
* one turn's abort must never tear down a connection another turn/user depends
* on. §6.1 step 7/M4 mention `signal` on the admission, but §6.5/§7.3 require
* the abort to leave an in-flight admission running — the latter is the
* governing reading, so the signal is wired to the wait only.
*/
signal?: AbortSignal
}
/**
* Outcome of one `bootstrapUserCatalog` call (§6.1). Every count is a disjoint
* tally of how a candidate was resolved; `skipped` breaks the non-admitted ones
* down by reason so the operator can tell a kill-switch (`disabled`) from a
* budget cap (`budget_exhausted`) from a definitive absence (`absent`).
*/
export interface McpCatalogBootstrapSummary {
/** Servers that survived the M12–M16 skip gates (i.e. were probe candidates). */
candidates: number
/** Coordinates actually sent to the `grants/exists` batch. */
probed: number
/** Admissions that resolved within the wait budget. */
admitted: number
/** Admissions still in flight when the wait budget expired (they continue). */
pending: number
skipped: Partial<
Record<
| 'disabled'
| 'no_user'
| 'no_factory'
| 'no_checker'
| 'manager_closed'
| 'gate'
| 'live'
| 'in_flight'
| 'cap'
| 'absent_cached'
| 'failed_cached'
| 'budget_exhausted'
| 'probe_unknown'
| 'absent'
| 'admission_failed',
number
>
>
/** Milliseconds the call waited on admissions. */
waitedMs: number
/** The wait budget expired with admissions still pending. */
timedOut: boolean
}
/** The partition a bootstrap candidate would open, keyed by its grant coordinate. */
export interface BootstrapCandidate {
key: string
kind: 'user' | 'shared'
serverName: string
}
/** Input to the pure `selectBootstrapCandidates` classifier (§6.1). */
export interface SelectBootstrapCandidatesInput {
userId: string
infos: Iterable<McpServerInfo>
/** clients.has — a live partition already exists for the key (M13). */
hasLive: (key: string) => boolean
/** ensureInFlight.has || pendingAdmissions.has — an admission is under way (M13). */
hasInFlight: (key: string) => boolean
/** A token-provider factory is wired (oauth-context is a candidate only then, M7). */
factoryPresent: boolean
/** Live per-user partition count (partitionLastUsed.size) for the cap check (M14). */
userPartitionCount: number
/** LRU cap on per-user partitions; undefined disables the cap. */
userPartitionMax: number | undefined
}
/** Output of `selectBootstrapCandidates`: what to probe and how to correlate it. */
export interface BootstrapCandidateSelection {
queries: GrantExistsQuery[]
/** grant coordinate → the partition to open on `exists:true`. */
byCoord: Map<string, BootstrapCandidate>
skipped: Partial<Record<'gate' | 'live' | 'in_flight' | 'cap', number>>
}
interface PendingAdmission {
attempt: symbol
client: McpClient
serverName: string
}
interface ClientRetirement {
cleanup: McpDetachedServerCleanup
claimed: boolean
}
// ─── ClientKey / principal ──────────────────────────────────────────────────
//
// SHARED is a STRUCTURAL sentinel (a tagged tuple), never a bare literal a real
// userId could collide with. Serialization tags the shared/user discriminator as
// the first tuple element, so no userId string can serialize to a SHARED key
// (property-tested in managerPartition.test.ts, T2).
export type McpPrincipal =
| { readonly kind: 'shared' }
| { readonly kind: 'user'; readonly userId: string }
export const SHARED_PRINCIPAL: McpPrincipal = { kind: 'shared' }
export function userPrincipal(userId: string): McpPrincipal {
return { kind: 'user', userId }
}
/** Serialize a (serverName, principal) pair into a stable Map key string. */
export function serializeClientKey(serverName: string, principal: McpPrincipal): string {
return principal.kind === 'shared'
? JSON.stringify(['s', serverName])
: JSON.stringify(['u', serverName, principal.userId])
}
/** Recover the serverName from a serialized ClientKey. */
export function serverNameFromClientKey(key: string): string {
return (JSON.parse(key) as [string, string, string?])[1]
}
/**
* Recover the principal from a serialized ClientKey — inverse of the
* discriminator `serializeClientKey` writes (`'u'` → user, else SHARED). Used by
* the grant-sweep to tell a per-user partition from the token-less SHARED
* representative of an oauth-user server.
*/
export function principalFromClientKey(key: string): McpPrincipal {
const parsed = JSON.parse(key) as ['s' | 'u', string, string?]
return parsed[0] === 'u' && typeof parsed[2] === 'string'
? userPrincipal(parsed[2])
: SHARED_PRINCIPAL
}
/** The OAuth flavor of a live partition the hot-revocation sweep re-validates. */
export type OAuthPartitionFlavor = 'oauth-user' | 'oauth-context'
/**
* A live OAuth partition exposed to the hot-revocation grant-sweep (mini-spec
* 13 §4.1). `key` is the serialized ClientKey the sweep hands back to
* `evictRevokedPartitions`; the coordinates address the grant in control-api
* (`userId` present only for `oauth-user`).
*/
export interface LiveOAuthPartition {
flavor: OAuthPartitionFlavor
serverName: string
userId?: string
key: string
}
/** Factory that builds a per-connection token provider for an oauth server. */
export type McpTokenProviderFactory = (
server: McpServerInfo,
principal: McpPrincipal
) => McpTokenProvider
export class McpManager {
// Live connections, keyed by serialized ClientKey.
private clients: Map<string, McpClient> = new Map()
// Desired-state projection is per-serverName (identical across partitions).
private serverInfos: Map<string, McpServerInfo> = new Map()
// In-flight admissions, keyed by serialized ClientKey.
private pendingAdmissions: Map<string, PendingAdmission> = new Map()
// serverName → set of live ClientKeys. Restores serverName-level reasoning
// (catalog dedup, connected-servers projection, server-wide purge).
private byServer: Map<string, Set<string>> = new Map()
// Coalesces concurrent lazy admissions for the same per-user partition.
private ensureInFlight: Map<string, Promise<void>> = new Map()
// Last-used timestamp for per-user partitions (LRU + TTL eviction).
private partitionLastUsed: Map<string, number> = new Map()
// In-flight tool-call tokens per per-user partition. A UNIQUE token per call
// (not a bare counter): each call removes only its own token, so a completing
// call can never zero a *re-admitted* partition's in-flight state after a
// force-eviction (the H1 re-admission race). A partition with a live call is
// NEVER evicted by the idle sweep (mini-spec §6 — respect in-flight).
private partitionInFlight: Map<string, Set<symbol>> = new Map()
private clientRetirements = new WeakMap<McpClient, ClientRetirement>()
private lifecycleEpoch = 0
private closed = false
private proxyUrl?: string
private statusTracker: ServerStatusTracker
private tokenProviderFactory?: McpTokenProviderFactory
// One probe instance per manager (bootstrap + SHARED gate share it). Undefined
// when no grant checker (or its config) is injected → lazy admission as today.
private readonly grantProbe?: GrantProbe
// Kill-switch: the probe is built whenever a checker + config are present, but
// the SHARED gate (and bootstrap) only engage while enabled. `enabled=false`
// ⇒ every rail falls back to today's eager/lazy admission.
private readonly catalogBootstrapEnabled: boolean
// Probe/admission timeouts for bootstrap-initiated connections. Kept alongside
// grantProbe (which owns the negative-cache/backoff knobs) because the manager
// — not the probe — owns the wait budget and the per-connection timeout.
private readonly catalogBootstrap?: McpCatalogBootstrapConfig
// LRU cap the bootstrap honors so it never thrashes admit→evict above the cap.
private readonly userPartitionMax?: number
// Injected clock (tests) so bootstrap metric labeling reads the same time base
// the probe policy decides on. Production uses Date.now.
private readonly now: () => number
// Coalesces concurrent bootstraps for the SAME user onto one plan/probe/admit
// (§6.1 step 2). Cleared on flota reset so a fresh manager starts empty.
private readonly bootstrapInFlight: Map<string, Promise<McpCatalogBootstrapSummary>> = new Map()
// Remote oauth-context servers currently registered WITHOUT an authenticated
// SHARED connection (no grant yet). Membership drives the once-per-transition
// `mcp_oauth_shared_awaiting_grant` log — discovery re-probes every poll, so
// logging per poll would be noise; we log only on entering/leaving the set.
private readonly sharedAwaitingGrant: Set<string> = new Set()
constructor(
proxyUrl?: string,
statusTracker?: ServerStatusTracker,
tokenProviderFactory?: McpTokenProviderFactory,
options?: McpManagerOptions
) {
this.proxyUrl = proxyUrl
this.statusTracker = statusTracker ?? new ServerStatusTracker()
this.tokenProviderFactory = tokenProviderFactory
// A probe is inert without both a checker and its policy config; either
// absent → stay on today's lazy admission (bootstrap and SHARED gate off).
this.now = options?.now ?? Date.now
this.grantProbe =
options?.grantExistence && options?.catalogBootstrap
? createGrantProbe(options.grantExistence, options.catalogBootstrap, this.now)
: undefined
this.catalogBootstrapEnabled = options?.catalogBootstrap?.enabled ?? false
this.catalogBootstrap = options?.catalogBootstrap
this.userPartitionMax = options?.userPartitionMax
if (proxyUrl) {
logger.info({ component: 'McpManager' }, 'Proxy mode enabled')
}
}
/**
* The shared status tracker. Exposed so higher layers (status endpoint,
* background heartbeat) can read snapshots and touch entries.
*/
get status(): ServerStatusTracker {
return this.statusTracker
}
private sharedKey(serverName: string): string {
return serializeClientKey(serverName, SHARED_PRINCIPAL)
}
private userKey(serverName: string, userId: string): string {
return serializeClientKey(serverName, userPrincipal(userId))
}
/**
* The representative connection for a serverName: prefer the SHARED partition
* (token-less for oauth-user, the sole connection for static/oauth-context),
* else any live per-user partition. Powers catalog dedup by serverName.
*/
private representativeClient(serverName: string): McpClient | undefined {
const shared = this.clients.get(this.sharedKey(serverName))
if (shared) return shared
const keys = this.byServer.get(serverName)
if (keys) {
for (const key of keys) {
const client = this.clients.get(key)
if (client) return client
}
}
return undefined
}
/**
* The representative to probe on behalf of the PLATFORM (background
* heartbeat) — the SHARED partition ONLY, with NO per-user fallback.
*
* A platform probe (`probeTools`) makes a REAL network call. It must never be
* charged to a user's OAuth token/quota, so it may only ride the SHARED
* connection:
* - static / oauth-context / the token-less oauth-user SHARED representative
* all live under the SHARED key, so they are still probed here.
* - the sole case skipped is the pathological "SHARED down + a per-user
* partition alive" — there `representativeClient` would fall back to a
* per-user client and spend that user's identity on a platform probe.
* Such a server simply goes un-refreshed this round (status left stale)
* rather than being probed with a user's token.
*
* Catalog reads (`getAllTools` / `describeCapabilities`) intentionally KEEP
* the per-user fallback: they read `client.availableTools` from cache — no
* network, no token spent — so falling back keeps the catalog populated.
*/
private probeRepresentativeClient(serverName: string): McpClient | undefined {
return this.clients.get(this.sharedKey(serverName))
}
/**
* A remote oauth server's status is NEVER derived from a platform probe. A
* probe (`probeTools`) is a real tools/list round-trip, and a spec-compliant
* remote 401s on any token-less request while an authenticated probe would
* spend a user's / context's OAuth token on a platform heartbeat (mini-spec 19
* §D-6). Its status is derived from the authenticated per-user catalog cache
* (a pure in-memory read) plus the `connect_required` marker set at call time.
*/
private isRemoteOauthServer(serverName: string): boolean {
const info = this.serverInfos.get(serverName)
return info?.remote === true && this.isOauthServer(info)
}
private isOauthServer(serverConfig: McpServerInfo): boolean {
return serverConfig.authKind === 'oauth-user' || serverConfig.authKind === 'oauth-context'
}
private buildTokenProvider(
serverConfig: McpServerInfo,
principal: McpPrincipal,
eagerToken: string | undefined
): McpTokenProvider | undefined {
if (this.isOauthServer(serverConfig)) {
// oauth: JIT resolution via the injected factory (broker per-user /
// per-context, or token-less representative).
if (this.tokenProviderFactory) {
return this.tokenProviderFactory(serverConfig, principal)
}
// No factory (dev/tests). Behavior forks on remote vs local, because their
// catalog preconditions are opposite:
// - LOCAL (v1 class-a, mini-spec 03 §5): the server serves
// initialize/tools/list WITHOUT auth and only 401s on tools/call, so a
// token-less SHARED representative still populates the catalog. Preserve
// that — `staticTokenProvider(undefined)`.
// - REMOTE (mini-spec 19 §D-6): a spec-compliant upstream 401s already at
// `initialize`. A token-less provider here would let a dev server that
// does NOT enforce auth connect and populate a token-less catalog,
// MASKING the authenticated rail (and, worse, hiding that the same
// server 401s in prod). Fail closed: return no provider. `undefined`
// resolves to a token-less connection, but the eager-admission path
// (addServer) never treats a remote-oauth SHARED representative as a
// "populated catalog" — see the remote-oauth branch there.
return serverConfig.remote ? undefined : staticTokenProvider(undefined)
}
// static/none/bearer/basic/apiKey: preserve today's frozen-token behavior.
return staticTokenProvider(eagerToken)
}
private installConnectedClient(
key: string,
serverConfig: McpServerInfo,
client: McpClient,
ownsServerStatus: boolean
): void {
this.clients.set(key, client)
this.serverInfos.set(serverConfig.name, serverConfig)
let keys = this.byServer.get(serverConfig.name)
if (!keys) {
keys = new Set()
this.byServer.set(serverConfig.name, keys)
}
keys.add(key)
// Only the SHARED/eager owner writes the per-serverName status (symmetric
// with the gated markConnecting / recordAdmissionFailure in
// connectAndInstall). A per-user admission must NOT flip the representative's
// status to `connected` with its own partition's toolCount — a caller reading
// the per-serverName status would then see a down representative as connected.
if (ownsServerStatus) {
this.statusTracker.markConnected(serverConfig.name, client.availableTools.length)
}
}
private claimClientCleanup(client: McpClient): McpDetachedServerCleanup | undefined {
let retirement = this.clientRetirements.get(client)
if (!retirement) {
const cleanupClient = client.retire()
let cleanupPromise: Promise<void> | undefined
retirement = {
claimed: false,
cleanup: () => {
if (!cleanupPromise) {
try {
cleanupPromise = Promise.resolve(cleanupClient())
} catch (error) {
cleanupPromise = Promise.reject(error)
}
}
return cleanupPromise
},
}
this.clientRetirements.set(client, retirement)
}
if (retirement.claimed) return undefined
retirement.claimed = true
return retirement.cleanup
}
private scheduleClientCleanup(client: McpClient, control: McpAdmissionControl): Promise<void> {
const cleanup = this.claimClientCleanup(client)
if (!cleanup) return Promise.resolve()
if (control.scheduleCleanup) {
control.scheduleCleanup(cleanup)
return Promise.resolve()
}
return cleanup().catch(() => undefined)
}
/**
* Retain an admission failure for inventory/status reporting when failure
* occurs before McpClient.connect (for example, auth-token discovery).
*/
recordAdmissionFailure(serverConfig: McpServerInfo, error: unknown): void {
if (this.closed) return
this.serverInfos.set(serverConfig.name, serverConfig)
this.statusTracker.markFailed(serverConfig.name, error)
}
/**
* Add and connect to an MCP server (SHARED partition — the eager coordinator
* path). LOCAL oauth-user servers admit only the token-less representative
* here; per-user partitions are lazily admitted on demand via callTool. REMOTE
* oauth servers open NO token-less SHARED representative (mini-spec 19 §D-6, see
* the remote-oauth branch below): the catalog is authenticated per-user (or, for
* oauth-context, over the authenticated SHARED grant).
*/
async addServer(
serverConfig: McpServerInfo,
authToken?: string,
control: McpAdmissionControl = {}
): Promise<McpAdmissionOutcome> {
const admissionLifecycleEpoch = this.lifecycleEpoch
const externalIsCurrent = control.isCurrent ?? (() => true)
const isCurrent = (): boolean =>
!this.closed && this.lifecycleEpoch === admissionLifecycleEpoch && externalIsCurrent()
if (!isCurrent()) return 'stale'
// Skip disabled servers — operator intent, not infra failure.
if (!serverConfig.enabled) {
logger.info(
{ component: 'McpManager', serverName: serverConfig.name },
'Skipping disabled server'
)
this.serverInfos.set(serverConfig.name, serverConfig)
this.statusTracker.markDisabled(serverConfig.name)
control.onCommit?.()
return 'applied'
}
// Explicitly non-authoritative readiness is a fail-closed admission
// signal, not permission to open a new connection.
if (serverConfig.status?.authoritative === false) {
logger.info(
{ component: 'McpManager', serverName: serverConfig.name },
'Skipping server with non-authoritative readiness'
)
this.serverInfos.set(serverConfig.name, serverConfig)
this.statusTracker.markNotReady(serverConfig.name, serverConfig.status?.message)
control.onCommit?.()
return 'applied'
}
// Skip servers that aren't ready yet — transient infra, surfaced as not_ready.
if (!serverConfig.status?.ready) {
logger.info(
{ component: 'McpManager', serverName: serverConfig.name },
'Skipping server not ready'
)
this.serverInfos.set(serverConfig.name, serverConfig)
this.statusTracker.markNotReady(serverConfig.name, serverConfig.status?.message)
control.onCommit?.()
return 'applied'
}
// ── Remote oauth: no token-less SHARED representative (mini-spec 19 §D-6) ──
// A LOCAL oauth server serves initialize/tools/list unauthenticated, so the
// eager SHARED representative can populate the catalog token-less. A REMOTE
// spec-compliant server 401s already at `initialize`, so that representative
// CANNOT be token-less. Handle the eager SHARED admission by flavor:
// - oauth-user: no shared grant exists → the catalog is authenticated
// PER-USER (each user's partition does the authenticated initialize/
// tools/list, admitted lazily on first tool call, surfaced through the
// per-user representative fallback in representativeClient/getAllTools).
// - no factory (dev/tests): nothing can authenticate the representative →
// fail closed rather than degrade to a masking token-less catalog.
// Both register the server (so per-user admission and status derive from it)
// but open no SHARED connection.
// - oauth-context WITH a factory is the one flavor whose SHARED
// representative CAN authenticate (on its context grant) and IS the
// catalog — but only once that grant exists. Opening it before the grant
// lands connects token-less against a lenient upstream and caches a
// token-less catalog. So when the grant gate is engaged we probe first:
// the SHARED opens ONLY on a confirmed grant; without one the server is
// registered connected-with-0-tools and discovery re-probes each poll,
// admitting the SHARED within one poll of the grant landing.
if (serverConfig.remote && this.isOauthServer(serverConfig)) {
const sharedAuthenticates =
serverConfig.authKind === 'oauth-context' && this.tokenProviderFactory !== undefined
if (!sharedAuthenticates) {
const installed = this.serverInfos.get(serverConfig.name)
if (installed && JSON.stringify(installed) === JSON.stringify(serverConfig)) {
control.onCommit?.()
return 'applied'
}
// Config changed (or first registration): drop any live per-user
// partitions so they rebuild lazily against the new revision — the remote
// analogue of replaceServer's per-user eviction. There is no SHARED
// representative to keep, so keepKey matches nothing.
this.evictPartitionsExcept(serverConfig.name, this.sharedKey(serverConfig.name), control)
// A server that WAS a gated oauth-context (and may be awaiting a grant) is
// no longer one here — it must leave the awaiting-grant set.
this.leaveSharedAwaitingGrant(serverConfig.name)
this.serverInfos.set(serverConfig.name, serverConfig)
// Reflect whatever the (per-user) representative catalog already holds —
// 0 right after an eviction / until a user with a live grant connects. The
// heartbeat refreshes this from the authenticated per-user catalog, never
// a token-less probe.
this.statusTracker.markConnected(
serverConfig.name,
this.representativeClient(serverConfig.name)?.availableTools.length ?? 0
)
control.onCommit?.()
return 'applied'
}
// oauth-context + factory: gate the SHARED admission on a confirmed grant
// when the probe is engaged. No probe / kill-switch off ⇒ fall through to
// the eager path below (today's behavior, M10 by absence).
if (this.grantProbe && this.catalogBootstrapEnabled) {
return this.addGatedSharedOauthContext(serverConfig, authToken, control)
}
}
// Reaching the general SHARED path means this is NOT a gated oauth-context
// this poll (a re-purpose to static/none/local-oauth, or the gate off). A
// gated oauth-context still awaiting its grant returns above, so this never
// clears a legitimately-awaiting server — only re-purposed ones.
this.leaveSharedAwaitingGrant(serverConfig.name)
const key = this.sharedKey(serverConfig.name)
// Check if already connected (SHARED partition)
if (this.clients.has(key)) {
const installed = this.serverInfos.get(serverConfig.name)
if (installed && JSON.stringify(installed) === JSON.stringify(serverConfig)) {
logger.info(
{ component: 'McpManager', serverName: serverConfig.name },
'Server already connected'
)
control.onCommit?.()
return 'applied'
}
return this.replaceServer(serverConfig, authToken, control)
}
const tokenProvider = this.buildTokenProvider(serverConfig, SHARED_PRINCIPAL, authToken)
return this.connectAndInstall(key, serverConfig, tokenProvider, control, true)
}
/**
* SHARED oauth-context admission gated on a confirmed grant (§6.2). A live
* client or an in-flight admission short-circuits (M13, idempotent + coalesced);
* otherwise a single-coordinate probe decides: a confirmed grant (or an
* indeterminate probe, fail-open like today) opens the authenticated SHARED;
* a definitive absence registers the server connected-with-0-tools and opens
* NOTHING, so no token-less catalog is cached and discovery re-probes each poll.
*/
private async addGatedSharedOauthContext(
serverConfig: McpServerInfo,
authToken: string | undefined,
control: McpAdmissionControl
): Promise<McpAdmissionOutcome> {
const key = this.sharedKey(serverConfig.name)
// A live SHARED representative is today's path: same config → no-op,
// changed → replace (never re-probes a connection that already exists).
if (this.clients.has(key)) {
const installed = this.serverInfos.get(serverConfig.name)
if (installed && JSON.stringify(installed) === JSON.stringify(serverConfig)) {
control.onCommit?.()
return 'applied'
}
return this.replaceServer(serverConfig, authToken, control)
}
// A SHARED admission already in flight (a concurrent discovery poll or the
// turn bootstrap): coalesce onto it instead of probing/opening a second
// connection (M13). If it landed a client we are done; otherwise it failed or
// was absent, so fall through to a fresh probe.
const inFlight = this.ensureInFlight.get(key)
if (inFlight) {
await inFlight
if (this.clients.has(key)) {
control.onCommit?.()
return 'applied'
}
}
const verdict = await this.grantProbe!.probeOne({ mcpServerName: serverConfig.name })
if (verdict === 'absent') {
return this.registerSharedAwaitingGrant(serverConfig, control)
}
// 'present' (grant confirmed) or 'unknown' (probe indeterminate → fail-open,
// connect as today: a down control-api must not empty the catalog).
return this.admitSharedOauthContext(serverConfig, control)
}
/**
* Open the authenticated SHARED oauth-context representative and register the
* admission under the SHARED key so a concurrent discovery poll or the turn
* bootstrap coalesces onto this single connection. `evictPartitionsExcept`
* preserves this key (keepKey = sharedKey), so a config-change eviction never
* orphans the in-flight admission.
*/
private admitSharedOauthContext(
serverConfig: McpServerInfo,
control: McpAdmissionControl,
options?: McpToolCallOptions
): Promise<McpAdmissionOutcome> {
const key = this.sharedKey(serverConfig.name)
// Coalesce at admission time, not only at selection. Both callers — the
// discovery gate (addGatedSharedOauthContext) and the eager bootstrap —
// pre-check live/in-flight, then `await` a grant probe (an RTT) before
// reaching here; in that gap the OTHER caller can land or start the SHARED
// connection. Without this re-check each would open its own, burning a
// broker-bucket unit and breaking §6.5's single-SHARED-connection guarantee
// (pendingAdmissions still fences the double-install, but not the connect).
// A live representative → adopt it; an in-flight admission → hang off it.
if (this.clients.has(key)) {
control.onCommit?.()
return Promise.resolve('applied')
}
const coalesced = this.ensureInFlight.get(key)
if (coalesced) {
return coalesced.then(() => {
if (this.clients.has(key)) {
control.onCommit?.()
return 'applied'
}
// The admission we adopted settled without a live client (it failed or
// was superseded). Surface it as a failure so the bootstrap tracker
// caches it and warns, rather than opening a second connection here.
throw new Error(`SHARED oauth-context admission did not connect: ${serverConfig.name}`)
})
}
this.exitSharedAwaitingGrant(serverConfig.name)
const tokenProvider = this.buildTokenProvider(serverConfig, SHARED_PRINCIPAL, undefined)
const admission = this.connectAndInstall(
key,
serverConfig,
tokenProvider,
control,
true,
options
)
const tracked = admission.then(
() => undefined,
() => undefined
)
const cleared = tracked.finally(() => {
if (this.ensureInFlight.get(key) === cleared) this.ensureInFlight.delete(key)
})
this.ensureInFlight.set(key, cleared)
return admission
}
/**
* No confirmed grant (M9): register the server connected-with-0-tools without
* opening any SHARED connection — the exact shape an oauth-user server shows
* with no live per-user partition. On a config change, evict stale partitions
* (mirror of the fail-closed remote-oauth branch) and refresh serverInfos/status;
* on an unchanged re-poll, just commit. The catalog stays empty until the grant
* lands and discovery/bootstrap re-probes.
*/
private registerSharedAwaitingGrant(
serverConfig: McpServerInfo,
control: McpAdmissionControl
): McpAdmissionOutcome {
const installed = this.serverInfos.get(serverConfig.name)
if (!(installed && JSON.stringify(installed) === JSON.stringify(serverConfig))) {
this.evictPartitionsExcept(serverConfig.name, this.sharedKey(serverConfig.name), control)
this.serverInfos.set(serverConfig.name, serverConfig)
this.statusTracker.markConnected(serverConfig.name, 0)
}
this.enterSharedAwaitingGrant(serverConfig.name)
control.onCommit?.()
return 'applied'
}
/** Log (once) that a SHARED oauth-context is now awaiting its grant. */
private enterSharedAwaitingGrant(serverName: string): void {
if (this.sharedAwaitingGrant.has(serverName)) return
this.sharedAwaitingGrant.add(serverName)
catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
logger.info(
{
component: 'McpManager',
event: 'mcp_oauth_shared_awaiting_grant',
serverName,
state: 'entered',
},
'SHARED oauth-context awaiting grant; no token-less connection opened'
)
}
/** Log (once) that a SHARED oauth-context left the awaiting-grant state. */
private exitSharedAwaitingGrant(serverName: string): void {
if (!this.sharedAwaitingGrant.delete(serverName)) return
catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
logger.info(
{
component: 'McpManager',
event: 'mcp_oauth_shared_awaiting_grant',
serverName,
state: 'resolved',
},
'SHARED oauth-context grant confirmed; opening authenticated connection'
)
}
/**
* Silently drop a server from the awaiting-grant set when it is removed or
* re-purposed (config no longer a gated oauth-context). Distinct from
* exitSharedAwaitingGrant, whose 'resolved' log asserts a grant was confirmed —
* which did NOT happen here — so this transition is not logged. Leaving the
* membership behind would both retain a stale entry and suppress the next
* genuine 'entered' log after a re-registration.
*/
private leaveSharedAwaitingGrant(serverName: string): void {
if (this.sharedAwaitingGrant.delete(serverName)) {
catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
}
}
/**
* Shared connect+install core reused by the eager SHARED path (addServer) and
* the lazy per-user path (ensureClient) — one fencing machine (D4), keyed by
* ClientKey. `ownsServerStatus` gates the per-serverName status writes so a
* per-user admission failure never flips a healthy representative to failed.
*/
private async connectAndInstall(
key: string,
serverConfig: McpServerInfo,
tokenProvider: McpTokenProvider | undefined,
control: McpAdmissionControl,
ownsServerStatus: boolean,
options?: McpToolCallOptions
): Promise<McpAdmissionOutcome> {
const admissionLifecycleEpoch = this.lifecycleEpoch
const externalIsCurrent = control.isCurrent ?? (() => true)
const isCurrent = (): boolean =>
!this.closed && this.lifecycleEpoch === admissionLifecycleEpoch && externalIsCurrent()
if (!isCurrent()) return 'stale'
const client = new McpClient(serverConfig, tokenProvider, this.proxyUrl)
const attempt = Symbol(key)
this.pendingAdmissions.set(key, { attempt, client, serverName: serverConfig.name })
if (ownsServerStatus) this.statusTracker.markConnecting(serverConfig.name)
try {
await client.connect(options ?? {})
if (!isCurrent() || this.pendingAdmissions.get(key)?.attempt !== attempt) {
await this.scheduleClientCleanup(client, control)
this.discardPendingAdmission(key, attempt, serverConfig.name)
return 'stale'
}
this.pendingAdmissions.delete(key)
this.installConnectedClient(key, serverConfig, client, ownsServerStatus)
control.onCommit?.()
logger.info(
{
component: 'McpManager',
serverName: serverConfig.name,
partition: ownsServerStatus ? 'shared' : 'per-user',
},
'Installed client for server'
)
return 'applied'
} catch (error) {
if (!isCurrent() || this.pendingAdmissions.get(key)?.attempt !== attempt) {
await this.scheduleClientCleanup(client, control)
this.discardPendingAdmission(key, attempt, serverConfig.name)
return 'stale'
}
this.pendingAdmissions.delete(key)
logger.error(
{
component: 'McpManager',
serverName: serverConfig.name,
partition: ownsServerStatus ? 'shared' : 'per-user',
err: error,
},
'Failed to install client for server'
)
// Only the SHARED/eager path owns the per-serverName status transition. A
// per-user admission failure must leave the representative's status intact.
if (ownsServerStatus) this.recordAdmissionFailure(serverConfig, error)
throw error
}
}
/**
* Connect a ready desired revision before committing it over the current
* SHARED connection. A candidate failure leaves the previous client, tools,
* serverInfo, and connected status untouched. On commit, per-user partitions
* of the same server are evicted (the desired config changed for everyone;
* they rebuild lazily with the new revision).
*/
async replaceServer(
serverConfig: McpServerInfo,
authToken?: string,
control: McpAdmissionControl = {}
): Promise<McpAdmissionOutcome> {
const admissionLifecycleEpoch = this.lifecycleEpoch
const externalIsCurrent = control.isCurrent ?? (() => true)
const isCurrent = (): boolean =>
!this.closed && this.lifecycleEpoch === admissionLifecycleEpoch && externalIsCurrent()
if (!isCurrent()) return 'stale'
const key = this.sharedKey(serverConfig.name)
const previousClient = this.clients.get(key)
if (!previousClient) {
return this.addServer(serverConfig, authToken, control)
}
const tokenProvider = this.buildTokenProvider(serverConfig, SHARED_PRINCIPAL, authToken)
const candidate = new McpClient(serverConfig, tokenProvider, this.proxyUrl)
const attempt = Symbol(key)
this.pendingAdmissions.set(key, { attempt, client: candidate, serverName: serverConfig.name })
try {
await candidate.connect()
} catch (error) {
await this.scheduleClientCleanup(candidate, control)
if (!isCurrent() || this.pendingAdmissions.get(key)?.attempt !== attempt) {
this.discardPendingAdmission(key, attempt, serverConfig.name)
return 'stale'
}
this.pendingAdmissions.delete(key)
throw error
}
if (!isCurrent() || this.pendingAdmissions.get(key)?.attempt !== attempt) {
await this.scheduleClientCleanup(candidate, control)
this.discardPendingAdmission(key, attempt, serverConfig.name)
return 'stale'
}
this.pendingAdmissions.delete(key)
// Commit is synchronous: readers see either the complete previous
// revision or the complete connected candidate, never a missing server.
// replaceServer is always the SHARED/eager coordinator path → owns status.
this.installConnectedClient(key, serverConfig, candidate, true)
control.onCommit?.()
// The desired config changed for the whole server → per-user partitions are
// stale. Evict them (and any in-flight lazy admission) so they rebuild lazily.
this.evictPartitionsExcept(serverConfig.name, key, control)
await this.scheduleClientCleanup(previousClient, control)
logger.info({ component: 'McpManager', serverName: serverConfig.name }, 'Replaced server')
return 'applied'
}
/**
* Lazily admit a per-user partition for an oauth-user server, reusing the same
* connect+install fencing as the eager path. Concurrent callers for the same
* partition coalesce onto one admission.
*/
private async ensureClient(
serverName: string,
userId: string,
options?: McpToolCallOptions
): Promise<void> {
const key = this.userKey(serverName, userId)
if (this.clients.has(key)) return
const existing = this.ensureInFlight.get(key)
if (existing) return existing
const info = this.serverInfos.get(serverName)
if (!info) {
throw new Error(`MCP server not connected: ${serverName}`)
}
const tokenProvider = this.buildTokenProvider(info, userPrincipal(userId), undefined)
const admission = this.connectAndInstall(key, info, tokenProvider, {}, false, options).then(
() => undefined
)
const tracked = admission.finally(() => {
if (this.ensureInFlight.get(key) === tracked) this.ensureInFlight.delete(key)
})
this.ensureInFlight.set(key, tracked)
return tracked
}
/**
* Evict every per-user partition of a server (and any in-flight lazy
* admission), except an optional key to keep (the SHARED representative).
*/
private evictPartitionsExcept(
serverName: string,
keepKey: string,
control: McpAdmissionControl
): void {
const keys = this.byServer.get(serverName)
if (keys) {
for (const key of [...keys]) {
if (key === keepKey) continue
const client = this.clients.get(key)
this.clients.delete(key)
keys.delete(key)
this.partitionLastUsed.delete(key)
this.partitionInFlight.delete(key)
if (client) void this.scheduleClientCleanup(client, control)
}
if (keys.size === 0) this.byServer.delete(serverName)
}
for (const [key, pending] of [...this.pendingAdmissions]) {
if (pending.serverName === serverName && key !== keepKey) {
this.pendingAdmissions.delete(key)
void this.scheduleClientCleanup(pending.client, control)
}
}
for (const key of [...this.ensureInFlight.keys()]) {
if (key !== keepKey && serverNameFromClientKey(key) === serverName) {
this.ensureInFlight.delete(key)
}
}
}
/**
* Evict a single per-user partition (terminal auth failure). SHARED/static
* partitions and per-serverName status/serverInfo are left untouched — a later
* call re-admits the partition lazily.
*/
private evictPartition(key: string): void {
const client = this.clients.get(key)
const serverName = serverNameFromClientKey(key)
this.clients.delete(key)
this.partitionLastUsed.delete(key)
this.partitionInFlight.delete(key)
this.ensureInFlight.delete(key)
const keys = this.byServer.get(serverName)
if (keys) {
keys.delete(key)
if (keys.size === 0) this.byServer.delete(serverName)
}
if (client) {
const cleanup = this.claimClientCleanup(client)
if (cleanup) void cleanup().catch(() => undefined)
}
}
/**
* Evict idle per-user partitions (LRU + TTL). SHARED/static/representative
* partitions are exempt (snapshot-driven, never evicted here). Bounds
* revocation latency for oauth-user grants and caps memory. Returns the count
* evicted.
*/
evictIdleUserPartitions(maxIdleMs: number, maxUserPartitions?: number): number {
const now = Date.now()
let evicted = 0
// TTL pass — never evict a partition with a live tool call.
for (const [key, lastUsed] of [...this.partitionLastUsed]) {
if (this.partitionHasInFlightCall(key)) continue
if (now - lastUsed >= maxIdleMs) {
this.evictPartition(key)
evicted += 1
}
}
// LRU-cap pass — same in-flight exemption; only idle partitions are candidates.
if (maxUserPartitions !== undefined && this.partitionLastUsed.size > maxUserPartitions) {
const ordered = [...this.partitionLastUsed.entries()]
.filter(([key]) => !this.partitionHasInFlightCall(key))
.sort((a, b) => a[1] - b[1])
const overflow = this.partitionLastUsed.size - maxUserPartitions
for (let i = 0; i < overflow && i < ordered.length; i++) {
this.evictPartition(ordered[i][0])
evicted += 1
}
}
return evicted
}
/**
* List the live OAuth partitions whose grant the hot-revocation sweep must
* re-validate against control-api (mini-spec 13 §4.1). Read-only and additive
* — no live state is mutated. By flavor:
* - `oauth-user`: every per-user partition (the token-less SHARED
* representative carries no grant, so it is deliberately skipped).
* - `oauth-context`: the single SHARED partition, which itself holds the
* context grant (control-api keys it by the server's contextRef).
* `static`/`none` servers contribute nothing.
*/
listLiveOAuthPartitions(): LiveOAuthPartition[] {
const partitions: LiveOAuthPartition[] = []
for (const [serverName, keys] of this.byServer) {
const authKind = this.serverInfos.get(serverName)?.authKind
if (authKind === 'oauth-user') {
for (const key of keys) {
const principal = principalFromClientKey(key)
if (principal.kind === 'user') {
partitions.push({ flavor: 'oauth-user', serverName, userId: principal.userId, key })
}
}
} else if (authKind === 'oauth-context') {
const key = this.sharedKey(serverName)
if (this.clients.has(key)) {
partitions.push({ flavor: 'oauth-context', serverName, key })
}
}
}
return partitions
}
/**
* Evict the OAuth partitions whose grant control-api reports absent (the
* hot-revocation grant-sweep, mini-spec 13 §4.2). The caller supplies only the
* keys of `exists:false` partitions; this reuses `evictPartition` under the
* SAME in-flight exemption as `evictIdleUserPartitions` — a partition with a
* live tool call is SKIPPED this tick and re-evaluated on the next sweep
* (~30s) once `partitionInFlight` drains, never queued for deferred eviction.
* Idempotent: a key with no live partition is a no-op and is not counted.
* Returns the count evicted.
*/
evictRevokedPartitions(keys: Iterable<string>): number {
let evicted = 0
for (const key of keys) {
// Respect in-flight, via the SAME predicate the idle sweep uses: the
// active call started authorized moments ago.
if (this.partitionHasInFlightCall(key)) continue
// Idempotent: nothing live under this key (already evicted / re-keyed).
if (!this.clients.has(key)) continue
this.evictPartition(key)
evicted += 1
}
return evicted
}
/**
* The in-flight exemption predicate shared by every eviction sweep (idle +
* grant-revocation): a partition with ≥1 live tool-call token is NEVER evicted.
* Single-sourced so the two sweeps can never drift on this invariant (D4).
*/
private partitionHasInFlightCall(key: string): boolean {
return (this.partitionInFlight.get(key)?.size ?? 0) > 0
}
private releaseInFlight(key: string, token: symbol): void {
// Remove only THIS call's token. If the partition was force-evicted and
// re-admitted under a fresh Set, this no-ops on the new Set instead of
// clearing another live call's in-flight state.
const set = this.partitionInFlight.get(key)
if (!set) return
set.delete(token)
if (set.size === 0) this.partitionInFlight.delete(key)
}
/**
* Revoke an MCP server synchronously, returning bounded async cleanup.
*
* Purges EVERY ClientKey that shares this serverName (SHARED representative +
* all per-user partitions) — a config change / delete affects everyone. This
* is the revocation path. Callers can detach every server in an authoritative
* revocation snapshot before awaiting any potentially slow disconnect.
*/
detachServer(serverName: string): McpDetachedServerCleanup {
const clients = new Set<McpClient>()
const liveKeys = this.byServer.get(serverName)
if (liveKeys) {
for (const key of liveKeys) {
const client = this.clients.get(key)
if (client) clients.add(client)
this.clients.delete(key)
this.partitionLastUsed.delete(key)
this.partitionInFlight.delete(key)
}
}
for (const [key, pending] of [...this.pendingAdmissions]) {
if (pending.serverName === serverName) {
clients.add(pending.client)
this.pendingAdmissions.delete(key)
}
}
for (const key of [...this.ensureInFlight.keys()]) {
if (serverNameFromClientKey(key) === serverName) this.ensureInFlight.delete(key)
}
const cleanups = [...clients]
.map(client => this.claimClientCleanup(client))
.filter((cleanup): cleanup is McpDetachedServerCleanup => cleanup !== undefined)
this.byServer.delete(serverName)
this.serverInfos.delete(serverName)
this.statusTracker.remove(serverName)
this.leaveSharedAwaitingGrant(serverName)
return async () => {
await this.runDetachedCleanups(cleanups)
logger.info({ component: 'McpManager', serverName }, 'Removed server')
}
}
/**
* Remove and disconnect from an MCP server.
*/
async removeServer(serverName: string): Promise<void> {
await this.detachServer(serverName)()
}
/**
* Disconnect from all MCP servers. Subsystem reset — per spec §4.5 this is
* the one path that allows `connecting` to reappear afterwards.
*/
async disconnectAll(): Promise<void> {
await this.runDetachedCleanups(this.detachAllServers())
}
/**
* Permanently close this manager. Unlike disconnectAll(), a closed manager
* cannot be authorized again by a delayed poll or direct admission.
*/
async close(scheduleCleanup?: (cleanup: McpDetachedServerCleanup) => void): Promise<void> {
if (this.closed) return
this.closed = true
const cleanups = this.detachAllServers()
if (scheduleCleanup) {
for (const cleanup of cleanups) {
scheduleCleanup(cleanup)
}
return
}
await this.runDetachedCleanups(cleanups)
}
private detachAllServers(): McpDetachedServerCleanup[] {
this.lifecycleEpoch += 1
const serverNames = [
...new Set([
...this.byServer.keys(),
...[...this.pendingAdmissions.values()].map(p => p.serverName),
]),
]
const cleanups = serverNames.map(name => this.detachServer(name))
this.serverInfos.clear()
this.pendingAdmissions.clear()
this.byServer.clear()
this.ensureInFlight.clear()
this.partitionLastUsed.clear()
this.partitionInFlight.clear()
this.sharedAwaitingGrant.clear()
catalogBootstrapMetrics.awaitingGrant(0)
this.bootstrapInFlight.clear()
this.grantProbe?.reset()
this.statusTracker.reset()
return cleanups
}
private async runDetachedCleanups(cleanups: McpDetachedServerCleanup[]): Promise<void> {
let firstError: unknown
let cleanupFailed = false
for (const cleanup of cleanups) {
try {
await cleanup()
} catch (error) {
if (!cleanupFailed) firstError = error
cleanupFailed = true
}
}
if (cleanupFailed) {
throw firstError
}
}
/**
* Get all available tools from all connected servers.
*
* Deduplicated by serverName via the representative connection: N per-user
* partitions of one oauth server contribute the tool exactly once.
* Tool names are prefixed with server name to avoid conflicts.
*/
getAllTools(): McpTool[] {
const allTools: McpTool[] = []
for (const serverName of this.byServer.keys()) {
const client = this.representativeClient(serverName)
if (!client) continue
for (const tool of client.availableTools) {
// Prefix tool name with server name for uniqueness
allTools.push({
...tool,
name: `${serverName}__${tool.name}`,
})
}
}
return allTools
}
/**
* Build a description of all connected MCP servers and their tools.
* Used to give the LLM full context about its available capabilities.
* Deduplicated by serverName (representative connection).
*/
describeCapabilities(): string {
if (this.byServer.size === 0) {
return 'No MCP servers are currently connected. You cannot use any tools.'
}
const sections: string[] = []
for (const serverName of this.byServer.keys()) {
const client = this.representativeClient(serverName)
if (!client) continue
const info = this.serverInfos.get(serverName)
const description = info?.description || 'No description available.'
const tools = client.availableTools
let section = `### ${serverName}\n${description}`
if (tools.length > 0) {
section += `\n\nTools (${tools.length}):`
for (const tool of tools) {
const toolDesc = tool.description ? ` — ${tool.description}` : ''
section += `\n- ${serverName}__${tool.name}${toolDesc}`
}
} else {
section += '\n\nNo tools available from this server.'
}
sections.push(section)
}
return `## Available MCP Servers\n\nYou have access to ${this.byServer.size} MCP server(s) with ${this.getAllTools().length} tool(s) total.\nAlways use the full tool name (serverName__toolName) when calling tools.\n\n${sections.join('\n\n')}`
}
/**
* Call a tool by its full name (serverName__toolName).
*
* Dispatch by the server's flavor: static & oauth grantScope='context' route
* to the SHARED partition; oauth grantScope='user' routes to the caller's
* per-user partition (lazily admitted). oauth-user with no/anonymous userId is
* rejected fail-closed BEFORE any token is resolved.
*/
/**
* Fail-closed admission gates (enabled → authoritative → ready) shared by the
* lazy per-user callTool path and the eager catalog bootstrap. serverInfos
* retains disabled / not-ready / non-authoritative servers (each eager skip
* branch does serverInfos.set before returning), so without this gate a lazy
* per-user admission would connect and execute against a server the operator
* disabled or that is not ready. A non-admissible server is operator intent
* (disabled) or transient/unauthoritative infra (not_ready) — NOT an auth
* failure, so the messages stay distinct from the "Authentication required"
* cases. Mirrors the eager admission gates in addServer.
*/
private admissionGate(
info: McpServerInfo | undefined
): { ok: true } | { ok: false; error: string } {
return evaluateAdmissionGate(info)
}
/**
* Eagerly open the remote OAuth partitions this user already has a grant for,
* so the turn's catalog carries their tools BEFORE the LLM needs them (§6.1) —
* without a prior failed `callTool`. Idempotent and best-effort: steady-state
* (every partition already live) costs zero round-trips, a down control-api
* degrades to today's lazy admission, and a candidate whose admission overruns
* the wait budget keeps connecting in the background (the tool loop reflects it
* on its next iteration). Never touches per-serverName status (admissions run
* with ownsServerStatus=false), never evicts, never probes tools.
*/
async bootstrapUserCatalog(
userId: string,
options: McpCatalogBootstrapOptions = {}
): Promise<McpCatalogBootstrapSummary> {
// Step 1 — total preconditions, each naming its single reason. The three
// dependency skips stay distinct (a kill-switch is not a missing checker is
// not a missing factory) so an operator can tell why bootstrap did nothing.
if (!userId || userId === 'anonymous') return skipAllSummary('no_user')
if (this.closed) return skipAllSummary('manager_closed')
if (!this.catalogBootstrapEnabled) return skipAllSummary('disabled')
if (!this.tokenProviderFactory) return skipAllSummary('no_factory')
if (!this.grantProbe || !this.catalogBootstrap) return skipAllSummary('no_checker')
// Step 2 — coalesce concurrent bootstraps of the SAME user onto one run so N
// simultaneous turns issue one plan/probe and one admission per partition.
const existing = this.bootstrapInFlight.get(userId)
if (existing) return existing
const run = this.runCatalogBootstrap(userId, this.grantProbe, this.catalogBootstrap, options)
this.bootstrapInFlight.set(userId, run)
try {
return await run
} finally {
if (this.bootstrapInFlight.get(userId) === run) this.bootstrapInFlight.delete(userId)
}
}
private async runCatalogBootstrap(
userId: string,
probe: GrantProbe,
config: McpCatalogBootstrapConfig,
options: McpCatalogBootstrapOptions
): Promise<McpCatalogBootstrapSummary> {
const skipped: McpCatalogBootstrapSummary['skipped'] = {}
const bump = (reason: keyof McpCatalogBootstrapSummary['skipped'], n = 1): void => {
if (n > 0) skipped[reason] = (skipped[reason] ?? 0) + n
}
// Step 3 — classify candidates (pure). serverInfos values are always defined,
// so the gate inside never sees undefined.
const selection = selectBootstrapCandidates({
userId,
infos: this.serverInfos.values(),
hasLive: key => this.clients.has(key),
hasInFlight: key => this.ensureInFlight.has(key) || this.pendingAdmissions.has(key),
factoryPresent: this.tokenProviderFactory !== undefined,
userPartitionCount: this.partitionLastUsed.size,
userPartitionMax: this.userPartitionMax,
})
for (const [reason, count] of Object.entries(selection.skipped)) {
bump(reason as keyof McpCatalogBootstrapSummary['skipped'], count ?? 0)
}
const candidates = selection.queries.length
// Step 4 — nothing to probe (steady-state): zero round-trips, no log.
if (candidates === 0) {
catalogBootstrapMetrics.runFinished('noop', 0)
return {
candidates: 0,
probed: 0,
admitted: 0,
pending: 0,
skipped,
waitedMs: 0,
timedOut: false,
}
}
// Steps 5–6 — plan then execute the probe. The plan's non-ask buckets are
// cache/budget skips; a throw from execute (endpoint down/timeout/429) leaves
// every asked coordinate unknown — execute already backed off and released
// its in-flight reservations, so the turn simply proceeds on lazy admission.
const plan = probe.plan(selection.queries)
bump('absent_cached', plan.absentCached.length)
bump('failed_cached', plan.failedCached.length)
bump('budget_exhausted', plan.budgetExhausted.length)
bump('in_flight', plan.inFlight.length)
const probed = plan.ask.length
// Non-ask probe dispositions are known before execute (so a throw still
// counts them). budgetExhausted is all-or-nothing: paused (P4) vs over-budget
// (P6). The planner collapses both, so read the live pause to label them.
catalogBootstrapMetrics.probes({
cached_absent: plan.absentCached.length,
cached_failed: plan.failedCached.length,
...(probe.snapshot().pausedUntil > this.now()
? { paused: plan.budgetExhausted.length }
: { budget: plan.budgetExhausted.length }),
})
let results: GrantExistsResult[]
try {
results = await probe.execute(plan.ask)
} catch {
bump('probe_unknown', plan.ask.length)
catalogBootstrapMetrics.probes({ unknown: plan.ask.length })
catalogBootstrapMetrics.runFinished('error', 0)
const summary: McpCatalogBootstrapSummary = {
candidates,
probed,
admitted: 0,
pending: 0,
skipped,
waitedMs: 0,
timedOut: false,
}
this.logBootstrap(summary)
return summary
}
// Step 7 — decide per ASKED coordinate (the authoritative set of what we
// probed): admit on a confirmed grant, cache a definitive absence, treat a
// missing / non-boolean result as unknown. Never admit on a malformed datum.
const resultByCoord = new Map<string, GrantExistsResult>()
for (const r of results) resultByCoord.set(grantCoordKey(r.mcpServerName, r.userId), r)
const admissions: Array<{
coord: string
candidate: BootstrapCandidate
promise: Promise<unknown>
}> = []
let present = 0
let unknownResults = 0
let absentResults = 0
for (const query of plan.ask) {
const coord = grantCoordKey(query.mcpServerName, query.userId)
const candidate = selection.byCoord.get(coord)
const result = resultByCoord.get(coord)
if (!candidate || !result || typeof result.exists !== 'boolean') {
bump('probe_unknown')
unknownResults += 1
continue
}
if (result.exists) {
present += 1
const promise = this.admitBootstrapCandidate(userId, candidate, config)
if (promise) admissions.push({ coord, candidate, promise })
} else {
probe.recordAbsent(coord)
bump('absent')
absentResults += 1
}
}
catalogBootstrapMetrics.probes({
present,
absent: absentResults,
unknown: unknownResults,
})
// Step 9 — per-admission bookkeeping. Runs whenever each admission settles,
// even after the wait budget expires (the admission is coalesced in
// ensureInFlight and the loop's next iteration sees it). oauth-user stamps
// partitionLastUsed so a bootstrapped-but-never-called partition is still
// idle-evictable (M4). A rejected admission caches the failure (P2) and warns
// WITHOUT a token or body.
let admitted = 0
const pendingCoords = new Set(admissions.map(a => a.coord))
const tracked = admissions.map(a =>
a.promise.then(
() => {
pendingCoords.delete(a.coord)
admitted += 1
if (a.candidate.kind === 'user') this.partitionLastUsed.set(a.candidate.key, Date.now())
probe.recordAdmissionOk(a.coord)
catalogBootstrapMetrics.admission(
a.candidate.kind === 'user' ? 'per-user' : 'shared',
'ok'
)
},
(err: unknown) => {
pendingCoords.delete(a.coord)
probe.recordAdmissionFailure(a.coord)
bump('admission_failed')
catalogBootstrapMetrics.admission(
a.candidate.kind === 'user' ? 'per-user' : 'shared',
'failed'
)
logger.warn(
{
component: 'McpManager',
event: 'mcp_catalog_bootstrap_admission_failed',
serverName: a.candidate.serverName,
partition: a.candidate.kind === 'user' ? 'per-user' : 'shared',
err,
},
'MCP catalog bootstrap admission failed'
)
}
)
)
// Step 8 — bounded wait: resolve at min(all-settled, waitBudget). Pending
// admissions keep running; timedOut records that the budget cut the wait.
const start = Date.now()
let timedOut = false
if (tracked.length > 0) {
timedOut = await raceWithBudget(
Promise.allSettled(tracked),
config.waitBudgetMs,
options.signal
)
}
const summary: McpCatalogBootstrapSummary = {
candidates,
probed,
admitted,
pending: pendingCoords.size,
// Snapshot: background admission callbacks (`admission_failed`) keep mutating
// the live `skipped` after the wait budget/abort frees this summary, so the
// returned object must not alias the object those callbacks bump.
skipped: { ...skipped },
waitedMs: Date.now() - start,
timedOut,
}
// Step 10 — log the summary (candidates > 0 here; steady-state returned above).
// timedOut is read on the returned summary because a background admission may
// have flipped `admitted` between the wait and here without changing whether
// the budget cut the wait.
const runOutcome =
admissions.length === 0
? 'skipped'
: timedOut
? 'timed_out'
: admitted === admissions.length
? 'admitted'
: 'partial'
catalogBootstrapMetrics.runFinished(runOutcome, summary.waitedMs)
this.logBootstrap(summary)
return summary
}
/**
* Open one bootstrap candidate's partition, bounded ONLY by connectTimeoutMs
* (invariant 11) so a bootstrap-initiated connect can never inherit the SDK's
* ~25-min default and pin a coalesced `ensureInFlight` entry. The turn's
* `signal` is deliberately NOT threaded here: admissions are owned by the
* manager and coalesced (per-user via `ensureInFlight`, SHARED cross-user), so
* one turn's abort must never tear down a connection another turn/user depends
* on (§6.5/§7.3). The abort instead cuts only THIS turn's wait, in
* `raceWithBudget`. oauth-user rides the lazy per-user path
* (ownsServerStatus=false); oauth-context adopts the SHARED admission,
* coalesced with discovery on the SHARED key.
*/
private admitBootstrapCandidate(
userId: string,
candidate: BootstrapCandidate,
config: McpCatalogBootstrapConfig
): Promise<unknown> | undefined {
const connectOptions: McpToolCallOptions = { timeoutMs: config.connectTimeoutMs }
if (candidate.kind === 'user') {
return this.ensureClient(candidate.serverName, userId, connectOptions)
}
const info = this.serverInfos.get(candidate.serverName)
if (!info) return undefined
return this.admitSharedOauthContext(info, {}, connectOptions)
}
private logBootstrap(summary: McpCatalogBootstrapSummary): void {
logger.info(
{ component: 'McpManager', event: 'mcp_catalog_bootstrap', ...summary },
'MCP catalog bootstrap'
)
}
async callTool(
fullToolName: string,
args: Record<string, unknown>,
options: McpToolCallOptions = {}
): Promise<ToolCallResult> {
// Server identities come from Kubernetes metadata.name (RFC1123, no
// underscores). Only the first delimiter is ours; the remote tool name
// may itself contain '__' and must reach the client unchanged.
const delimiter = fullToolName.indexOf('__')
if (delimiter <= 0 || delimiter + 2 === fullToolName.length) {
return {
toolName: fullToolName,
result: { error: `Invalid tool name format: ${fullToolName}` },
isError: true,
}
}
const serverName = fullToolName.slice(0, delimiter)
const toolName = fullToolName.slice(delimiter + 2)
const info = this.serverInfos.get(serverName)
// Only the per-user oauth sabor gets per-user partitions. oauth-context and
// static both resolve on the SHARED representative (the else branch below).
const isOauthUser = info?.authKind === 'oauth-user'
// oauth-context serves over the SHARED partition, but that partition IS
// grant-sweep-evictable (listLiveOAuthPartitions exposes it). So it must also
// pin an in-flight token for the call's duration, or the grant-sweep could
// close the connection mid-call (respect-in-flight, spec §5). It does NOT
// pin partitionLastUsed — that map drives the idle sweep, which must never
// idle-evict a snapshot-driven oauth-context SHARED partition.
const isOauthContext = info?.authKind === 'oauth-context'
// Partitions the eviction sweeps can reclaim → the ones that must register
// an in-flight claim while a call is running.
const pinsInFlight = isOauthUser || isOauthContext
let key: string
if (isOauthUser) {
// options.userId originates from the authenticated session (`sender`,
// rpc-proxy's auth.sub). It is forwarded verbatim as the broker grant
// subject: mcp-host asserts the identity, and control-api is the
// authorization authority that resolves the grant against it (spec §U4
// must-fix). Fail-closed in practice — a non-sub principal matches no grant
// → 404/no_grant → no token. No authz is enforced here by design.
const userId = options.userId
// Fail-closed: never resolve a token (nor forward the call) without a
// concrete user identity on a per-user oauth server.
if (!userId || userId === 'anonymous') {
return {
toolName: fullToolName,
result: {
error: `Authentication required: ${serverName} needs a per-user connection, but no user identity was provided`,
},
isError: true,
}
}
key = this.userKey(serverName, userId)
// Fail-closed: replicate the eager addServer admission gates
// (enabled → authoritative → ready) before opening a per-user connection
// with the caller's OAuth token — see admissionGate for why.
const gate = this.admissionGate(info)
if (!gate.ok) {
return { toolName: fullToolName, result: { error: gate.error }, isError: true }
}
try {
await this.ensureClient(serverName, userId)
} catch (error) {
const result: ToolCallResult = {
toolName: fullToolName,
result: {
error: `Authentication required for ${serverName}: ${error instanceof Error ? error.message : 'connection failed'}`,
},
isError: true,
}
// A 401 during the per-user admission's `initialize` is the same
// re-consent signal as a 401 on `tools/call`; without the marker the
// desktop cannot offer the reconnect flow. A 403 (insufficient scope)
// stays unmarked (terminal), mirroring the McpAuthError path below.
if (isAuthError(error) && extractStructuredHttpStatus(error) === 401) {
result.connectRequired = { mcpServerName: serverName }
}
return result
}
} else {
key = this.sharedKey(serverName)
}
const client = this.clients.get(key)
if (!client) {
return {
toolName: fullToolName,
result: { error: `MCP server not found: ${serverName}` },
isError: true,
}
}
// Unique token identifying THIS call's in-flight claim on the partition.
const callToken = Symbol('mcp-call')
if (isOauthUser) {
// Stamp usage only once the partition is confirmed live (a 'stale'
// admission installs no client) so no phantom entry counts toward the cap.
// oauth-user ONLY: partitionLastUsed feeds the idle sweep (LRU + TTL) whose
// subjects are the per-user partitions; the oauth-context SHARED partition
// is deliberately absent from it (never idle-evicted).
this.partitionLastUsed.set(key, Date.now())
}
if (pinsInFlight) {
// Pin the partition against BOTH eviction sweeps (idle + grant-revocation)
// for the duration of this call — same register-on-enter / clear-in-finally
// pattern for oauth-user per-user and oauth-context SHARED alike.
let set = this.partitionInFlight.get(key)
if (!set) {
set = new Set()
this.partitionInFlight.set(key, set)
}
set.add(callToken)
}
try {
const result = await client.callTool(toolName, args, options)
if (isOauthUser) this.partitionLastUsed.set(key, Date.now())
return {
toolName: fullToolName,
result,
isError: false,
}
} catch (error) {
if (error instanceof McpAuthError) {
// Terminal auth failure — evict the per-user partition so a later call
// (e.g. after the user reconnects) re-admits fresh. Static/shared exempt.
if (isOauthUser) this.evictPartition(key)
// U5 reactive-consent marker: a live 401 on an oauth server (surfaced
// only AFTER the client's single forced-refresh retry, so already
// terminal-after-retry — no loop) means the user must (re)connect. Gate
// strictly on `authKind ∈ {oauth-user, oauth-context}` (covers BOTH
// per-user and shared grantScope='context' — the shared flavor bootstraps
// on its first user) AND `status===401`. A 403 (insufficient scope) is
// TERMINAL: no marker, no connect flow. static (secretRef) has no consent
// flow → excluded by the authKind check. The marker is attached as a typed
// field so the caller sees it (NOT flattened into the opaque error).
//
// NOTE: this gate reads `authKind` (the HCC v2 inventory field), NOT the
// pre-mini-spec-10 `auth.type` — `decodeMcpServer` populates `authKind`
// and never `auth`, so `info.auth?.type` is always undefined here and the
// old check silently disabled reactive consent for every oauth server.
if (
(info?.authKind === 'oauth-user' || info?.authKind === 'oauth-context') &&
error.status === 401
) {
return {
toolName: fullToolName,
result: { error: error.message },
isError: true,
connectRequired: { mcpServerName: serverName },
}
}
return {
toolName: fullToolName,
result: { error: error.message },
isError: true,
}
}
return {
toolName: fullToolName,
result: { error: error instanceof Error ? error.message : 'Unknown error' },
isError: true,
}
} finally {
if (pinsInFlight) this.releaseInFlight(key, callToken)
}
}
/**
* Probe every connected server's tool list and update the status tracker.
* Called by the background heartbeat to keep `observedAt` fresh and to
* classify refresh failures (spec §4.5 — stay connected, attach reason).
* Probes the SHARED representative per serverName (probeRepresentativeClient):
* a platform probe is a real network call and must never spend a user's OAuth
* token, so a server whose only live client is per-user is skipped this round.
*
* Probes a stable client snapshot. State is written only after every probe
* settles; an aborted round never mutates the last known server status.
*/
async refreshAllServerStatus(options: McpToolCallOptions = {}): Promise<McpStatusRefreshSummary> {
// One representative client per server. The ClientKey index partitions an
// oauth grantScope='user' server into per-user clients, but a status round
// probes each server once via its representative; iterating raw clients would
// double-count those partitions in the summary tally.
// Remote oauth servers are excluded from the probe set: their status is
// derived from the per-user catalog cache below, never a token-less/platform
// probe (see isRemoteOauthServer / mini-spec 19 §D-6).
const allServers = [...this.byServer.keys()]
const remoteOauthServers = allServers.filter(name => this.isRemoteOauthServer(name))
const entries = allServers
.filter(name => !this.isRemoteOauthServer(name))
.map(name => [name, this.probeRepresentativeClient(name)] as const)
.filter((entry): entry is readonly [string, McpClient] => entry[1] !== undefined)
if (options.signal?.aborted) {
return {
serverCount: entries.length,
succeeded: 0,
failed: 0,
toolCount: 0,
outputSchemaCount: 0,
aborted: true,
}
}
const results = await Promise.all(
entries.map(async ([name, client]) => {
try {
return {
name,
client,
// Probes share the round's abort signal directly: the heartbeat owns
// round-level cancellation and no per-probe cancellation capability
// exists, so wrapping the signal would only fake a seam.
result: await client.probeTools({
timeoutMs: options.timeoutMs,
signal: options.signal,
}),
}
} catch (error) {
return { name, client, result: { ok: false as const, error, stale: false } }
}
})
)
// A probe only counts toward the round's tally when its result is
// authoritative for the currently-installed client: never a client swapped
// out mid-round, never a stale-error probe that raced a reconnect. This is
// the same predicate the status commit below uses, so summary.succeeded /
// summary.failed match what was written — a benign mid-round swap can't
// inflate `failed` and flip the run's outcome on the series #148 watches.
const committed = results.filter(
({ name, client, result }) =>
this.probeRepresentativeClient(name) === client && !(!result.ok && result.stale)
)
const succeeded = committed.filter(({ result }) => result.ok).length
const summary: McpStatusRefreshSummary = {
serverCount: entries.length,
succeeded,
failed: committed.length - succeeded,
toolCount: committed.reduce(
(total, { result }) => total + (result.ok ? result.toolCount : 0),
0
),
outputSchemaCount: committed.reduce(
(total, { result }) => total + (result.ok ? result.outputSchemaCount : 0),
0
),
aborted: options.signal?.aborted === true,
}
if (summary.aborted) return summary
for (const { name, result } of committed) {
if (result.ok) {
this.statusTracker.updateToolCount(name, result.toolCount)
} else {
this.statusTracker.updateToolCount(name, 0, { refreshError: result.error })
}
}
// Remote oauth: refresh the tool count from the authenticated per-user catalog
// cache (representativeClient, no network) instead of a probe — the token-safe
// status derivation of mini-spec 19 §D-6. These servers are absent from the
// summary tally on purpose (no probe was issued, no token spent).
for (const name of remoteOauthServers) {
const representative = this.representativeClient(name)
if (representative) {
this.statusTracker.updateToolCount(name, representative.availableTools.length)
}
}
return summary
}
/**
* Get connected server names (projected from the ClientKey index).
*/
getConnectedServers(): string[] {
return [...this.byServer.keys()]
}
/**
* Get every server represented in manager inventory, including disabled,
* not-ready, and failed-admission entries that have no connected client.
*/
getKnownServers(): string[] {
return [
...new Set([
...this.serverInfos.keys(),
...[...this.pendingAdmissions.values()].map(p => p.serverName),
]),
]
}
private discardPendingAdmission(key: string, attempt: symbol, serverName: string): void {
if (this.pendingAdmissions.get(key)?.attempt !== attempt) return
this.pendingAdmissions.delete(key)
const hasLiveClient = (this.byServer.get(serverName)?.size ?? 0) > 0
if (!hasLiveClient && !this.serverInfos.has(serverName)) {
this.statusTracker.remove(serverName)
}
}
/**
* Check if any servers are connected.
*/
hasConnectedServers(): boolean {
return this.byServer.size > 0
}
}
/**
* Fail-closed admission gate (enabled → authoritative → ready), shared by the
* lazy per-user callTool path (`McpManager.admissionGate`) and the eager
* bootstrap classifier (`selectBootstrapCandidates`) so the two can never drift
* (D4). The error strings are load-bearing — `callTool` returns them verbatim.
*/
export function evaluateAdmissionGate(
info: McpServerInfo | undefined
): { ok: true } | { ok: false; error: string } {
if (!info?.enabled) {
return { ok: false, error: `MCP server disabled: ${info?.name}` }
}
if (info.status?.authoritative === false || !info.status?.ready) {
const detail = info.status?.message ? ` (${info.status.message})` : ''
return { ok: false, error: `MCP server not ready: ${info.name}${detail}` }
}
return { ok: true }
}
/**
* Pure classifier for `bootstrapUserCatalog` (§6.1, tables M1/M2/M7/M12–M14):
* which remote-oauth servers this user should probe a grant for, keyed by the
* partition each would open. No I/O, no manager state mutated — property-tested.
* A local/static server is silently out of scope (M1/M2); every other exclusion
* is tallied in `skipped`. `oauth-context` is a candidate only with a factory
* (M7); a live/in-flight partition (M13) or an over-cap per-user admission (M14)
* is skipped so steady-state yields zero candidates and zero round-trips.
*/
export function selectBootstrapCandidates(
input: SelectBootstrapCandidatesInput
): BootstrapCandidateSelection {
const {
userId,
infos,
hasLive,
hasInFlight,
factoryPresent,
userPartitionCount,
userPartitionMax,
} = input
const queries: GrantExistsQuery[] = []
const byCoord = new Map<string, BootstrapCandidate>()
const skipped: BootstrapCandidateSelection['skipped'] = {}
const bump = (reason: keyof BootstrapCandidateSelection['skipped']): void => {
skipped[reason] = (skipped[reason] ?? 0) + 1
}
for (const info of infos) {
// M1/M2 — only a remote oauth server is a bootstrap candidate.
if (info.remote !== true) continue
if (info.authKind !== 'oauth-user' && info.authKind !== 'oauth-context') continue
// M12 — replicate the lazy callTool admission gate. serverInfos values are
// always defined, so the gate never degrades to "disabled: undefined".
if (!evaluateAdmissionGate(info).ok) {
bump('gate')
continue
}
let key: string
let kind: 'user' | 'shared'
let query: GrantExistsQuery
if (info.authKind === 'oauth-user') {
key = serializeClientKey(info.name, userPrincipal(userId))
kind = 'user'
query = { mcpServerName: info.name, userId }
} else {
// M7 — an oauth-context SHARED can authenticate only through the factory.
if (!factoryPresent) continue
key = serializeClientKey(info.name, SHARED_PRINCIPAL)
kind = 'shared'
query = { mcpServerName: info.name }
}
// M13 — idempotency: a live partition or an in-flight admission is not a
// candidate. Steady-state ⇒ zero candidates ⇒ zero round-trips.
if (hasLive(key)) {
bump('live')
continue
}
if (hasInFlight(key)) {
bump('in_flight')
continue
}
// M14 — cap: never bootstrap a NEW per-user partition above the LRU cap. The
// lazy callTool path still admits (today's behavior); this only avoids
// bootstrap-induced admit→evict thrash above the cap.
if (
kind === 'user' &&
userPartitionMax !== undefined &&
userPartitionCount >= userPartitionMax
) {
bump('cap')
continue
}
queries.push(query)
byCoord.set(grantCoordKey(info.name, kind === 'user' ? userId : undefined), {
key,
kind,
serverName: info.name,
})
}
return { queries, byCoord, skipped }
}
/** A total-skip summary naming the single precondition that stopped bootstrap. */
function skipAllSummary(
reason: keyof McpCatalogBootstrapSummary['skipped']
): McpCatalogBootstrapSummary {
return {
candidates: 0,
probed: 0,
admitted: 0,
pending: 0,
skipped: { [reason]: 1 },
waitedMs: 0,
timedOut: false,
}
}
/**
* Resolve `false` when `settled` (a non-rejecting Promise, e.g.
* `Promise.allSettled`) resolves first, else `true` when the wait ends early —
* either `budgetMs` elapses OR `signal` aborts. The loser is left running: the
* abort (like the budget) cuts only the WAIT, never the manager-owned admissions
* behind `settled`, which keep connecting in the background (§6.5/§7.3). An
* already-aborted signal ends the wait on the next tick.
*/
function raceWithBudget(
settled: Promise<unknown>,
budgetMs: number,
signal?: AbortSignal
): Promise<boolean> {
return new Promise<boolean>(resolve => {
let done = false
const finishEarly = (): void => {
if (done) return
done = true
clearTimeout(timer)
signal?.removeEventListener('abort', finishEarly)
resolve(true)
}
const timer = setTimeout(finishEarly, budgetMs)
if (signal) {
if (signal.aborted) queueMicrotask(finishEarly)
else signal.addEventListener('abort', finishEarly, { once: true })
}
void settled.then(() => {
if (done) return
done = true
clearTimeout(timer)
signal?.removeEventListener('abort', finishEarly)
resolve(false)
})
})
}