Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(mcp-host): wire the per-turn catalog bootstrap into production (spec 13 phase 5)

Diego Zacarias223afd650b4d
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index f6200ec47..cdf849455 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.376",
+ "version": "0.1.377",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.376",
+ "version": "0.1.377",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 4d3f5cb58..089536792 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.376",
+ "version": "0.1.377",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/agent/__tests__/taskExecutor.test.ts b/mcp-host/src/agent/__tests__/taskExecutor.test.ts
index 7bb7bccbe..9baac6e2e 100644
--- a/mcp-host/src/agent/__tests__/taskExecutor.test.ts
+++ b/mcp-host/src/agent/__tests__/taskExecutor.test.ts
@@ -102,7 +102,19 @@ function createDeps(overrides?: Partial<TaskExecutorDeps>): TaskExecutorDeps {
completeSingleTurnWithTools: vi.fn(),
getProviderType: () => 'openai' as const,
} as any,
- mcpManager: { getAllTools: () => [], callTool: vi.fn() } as any,
+ mcpManager: {
+ getAllTools: () => [],
+ callTool: vi.fn(),
+ bootstrapUserCatalog: vi.fn(async () => ({
+ candidates: 0,
+ probed: 0,
+ admitted: 0,
+ pending: 0,
+ skipped: {},
+ waitedMs: 0,
+ timedOut: false,
+ })),
+ } as any,
workspaceService: undefined,
config: {
maxTaskDuration: 300000,
diff --git a/mcp-host/src/agent/__tests__/taskExecutorCatalogBootstrap.integration.test.ts b/mcp-host/src/agent/__tests__/taskExecutorCatalogBootstrap.integration.test.ts
new file mode 100644
index 000000000..8a711c7b4
--- /dev/null
+++ b/mcp-host/src/agent/__tests__/taskExecutorCatalogBootstrap.integration.test.ts
@@ -0,0 +1,238 @@
+/**
+ * Per-turn catalog bootstrap, at the task level (spec §7.3, invariant 1 + 7). A
+ * task whose sender holds a grant for a remote oauth-user server must see that
+ * server's tools in the FIRST turn's catalog — before any tool call — because
+ * `TaskExecutor.createToolRegistry` now bootstraps the caller's partitions before
+ * building the registry the loop reads. This is the observable, task-level repro
+ * of the per-user deadlock: at the parent commit the executor never bootstraps,
+ * so the tool is absent from the tools handed to the LLM and the assertion fails.
+ *
+ * Fixtures come from the shared broker helper (T1): `brokerWiring` serves the
+ * `/grants/exists` probe AND the `/user-token` mint from ONE grant store, and
+ * `remoteUpstream({strict:true})` is the real McpClient over a mocked SDK. The
+ * assertion is observable (T4): the tool definitions the provider receives, which
+ * are exactly `toolRegistry.listDefinitions()` for the turn.
+ */
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { config as appConfig } from '../../config'
+import { ConversationManager } from '../../core/conversation/conversation'
+import { SimpleEventEmitter } from '../../core/orchestration/eventEmitter'
+import { FinishReason } from '../../core/types'
+import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
+import type { SingleTurnProvider } from '../../llm/types'
+import { type RemoteUpstreamState, brokerWiring } from '../../mcp/__tests__/helpers/brokerWiring'
+import { checkGrantExistence } from '../../mcp/grantExistenceClient'
+import type { GrantExistenceChecker, McpCatalogBootstrapConfig } from '../../mcp/grantProbe'
+import { McpManager } from '../../mcp/manager'
+import type { Task } from '../../queue/types'
+import type { McpServerInfo } from '../../types'
+import { TaskExecutor, type TaskExecutorDeps } from '../taskExecutor'
+
+// Strict (spec-compliant) remote upstream: 401s at `initialize` when a token-less
+// request reaches an https target, so a tool only appears once the per-user grant
+// mints a Bearer. Shared hoisted state; each vi.mock factory dynamically imports
+// the builder (it runs before the file's static imports resolve).
+const sdk = vi.hoisted<RemoteUpstreamState>(() => ({
+ transports: [],
+ probeAuth: [],
+ callToolImpl: null,
+ toolCall401Count: 0,
+}))
+vi.mock('@modelcontextprotocol/sdk/client/index.js', async () => {
+ const { remoteUpstream } = await import('../../mcp/__tests__/helpers/brokerWiring')
+ return remoteUpstream({ strict: true }, sdk).clientModule
+})
+vi.mock('@modelcontextprotocol/sdk/client/streamableHttp.js', async () => {
+ const { remoteUpstream } = await import('../../mcp/__tests__/helpers/brokerWiring')
+ return remoteUpstream({ strict: true }, sdk).transportModule
+})
+vi.mock('@modelcontextprotocol/sdk/client/sse.js', async () => {
+ const { remoteUpstream } = await import('../../mcp/__tests__/helpers/brokerWiring')
+ return remoteUpstream({ strict: true }, sdk).sseModule
+})
+// The SDK is mocked, so no real transport fetch runs; keep the SSRF guard off DNS.
+vi.mock('../../core/net/ssrf', () => ({
+ SsrfBlockedError: class SsrfBlockedError extends Error {},
+ resolvePinnedPublicIp: vi.fn(async () => '203.0.113.10'),
+ pinnedFetch: vi.fn(() => vi.fn(async () => new Response('{}'))),
+}))
+
+function remoteOauthUserServer(name = 'calendar'): McpServerInfo {
+ return {
+ name,
+ contextRef: 'ctx-1',
+ transport: { type: 'streamableHttp', url: `https://${name}.example.com/mcp` },
+ authKind: 'oauth-user',
+ remote: true,
+ enabled: true,
+ status: { deployed: true, ready: true },
+ }
+}
+
+function bootstrapConfig(over: Partial<McpCatalogBootstrapConfig> = {}): McpCatalogBootstrapConfig {
+ return {
+ enabled: true,
+ waitBudgetMs: 4000,
+ probeTimeoutMs: 2000,
+ connectTimeoutMs: 8000,
+ negativeTtlMs: 15000,
+ failureTtlMs: 60000,
+ probesPerMin: 20,
+ backoffMs: 30000,
+ ...over,
+ }
+}
+
+function taskFor(sender: string): Task {
+ return {
+ id: `bootstrap-${sender}-${Math.random().toString(36).slice(2, 6)}`,
+ source: 'channel',
+ status: 'pending',
+ priority: 'normal',
+ createdAt: new Date(),
+ sourceMessage: {
+ sender,
+ content: 'Do the thing',
+ channelType: 'rpc',
+ channelId: 'isolated-channel',
+ messageId: `msg-${sender}`,
+ timestamp: new Date().toISOString(),
+ hostRef: 'fixture-host',
+ },
+ conversationHistory: [{ role: 'user', content: 'Do the thing', timestamp: new Date() }],
+ responseCallback: vi.fn(async () => {}),
+ }
+}
+
+/**
+ * A provider that records the tool names it is handed each turn and finishes
+ * immediately (no tool call, no approval). The recorded names ARE the turn's
+ * `toolRegistry.listDefinitions()`, so asserting on them is the observable proof
+ * the bootstrap populated the catalog before the loop read it.
+ */
+function recordingProvider(seenTools: string[][]): SingleTurnProvider {
+ return {
+ getProviderType: () => 'openai',
+ classifyError: () => {
+ throw new Error('unexpected classifyError')
+ },
+ completeSingleTurn: async () => {
+ throw new Error('unexpected non-tool completion')
+ },
+ completeSingleTurnWithTools: async (_messages: unknown, tools: Array<{ name: string }>) => {
+ seenTools.push(tools.map(t => t.name))
+ return {
+ content: 'done',
+ tool_calls: null,
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: FinishReason.Stop,
+ }
+ },
+ } as unknown as SingleTurnProvider
+}
+
+function depsFor(
+ manager: McpManager,
+ provider: SingleTurnProvider,
+ lifecycle: TaskLifecycle
+): TaskExecutorDeps {
+ return {
+ conversationManager: new ConversationManager(),
+ llmProvider: provider,
+ mcpManager: manager,
+ workspaceService: undefined,
+ modelName: 'test-model',
+ approvalConfig: undefined,
+ config: {
+ maxTaskDuration: 300000,
+ maxToolCallsPerTask: 10,
+ autoStart: true,
+ taskDelay: 0,
+ approvalTimeout: 300000,
+ },
+ coreEvents: new SimpleEventEmitter(),
+ cronScheduler: null,
+ taskLifecycle: lifecycle,
+ onApprovalNeeded: vi.fn(),
+ onComplete: vi.fn(),
+ onFail: vi.fn(),
+ dynamicEnvProvider: () => ({}),
+ } as unknown as TaskExecutorDeps
+}
+
+async function runTurn(
+ manager: McpManager,
+ sender: string,
+ seenTools: string[][]
+): Promise<{ deps: TaskExecutorDeps; executor: TaskExecutor }> {
+ const task = taskFor(sender)
+ const lifecycle = new TaskLifecycle()
+ lifecycle.register(task)
+ const deps = depsFor(manager, recordingProvider(seenTools), lifecycle)
+ const executor = new TaskExecutor(task, deps)
+ await executor.run()
+ return { deps, executor }
+}
+
+afterEach(() => {
+ sdk.transports = []
+ sdk.probeAuth = []
+ sdk.callToolImpl = null
+ sdk.toolCall401Count = 0
+})
+
+describe('per-turn catalog bootstrap at the task level (spec §7.3)', () => {
+ // ── invariant 1 (T3): first turn exposes the grant-backed tool ──
+ it("exposes alice's granted oauth-user tool in the first turn's catalog", async () => {
+ const grantStore = new Set(['calendar:alice']) // alice has a grant; nobody else
+ const { deps, factory } = brokerWiring(grantStore)
+ const grantExistence: GrantExistenceChecker = (queries, { timeoutMs }) =>
+ checkGrantExistence({ ...deps, timeoutMs }, queries)
+ const manager = new McpManager(undefined, undefined, factory, {
+ grantExistence,
+ catalogBootstrap: bootstrapConfig(),
+ })
+ // Discovery registers the server: oauth-user takes NO token-less SHARED, so it
+ // is absent from the catalog until a per-user partition opens.
+ await manager.addServer(remoteOauthUserServer())
+ expect(manager.getAllTools()).toEqual([])
+
+ const seenTools: string[][] = []
+ const { deps: runDeps } = await runTurn(manager, 'alice', seenTools)
+
+ expect(runDeps.onFail).not.toHaveBeenCalled()
+ // The tool is in the FIRST (only) turn's definitions — bootstrap admitted the
+ // per-user partition before the registry was read. Parent commit: absent.
+ expect(seenTools).toHaveLength(1)
+ expect(seenTools[0]).toContain('calendar__do')
+ })
+
+ // ── invariant 7: a broken probe never fails the turn, and stays backed off ──
+ it('a throwing grant probe leaves the turn intact and does not re-probe within backoff', async () => {
+ let clock = 1_000
+ const grantStore = new Set(['calendar:alice'])
+ const { deps, factory, existsCalls, failExistsWith } = brokerWiring(grantStore)
+ failExistsWith(500) // every /grants/exists 500s → checkGrantExistence throws
+ const grantExistence: GrantExistenceChecker = (queries, { timeoutMs }) =>
+ checkGrantExistence({ ...deps, timeoutMs }, queries)
+ const manager = new McpManager(undefined, undefined, factory, {
+ grantExistence,
+ catalogBootstrap: bootstrapConfig(),
+ now: () => clock,
+ })
+ await manager.addServer(remoteOauthUserServer())
+
+ const seenTools: string[][] = []
+ // Turn 1: the probe throws → backoff → the turn proceeds on lazy admission.
+ const { deps: t1 } = await runTurn(manager, 'alice', seenTools)
+ expect(t1.onFail).not.toHaveBeenCalled()
+ expect(seenTools[0]).not.toContain('calendar__do') // no partition opened
+ expect(existsCalls).toHaveLength(1) // one probe was attempted (it threw)
+
+ // Turn 2, still inside backoffMs: the plan is paused, so NO new probe POSTs.
+ clock += 5_000
+ const { deps: t2 } = await runTurn(manager, 'alice', seenTools)
+ expect(t2.onFail).not.toHaveBeenCalled()
+ expect(existsCalls).toHaveLength(1) // still one — the backoff suppressed turn 2
+ })
+})
diff --git a/mcp-host/src/agent/taskExecutor.ts b/mcp-host/src/agent/taskExecutor.ts
index 35027d96d..69ae2bff5 100644
--- a/mcp-host/src/agent/taskExecutor.ts
+++ b/mcp-host/src/agent/taskExecutor.ts
@@ -1889,6 +1889,36 @@ export class TaskExecutor {
return modelLine
}
+ /**
+ * Best-effort per-turn catalog bootstrap. Opens the caller's grant-backed
+ * remote OAuth partitions (per-user and SHARED oauth-context) so their tools
+ * are in the catalog this turn builds — a failure only forfeits that head start
+ * and falls back to lazy admission, so it NEVER throws. It runs once per task
+ * (createToolRegistry is cached by toolRegistryPromise), and the abort only cuts
+ * this turn's wait: the admissions belong to the manager, are coalesced there,
+ * and outlive the turn.
+ */
+ private async bootstrapMcpCatalog(): Promise<void> {
+ const manager = this.deps.mcpManager
+ const userId = this.task.sourceMessage?.sender
+ if (!manager || !userId) return
+ const started = Date.now()
+ try {
+ const summary = await withAbort(
+ () => manager.bootstrapUserCatalog(userId, { signal: this.abortController.signal }),
+ this.abortController.signal
+ )
+ if (summary.candidates > 0) {
+ logger.info({ taskId: this.taskId, ...summary }, 'MCP catalog bootstrap')
+ }
+ } catch (err) {
+ logger.warn(
+ { taskId: this.taskId, err, elapsedMs: Date.now() - started },
+ 'MCP catalog bootstrap failed; continuing with lazy admission'
+ )
+ }
+ }
+
private async buildToolRegistry(): Promise<{
registry: ToolRegistry
loopController: LoopController
@@ -1946,6 +1976,11 @@ export class TaskExecutor {
this.deps.llmProvider.getProviderType()
)
await registerDesktopTools(nativeRegistry)
+ // Eagerly open this caller's remote OAuth partitions before the catalog is
+ // read, so grant-backed tools exist in the very first turn instead of after a
+ // failed lazy call. Best-effort: a failure here must never invalidate the
+ // registry promise or the task.
+ await this.bootstrapMcpCatalog()
const mcpRegistry = this.deps.mcpManager
? // Thread the authenticated caller identity so oauth grantScope='user'
// tools dispatch to the caller's per-user partition (fail-closed when
diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.ts
index de5a4d44a..f7a028d4a 100644
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@ -1041,7 +1041,13 @@ async function initializeMcpServers(): Promise<void> {
new McpManager(
config.mcpProxyEnabled ? config.mcpProxyUrl : undefined,
undefined,
- createMcpTokenProviderFactory()
+ createMcpTokenProviderFactory(),
+ {
+ grantExistence: (queries, { timeoutMs }) =>
+ checkGrantExistence({ ...brokerTokenProviderDeps(), timeoutMs }, queries),
+ userPartitionMax: OAUTH_USER_PARTITION_MAX,
+ catalogBootstrap: config.mcpCatalogBootstrap,
+ }
),
getAuthToken: async (serverName, expectedRevision) => {
return ensureAuthenticatedContextMapperClient().getAuthToken(serverName, expectedRevision)
@@ -3097,7 +3103,13 @@ async function startDevMode(): Promise<void> {
mcpManager = new McpManager(
config.mcpProxyEnabled ? config.mcpProxyUrl : undefined,
undefined,
- createMcpTokenProviderFactory()
+ createMcpTokenProviderFactory(),
+ {
+ grantExistence: (queries, { timeoutMs }) =>
+ checkGrantExistence({ ...brokerTokenProviderDeps(), timeoutMs }, queries),
+ userPartitionMax: OAUTH_USER_PARTITION_MAX,
+ catalogBootstrap: config.mcpCatalogBootstrap,
+ }
)
if (config.devMcpServers && config.devMcpServers.length > 0) {
diff --git a/mcp-host/src/mcp/catalogBootstrapMetrics.ts b/mcp-host/src/mcp/catalogBootstrapMetrics.ts
new file mode 100644
index 000000000..a10d9cd64
--- /dev/null
+++ b/mcp-host/src/mcp/catalogBootstrapMetrics.ts
@@ -0,0 +1,130 @@
+import { Counter, Gauge, type Registry, register } from 'prom-client'
+
+/** Terminal disposition of one `bootstrapUserCatalog` run. */
+type RunOutcome = 'noop' | 'admitted' | 'partial' | 'timed_out' | 'skipped' | 'error'
+/** Per-coordinate probe disposition (the §6.3 buckets, plus the answer). */
+type ProbeResult =
+ | 'present'
+ | 'absent'
+ | 'unknown'
+ | 'cached_absent'
+ | 'cached_failed'
+ | 'budget'
+ | 'paused'
+type Partition = 'per-user' | 'shared'
+// 'stale' is declared for completeness but stays 0 today: the admission layer
+// resolves a superseded/fenced attempt as ok (per-user) or throws (shared), so
+// the bootstrap never labels an outcome 'stale'. Kept as a materialized-zero
+// series so the label exists if that bookkeeping is ever surfaced.
+type AdmissionOutcome = 'ok' | 'failed' | 'stale'
+
+/** Probe dispositions counted for one run; omitted/zero keys are not incremented. */
+export type ProbeResultCounts = Partial<Record<ProbeResult, number>>
+
+export interface McpCatalogBootstrapMetricsPort {
+ runFinished(outcome: RunOutcome, waitedMs: number): void
+ probes(counts: ProbeResultCounts): void
+ admission(partition: Partition, outcome: AdmissionOutcome): void
+ awaitingGrant(size: number): void
+}
+
+function counter<Label extends string>(
+ registry: Registry,
+ options: { name: string; help: string; labelNames?: Label[] }
+): Counter<Label> {
+ const existing = registry.getSingleMetric(options.name)
+ if (existing) return existing as Counter<Label>
+ return new Counter<Label>({ ...options, registers: [registry] })
+}
+
+function gauge<Label extends string>(
+ registry: Registry,
+ options: { name: string; help: string; labelNames?: Label[] }
+): Gauge<Label> {
+ const existing = registry.getSingleMetric(options.name)
+ if (existing) return existing as Gauge<Label>
+ return new Gauge<Label>({ ...options, registers: [registry] })
+}
+
+const RUN_OUTCOMES: RunOutcome[] = ['noop', 'admitted', 'partial', 'timed_out', 'skipped', 'error']
+const PROBE_RESULTS: ProbeResult[] = [
+ 'present',
+ 'absent',
+ 'unknown',
+ 'cached_absent',
+ 'cached_failed',
+ 'budget',
+ 'paused',
+]
+const PARTITIONS: Partition[] = ['per-user', 'shared']
+const ADMISSION_OUTCOMES: AdmissionOutcome[] = ['ok', 'failed', 'stale']
+
+export class McpCatalogBootstrapMetrics implements McpCatalogBootstrapMetricsPort {
+ private readonly runs: Counter<'outcome'>
+ private readonly probesTotal: Counter<'result'>
+ private readonly admissions: Counter<'partition' | 'outcome'>
+ private readonly waitMs: Gauge
+ private readonly awaiting: Gauge
+
+ constructor(registry: Registry = register) {
+ this.runs = counter(registry, {
+ name: 'clerum_mcp_catalog_bootstrap_runs_total',
+ help: 'Per-turn catalog bootstrap runs by terminal outcome.',
+ labelNames: ['outcome'],
+ })
+ this.probesTotal = counter(registry, {
+ name: 'clerum_mcp_catalog_bootstrap_probes_total',
+ help: 'Grant-existence probe coordinates by disposition (answer or skip bucket).',
+ labelNames: ['result'],
+ })
+ this.admissions = counter(registry, {
+ name: 'clerum_mcp_catalog_bootstrap_admissions_total',
+ help: 'Bootstrap-initiated partition admissions by partition kind and outcome.',
+ labelNames: ['partition', 'outcome'],
+ })
+ this.waitMs = gauge(registry, {
+ name: 'clerum_mcp_catalog_bootstrap_wait_ms',
+ help: 'Time the most recent bootstrap run waited on its admissions before continuing.',
+ })
+ this.awaiting = gauge(registry, {
+ name: 'clerum_mcp_oauth_shared_awaiting_grant',
+ help: 'SHARED oauth-context servers currently registered without a confirmed grant.',
+ })
+ this.initializeSeries()
+ }
+
+ /**
+ * Materialize every counter series at zero so a scrape distinguishes "none"
+ * from "not exported" — the same contract the status-heartbeat metrics keep
+ * so an alert on, say, `result="paused"` reads a real zero, not a gap.
+ */
+ private initializeSeries(): void {
+ for (const outcome of RUN_OUTCOMES) this.runs.inc({ outcome }, 0)
+ for (const result of PROBE_RESULTS) this.probesTotal.inc({ result }, 0)
+ for (const partition of PARTITIONS)
+ for (const outcome of ADMISSION_OUTCOMES) this.admissions.inc({ partition, outcome }, 0)
+ this.awaiting.set(0)
+ }
+
+ runFinished(outcome: RunOutcome, waitedMs: number): void {
+ this.runs.inc({ outcome })
+ this.waitMs.set(waitedMs)
+ }
+
+ probes(counts: ProbeResultCounts): void {
+ for (const result of PROBE_RESULTS) {
+ const n = counts[result]
+ if (n && n > 0) this.probesTotal.inc({ result }, n)
+ }
+ }
+
+ admission(partition: Partition, outcome: AdmissionOutcome): void {
+ this.admissions.inc({ partition, outcome })
+ }
+
+ awaitingGrant(size: number): void {
+ this.awaiting.set(size)
+ }
+}
+
+export const catalogBootstrapMetrics = new McpCatalogBootstrapMetrics()
diff --git a/mcp-host/src/mcp/manager.ts b/mcp-host/src/mcp/manager.ts
index 4340d497a..f499e55a4 100644
--- a/mcp-host/src/mcp/manager.ts
+++ b/mcp-host/src/mcp/manager.ts
@@ -9,6 +9,7 @@
*/
import { logger } from '../logger'
import { McpServerInfo, McpTool, ToolCallResult } from '../types'
+import { catalogBootstrapMetrics } from './catalogBootstrapMetrics'
import {
McpAuthError,
McpClient,
@@ -261,6 +262,9 @@ export class McpManager {
private readonly catalogBootstrap?: McpCatalogBootstrapConfig
// LRU cap the bootstrap honors so it never thrashes admit→evict above the cap.
private readonly userPartitionMax?: number
+ // Injected clock (tests) so bootstrap metric labeling reads the same time base
+ // the probe policy decides on. Production uses Date.now.
+ private readonly now: () => number
// Coalesces concurrent bootstraps for the SAME user onto one plan/probe/admit
// (§6.1 step 2). Cleared on flota reset so a fresh manager starts empty.
private readonly bootstrapInFlight: Map<string, Promise<McpCatalogBootstrapSummary>> = new Map()
@@ -281,13 +285,10 @@ export class McpManager {
this.tokenProviderFactory = tokenProviderFactory
// A probe is inert without both a checker and its policy config; either
// absent → stay on today's lazy admission (bootstrap and SHARED gate off).
+ this.now = options?.now ?? Date.now
this.grantProbe =
options?.grantExistence && options?.catalogBootstrap
- ? createGrantProbe(
- options.grantExistence,
- options.catalogBootstrap,
- options.now ?? Date.now
- )
+ ? createGrantProbe(options.grantExistence, options.catalogBootstrap, this.now)
: undefined
this.catalogBootstrapEnabled = options?.catalogBootstrap?.enabled ?? false
this.catalogBootstrap = options?.catalogBootstrap
@@ -749,6 +750,7 @@ export class McpManager {
private enterSharedAwaitingGrant(serverName: string): void {
if (this.sharedAwaitingGrant.has(serverName)) return
this.sharedAwaitingGrant.add(serverName)
+ catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
logger.info(
{
component: 'McpManager',
@@ -763,6 +765,7 @@ export class McpManager {
/** Log (once) that a SHARED oauth-context left the awaiting-grant state. */
private exitSharedAwaitingGrant(serverName: string): void {
if (!this.sharedAwaitingGrant.delete(serverName)) return
+ catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
logger.info(
{
component: 'McpManager',
@@ -783,7 +786,9 @@ export class McpManager {
* genuine 'entered' log after a re-registration.
*/
private leaveSharedAwaitingGrant(serverName: string): void {
- this.sharedAwaitingGrant.delete(serverName)
+ if (this.sharedAwaitingGrant.delete(serverName)) {
+ catalogBootstrapMetrics.awaitingGrant(this.sharedAwaitingGrant.size)
+ }
}
/**
@@ -1201,6 +1206,7 @@ export class McpManager {
this.partitionLastUsed.clear()
this.partitionInFlight.clear()
this.sharedAwaitingGrant.clear()
+ catalogBootstrapMetrics.awaitingGrant(0)
this.bootstrapInFlight.clear()
this.grantProbe?.reset()
this.statusTracker.reset()
@@ -1375,6 +1381,7 @@ export class McpManager {
const candidates = selection.queries.length
// Step 4 — nothing to probe (steady-state): zero round-trips, no log.
if (candidates === 0) {
+ catalogBootstrapMetrics.runFinished('noop', 0)
return {
candidates: 0,
probed: 0,
@@ -1396,12 +1403,24 @@ export class McpManager {
bump('budget_exhausted', plan.budgetExhausted.length)
bump('in_flight', plan.inFlight.length)
const probed = plan.ask.length
+ // Non-ask probe dispositions are known before execute (so a throw still
+ // counts them). budgetExhausted is all-or-nothing: paused (P4) vs over-budget
+ // (P6). The planner collapses both, so read the live pause to label them.
+ catalogBootstrapMetrics.probes({
+ cached_absent: plan.absentCached.length,
+ cached_failed: plan.failedCached.length,
+ ...(probe.snapshot().pausedUntil > this.now()
+ ? { paused: plan.budgetExhausted.length }
+ : { budget: plan.budgetExhausted.length }),
+ })
let results: GrantExistsResult[]
try {
results = await probe.execute(plan.ask)
} catch {
bump('probe_unknown', plan.ask.length)
+ catalogBootstrapMetrics.probes({ unknown: plan.ask.length })
+ catalogBootstrapMetrics.runFinished('error', 0)
const summary: McpCatalogBootstrapSummary = {
candidates,
probed,
@@ -1425,22 +1444,33 @@ export class McpManager {
candidate: BootstrapCandidate
promise: Promise<unknown>
}> = []
+ let present = 0
+ let unknownResults = 0
+ let absentResults = 0
for (const query of plan.ask) {
const coord = grantCoordKey(query.mcpServerName, query.userId)
const candidate = selection.byCoord.get(coord)
const result = resultByCoord.get(coord)
if (!candidate || !result || typeof result.exists !== 'boolean') {
bump('probe_unknown')
+ unknownResults += 1
continue
}
if (result.exists) {
+ present += 1
const promise = this.admitBootstrapCandidate(userId, candidate, config)
if (promise) admissions.push({ coord, candidate, promise })
} else {
probe.recordAbsent(coord)
bump('absent')
+ absentResults += 1
}
}
+ catalogBootstrapMetrics.probes({
+ present,
+ absent: absentResults,
+ unknown: unknownResults,
+ })
// Step 9 — per-admission bookkeeping. Runs whenever each admission settles,
// even after the wait budget expires (the admission is coalesced in
@@ -1457,11 +1487,19 @@ export class McpManager {
admitted += 1
if (a.candidate.kind === 'user') this.partitionLastUsed.set(a.candidate.key, Date.now())
probe.recordAdmissionOk(a.coord)
+ catalogBootstrapMetrics.admission(
+ a.candidate.kind === 'user' ? 'per-user' : 'shared',
+ 'ok'
+ )
},
(err: unknown) => {
pendingCoords.delete(a.coord)
probe.recordAdmissionFailure(a.coord)
bump('admission_failed')
+ catalogBootstrapMetrics.admission(
+ a.candidate.kind === 'user' ? 'per-user' : 'shared',
+ 'failed'
+ )
logger.warn(
{
component: 'McpManager',
@@ -1501,6 +1539,18 @@ export class McpManager {
timedOut,
}
// Step 10 — log the summary (candidates > 0 here; steady-state returned above).
+ // timedOut is read on the returned summary because a background admission may
+ // have flipped `admitted` between the wait and here without changing whether
+ // the budget cut the wait.
+ const runOutcome =
+ admissions.length === 0
+ ? 'skipped'
+ : timedOut
+ ? 'timed_out'
+ : admitted === admissions.length
+ ? 'admitted'
+ : 'partial'
+ catalogBootstrapMetrics.runFinished(runOutcome, summary.waitedMs)
this.logBootstrap(summary)
return summary
}
diff --git a/mcp-host/src/server.ts b/mcp-host/src/server.ts
index 9d45efb78..25125dc44 100644
--- a/mcp-host/src/server.ts
+++ b/mcp-host/src/server.ts
@@ -11,6 +11,7 @@ import { register } from 'prom-client'
import { readOpenedArtifactBuffer, redactArtifactForDelivery } from './artifacts/artifactBytes'
import type { ArtifactSecretEntry } from './artifacts/artifactRedaction'
import type { RuntimeLifecycleGate } from './lifecycle/statelessHeartbeat'
+import './mcp/catalogBootstrapMetrics'
import './mcp/statusHeartbeatMetrics'
import './observability/processMetrics'
import { requireScope } from './server/authMiddleware'