Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(mcp-host): add grant-probe policy module and bootstrap config (spec 13 phase 1)

Diego Zacariasa6d3ae9d197a
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 412f0840f..38ed97aaf 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.372",
+ "version": "0.1.373",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.372",
+ "version": "0.1.373",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index c6d454df5..4f4971423 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.372",
+ "version": "0.1.373",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/config.ts b/mcp-host/src/config.ts
index 2f66223df..a4222c955 100644
--- a/mcp-host/src/config.ts
+++ b/mcp-host/src/config.ts
@@ -10,6 +10,7 @@ import {
parseCodexToolPresentation,
} from './core/orchestration/toolPresentationPolicy'
import { ALL_PROVIDERS, type LlmProvider, descriptorFor, isLlmProvider } from './llm/registryCore'
+import type { McpCatalogBootstrapConfig } from './mcp/grantProbe'
import { HostSpec, McpServerInfo, MemoryConfig, ModelConfig, PersonalizationConfig } from './types'
/** Route families projected by WRC into a recipe-bound mcp-host. */
@@ -109,6 +110,12 @@ export interface Config {
// scheduler in-flight forever or cause overlapping rounds.
mcpStatusHeartbeatTimeoutMs: number
+ // Eager catalog-bootstrap probe policy. Bounds how often a user turn / the
+ // SHARED oauth-context gate asks control-api "does this grant exist?" so the
+ // probe cannot starve the host's own token issuance on the shared 60/min
+ // bucket. `enabled` is the kill-switch back to today's lazy admission.
+ mcpCatalogBootstrap: McpCatalogBootstrapConfig
+
// Agent configuration
agentTaskDelay: number
agentMaxTaskDuration: number
@@ -538,6 +545,57 @@ export function validateHccAuthorityTiming(
}
}
+/**
+ * Parse one positive-integer knob, failing startup closed on a set-but-invalid
+ * value rather than silently falling back — a mistyped probe budget must be
+ * visible at boot, not degrade the shared rate-limit bucket in production.
+ */
+function parseBootstrapNumber(key: string, defaultValue: number, min = 1): number {
+ const raw = process.env[key]
+ const value = raw === undefined ? defaultValue : Number(raw)
+ if (!Number.isFinite(value) || value < min) {
+ throw new Error(`${key} must be a finite number >= ${min} (got '${raw}')`)
+ }
+ return value
+}
+
+// control-api rate-limits `user-token` AND `grants/exists` under one shared
+// `mcp_oauth_broker` bucket (default 60/min per host). If the per-turn probe
+// budget could reach that ceiling, probes alone could starve token issuance —
+// the exact host self-DoS this module exists to prevent. Guard against an
+// operator setting probesPerMin at or above the bucket; mirrors control-api's
+// CONTROL_API_OAUTH_BROKER_RL_PER_MIN default.
+const OAUTH_BROKER_SHARED_BUCKET_PER_MIN = 60
+
+/**
+ * Build the eager catalog-bootstrap probe policy from env. Fail-fast on a probe
+ * timeout that meets or exceeds the wait budget: a probe slower than the whole
+ * turn budget can never complete in time, so the config is nonsensical.
+ */
+export function buildMcpCatalogBootstrapConfig(): McpCatalogBootstrapConfig {
+ const cfg: McpCatalogBootstrapConfig = {
+ enabled: getEnvBool('MCP_CATALOG_BOOTSTRAP_ENABLED', true),
+ waitBudgetMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_WAIT_BUDGET_MS', 4000),
+ probeTimeoutMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_PROBE_TIMEOUT_MS', 2000),
+ connectTimeoutMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_CONNECT_TIMEOUT_MS', 8000),
+ negativeTtlMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_NEGATIVE_TTL_MS', 15000),
+ failureTtlMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_FAILURE_TTL_MS', 60000),
+ probesPerMin: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_PROBES_PER_MIN', 20),
+ backoffMs: parseBootstrapNumber('MCP_CATALOG_BOOTSTRAP_BACKOFF_MS', 30000),
+ }
+ if (cfg.probeTimeoutMs >= cfg.waitBudgetMs) {
+ throw new Error(
+ `MCP_CATALOG_BOOTSTRAP_PROBE_TIMEOUT_MS (${cfg.probeTimeoutMs}ms) must be less than MCP_CATALOG_BOOTSTRAP_WAIT_BUDGET_MS (${cfg.waitBudgetMs}ms)`
+ )
+ }
+ if (cfg.probesPerMin >= OAUTH_BROKER_SHARED_BUCKET_PER_MIN) {
+ throw new Error(
+ `MCP_CATALOG_BOOTSTRAP_PROBES_PER_MIN (${cfg.probesPerMin}) must be less than the shared mcp_oauth_broker bucket (${OAUTH_BROKER_SHARED_BUCKET_PER_MIN}/min) so probes cannot starve user-token issuance`
+ )
+ }
+ return cfg
+}
+
// In dev mode, try CLERUM_HOST_CONFIG first, then fall back to building from env vars
function getDevHostConfig(): HostSpec | undefined {
if (!devMode) return undefined
@@ -1050,4 +1108,7 @@ export const config: Config = {
}
return { enabled, ...seed }
})(),
+
+ // Eager catalog-bootstrap probe policy (fail-fast validated at load).
+ mcpCatalogBootstrap: buildMcpCatalogBootstrapConfig(),
}
diff --git a/mcp-host/src/mcp/__tests__/grantProbe.property.test.ts b/mcp-host/src/mcp/__tests__/grantProbe.property.test.ts
new file mode 100644
index 000000000..0ba82532b
--- /dev/null
+++ b/mcp-host/src/mcp/__tests__/grantProbe.property.test.ts
@@ -0,0 +1,163 @@
+import { describe, expect, it } from 'vitest'
+import fc from 'fast-check'
+import type { GrantExistsQuery } from '../grantExistenceClient'
+import { type GrantProbeState, type McpCatalogBootstrapConfig, planProbe } from '../grantProbe'
+
+function coord(mcpServerName: string, userId?: string): string {
+ return JSON.stringify([mcpServerName, userId ?? null])
+}
+function coordOf(q: GrantExistsQuery): string {
+ return coord(q.mcpServerName, q.userId)
+}
+
+const SERVERS = ['s1', 's2', 's3']
+const USERS: (string | undefined)[] = ['u1', 'u2', undefined]
+
+const arbQuery: fc.Arbitrary<GrantExistsQuery> = fc
+ .record({
+ mcpServerName: fc.constantFrom(...SERVERS),
+ userId: fc.constantFrom(...USERS),
+ })
+ .map(({ mcpServerName, userId }) =>
+ userId === undefined ? { mcpServerName } : { mcpServerName, userId }
+ )
+
+const arbQueries = fc.array(arbQuery, { maxLength: 8 })
+
+const arbConfig: fc.Arbitrary<McpCatalogBootstrapConfig> = fc.record({
+ enabled: fc.constant(true),
+ waitBudgetMs: fc.constant(4000),
+ probeTimeoutMs: fc.constant(2000),
+ connectTimeoutMs: fc.constant(8000),
+ negativeTtlMs: fc.integer({ min: 1, max: 100_000 }),
+ failureTtlMs: fc.integer({ min: 1, max: 100_000 }),
+ probesPerMin: fc.integer({ min: 1, max: 5 }),
+ backoffMs: fc.integer({ min: 1, max: 100_000 }),
+})
+
+/** Build an arbitrary state whose caches reference some of the addressable coordinates. */
+function arbState(now: number): fc.Arbitrary<GrantProbeState> {
+ const allCoords = SERVERS.flatMap(s => USERS.map(u => coord(s, u)))
+ const arbCoordSubset = fc.subarray(allCoords)
+ const arbCoordTtlMap = fc.array(
+ fc.tuple(fc.constantFrom(...allCoords), fc.integer({ min: now - 50_000, max: now + 50_000 })),
+ { maxLength: allCoords.length }
+ )
+ return fc.record({
+ absentUntil: arbCoordTtlMap.map(entries => new Map(entries)),
+ failedUntil: arbCoordTtlMap.map(entries => new Map(entries)),
+ inFlight: arbCoordSubset.map(cs => new Set(cs)),
+ pausedUntil: fc.integer({ min: now - 50_000, max: now + 50_000 }),
+ bucket: fc.record({
+ windowStartMs: fc.integer({ min: now - 120_000, max: now }),
+ used: fc.integer({ min: 0, max: 8 }),
+ }),
+ })
+}
+
+function cloneState(s: GrantProbeState): GrantProbeState {
+ return {
+ absentUntil: new Map(s.absentUntil),
+ failedUntil: new Map(s.failedUntil),
+ inFlight: new Set(s.inFlight),
+ pausedUntil: s.pausedUntil,
+ bucket: { ...s.bucket },
+ }
+}
+
+const NOW = 1_000_000
+
+describe('planProbe — property-based (T2)', () => {
+ it('never asks a coordinate that is in-flight, negatively cached, failure-cached, or while paused', () => {
+ fc.assert(
+ fc.property(arbState(NOW), arbQueries, arbConfig, (state, queries, config) => {
+ const r = planProbe(state, NOW, queries, config)
+ const paused = NOW < state.pausedUntil
+ for (const q of r.ask) {
+ const c = coordOf(q)
+ expect(state.inFlight.has(c)).toBe(false)
+ expect(NOW < (state.failedUntil.get(c) ?? 0)).toBe(false)
+ expect(NOW < (state.absentUntil.get(c) ?? 0)).toBe(false)
+ }
+ if (paused) expect(r.ask).toHaveLength(0)
+ })
+ )
+ })
+
+ it('respects the request budget: a plan asks (issues one batch) only when a window unit is free', () => {
+ fc.assert(
+ fc.property(arbState(NOW), arbQueries, arbConfig, (state, queries, config) => {
+ const r = planProbe(state, NOW, queries, config)
+ const windowExpired = NOW - state.bucket.windowStartMs >= 60_000
+ const effectiveUsed = windowExpired ? 0 : state.bucket.used
+ if (r.ask.length > 0) {
+ expect(effectiveUsed).toBeLessThan(config.probesPerMin)
+ }
+ })
+ )
+ })
+
+ it('is idempotent: re-planning the same queries against the resulting state asks nothing new', () => {
+ fc.assert(
+ fc.property(arbState(NOW), arbQueries, arbConfig, (state, queries, config) => {
+ const first = planProbe(state, NOW, queries, config)
+ const second = planProbe(first.state, NOW, queries, config)
+ expect(second.ask).toHaveLength(0)
+ })
+ )
+ })
+
+ it('partitions every distinct coordinate into exactly one bucket', () => {
+ fc.assert(
+ fc.property(arbState(NOW), arbQueries, arbConfig, (state, queries, config) => {
+ const r = planProbe(state, NOW, queries, config)
+ const distinct = new Set(queries.map(coordOf))
+ const buckets = [
+ ...r.ask.map(coordOf),
+ ...r.absentCached,
+ ...r.failedCached,
+ ...r.budgetExhausted,
+ ...r.inFlight,
+ ]
+ // No coordinate appears in two buckets.
+ expect(new Set(buckets).size).toBe(buckets.length)
+ // The buckets cover exactly the distinct coordinates.
+ expect(new Set(buckets)).toEqual(distinct)
+ expect(buckets.length).toBe(distinct.size)
+ })
+ )
+ })
+
+ it('does not mutate the input state', () => {
+ fc.assert(
+ fc.property(arbState(NOW), arbQueries, arbConfig, (state, queries, config) => {
+ const before = cloneState(state)
+ planProbe(state, NOW, queries, config)
+ expect(state.inFlight).toEqual(before.inFlight)
+ expect(state.bucket).toEqual(before.bucket)
+ expect(state.absentUntil).toEqual(before.absentUntil)
+ expect(state.failedUntil).toEqual(before.failedUntil)
+ expect(state.pausedUntil).toBe(before.pausedUntil)
+ })
+ )
+ })
+
+ it('is temporally monotone: advancing now never turns an asked coordinate into a skipped one', () => {
+ fc.assert(
+ fc.property(
+ arbState(NOW),
+ arbQueries,
+ arbConfig,
+ fc.integer({ min: 0, max: 200_000 }),
+ (state, queries, config, delta) => {
+ const early = planProbe(cloneState(state), NOW, queries, config)
+ const late = planProbe(cloneState(state), NOW + delta, queries, config)
+ const lateAsk = new Set(late.ask.map(coordOf))
+ for (const q of early.ask) {
+ expect(lateAsk.has(coordOf(q))).toBe(true)
+ }
+ }
+ )
+ )
+ })
+})
diff --git a/mcp-host/src/mcp/__tests__/grantProbe.test.ts b/mcp-host/src/mcp/__tests__/grantProbe.test.ts
new file mode 100644
index 000000000..32989160b
--- /dev/null
+++ b/mcp-host/src/mcp/__tests__/grantProbe.test.ts
@@ -0,0 +1,226 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ type GrantExistsQuery,
+ type GrantExistsResult,
+ checkGrantExistence,
+} from '../grantExistenceClient'
+import {
+ type GrantExistenceChecker,
+ type McpCatalogBootstrapConfig,
+ createGrantProbe,
+} from '../grantProbe'
+
+function makeConfig(overrides: Partial<McpCatalogBootstrapConfig> = {}): McpCatalogBootstrapConfig {
+ return {
+ enabled: true,
+ waitBudgetMs: 4000,
+ probeTimeoutMs: 2000,
+ connectTimeoutMs: 8000,
+ negativeTtlMs: 15000,
+ failureTtlMs: 60000,
+ probesPerMin: 20,
+ backoffMs: 30000,
+ ...overrides,
+ }
+}
+
+function coord(mcpServerName: string, userId?: string): string {
+ return JSON.stringify([mcpServerName, userId ?? null])
+}
+
+/**
+ * T1 — the checker is the REAL `checkGrantExistence` served by a fake gateway
+ * whose grant table is a plain Set of coordinates. `exists` is therefore parsed
+ * from a real `{results}` body by the production client, never hand-authored as a
+ * `GrantExistsResult`. The gateway echoes the query coordinates exactly as
+ * control-api does so the client correlates by tuple.
+ */
+function checkerBackedByGrants(grants: Set<string>): {
+ checker: GrantExistenceChecker
+ posts: () => number
+} {
+ const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
+ const body = JSON.parse(String(init.body)) as { queries: GrantExistsQuery[] }
+ const results: GrantExistsResult[] = body.queries.map(q => ({
+ mcpServerName: q.mcpServerName,
+ ...(q.userId !== undefined ? { userId: q.userId } : {}),
+ exists: grants.has(coord(q.mcpServerName, q.userId)),
+ }))
+ return { status: 200, json: async () => ({ results }) } as unknown as Response
+ })
+ const checker: GrantExistenceChecker = (queries, opts) =>
+ checkGrantExistence(
+ {
+ gatewayUrl: () => 'http://gw',
+ controlToken: () => 'ctl',
+ fetchImpl: fetchImpl as unknown as typeof fetch,
+ timeoutMs: opts.timeoutMs,
+ },
+ queries
+ )
+ return { checker, posts: () => fetchImpl.mock.calls.length }
+}
+
+/** Drive one bootstrap-style pass for a single coordinate: plan → execute → record. */
+async function bootstrapOnce(
+ probe: ReturnType<typeof createGrantProbe>,
+ query: GrantExistsQuery
+): Promise<'asked-present' | 'asked-absent' | 'skipped'> {
+ const plan = probe.plan([query])
+ if (plan.ask.length === 0) return 'skipped'
+ const results = await probe.execute(plan.ask)
+ const key = coord(query.mcpServerName, query.userId)
+ const match = results.find(r => coord(r.mcpServerName, r.userId) === key)
+ if (match?.exists) {
+ probe.recordAdmissionOk(key)
+ return 'asked-present'
+ }
+ probe.recordAbsent(key)
+ return 'asked-absent'
+}
+
+describe('grantProbe — per-minute request budget (invariant 8)', () => {
+ it('with probesPerMin=2, three grant-less users in the same minute cost 2 POSTs; the third is budget-exhausted; the next minute resumes', async () => {
+ let clock = 1_000_000
+ const { checker, posts } = checkerBackedByGrants(new Set()) // nobody has a grant
+ const probe = createGrantProbe(checker, makeConfig({ probesPerMin: 2 }), () => clock)
+
+ const users = ['u1', 'u2', 'u3'].map(u => ({ mcpServerName: 'calendar', userId: u }))
+
+ // First two users each cost one POST (each is a distinct coordinate/batch).
+ expect(await bootstrapOnce(probe, users[0])).toBe('asked-absent')
+ expect(await bootstrapOnce(probe, users[1])).toBe('asked-absent')
+
+ // The third user is over the local budget for this window: no POST at all.
+ const third = probe.plan([users[2]])
+ expect(third.ask).toHaveLength(0)
+ expect(third.budgetExhausted).toEqual([coord('calendar', 'u3')])
+ expect(posts()).toBe(2)
+
+ // Next minute: the window resets and u3 (never absent-cached) is probed.
+ clock += 60_000
+ expect(await bootstrapOnce(probe, users[2])).toBe('asked-absent')
+ expect(posts()).toBe(3)
+ })
+})
+
+describe('grantProbe — negative cache TTL (invariant 9)', () => {
+ it('an exists:false result suppresses a second probe until negativeTtlMs elapses', async () => {
+ let clock = 5_000_000
+ const { checker, posts } = checkerBackedByGrants(new Set())
+ const probe = createGrantProbe(checker, makeConfig({ negativeTtlMs: 15_000 }), () => clock)
+ const query = { mcpServerName: 'gh', userId: 'alice' }
+
+ // First probe: absent → cached negative.
+ expect(await bootstrapOnce(probe, query)).toBe('asked-absent')
+ expect(posts()).toBe(1)
+
+ // Before the TTL: no probe, reported as absent-cached.
+ clock += 14_999
+ const cached = probe.plan([query])
+ expect(cached.ask).toHaveLength(0)
+ expect(cached.absentCached).toEqual([coord('gh', 'alice')])
+ expect(posts()).toBe(1)
+
+ // After the TTL: probed again.
+ clock += 2
+ expect(await bootstrapOnce(probe, query)).toBe('asked-absent')
+ expect(posts()).toBe(2)
+ })
+})
+
+describe('grantProbe — supporting policy behaviors', () => {
+ it('deduplicates a coordinate already in flight (P1)', () => {
+ const probe = createGrantProbe(
+ async () => [],
+ makeConfig(),
+ () => 1
+ )
+ const query = { mcpServerName: 'gh', userId: 'alice' }
+ const first = probe.plan([query])
+ expect(first.ask).toHaveLength(1)
+ // Not yet executed: the coordinate stays in flight, so a concurrent plan skips it.
+ const second = probe.plan([query])
+ expect(second.ask).toHaveLength(0)
+ expect(second.inFlight).toEqual([coord('gh', 'alice')])
+ })
+
+ it('a checker throw enters global backoff and suppresses the next probe (P4)', async () => {
+ let clock = 2_000_000
+ const checker = vi.fn(async () => {
+ throw new Error('gateway down')
+ })
+ const probe = createGrantProbe(checker, makeConfig({ backoffMs: 30_000 }), () => clock)
+ const query = { mcpServerName: 'gh' }
+
+ const plan = probe.plan([query])
+ await expect(probe.execute(plan.ask)).rejects.toThrow(/gateway down/)
+ // In-flight was released so a retry is possible once the backoff clears.
+ expect(probe.snapshot().inFlight).toBe(0)
+
+ // Within backoff: paused, no checker call.
+ clock += 29_999
+ const paused = probe.plan([query])
+ expect(paused.ask).toHaveLength(0)
+ expect(paused.budgetExhausted).toEqual([coord('gh')])
+
+ // After backoff: probes again.
+ clock += 2
+ const resumed = probe.plan([query])
+ expect(resumed.ask).toHaveLength(1)
+ expect(checker).toHaveBeenCalledTimes(1)
+ })
+
+ it('probeOne returns present/absent/unknown from the injected checker', async () => {
+ const grants = new Set([coord('slack')])
+ const { checker } = checkerBackedByGrants(grants)
+ const probe = createGrantProbe(checker, makeConfig(), () => 1)
+
+ expect(await probe.probeOne({ mcpServerName: 'slack' })).toBe('present')
+ expect(await probe.probeOne({ mcpServerName: 'notion' })).toBe('absent')
+ })
+
+ it('probeOne is unknown (not absent) when the endpoint throws — fail-open', async () => {
+ const probe = createGrantProbe(
+ async () => {
+ throw new Error('boom')
+ },
+ makeConfig(),
+ () => 1
+ )
+ expect(await probe.probeOne({ mcpServerName: 'slack' })).toBe('unknown')
+ })
+
+ it('caps the planned ask to the remaining budget so a huge candidate set cannot exceed probesPerMin requests', async () => {
+ const checker = vi.fn(async (queries: GrantExistsQuery[]) =>
+ queries.map(q => ({ mcpServerName: q.mcpServerName, userId: q.userId, exists: false }))
+ )
+ const probe = createGrantProbe(checker, makeConfig({ probesPerMin: 2 }), () => 1)
+ // 2500 distinct coordinates with a 2-unit budget: at ≤1000/chunk the plan may
+ // authorize at most 2000, so executing the ask stays within the 2 requests the
+ // budget accounts for — the remaining 500 spill to budgetExhausted.
+ const queries: GrantExistsQuery[] = Array.from({ length: 2500 }, (_, i) => ({
+ mcpServerName: `s${i}`,
+ }))
+ const plan = probe.plan(queries)
+ expect(plan.ask).toHaveLength(2000)
+ expect(plan.budgetExhausted).toHaveLength(500)
+ await probe.execute(plan.ask)
+ expect(checker).toHaveBeenCalledTimes(2)
+ expect(probe.snapshot().bucketUsed).toBe(2)
+ })
+
+ it('execute chunks the ask into ≤MAX_EXISTS_BATCH batches and charges one budget unit per batch', async () => {
+ const checker = vi.fn(async (queries: GrantExistsQuery[]) =>
+ queries.map(q => ({ mcpServerName: q.mcpServerName, userId: q.userId, exists: false }))
+ )
+ const probe = createGrantProbe(checker, makeConfig({ probesPerMin: 5 }), () => 1)
+ const ask: GrantExistsQuery[] = Array.from({ length: 1500 }, (_, i) => ({
+ mcpServerName: `s${i}`,
+ }))
+ await probe.execute(ask)
+ // 1500 coordinates → two HTTP requests → two budget units.
+ expect(checker).toHaveBeenCalledTimes(2)
+ expect(probe.snapshot().bucketUsed).toBe(2)
+ })
+})
diff --git a/mcp-host/src/mcp/grantProbe.ts b/mcp-host/src/mcp/grantProbe.ts
new file mode 100644
index 000000000..8c27996b9
--- /dev/null
+++ b/mcp-host/src/mcp/grantProbe.ts
@@ -0,0 +1,347 @@
+/**
+ * Grant-existence probe policy — the decision module that bounds how often
+ * mcp-host asks control-api "does this grant exist?" while opening remote OAuth
+ * partitions eagerly (per-user catalog bootstrap and the SHARED oauth-context
+ * gate). It sits in front of `checkGrantExistence` (grantExistenceClient.ts) and
+ * exists because control-api meters `user-token` AND `grants/exists` on the SAME
+ * 60/min-per-host bucket: an unbounded probe-per-turn could starve the host's own
+ * token issuance. The policy is: negative cache, admission-failure cache, in-flight
+ * dedup, a local per-minute request budget, and a global backoff after an endpoint
+ * failure.
+ *
+ * `planProbe` is a PURE function of `(state, now, queries, config)` — it is the
+ * property-tested core; `createGrantProbe` wraps it with the injected HTTP checker
+ * and the mutable per-manager state. This module deliberately imports NOTHING at
+ * runtime (config arrives by parameter, the checker is injected) so config.ts can
+ * import its `McpCatalogBootstrapConfig` type without a cycle.
+ */
+import type { GrantExistsQuery, GrantExistsResult } from './grantExistenceClient'
+
+/**
+ * The 8 knobs of the eager-bootstrap probe policy. Exported from THIS module (not
+ * config.ts) so config.ts can import the type without an import cycle through the
+ * grant-existence client and the manager.
+ */
+export interface McpCatalogBootstrapConfig {
+ /** Kill-switch back to today's lazy admission (turn bootstrap AND SHARED gate). */
+ enabled: boolean
+ /** Max a turn waits for its admissions before continuing with what landed. */
+ waitBudgetMs: number
+ /** Per-probe HTTP timeout; must stay below waitBudgetMs. */
+ probeTimeoutMs: number
+ /** Timeout handed to a bootstrap-initiated `connect()` so it cannot hang for the SDK default. */
+ connectTimeoutMs: number
+ /** Negative-cache TTL for a definitive `exists:false`. */
+ negativeTtlMs: number
+ /** Cache TTL after an admission fails despite a present grant. */
+ failureTtlMs: number
+ /** Local per-minute request budget (HTTP POSTs, not coordinates). */
+ probesPerMin: number
+ /** Global pause after the endpoint throws/times out/429s. */
+ backoffMs: number
+}
+
+/**
+ * The checker `createGrantProbe` calls to actually reach control-api. In
+ * production `main.ts` wires this to `checkGrantExistence` with the broker deps
+ * and a short timeout; tests inject a controlled fake. Throwing on any failure
+ * mode (network/timeout/non-200 incl. 429/malformed body) is the contract the
+ * policy relies on to trigger backoff — it mirrors `checkGrantExistence`.
+ */
+export type GrantExistenceChecker = (
+ queries: GrantExistsQuery[],
+ opts: { timeoutMs: number }
+) => Promise<GrantExistsResult[]>
+
+interface ProbeBucket {
+ windowStartMs: number
+ used: number
+}
+
+/** Mutable policy state; one instance per `McpManager`. */
+export interface GrantProbeState {
+ /** Negative cache: coordinate → epoch ms until which "no grant" holds. */
+ absentUntil: Map<string, number>
+ /** Admission-failure cache: coordinate → epoch ms until which we skip re-admitting. */
+ failedUntil: Map<string, number>
+ /** Coordinates already asked and awaiting a response (dedup across concurrent turns). */
+ inFlight: Set<string>
+ /** Global backoff: no probe is issued while `now < pausedUntil`. */
+ pausedUntil: number
+ /** Local per-minute request budget over a fixed 60s window. */
+ bucket: ProbeBucket
+}
+
+/** The buckets `planProbe` sorts each coordinate into. `ask` carries the queries to send; the rest are coordinate keys. */
+export interface ProbePlan {
+ ask: GrantExistsQuery[]
+ absentCached: string[]
+ failedCached: string[]
+ budgetExhausted: string[]
+ inFlight: string[]
+}
+
+export interface PlanProbeResult extends ProbePlan {
+ /** The new state after planning: asked coordinates recorded in `inFlight`, window reset if stale. */
+ state: GrantProbeState
+}
+
+/** A probe verdict for a single coordinate (used by the SHARED oauth-context gate). */
+export type ProbeVerdict = 'present' | 'absent' | 'unknown'
+
+/** Fixed 1-minute window, aligned with control-api's per-minute limiter. */
+const PROBE_WINDOW_MS = 60_000
+
+/** control-api caps a single `grants/exists` batch at this many coordinates (routes/mcpOauth.ts). */
+export const MAX_EXISTS_BATCH = 1000
+
+/**
+ * Stable coordinate key. MUST match the convention of
+ * `grantExistenceClient.ts`'s `coordKey`/`selectRevokedPartitionKeys`
+ * (`oauth-context` carries no userId → normalized to null) so the probe cache
+ * and the revocation sweep agree on what "a grant" is.
+ */
+function coordKey(mcpServerName: string, userId?: string): string {
+ return JSON.stringify([mcpServerName, userId ?? null])
+}
+
+function coordOf(query: GrantExistsQuery): string {
+ return coordKey(query.mcpServerName, query.userId)
+}
+
+export function emptyProbeState(): GrantProbeState {
+ return {
+ absentUntil: new Map(),
+ failedUntil: new Map(),
+ inFlight: new Set(),
+ pausedUntil: 0,
+ bucket: { windowStartMs: 0, used: 0 },
+ }
+}
+
+/**
+ * Pure policy core (§6.3 table). Evaluates each DISTINCT coordinate of `queries`
+ * against the state in the fixed order P1→P7 — the first row that applies wins —
+ * and returns the new state plus the plan. It NEVER mutates its input state and
+ * NEVER performs I/O. It records asked coordinates in the returned `inFlight`
+ * (dedup) but does NOT charge the request budget: the bucket is charged per HTTP
+ * request by `execute` (P7). `execute` splits the `ask` into ≤MAX_EXISTS_BATCH
+ * chunks and each chunk is one limiter unit, so `plan` only authorizes as many
+ * coordinates as the remaining window budget can pay for in chunks — the rest
+ * spill to `budgetExhausted` (P6). This keeps the local budget an actual bound on
+ * requests, not just on plans.
+ */
+export function planProbe(
+ state: GrantProbeState,
+ now: number,
+ queries: readonly GrantExistsQuery[],
+ config: McpCatalogBootstrapConfig
+): PlanProbeResult {
+ // Clone so the function stays pure; callers replace their state with the result.
+ const absentUntil = new Map(state.absentUntil)
+ const failedUntil = new Map(state.failedUntil)
+ const inFlight = new Set(state.inFlight)
+ const pausedUntil = state.pausedUntil
+ // A window that has aged out (or never started) resets before any decision, so
+ // budget checks in this plan see the current window. windowStartMs=0 on a fresh
+ // state resets on the first real `now`.
+ const windowExpired = now - state.bucket.windowStartMs >= PROBE_WINDOW_MS
+ const bucket: ProbeBucket = windowExpired ? { windowStartMs: now, used: 0 } : { ...state.bucket }
+
+ const plan: ProbePlan = {
+ ask: [],
+ absentCached: [],
+ failedCached: [],
+ budgetExhausted: [],
+ inFlight: [],
+ }
+
+ const paused = now < pausedUntil
+
+ // De-duplicate by coordinate: two queries for the same coordinate are one probe.
+ const seen = new Set<string>()
+ const candidates: GrantExistsQuery[] = []
+ for (const query of queries) {
+ const coord = coordOf(query)
+ if (seen.has(coord)) continue
+ seen.add(coord)
+
+ if (inFlight.has(coord)) {
+ plan.inFlight.push(coord) // P1
+ } else if (now < (failedUntil.get(coord) ?? 0)) {
+ plan.failedCached.push(coord) // P2
+ } else if (now < (absentUntil.get(coord) ?? 0)) {
+ plan.absentCached.push(coord) // P3
+ } else if (paused) {
+ plan.budgetExhausted.push(coord) // P4 (global backoff)
+ } else {
+ candidates.push(query) // P5/P6/P7 decided below
+ }
+ }
+
+ if (candidates.length > 0) {
+ // `execute` charges one bucket unit per ≤MAX_EXISTS_BATCH chunk, so the plan
+ // may authorize at most `remaining units × chunk size` coordinates; the rest
+ // spill to budgetExhausted (P6). Reserving in `inFlight` happens only for the
+ // authorized slice, since only those coordinates reach `execute`.
+ const remainingUnits = Math.max(0, config.probesPerMin - bucket.used)
+ const askCapacity = remainingUnits * MAX_EXISTS_BATCH
+ candidates.forEach((query, i) => {
+ if (i < askCapacity) {
+ plan.ask.push(query) // P7
+ inFlight.add(coordOf(query))
+ } else {
+ plan.budgetExhausted.push(coordOf(query)) // P6: over the window budget
+ }
+ })
+ }
+
+ return {
+ ...plan,
+ state: { absentUntil, failedUntil, inFlight, pausedUntil, bucket },
+ }
+}
+
+/** Stateful probe over an injected checker. One instance per `McpManager`. */
+export interface GrantProbe {
+ /**
+ * Plan the batch for `queries` (advances state: reserves in-flight, resets a stale window).
+ * Contract: every non-empty `ask` this returns MUST be handed to `execute` (which clears the
+ * in-flight reservation on success or throw). Dropping an `ask` leaks its coordinates in
+ * `inFlight`, and P1 would then skip them forever until `reset()`.
+ */
+ plan(queries: readonly GrantExistsQuery[]): ProbePlan
+ /** Send the planned `ask` in ≤MAX_EXISTS_BATCH chunks; charges the bucket per HTTP request. Throws (after backoff + in-flight cleanup) if the checker throws. */
+ execute(ask: GrantExistsQuery[]): Promise<GrantExistsResult[]>
+ /** Convenience for the single-coordinate SHARED gate: plan+execute+interpret one query. */
+ probeOne(query: GrantExistsQuery, nowMs?: number): Promise<ProbeVerdict>
+ /** Cache a definitive `exists:false` and clear the coordinate's in-flight mark. */
+ recordAbsent(coord: string, nowMs?: number): void
+ /** Cache an admission that failed despite a present grant. */
+ recordAdmissionFailure(coord: string, nowMs?: number): void
+ /** Clear the failure/absent caches for a coordinate whose admission succeeded. */
+ recordAdmissionOk(coord: string): void
+ /** Enter global backoff after an endpoint failure. */
+ recordFailure(nowMs?: number): void
+ /** Drop all state (new manager / tests). */
+ reset(): void
+ /** Read-only view for metrics/tests. */
+ snapshot(): {
+ inFlight: number
+ absentCached: number
+ failedCached: number
+ pausedUntil: number
+ bucketUsed: number
+ windowStartMs: number
+ }
+}
+
+export function createGrantProbe(
+ checker: GrantExistenceChecker,
+ config: McpCatalogBootstrapConfig,
+ now: () => number = Date.now
+): GrantProbe {
+ let state = emptyProbeState()
+
+ const plan = (queries: readonly GrantExistsQuery[]): ProbePlan => {
+ const result = planProbe(state, now(), queries, config)
+ state = result.state
+ return {
+ ask: result.ask,
+ absentCached: result.absentCached,
+ failedCached: result.failedCached,
+ budgetExhausted: result.budgetExhausted,
+ inFlight: result.inFlight,
+ }
+ }
+
+ const recordFailure = (nowMs: number = now()): void => {
+ state.pausedUntil = nowMs + config.backoffMs
+ }
+
+ const execute = async (ask: GrantExistsQuery[]): Promise<GrantExistsResult[]> => {
+ if (ask.length === 0) return []
+ const results: GrantExistsResult[] = []
+ try {
+ for (let i = 0; i < ask.length; i += MAX_EXISTS_BATCH) {
+ const chunk = ask.slice(i, i + MAX_EXISTS_BATCH)
+ // One HTTP request costs one limiter unit at control-api (P7).
+ state.bucket.used += 1
+ const chunkResults = await checker(chunk, { timeoutMs: config.probeTimeoutMs })
+ results.push(...chunkResults)
+ }
+ } catch (err) {
+ // Endpoint failure: release every reserved coordinate and back off globally.
+ for (const query of ask) state.inFlight.delete(coordOf(query))
+ recordFailure()
+ throw err
+ }
+ // The request completed for every asked coordinate; the caller records the
+ // per-coordinate outcome (absent / admission ok / admission failed).
+ for (const query of ask) state.inFlight.delete(coordOf(query))
+ return results
+ }
+
+ const recordAbsent = (coord: string, nowMs: number = now()): void => {
+ state.inFlight.delete(coord)
+ state.absentUntil.set(coord, nowMs + config.negativeTtlMs)
+ }
+
+ const recordAdmissionFailure = (coord: string, nowMs: number = now()): void => {
+ state.inFlight.delete(coord)
+ state.failedUntil.set(coord, nowMs + config.failureTtlMs)
+ }
+
+ const recordAdmissionOk = (coord: string): void => {
+ state.inFlight.delete(coord)
+ state.failedUntil.delete(coord)
+ state.absentUntil.delete(coord)
+ }
+
+ const probeOne = async (
+ query: GrantExistsQuery,
+ nowMs: number = now()
+ ): Promise<ProbeVerdict> => {
+ const planned = planProbe(state, nowMs, [query], config)
+ state = planned.state
+ const coord = coordOf(query)
+ if (planned.ask.length === 0) {
+ // A definitive negative cache is a real "absent"; every other skip
+ // (in-flight, failure cache, backoff, budget) is "unknown" — we do not
+ // know the grant's status, so the caller must not treat it as absent.
+ return planned.absentCached.includes(coord) ? 'absent' : 'unknown'
+ }
+ try {
+ const results = await execute(planned.ask)
+ const match = results.find(r => coordOf(r) === coord)
+ if (!match || typeof match.exists !== 'boolean') return 'unknown'
+ if (match.exists) return 'present'
+ recordAbsent(coord, nowMs)
+ return 'absent'
+ } catch {
+ // execute already backed off and cleared in-flight; unknown is fail-open.
+ return 'unknown'
+ }
+ }
+
+ return {
+ plan,
+ execute,
+ probeOne,
+ recordAbsent,
+ recordAdmissionFailure,
+ recordAdmissionOk,
+ recordFailure,
+ reset: () => {
+ state = emptyProbeState()
+ },
+ snapshot: () => ({
+ inFlight: state.inFlight.size,
+ absentCached: state.absentUntil.size,
+ failedCached: state.failedUntil.size,
+ pausedUntil: state.pausedUntil,
+ bucketUsed: state.bucket.used,
+ windowStartMs: state.bucket.windowStartMs,
+ }),
+ }
+}