Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, vi } from 'vitest'
import { config } from '../config'
import { ContextMapperRequestError } from '../contextMapperClient'
import {
admitDevelopmentMcpServers,
createCoalescedPollRunner,
createMcpAuthorityPollFailureOptions,
createMcpAuthorityStalenessDeadline,
isMcpAuthorityStale,
startContextMapperPolling,
stopContextMapperPolling,
} from '../main'
import {
AuthoritativeMcpFleetCoordinator,
pollAuthoritativeMcpSnapshotIfCurrent,
reconcileAuthoritativeMcpSnapshot,
replaceAuthoritativeMcpFleet,
runAuthoritativeMcpInitialization,
} from '../mcp/authoritativeFleet'
import { handleMcpAuthorityPollFailure } from '../mcp/authorityLifecycle'
import { McpManager } from '../mcp/manager'
import type { McpServerInfo } from '../types'
function readyServer(overrides: Partial<McpServerInfo> = {}): McpServerInfo {
return {
name: 'secured-server',
contextRef: 'production',
transport: { type: 'streamableHttp', url: 'http://secured-server.test/mcp' },
auth: { type: 'bearer', secretRef: 'secured-server-auth' },
enabled: true,
// Mirror the real producer contract: HCC's reconciler.getStatus() (surfaced
// through k8sClient.toServerInfo) always stamps `authoritative` explicitly —
// a ready, identity-matched snapshot carries authoritative: true. The legacy
// shape that omitted it is never emitted for a live server, so the default
// fixture must carry it too.
status: { deployed: true, ready: true, authoritative: true },
...overrides,
}
}
function deferred<T = void>(): {
promise: Promise<T>
resolve: (value: T) => void
reject: (reason?: unknown) => void
} {
let resolve!: (value: T) => void
let reject!: (reason?: unknown) => void
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise
reject = rejectPromise
})
return { promise, resolve, reject }
}
function appliedAdmission(
effect?: (server: McpServerInfo, authToken?: string) => void | Promise<void>
) {
return vi.fn(
async (
server: McpServerInfo,
authToken?: string,
control?: Parameters<McpManager['addServer']>[2]
) => {
await effect?.(server, authToken)
if (control?.isCurrent?.() === false) return 'stale' as const
control?.onCommit?.()
return 'applied' as const
}
)
}
describe('admitDevelopmentMcpServers', () => {
it('continues admitting peers when one development server is unavailable', async () => {
const firstServer = readyServer({ name: 'first-server', auth: { type: 'none' } })
const failedServer = readyServer({ name: 'failed-server', auth: { type: 'none' } })
const thirdServer = readyServer({ name: 'third-server', auth: { type: 'none' } })
const addServer = appliedAdmission(async server => {
if (server.name === failedServer.name) {
throw new Error('MCP connect failed')
}
})
await expect(
admitDevelopmentMcpServers([firstServer, failedServer, thirdServer], { addServer })
).resolves.toBeUndefined()
expect(addServer).toHaveBeenCalledTimes(3)
expect(addServer).toHaveBeenNthCalledWith(1, firstServer)
expect(addServer).toHaveBeenNthCalledWith(2, failedServer)
expect(addServer).toHaveBeenNthCalledWith(3, thirdServer)
})
})
describe('runAuthoritativeMcpInitialization', () => {
it('obtains an authoritative snapshot before replacing the current fleet', async () => {
const effects: string[] = []
const client = {
healthCheck: vi.fn(async () => {
effects.push('ready')
return true
}),
listServersForHost: vi.fn(async () => {
effects.push('snapshot')
return []
}),
}
await runAuthoritativeMcpInitialization({
client,
replaceFleet: async servers => {
effects.push('replace')
expect(servers).toEqual([])
},
sleep: vi.fn(),
maxRetries: 2,
})
expect(effects).toEqual(['ready', 'snapshot', 'replace'])
})
it('preserves the prior fleet when authoritative discovery rejects', async () => {
const priorManager = { id: 'prior-manager' }
const priorState = new Map([['existing-server', 'existing-state']])
let currentManager = priorManager
let currentState = priorState
const discoveryError = new Error('HTTP 503: Service Unavailable')
const replaceFleet = vi.fn(async () => {
currentManager = { id: 'replacement-manager' }
currentState = new Map()
})
await expect(
runAuthoritativeMcpInitialization({
client: {
healthCheck: vi.fn().mockResolvedValue(true),
listServersForHost: vi.fn().mockRejectedValue(discoveryError),
},
replaceFleet,
sleep: vi.fn(),
maxRetries: 2,
})
).rejects.toBe(discoveryError)
expect(replaceFleet).not.toHaveBeenCalled()
expect(currentManager).toBe(priorManager)
expect(currentState).toBe(priorState)
})
it.each(['shutdown', 'a newer initialization'] as const)(
'drops delayed initial discovery after %s while the manager is still null',
async invalidation => {
const discoveryStarted = deferred()
const delayedDiscovery = deferred<McpServerInfo[]>()
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
let shuttingDown = false
let currentGeneration = 1
const initializationGeneration = currentGeneration
let currentManager: typeof candidateManager | null = null
const installFleet = vi.fn((manager: typeof candidateManager) => {
currentManager = manager
})
const startPolling = vi.fn()
const createManager = vi.fn(() => candidateManager)
const replaceFleet = vi.fn((servers: McpServerInfo[]) =>
replaceAuthoritativeMcpFleet({
servers,
previousManager: currentManager,
createManager,
getAuthToken: vi.fn(),
installFleet,
onColdStartPublished: startPolling,
isFleetLifecycleCurrent: () =>
!shuttingDown && currentGeneration === initializationGeneration,
})
)
const initialization = runAuthoritativeMcpInitialization({
client: {
healthCheck: vi.fn().mockResolvedValue(true),
listServersForHost: vi.fn(async () => {
discoveryStarted.resolve()
return delayedDiscovery.promise
}),
},
replaceFleet,
isCurrent: () => !shuttingDown && currentGeneration === initializationGeneration,
})
await discoveryStarted.promise
expect(currentManager).toBeNull()
if (invalidation === 'shutdown') {
shuttingDown = true
}
currentGeneration += 1
delayedDiscovery.resolve([readyServer({ auth: { type: 'none' } })])
await initialization
expect(replaceFleet).not.toHaveBeenCalled()
expect(createManager).not.toHaveBeenCalled()
expect(installFleet).not.toHaveBeenCalled()
expect(startPolling).not.toHaveBeenCalled()
expect(currentManager).toBeNull()
}
)
it('fails explicitly after readiness retry exhaustion without replacing the prior fleet', async () => {
const replaceFleet = vi.fn()
const sleep = vi.fn().mockResolvedValue(undefined)
const healthCheck = vi.fn().mockResolvedValue(false)
const listServersForHost = vi.fn()
await expect(
runAuthoritativeMcpInitialization({
client: { healthCheck, listServersForHost },
replaceFleet,
sleep,
maxRetries: 2,
})
).rejects.toThrow('Context Mapper was not ready after 2 attempts')
expect(healthCheck).toHaveBeenCalledTimes(2)
expect(sleep).toHaveBeenCalledTimes(1)
expect(listServersForHost).not.toHaveBeenCalled()
expect(replaceFleet).not.toHaveBeenCalled()
})
})
describe('createCoalescedPollRunner', () => {
it('coalesces overlapping ticks into one bounded trailing poll', async () => {
let resolveFirst!: () => void
let resolveSecond!: () => void
const poll = vi
.fn()
.mockImplementationOnce(() => new Promise<void>(resolve => (resolveFirst = resolve)))
.mockImplementationOnce(() => new Promise<void>(resolve => (resolveSecond = resolve)))
.mockResolvedValue(undefined)
const runner = createCoalescedPollRunner(poll)
runner.trigger()
runner.trigger()
runner.trigger()
expect(poll).toHaveBeenCalledTimes(1)
resolveFirst()
await vi.waitFor(() => expect(poll).toHaveBeenCalledTimes(2))
resolveSecond()
await vi.waitFor(() => expect(poll).toHaveBeenCalledTimes(2))
runner.trigger()
await vi.waitFor(() => expect(poll).toHaveBeenCalledTimes(3))
})
it('drops a pending trailing poll after stop', async () => {
let resolveFirst!: () => void
const poll = vi
.fn()
.mockImplementationOnce(() => new Promise<void>(resolve => (resolveFirst = resolve)))
const runner = createCoalescedPollRunner(poll)
runner.trigger()
runner.trigger()
runner.stop()
resolveFirst()
await Promise.resolve()
await Promise.resolve()
expect(poll).toHaveBeenCalledTimes(1)
})
})
describe('AuthoritativeMcpFleetCoordinator scheduling', () => {
it('prunes obsolete unique-name admissions while the global permit is saturated', async () => {
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const manager = {}
const blocker = readyServer({ name: 'blocker', auth: { type: 'none' } })
const blockerLease = coordinator.publishSnapshot(manager, [blocker])
const blockerStarted = deferred()
const releaseBlocker = deferred()
const blockerRun = coordinator.runAdmission(manager, blocker.name, blockerLease, async () => {
blockerStarted.resolve()
await releaseBlocker.promise
})
await blockerStarted.promise
const executed: string[] = []
const scheduled: Array<Promise<void>> = []
for (let revision = 0; revision < 50; revision += 1) {
const server = readyServer({
name: `churn-${revision}`,
auth: { type: 'none' },
})
const lease = coordinator.publishSnapshot(manager, [blocker, server])
scheduled.push(
coordinator.runAdmission(manager, server.name, lease, async isCurrent => {
expect(isCurrent()).toBe(true)
executed.push(server.name)
})
)
}
const scheduler = coordinator as unknown as {
admissionWaiters: unknown[]
admissions: WeakMap<object, Map<string, unknown>>
admissionTasks: Set<Promise<void>>
}
const queuedBeforeRelease = scheduler.admissionWaiters.length
const slotsBeforeRelease = scheduler.admissions.get(manager)?.size ?? 0
const tasksBeforeRelease = scheduler.admissionTasks.size
await Promise.all(scheduled.slice(0, -1))
expect(executed).toEqual([])
releaseBlocker.resolve()
await Promise.all([blockerRun, ...scheduled])
expect({
queuedBeforeRelease,
slotsBeforeRelease,
tasksBeforeRelease,
}).toEqual({
queuedBeforeRelease: 1,
slotsBeforeRelease: 2,
tasksBeforeRelease: 2,
})
expect(executed).toEqual(['churn-49'])
})
it('replaces stale queued epochs with only the latest admission per server', async () => {
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const manager = {}
const blocker = readyServer({ name: 'blocker', auth: { type: 'none' } })
const blockerLease = coordinator.publishSnapshot(manager, [blocker])
const blockerStarted = deferred()
const releaseBlocker = deferred()
const blockerRun = coordinator.runAdmission(manager, blocker.name, blockerLease, async () => {
blockerStarted.resolve()
await releaseBlocker.promise
})
await blockerStarted.promise
const executedRevisions: number[] = []
const scheduled: Array<Promise<void>> = []
for (let revision = 0; revision < 25; revision += 1) {
const target = readyServer({
name: 'target',
auth: { type: 'none' },
transport: {
type: 'streamableHttp',
url: `http://target.test/revision-${revision}`,
},
})
const lease = coordinator.publishSnapshot(manager, [blocker, target])
scheduled.push(
coordinator.runAdmission(manager, target.name, lease, async isCurrent => {
expect(isCurrent()).toBe(true)
executedRevisions.push(revision)
})
)
}
expect(executedRevisions).toEqual([])
releaseBlocker.resolve()
await Promise.all([blockerRun, ...scheduled])
expect(executedRevisions).toEqual([24])
})
it('does not let a delayed poll publish authority after manager shutdown', async () => {
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const manager = {}
const server = readyServer({ name: 'closed-server', auth: { type: 'none' } })
coordinator.publishSnapshot(manager, [server])
const delayedSnapshot = deferred<McpServerInfo[]>()
const admission = vi.fn()
const delayedPoll = delayedSnapshot.promise.then(async servers => {
const delayedLease = coordinator.publishSnapshot(manager, servers)
await coordinator.runAdmission(manager, server.name, delayedLease, admission)
})
coordinator.closeManager(manager)
delayedSnapshot.resolve([server])
await delayedPoll
expect(admission).not.toHaveBeenCalled()
})
it('abandons a hung cleanup permit so later cleanup can progress', async () => {
vi.useFakeTimers()
try {
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const neverSettles = new Promise<void>(() => {})
const laterCleanup = vi.fn().mockResolvedValue(undefined)
coordinator.scheduleCleanup(() => neverSettles)
coordinator.scheduleCleanup(laterCleanup)
await Promise.resolve()
expect(laterCleanup).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(30_000)
await vi.waitFor(() => expect(laterCleanup).toHaveBeenCalledTimes(1))
await coordinator.drainCleanups()
} finally {
vi.useRealTimers()
}
})
it('observes manager-scheduled cleanup rejection and drains the tracked task', async () => {
const { McpClient } = await import('../mcp/client')
const error = new Error('cleanup rejected')
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const connect = vi.spyOn(McpClient.prototype, 'connect').mockResolvedValue(undefined)
const retire = vi.spyOn(McpClient.prototype, 'retire').mockReturnValue(async () => {
throw error
})
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const manager = new McpManager()
const server = readyServer({ name: 'cleanup-error-server', auth: { type: 'none' } })
await manager.addServer(server)
await manager.close(cleanup => coordinator.scheduleCleanup(cleanup))
await coordinator.drainCleanups()
expect(errorSpy).toHaveBeenCalledWith('[Main] MCP detached client cleanup failed:', error)
expect(retire).toHaveBeenCalledTimes(1)
connect.mockRestore()
retire.mockRestore()
errorSpy.mockRestore()
})
it('waits for a closing admission to schedule its late cleanup before shutdown drain returns', async () => {
const { McpClient } = await import('../mcp/client')
const connectStarted = deferred()
const releaseConnect = deferred()
const connect = vi.spyOn(McpClient.prototype, 'connect').mockImplementation(async function (
this: any
) {
connectStarted.resolve()
await releaseConnect.promise
this.connected = true
this.tools = [{ name: 'late-tool', inputSchema: {}, serverName: 'late-server' }]
})
const retire = vi.spyOn(McpClient.prototype, 'retire').mockReturnValue(async () => undefined)
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true)
const manager = new McpManager()
const server = readyServer({ name: 'late-server', auth: { type: 'none' } })
const lease = coordinator.publishSnapshot(manager, [server])
const admission = coordinator.runAdmission(manager, server.name, lease, async isCurrent => {
await manager.addServer(server, undefined, {
isCurrent,
onCommit: vi.fn(),
scheduleCleanup: cleanup => coordinator.scheduleCleanup(cleanup),
})
})
await connectStarted.promise
coordinator.closeManager(manager)
await manager.close(cleanup => coordinator.scheduleCleanup(cleanup))
let drainReturned = false
const drain = coordinator.drainForShutdown().then(() => {
drainReturned = true
})
await Promise.resolve()
expect(drainReturned).toBe(false)
releaseConnect.resolve()
await Promise.all([admission, drain])
expect(connect).toHaveBeenCalledTimes(1)
expect(retire).toHaveBeenCalledTimes(1)
expect(manager.getConnectedServers()).toEqual([])
expect(manager.getKnownServers()).toEqual([])
connect.mockRestore()
retire.mockRestore()
})
it('bounds shutdown admission drain when transport connect never settles', async () => {
vi.useFakeTimers()
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
const coordinator = new AuthoritativeMcpFleetCoordinator(1, 1, true, 30_000, 25)
const manager = {}
const server = readyServer({ name: 'never-settles', auth: { type: 'none' } })
const lease = coordinator.publishSnapshot(manager, [server])
void coordinator.runAdmission(manager, server.name, lease, () => new Promise<void>(() => {}))
await Promise.resolve()
coordinator.closeManager(manager)
const drain = coordinator.drainForShutdown()
await vi.advanceTimersByTimeAsync(25)
await drain
expect(warnSpy).toHaveBeenCalledWith(
'[Main] MCP admission drain exceeded 25ms; abandoning transport wait'
)
} finally {
warnSpy.mockRestore()
vi.useRealTimers()
}
})
})
describe('authoritative poll publication fence', () => {
it('drops a delayed snapshot when shutdown or a manager swap makes the poll stale', async () => {
const pollStarted = deferred()
const delayedSnapshot = deferred<{ servers: McpServerInfo[] }>()
let current = true
const reconcile = vi.fn().mockResolvedValue(undefined)
const poll = pollAuthoritativeMcpSnapshotIfCurrent({
poll: async () => {
pollStarted.resolve()
return delayedSnapshot.promise
},
isCurrent: () => current,
reconcile,
})
await pollStarted.promise
current = false
delayedSnapshot.resolve({
servers: [readyServer({ name: 'stale-post-fetch-server' })],
})
await poll
expect(reconcile).not.toHaveBeenCalled()
})
})
describe('context-mapper polling lifecycle', () => {
it('wires the live manager and configured staleness ceiling into poll failure handling', () => {
const manager = {}
const lastSuccessAt = 1_000
const staleNow = lastSuccessAt + config.hccAuthorityMaxStalenessMs
const revoke = vi.fn()
const onCallerAuthorizationRejected = vi.fn()
const onInventoryAuthorityRevoked = vi.fn()
const onUnavailable = vi.fn()
const options = createMcpAuthorityPollFailureOptions({
getManager: () => manager,
lastSuccessAt: () => lastSuccessAt,
now: () => staleNow,
revoke,
onCallerAuthorizationRejected,
onInventoryAuthorityRevoked,
onUnavailable,
})
expect(options.maxStalenessMs).toBe(config.hccAuthorityMaxStalenessMs)
const disposition = handleMcpAuthorityPollFailure(
new ContextMapperRequestError(503, 'inventory', false),
options
)
expect(disposition).toBe('authority_stale')
expect(revoke).toHaveBeenCalledWith('authority_stale', true)
expect(onUnavailable).toHaveBeenCalledTimes(1)
expect(onCallerAuthorizationRejected).not.toHaveBeenCalled()
expect(onInventoryAuthorityRevoked).not.toHaveBeenCalled()
const noManagerRevoke = vi.fn()
const noManagerDisposition = handleMcpAuthorityPollFailure(
new ContextMapperRequestError(503, 'inventory', false),
createMcpAuthorityPollFailureOptions({
getManager: () => null,
lastSuccessAt: () => lastSuccessAt,
now: () => staleNow,
revoke: noManagerRevoke,
onCallerAuthorizationRejected: vi.fn(),
onInventoryAuthorityRevoked: vi.fn(),
onUnavailable: vi.fn(),
})
)
expect(noManagerDisposition).toBe('unavailable')
expect(noManagerRevoke).not.toHaveBeenCalled()
})
it('keeps exactly one interval producer when polling is started again', () => {
vi.useFakeTimers()
try {
startContextMapperPolling()
expect(vi.getTimerCount()).toBe(1)
startContextMapperPolling()
expect(vi.getTimerCount()).toBe(1)
} finally {
stopContextMapperPolling()
vi.useRealTimers()
}
})
it('bounds transient authority preservation to the configured staleness window', () => {
expect(isMcpAuthorityStale(1_000, 60_999, 60_000)).toBe(false)
expect(isMcpAuthorityStale(1_000, 61_000, 60_000)).toBe(true)
expect(isMcpAuthorityStale(0, 1_000_000, 60_000)).toBe(false)
expect(isMcpAuthorityStale(1_000, 61_000, 0)).toBe(false)
})
it('revokes at the absolute staleness deadline without waiting for another poll', () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-08-16T00:00:00.000Z'))
try {
const expired = vi.fn()
const deadline = createMcpAuthorityStalenessDeadline(60_000, expired)
deadline.recordSuccess()
vi.advanceTimersByTime(59_999)
expect(expired).not.toHaveBeenCalled()
vi.advanceTimersByTime(1)
expect(expired).toHaveBeenCalledTimes(1)
} finally {
vi.clearAllTimers()
vi.useRealTimers()
}
})
it('moves the absolute deadline forward only after a newer authoritative success', () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-08-16T00:00:00.000Z'))
try {
const expired = vi.fn()
const deadline = createMcpAuthorityStalenessDeadline(60_000, expired)
deadline.recordSuccess()
vi.advanceTimersByTime(50_000)
deadline.recordSuccess()
vi.advanceTimersByTime(10_000)
expect(expired).not.toHaveBeenCalled()
vi.advanceTimersByTime(50_000)
expect(expired).toHaveBeenCalledTimes(1)
} finally {
vi.clearAllTimers()
vi.useRealTimers()
}
})
it('cancels the absolute deadline when authority is withdrawn explicitly', () => {
vi.useFakeTimers()
try {
const expired = vi.fn()
const deadline = createMcpAuthorityStalenessDeadline(60_000, expired)
deadline.recordSuccess()
deadline.clear()
vi.advanceTimersByTime(60_000)
expect(expired).not.toHaveBeenCalled()
} finally {
vi.clearAllTimers()
vi.useRealTimers()
}
})
})
describe('replaceAuthoritativeMcpFleet', () => {
it('publishes a cold manager and healthy peer before a slow peer finishes admission', async () => {
const slowServer = readyServer({ name: 'slow-server', auth: { type: 'none' } })
const healthyServer = readyServer({ name: 'healthy-server', auth: { type: 'none' } })
const slowStarted = deferred()
const releaseSlow = deferred()
const connected = new Set<string>()
const initializationGeneration = 1
const currentGeneration = initializationGeneration
const candidateManager = {
addServer: appliedAdmission(async (server: McpServerInfo) => {
if (server.name === slowServer.name) {
slowStarted.resolve()
await releaseSlow.promise
}
connected.add(server.name)
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
let installedManager: typeof candidateManager | undefined
let installedState: Map<string, string> | undefined
const initialization = replaceAuthoritativeMcpFleet({
servers: [slowServer, healthyServer],
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
maxConcurrency: 2,
installFleet: (manager, serverState) => {
installedManager = manager
installedState = serverState
},
isFleetLifecycleCurrent: () => currentGeneration === initializationGeneration,
})
try {
await slowStarted.promise
await Promise.resolve()
await Promise.resolve()
expect(installedManager).toBe(candidateManager)
expect(candidateManager.addServer).toHaveBeenCalledTimes(2)
expect(connected).toContain(healthyServer.name)
expect(installedState?.get(healthyServer.name)).toBe(JSON.stringify(healthyServer))
expect(connected).not.toContain(slowServer.name)
expect(installedState?.has(slowServer.name)).toBe(false)
} finally {
releaseSlow.resolve()
await initialization
}
expect(connected).toEqual(new Set([healthyServer.name, slowServer.name]))
expect(installedState?.get(slowServer.name)).toBe(JSON.stringify(slowServer))
})
it('bounds concurrent cold-start admission without serializing the fleet', async () => {
const servers = Array.from({ length: 5 }, (_, index) =>
readyServer({ name: `server-${index}`, auth: { type: 'none' } })
)
const firstAdmissionStarted = deferred()
const releaseAdmissions = deferred()
let active = 0
let maxActive = 0
const candidateManager = {
addServer: appliedAdmission(async () => {
active += 1
maxActive = Math.max(maxActive, active)
firstAdmissionStarted.resolve()
await releaseAdmissions.promise
active -= 1
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const initialization = replaceAuthoritativeMcpFleet({
servers,
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
maxConcurrency: 2,
installFleet: vi.fn(),
})
try {
await firstAdmissionStarted.promise
await Promise.resolve()
await Promise.resolve()
expect(candidateManager.addServer).toHaveBeenCalledTimes(2)
expect(maxActive).toBe(2)
} finally {
releaseAdmissions.resolve()
await initialization
}
expect(candidateManager.addServer).toHaveBeenCalledTimes(servers.length)
expect(maxActive).toBe(2)
})
it('rejects invalid cold-start concurrency before publishing a candidate', async () => {
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const createManager = vi.fn(() => candidateManager)
const installFleet = vi.fn()
await expect(
replaceAuthoritativeMcpFleet({
servers: [readyServer({ auth: { type: 'none' } })],
previousManager: null,
createManager,
getAuthToken: vi.fn(),
installFleet,
maxConcurrency: 0,
})
).rejects.toThrow('MCP fleet reconciliation concurrency must be a positive integer')
expect(createManager).not.toHaveBeenCalled()
expect(installFleet).not.toHaveBeenCalled()
expect(candidateManager.addServer).not.toHaveBeenCalled()
expect(candidateManager.disconnectAll).not.toHaveBeenCalled()
})
it('keeps polling live after cold publish and prevents revoked in-flight peers from resurfacing', async () => {
const slowServer = readyServer({ name: 'slow-server', auth: { type: 'none' } })
const healthyServer = readyServer({ name: 'healthy-server', auth: { type: 'none' } })
const slowStarted = deferred()
const releaseSlow = deferred()
const connected = new Set<string>()
const known = new Set<string>()
const coordinator = new AuthoritativeMcpFleetCoordinator(2, 2, true)
const candidateManager = {
addServer: vi.fn(
async (
server: McpServerInfo,
_authToken?: string,
control?: { isCurrent(): boolean; onCommit(): void }
) => {
if (server.name === slowServer.name) {
slowStarted.resolve()
await releaseSlow.promise
}
if (control && !control.isCurrent()) return 'stale'
connected.add(server.name)
known.add(server.name)
control?.onCommit()
return 'applied'
}
),
replaceServer: appliedAdmission(),
removeServer: vi.fn(async (name: string) => {
connected.delete(name)
known.delete(name)
}),
detachServer: vi.fn((name: string) => {
connected.delete(name)
known.delete(name)
return vi.fn().mockResolvedValue(undefined)
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => [...connected]),
getKnownServers: vi.fn(() => [...known]),
recordAdmissionFailure: vi.fn(),
}
let installedState: Map<string, string> | undefined
const pollingStarted = vi.fn()
let initializationSettled = false
const initialization = replaceAuthoritativeMcpFleet({
servers: [slowServer, healthyServer],
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet: (_manager, serverState) => {
installedState = serverState
},
coordinator,
onColdStartPublished: pollingStarted,
maxConcurrency: 2,
}).finally(() => {
initializationSettled = true
})
await slowStarted.promise
await vi.waitFor(() => expect(connected).toContain(healthyServer.name))
expect(pollingStarted).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(initializationSettled).toBe(true))
await reconcileAuthoritativeMcpSnapshot({
servers: [slowServer, healthyServer],
manager: candidateManager,
serverState: installedState!,
getAuthToken: vi.fn(),
coordinator,
maxConcurrency: 2,
})
await reconcileAuthoritativeMcpSnapshot({
servers: [slowServer],
manager: candidateManager,
serverState: installedState!,
getAuthToken: vi.fn(),
coordinator,
maxConcurrency: 2,
})
expect(connected).toEqual(new Set())
expect(installedState).toEqual(new Map())
releaseSlow.resolve()
await initialization
expect(connected).toEqual(new Set([slowServer.name]))
expect(known).toEqual(new Set([slowServer.name]))
expect(installedState).toEqual(new Map([[slowServer.name, JSON.stringify(slowServer)]]))
})
it('coalesces an identical poll onto a cold admission and retries after its failure', async () => {
const server = readyServer({ name: 'retry-server', auth: { type: 'none' } })
const firstStarted = deferred()
const releaseFirst = deferred()
const coordinator = new AuthoritativeMcpFleetCoordinator(2, 2, true)
let attempt = 0
const known = new Set<string>()
const manager = {
addServer: vi.fn(
async (
desired: McpServerInfo,
_authToken?: string,
control?: { isCurrent(): boolean; onCommit(): void }
) => {
attempt += 1
if (attempt === 1) {
firstStarted.resolve()
await releaseFirst.promise
throw new Error('first connect failed')
}
if (control && !control.isCurrent()) return 'stale'
known.add(desired.name)
control?.onCommit()
return 'applied'
}
),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => [...known]),
recordAdmissionFailure: vi.fn((desired: McpServerInfo) => known.add(desired.name)),
}
let state: Map<string, string> | undefined
const initialization = replaceAuthoritativeMcpFleet({
servers: [server],
previousManager: null,
createManager: () => manager,
getAuthToken: vi.fn(),
installFleet: (_manager, serverState) => {
state = serverState
},
coordinator,
})
await firstStarted.promise
for (let poll = 0; poll < 25; poll += 1) {
await reconcileAuthoritativeMcpSnapshot({
servers: [server],
manager,
serverState: state!,
getAuthToken: vi.fn(),
coordinator,
})
}
expect(manager.addServer).toHaveBeenCalledTimes(1)
releaseFirst.resolve()
await initialization
await vi.waitFor(() =>
expect(
(
coordinator as unknown as {
admissionTasks: Set<Promise<void>>
}
).admissionTasks.size
).toBe(0)
)
expect(state).toEqual(new Map())
await reconcileAuthoritativeMcpSnapshot({
servers: [server],
manager,
serverState: state!,
getAuthToken: vi.fn(),
coordinator,
})
await vi.waitFor(() => expect(manager.addServer).toHaveBeenCalledTimes(2))
expect(state?.get(server.name)).toBe(JSON.stringify(server))
})
it('does not publish an auth failure after a newer snapshot deletes the server', async () => {
const server = readyServer({ name: 'stale-auth-server' })
const authStarted = deferred()
const rejectAuth = deferred<string | undefined>()
const coordinator = new AuthoritativeMcpFleetCoordinator(2, 2, true)
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
let state: Map<string, string> | undefined
const initialization = replaceAuthoritativeMcpFleet({
servers: [server],
previousManager: null,
createManager: () => manager,
getAuthToken: vi.fn(async () => {
authStarted.resolve()
return rejectAuth.promise
}),
installFleet: (_manager, serverState) => {
state = serverState
},
coordinator,
})
await authStarted.promise
await reconcileAuthoritativeMcpSnapshot({
servers: [],
manager,
serverState: state!,
getAuthToken: vi.fn(),
coordinator,
})
rejectAuth.reject(new Error('stale auth failure'))
await initialization
expect(manager.recordAdmissionFailure).not.toHaveBeenCalled()
expect(manager.addServer).not.toHaveBeenCalled()
expect(state).toEqual(new Map())
})
it('removes a failed-only server when the next authoritative snapshot deletes it', async () => {
const failedServer = readyServer({ name: 'failed-only-server', auth: { type: 'none' } })
const connectError = new Error('MCP connect failed')
const knownServers = new Set<string>()
const candidateManager = {
addServer: vi.fn(async (server: McpServerInfo) => {
knownServers.add(server.name)
throw connectError
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn(async (serverName: string) => {
knownServers.delete(serverName)
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => [...knownServers]),
recordAdmissionFailure: vi.fn((server: McpServerInfo) => {
knownServers.add(server.name)
}),
}
let installedState: Map<string, string> | undefined
await replaceAuthoritativeMcpFleet({
servers: [failedServer],
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet: (_manager, serverState) => {
installedState = serverState
},
})
expect(installedState).toEqual(new Map())
expect(candidateManager.getKnownServers()).toEqual([failedServer.name])
await reconcileAuthoritativeMcpSnapshot({
servers: [],
manager: candidateManager,
serverState: installedState!,
getAuthToken: vi.fn(),
})
expect(candidateManager.removeServer).toHaveBeenCalledWith(failedServer.name)
expect(candidateManager.getKnownServers()).toEqual([])
})
it('publishes healthy servers, retains the failed admission status, and leaves its revision retryable', async () => {
const firstServer = readyServer({ name: 'first-server', auth: { type: 'none' } })
const failedServer = readyServer({ name: 'failed-server', auth: { type: 'none' } })
const thirdServer = readyServer({ name: 'third-server', auth: { type: 'none' } })
const connectError = new Error('MCP connect failed')
const candidateManager = {
addServer: appliedAdmission(async server => {
if (server.name === failedServer.name) throw connectError
}),
disconnectAll: vi.fn(),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
await replaceAuthoritativeMcpFleet({
servers: [firstServer, failedServer, thirdServer],
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet,
})
expect(candidateManager.addServer).toHaveBeenCalledTimes(3)
// addServer owns connection-failure status; the fleet helper must not
// duplicate that transition.
expect(candidateManager.recordAdmissionFailure).not.toHaveBeenCalled()
expect(installFleet).toHaveBeenCalledWith(
candidateManager,
new Map([
[firstServer.name, JSON.stringify(firstServer)],
[thirdServer.name, JSON.stringify(thirdServer)],
])
)
expect(candidateManager.disconnectAll).not.toHaveBeenCalled()
})
it('publishes a failed auth status without recording its revision', async () => {
const authError = new Error('HTTP 503: Service Unavailable')
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
const server = readyServer()
await replaceAuthoritativeMcpFleet({
servers: [server],
previousManager: null,
createManager: () => candidateManager,
getAuthToken: vi.fn().mockRejectedValue(authError),
installFleet,
})
expect(candidateManager.addServer).not.toHaveBeenCalled()
expect(candidateManager.recordAdmissionFailure).toHaveBeenCalledWith(server, authError)
expect(installFleet).toHaveBeenCalledWith(candidateManager, new Map())
expect(candidateManager.disconnectAll).not.toHaveBeenCalled()
})
it('preserves an existing fleet when a replacement candidate has an admission failure', async () => {
const connectError = new Error('MCP connect failed')
const firstServer = readyServer({ name: 'first-server', auth: { type: 'none' } })
const failingServer = readyServer({ name: 'failing-server', auth: { type: 'none' } })
const previousManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
}
const candidateManager = {
addServer: appliedAdmission(async server => {
if (server.name === failingServer.name) throw connectError
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
await expect(
replaceAuthoritativeMcpFleet({
servers: [firstServer, failingServer],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet,
})
).rejects.toBe(connectError)
expect(candidateManager.addServer).toHaveBeenNthCalledWith(
1,
firstServer,
undefined,
expect.any(Object)
)
expect(candidateManager.addServer).toHaveBeenNthCalledWith(
2,
failingServer,
undefined,
expect.any(Object)
)
expect(candidateManager.recordAdmissionFailure).not.toHaveBeenCalled()
expect(installFleet).not.toHaveBeenCalled()
expect(candidateManager.disconnectAll).toHaveBeenCalledTimes(1)
expect(previousManager.disconnectAll).not.toHaveBeenCalled()
})
it('rejects a replacement candidate superseded by a newer live snapshot', async () => {
const previousServer = readyServer({ name: 'previous-server', auth: { type: 'none' } })
const candidateServer = readyServer({ name: 'candidate-server', auth: { type: 'none' } })
const newerServer = readyServer({ name: 'newer-server', auth: { type: 'none' } })
const previousManager = { disconnectAll: vi.fn().mockResolvedValue(undefined) }
const coordinator = new AuthoritativeMcpFleetCoordinator(2, 2, true)
coordinator.publishSnapshot(previousManager, [previousServer])
const candidateStarted = deferred()
const releaseCandidate = deferred()
const candidateManager = {
addServer: appliedAdmission(async () => {
candidateStarted.resolve()
await releaseCandidate.promise
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
const replacement = replaceAuthoritativeMcpFleet({
servers: [candidateServer],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet,
coordinator,
})
await candidateStarted.promise
coordinator.publishSnapshot(previousManager, [newerServer])
releaseCandidate.resolve()
await expect(replacement).rejects.toThrow(
'MCP fleet replacement candidate was superseded before commit'
)
expect(installFleet).not.toHaveBeenCalled()
expect(candidateManager.disconnectAll).toHaveBeenCalledTimes(1)
expect(previousManager.disconnectAll).not.toHaveBeenCalled()
})
it('commits a replacement candidate across an identical live poll', async () => {
const previousServer = readyServer({ name: 'previous-server', auth: { type: 'none' } })
const candidateServer = readyServer({ name: 'candidate-server', auth: { type: 'none' } })
const previousManager = { disconnectAll: vi.fn().mockResolvedValue(undefined) }
const coordinator = new AuthoritativeMcpFleetCoordinator(2, 2, true)
coordinator.publishSnapshot(previousManager, [previousServer])
const candidateStarted = deferred()
const releaseCandidate = deferred()
const candidateManager = {
addServer: appliedAdmission(async () => {
candidateStarted.resolve()
await releaseCandidate.promise
}),
disconnectAll: vi.fn().mockResolvedValue(undefined),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
const replacement = replaceAuthoritativeMcpFleet({
servers: [candidateServer],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet,
coordinator,
})
await candidateStarted.promise
coordinator.publishSnapshot(previousManager, [previousServer])
releaseCandidate.resolve()
await replacement
expect(installFleet).toHaveBeenCalledWith(
candidateManager,
new Map([[candidateServer.name, JSON.stringify(candidateServer)]])
)
expect(candidateManager.disconnectAll).not.toHaveBeenCalled()
expect(previousManager.disconnectAll).toHaveBeenCalledTimes(1)
})
it('installs an HTTP 200 empty fleet before retiring the prior fleet', async () => {
const effects: string[] = []
const previousManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn(async () => {
effects.push('retire-prior')
}),
}
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn(),
recordAdmissionFailure: vi.fn(),
}
await replaceAuthoritativeMcpFleet({
servers: [],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet: (manager, serverState) => {
effects.push('install')
expect(manager).toBe(candidateManager)
expect(serverState).toEqual(new Map())
},
})
expect(effects).toEqual(['install', 'retire-prior'])
})
it('records an intentional not-ready admission without treating it as a hard failure', async () => {
const notReady = readyServer({
status: { deployed: true, ready: false, message: 'Deployment is progressing' },
})
const previousManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
}
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn(),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
await replaceAuthoritativeMcpFleet({
servers: [notReady],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(() => {
throw new Error('auth must not be fetched until the server is ready for admission')
}),
installFleet,
})
expect(candidateManager.addServer).toHaveBeenCalledWith(notReady, undefined, expect.any(Object))
expect(installFleet).toHaveBeenCalledWith(
candidateManager,
new Map([[notReady.name, JSON.stringify(notReady)]])
)
expect(previousManager.disconnectAll).toHaveBeenCalledTimes(1)
})
it('records but never admits an explicitly non-authoritative ready server', async () => {
const nonAuthoritative = readyServer({
status: {
deployed: true,
ready: true,
authoritative: false,
message: 'Status identity could not be verified',
},
})
const previousManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockResolvedValue(undefined),
}
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn(),
recordAdmissionFailure: vi.fn(),
}
const getAuthToken = vi.fn()
const installFleet = vi.fn()
await replaceAuthoritativeMcpFleet({
servers: [nonAuthoritative],
previousManager,
createManager: () => candidateManager,
getAuthToken,
installFleet,
})
expect(getAuthToken).not.toHaveBeenCalled()
expect(candidateManager.addServer).toHaveBeenCalledWith(
nonAuthoritative,
undefined,
expect.any(Object)
)
expect(installFleet).toHaveBeenCalledWith(
candidateManager,
new Map([[nonAuthoritative.name, JSON.stringify(nonAuthoritative)]])
)
})
it('publishes a real not-ready health entry for a non-authoritative cold snapshot', async () => {
const nonAuthoritative = readyServer({
status: {
deployed: true,
ready: true,
authoritative: false,
message: 'Status identity could not be verified',
},
})
let installedManager: McpManager | undefined
await replaceAuthoritativeMcpFleet({
servers: [nonAuthoritative],
previousManager: null,
createManager: () => new McpManager(),
getAuthToken: vi.fn(),
installFleet: manager => {
installedManager = manager
},
})
expect(installedManager?.getConnectedServers()).toEqual([])
expect(installedManager?.status.snapshot()).toEqual([
expect.objectContaining({
name: nonAuthoritative.name,
state: 'failed',
expected: true,
reason: 'not_ready',
message: 'Status identity could not be verified',
}),
])
})
it('keeps the committed fleet when retiring the prior manager fails', async () => {
const cleanupError = new Error('disconnect failed')
const previousManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn().mockRejectedValue(cleanupError),
}
const candidateManager = {
addServer: appliedAdmission(),
disconnectAll: vi.fn(),
recordAdmissionFailure: vi.fn(),
}
const installFleet = vi.fn()
await expect(
replaceAuthoritativeMcpFleet({
servers: [],
previousManager,
createManager: () => candidateManager,
getAuthToken: vi.fn(),
installFleet,
})
).resolves.toBeUndefined()
expect(installFleet).toHaveBeenCalledWith(candidateManager, new Map())
expect(previousManager.disconnectAll).toHaveBeenCalledTimes(1)
expect(candidateManager.disconnectAll).not.toHaveBeenCalled()
})
})
describe('reconcileAuthoritativeMcpSnapshot', () => {
it('detaches every deletion synchronously while bounding deferred cleanup', async () => {
const deletedServers = Array.from({ length: 3 }, (_, index) =>
readyServer({ name: `deleted-${index}`, auth: { type: 'none' } })
)
const connected = new Set(deletedServers.map(server => server.name))
const known = new Set(connected)
const serverState = new Map(deletedServers.map(server => [server.name, JSON.stringify(server)]))
const releaseCleanup = deferred()
let activeCleanup = 0
let maxActiveCleanup = 0
const cleanupStarted = vi.fn()
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
detachServer: vi.fn((name: string) => {
connected.delete(name)
known.delete(name)
return async () => {
activeCleanup += 1
maxActiveCleanup = Math.max(maxActiveCleanup, activeCleanup)
cleanupStarted()
await releaseCleanup.promise
activeCleanup -= 1
}
}),
getConnectedServers: vi.fn(() => [...connected]),
getKnownServers: vi.fn(() => [...known]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [],
manager,
serverState,
getAuthToken: vi.fn(),
coordinator: new AuthoritativeMcpFleetCoordinator(2, 2, true),
})
expect(manager.detachServer).toHaveBeenCalledTimes(deletedServers.length)
expect(connected).toEqual(new Set())
expect(known).toEqual(new Set())
expect(serverState).toEqual(new Map())
expect(cleanupStarted).toHaveBeenCalledTimes(2)
expect(maxActiveCleanup).toBe(2)
releaseCleanup.resolve()
await vi.waitFor(() => expect(cleanupStarted).toHaveBeenCalledTimes(deletedServers.length))
expect(maxActiveCleanup).toBe(2)
})
it('prioritizes authoritative deletions when every admission worker is slow', async () => {
const slowServers = [
readyServer({ name: 'slow-server-1', auth: { type: 'none' } }),
readyServer({ name: 'slow-server-2', auth: { type: 'none' } }),
]
const deletedServer = readyServer({ name: 'deleted-server', auth: { type: 'none' } })
const serverState = new Map([[deletedServer.name, JSON.stringify(deletedServer)]])
const allAdmissionsStarted = deferred()
const releaseAdmissions = deferred()
let startedAdmissions = 0
const manager = {
addServer: appliedAdmission(async () => {
startedAdmissions += 1
if (startedAdmissions === slowServers.length) {
allAdmissionsStarted.resolve()
}
await releaseAdmissions.promise
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => [deletedServer.name]),
recordAdmissionFailure: vi.fn(),
}
const reconciliation = reconcileAuthoritativeMcpSnapshot({
servers: slowServers,
manager,
serverState,
getAuthToken: vi.fn(),
maxConcurrency: slowServers.length,
})
try {
await allAdmissionsStarted.promise
expect(manager.removeServer).toHaveBeenCalledWith(deletedServer.name)
expect(serverState.has(deletedServer.name)).toBe(false)
} finally {
releaseAdmissions.resolve()
await reconciliation
}
})
it('reconciles healthy peers and authoritative deletions while another peer is slow', async () => {
const slowServer = readyServer({ name: 'slow-server', auth: { type: 'none' } })
const healthyServer = readyServer({ name: 'healthy-server', auth: { type: 'none' } })
const deletedServer = readyServer({ name: 'deleted-server', auth: { type: 'none' } })
const serverState = new Map([[deletedServer.name, JSON.stringify(deletedServer)]])
const slowStarted = deferred()
const releaseSlow = deferred()
const knownServers = new Set([deletedServer.name])
const manager = {
addServer: appliedAdmission(async (server: McpServerInfo) => {
if (server.name === slowServer.name) {
slowStarted.resolve()
await releaseSlow.promise
}
knownServers.add(server.name)
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn(async (serverName: string) => {
knownServers.delete(serverName)
}),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => [...knownServers]),
recordAdmissionFailure: vi.fn(),
}
const reconciliation = reconcileAuthoritativeMcpSnapshot({
servers: [slowServer, healthyServer],
manager,
serverState,
getAuthToken: vi.fn(),
maxConcurrency: 2,
})
try {
await slowStarted.promise
await vi.waitFor(() => {
expect(manager.addServer).toHaveBeenCalledWith(healthyServer, undefined, expect.any(Object))
expect(manager.removeServer).toHaveBeenCalledWith(deletedServer.name)
})
expect(serverState.get(healthyServer.name)).toBe(JSON.stringify(healthyServer))
expect(serverState.has(deletedServer.name)).toBe(false)
expect(knownServers).not.toContain(deletedServer.name)
} finally {
releaseSlow.resolve()
await reconciliation
}
expect(serverState.get(slowServer.name)).toBe(JSON.stringify(slowServer))
})
it('bounds concurrent authoritative snapshot effects without serializing distinct servers', async () => {
const servers = Array.from({ length: 5 }, (_, index) =>
readyServer({ name: `snapshot-server-${index}`, auth: { type: 'none' } })
)
const firstEffectStarted = deferred()
const releaseEffects = deferred()
let active = 0
let maxActive = 0
const manager = {
addServer: appliedAdmission(async () => {
active += 1
maxActive = Math.max(maxActive, active)
firstEffectStarted.resolve()
await releaseEffects.promise
active -= 1
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
const reconciliation = reconcileAuthoritativeMcpSnapshot({
servers,
manager,
serverState: new Map(),
getAuthToken: vi.fn(),
maxConcurrency: 2,
})
try {
await firstEffectStarted.promise
await Promise.resolve()
await Promise.resolve()
expect(manager.addServer).toHaveBeenCalledTimes(2)
expect(maxActive).toBe(2)
} finally {
releaseEffects.resolve()
await reconciliation
}
expect(manager.addServer).toHaveBeenCalledTimes(servers.length)
expect(maxActive).toBe(2)
})
it('rejects invalid snapshot concurrency before reading or mutating manager state', async () => {
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
await expect(
reconcileAuthoritativeMcpSnapshot({
servers: [readyServer({ auth: { type: 'none' } })],
manager,
serverState: new Map(),
getAuthToken: vi.fn(),
maxConcurrency: 0,
})
).rejects.toThrow('MCP fleet reconciliation concurrency must be a positive integer')
expect(manager.getConnectedServers).not.toHaveBeenCalled()
expect(manager.getKnownServers).not.toHaveBeenCalled()
expect(manager.addServer).not.toHaveBeenCalled()
expect(manager.replaceServer).not.toHaveBeenCalled()
expect(manager.removeServer).not.toHaveBeenCalled()
})
it('connects a ready desired candidate before retiring the previous connection', async () => {
const previous = readyServer({ auth: { type: 'none' } })
const modified = readyServer({
auth: { type: 'none' },
transport: { type: 'streamableHttp', url: 'http://replacement.test/mcp' },
status: { deployed: true, ready: true, authoritative: true },
})
const connectError = new Error('replacement connect failed')
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(
vi.fn().mockRejectedValueOnce(connectError).mockResolvedValue(undefined)
),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
const options = {
servers: [modified],
manager,
serverState,
getAuthToken: vi.fn(),
}
await expect(reconcileAuthoritativeMcpSnapshot(options)).resolves.toBeUndefined()
expect(manager.replaceServer).toHaveBeenCalledWith(modified, undefined, expect.any(Object))
expect(manager.removeServer).not.toHaveBeenCalled()
expect(manager.addServer).not.toHaveBeenCalled()
expect(manager.recordAdmissionFailure).not.toHaveBeenCalled()
expect(serverState.get(previous.name)).toBe(JSON.stringify(previous))
await expect(reconcileAuthoritativeMcpSnapshot(options)).resolves.toBeUndefined()
expect(manager.replaceServer).toHaveBeenCalledTimes(2)
expect(serverState.get(modified.name)).toBe(JSON.stringify(modified))
})
it('revokes the old bearer before admitting a changed credential revision', async () => {
const previous = readyServer({
auth: undefined,
authRequired: true,
credentialRevision: 'credential-revision-1',
})
const rotated = readyServer({
auth: undefined,
authRequired: true,
credentialRevision: 'credential-revision-2',
})
const connected = new Set([previous.name])
const effects: string[] = []
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(async (_server, authToken) => {
effects.push(`replace:${authToken}`)
connected.add(rotated.name)
}),
removeServer: vi.fn(),
detachServer: vi.fn((name: string) => {
effects.push('detach')
connected.delete(name)
return vi.fn().mockResolvedValue(undefined)
}),
getConnectedServers: vi.fn(() => [...connected]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
const getAuthToken = vi.fn(async () => {
effects.push('credential')
return 'rotated-token'
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
await reconcileAuthoritativeMcpSnapshot({
servers: [rotated],
manager,
serverState,
getAuthToken,
})
expect(effects[0]).toBe('detach')
expect(getAuthToken).toHaveBeenCalledWith(rotated.name, 'credential-revision-2')
expect(effects).toEqual(['detach', 'credential', 'replace:rotated-token'])
expect(serverState.get(rotated.name)).toBe(JSON.stringify(rotated))
})
it('does not tear down a healthy connection for a status-only readiness degradation', async () => {
const previous = readyServer()
const degraded = readyServer({
status: {
deployed: false,
ready: false,
authoritative: false,
message: 'Deployment status unknown',
},
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [degraded],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(manager.removeServer).not.toHaveBeenCalled()
expect(manager.addServer).not.toHaveBeenCalled()
expect(serverState.get(previous.name)).toBe(JSON.stringify(degraded))
})
it('disconnects and marks not-ready when the false status is authoritative', async () => {
const effects: string[] = []
const previous = readyServer()
const notReady = readyServer({
status: {
deployed: true,
ready: false,
authoritative: true,
message: 'Secret validation failed',
},
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(async () => {
effects.push('mark-not-ready')
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn(async () => {
effects.push('remove')
}),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [notReady],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(effects).toEqual(['remove', 'mark-not-ready'])
expect(manager.addServer).toHaveBeenCalledWith(notReady, undefined, expect.any(Object))
expect(serverState.get(previous.name)).toBe(JSON.stringify(notReady))
})
it.each([
[
'disabled',
readyServer({
enabled: false,
status: { deployed: true, ready: true, authoritative: true },
}),
],
[
'authoritatively not-ready',
readyServer({
status: { deployed: true, ready: false, authoritative: true },
}),
],
// Absent `authoritative` must keep the legacy fail-closed contract on the
// synchronous revoke path too. Without this row the condition can be
// narrowed to `authoritative === true` and every other test still passes.
[
'not-ready without an authoritative field',
readyServer({
status: { deployed: false, ready: false, message: 'Deployment not ready' },
}),
],
])(
'synchronously detaches a connected %s server before asynchronous reconciliation',
async (_reason, next) => {
const previous = readyServer()
const detachServer = vi.fn(() => vi.fn().mockResolvedValue(undefined))
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn().mockResolvedValue(undefined),
detachServer,
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
const reconciliation = reconcileAuthoritativeMcpSnapshot({
servers: [next],
manager,
serverState: new Map([[previous.name, JSON.stringify(previous)]]),
getAuthToken: vi.fn(),
})
expect(detachServer).toHaveBeenCalledWith(previous.name)
await reconciliation
}
)
it('preserves the legacy fail-closed revocation contract when HCC omits authoritative', async () => {
const previous = readyServer()
const legacyNotReady = readyServer({
status: { deployed: false, ready: false, message: 'Deployment not ready' },
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [legacyNotReady],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(manager.removeServer).toHaveBeenCalledWith(previous.name)
expect(manager.addServer).toHaveBeenCalledWith(
legacyNotReady,
undefined,
expect.objectContaining({ isCurrent: expect.any(Function) })
)
expect(serverState.get(previous.name)).toBe(JSON.stringify(legacyNotReady))
})
it('admits an unconnected server when its observed readiness becomes true', async () => {
const notReady = readyServer({
auth: { type: 'none' },
status: { deployed: true, ready: false },
})
const ready = readyServer({
auth: { type: 'none' },
status: { deployed: true, ready: true, authoritative: true },
})
const serverState = new Map([[notReady.name, JSON.stringify(notReady)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => [notReady.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [ready],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(manager.removeServer).not.toHaveBeenCalled()
expect(manager.addServer).toHaveBeenCalledWith(ready, undefined, expect.any(Object))
expect(serverState.get(ready.name)).toBe(JSON.stringify(ready))
})
it('records but never admits a new explicitly non-authoritative ready snapshot', async () => {
const nonAuthoritative = readyServer({
status: {
deployed: true,
ready: true,
authoritative: false,
message: 'Status identity could not be verified',
},
})
const serverState = new Map<string, string>()
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
const getAuthToken = vi.fn()
await reconcileAuthoritativeMcpSnapshot({
servers: [nonAuthoritative],
manager,
serverState,
getAuthToken,
})
expect(getAuthToken).not.toHaveBeenCalled()
expect(manager.addServer).toHaveBeenCalledWith(nonAuthoritative, undefined, expect.any(Object))
expect(manager.removeServer).not.toHaveBeenCalled()
expect(serverState.get(nonAuthoritative.name)).toBe(JSON.stringify(nonAuthoritative))
})
it('retires a changed desired revision without admitting non-authoritative status', async () => {
const previous = readyServer()
const modified = readyServer({
transport: { type: 'streamableHttp', url: 'http://replacement.test/mcp' },
status: {
deployed: true,
ready: true,
authoritative: false,
message: 'Status identity could not be verified',
},
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
const getAuthToken = vi.fn()
await reconcileAuthoritativeMcpSnapshot({
servers: [modified],
manager,
serverState,
getAuthToken,
})
expect(manager.removeServer).toHaveBeenCalledWith(previous.name)
expect(getAuthToken).not.toHaveBeenCalled()
expect(manager.addServer).toHaveBeenCalledWith(modified, undefined, expect.any(Object))
expect(serverState.get(modified.name)).toBe(JSON.stringify(modified))
})
it('does not reconnect when desired objects differ only by key insertion order', async () => {
const current = readyServer({
auth: { type: 'bearer', secretRef: 'secured-server-auth', secretKey: 'token' },
})
const previousWithDifferentKeyOrder = {
enabled: true,
transport: {
url: 'http://secured-server.test/mcp',
type: 'streamableHttp',
},
contextRef: 'production',
name: 'secured-server',
auth: { secretKey: 'token', secretRef: 'secured-server-auth', type: 'bearer' },
status: { ready: true, deployed: true },
} as McpServerInfo
const serverState = new Map([[current.name, JSON.stringify(previousWithDifferentKeyOrder)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => [current.name]),
getKnownServers: vi.fn(() => [current.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [current],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(manager.replaceServer).not.toHaveBeenCalled()
expect(manager.removeServer).not.toHaveBeenCalled()
expect(manager.addServer).not.toHaveBeenCalled()
expect(serverState.get(current.name)).toBe(JSON.stringify(current))
})
it('retains reconnect semantics for a desired transport change', async () => {
const effects: string[] = []
const previous = readyServer({ auth: { type: 'none' } })
const modified = readyServer({
auth: { type: 'none' },
transport: { type: 'streamableHttp', url: 'http://replacement.test/mcp' },
status: {
deployed: false,
ready: false,
authoritative: true,
message: 'Deployment not ready',
},
})
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(async () => {
effects.push('add')
}),
replaceServer: appliedAdmission(),
removeServer: vi.fn(async () => {
effects.push('remove')
}),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [modified],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(effects).toEqual(['remove', 'add'])
expect(serverState.get(modified.name)).toBe(JSON.stringify(modified))
})
it('retains authoritative deletion semantics', async () => {
const previous = readyServer({ auth: { type: 'none' } })
const serverState = new Map([[previous.name, JSON.stringify(previous)]])
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn().mockResolvedValue(undefined),
getConnectedServers: vi.fn(() => [previous.name]),
getKnownServers: vi.fn(() => [previous.name]),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [],
manager,
serverState,
getAuthToken: vi.fn(),
})
expect(manager.removeServer).toHaveBeenCalledWith(previous.name)
expect(serverState.has(previous.name)).toBe(false)
})
it('does not record a failed connection twice and leaves its revision retryable', async () => {
const server = readyServer({ auth: { type: 'none' } })
const serverState = new Map<string, string>()
const connectError = new Error('MCP connect failed')
const manager = {
addServer: appliedAdmission(vi.fn().mockRejectedValueOnce(connectError)),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
const options = {
servers: [server],
manager,
serverState,
getAuthToken: vi.fn(),
}
await expect(reconcileAuthoritativeMcpSnapshot(options)).resolves.toBeUndefined()
expect(serverState.has(server.name)).toBe(false)
expect(manager.recordAdmissionFailure).not.toHaveBeenCalled()
await expect(reconcileAuthoritativeMcpSnapshot(options)).resolves.toBeUndefined()
expect(manager.addServer).toHaveBeenCalledTimes(2)
expect(serverState.get(server.name)).toBe(JSON.stringify(server))
})
it('records an auth-discovery failure once and leaves its revision retryable', async () => {
const server = readyServer()
const serverState = new Map<string, string>()
const authError = new Error('auth discovery failed')
const manager = {
addServer: appliedAdmission(),
replaceServer: appliedAdmission(),
removeServer: vi.fn(),
getConnectedServers: vi.fn(() => []),
getKnownServers: vi.fn(() => []),
recordAdmissionFailure: vi.fn(),
}
await reconcileAuthoritativeMcpSnapshot({
servers: [server],
manager,
serverState,
getAuthToken: vi.fn().mockRejectedValue(authError),
})
expect(manager.addServer).not.toHaveBeenCalled()
expect(manager.recordAdmissionFailure).toHaveBeenCalledTimes(1)
expect(manager.recordAdmissionFailure).toHaveBeenCalledWith(server, authError)
expect(serverState.has(server.name)).toBe(false)
})
})