Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
asAppsApi,
asCoreApi,
asNetworkingApi,
asRbacApi,
createMockAppsApi,
createMockCoreApi,
createMockNetworkingApi,
createMockRbacApi,
} from '../test/__fixtures__/testMocks'
import {
HostFleetReconcileError,
HostReconciler,
destructiveCleanupAllowed,
} from './hostReconciler'
import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'
import { registry } from './metrics'
import { HostCRD } from './types'
function deferred<T = void>(): {
promise: Promise<T>
resolve: (value: T | PromiseLike<T>) => void
reject: (reason?: unknown) => void
} {
let resolve!: (value: T | PromiseLike<T>) => void
let reject!: (reason?: unknown) => void
const promise = new Promise<T>((res, rej) => {
resolve = res
reject = rej
})
return { promise, resolve, reject }
}
async function readLabeledCounter(name: string, labels: Record<string, string>): Promise<number> {
const metric = registry.getSingleMetric(name)
if (!metric) throw new Error(`${name} is not registered`)
const snapshot = await metric.get()
return (
snapshot.values.find(entry =>
Object.entries(labels).every(([key, value]) => entry.labels[key] === value)
)?.value ?? 0
)
}
vi.mock('./config', () => ({
config: {
hostNamespace: 'mcp-host',
hostWorkspaceStorageClassName: 'standard',
hostWorkspaceStorageSize: '1Gi',
hostWorkspacePath: '/workspace',
hostImage: 'clerum/mcp-host:test',
desktopImage: 'clerum/mcp-host-desktop:test',
hostImagePullPolicy: 'IfNotPresent',
hostImagePullSecretName: '',
hostPort: 8080,
gfsNamespace: 'gfs',
gfscPort: 8087,
desktopPort: 3000,
rpcProxyNamespace: 'rpc-proxy',
channelsNamespace: 'channels',
hostK8sRequestTimeoutMs: 30_000,
hostFullReconcileConcurrency: 2,
mcpHostGatewayUrl: 'http://mcp-host-gateway',
hostResources: {
requests: { memory: '128Mi', cpu: '50m' },
limits: { memory: '256Mi', cpu: '200m' },
},
desktopResources: {
requests: { memory: '256Mi', cpu: '100m' },
limits: { memory: '512Mi', cpu: '500m' },
},
},
}))
vi.mock('./gfsHostBinding', () => ({
mintHostGfsToken: vi
.fn()
.mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => ({
token: 'gfs-runtime-value',
expiresInSeconds: 300,
subject: `host:1st:${namespace}/${name}`,
})),
}))
function makeHost(overrides: Partial<HostCRD['spec']> & { name?: string } = {}): HostCRD {
const { name, ...specOverrides } = overrides
return {
name: name ?? 'chatllm',
namespace: 'mcp-host',
spec: {
host: name ?? 'chatllm',
contextRef: 'ctx1',
secretRef: 'llm-secret',
...specOverrides,
},
}
}
function hccOwnedHostResource(hostName = 'chatllm') {
return {
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': hostName,
},
resourceVersion: '1',
},
}
}
describe('HostReconciler secret fail-closed cleanup', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const networkingApi = createMockNetworkingApi()
const rbacApi = createMockRbacApi()
let reconciler: HostReconciler
beforeEach(() => {
vi.clearAllMocks()
reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
})
})
it('removes runtime endpoints and RBAC on missing Host secret but preserves PVC', async () => {
const missing = new Error('missing') as Error & { code?: number }
missing.code = 404
coreApi.readNamespacedSecret.mockRejectedValueOnce(missing).mockResolvedValueOnce({
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'chatllm',
},
},
})
appsApi.readNamespacedDeployment.mockResolvedValue(hccOwnedHostResource())
coreApi.readNamespacedService.mockResolvedValue(hccOwnedHostResource())
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(hccOwnedHostResource())
rbacApi.readNamespacedRoleBinding.mockResolvedValue(hccOwnedHostResource())
rbacApi.readNamespacedRole.mockResolvedValue(hccOwnedHostResource())
coreApi.readNamespacedServiceAccount.mockResolvedValue(hccOwnedHostResource())
await reconciler.reconcile(makeHost({ name: 'chatllm' }))
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
name: 'chatllm',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'chatllm',
namespace: 'mcp-host',
})
expect(rbacApi.deleteNamespacedRole).toHaveBeenCalled()
expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'host-chatllm-mcp-host-runtime-tokens',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
})
it('does not delete host resources missing HCC ownership labels', async () => {
const missing = new Error('missing') as Error & { code?: number }
missing.code = 404
coreApi.readNamespacedSecret.mockRejectedValueOnce(missing).mockResolvedValueOnce({
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'chatllm',
},
},
})
const userOwned = { metadata: { labels: { 'clerum.io/managed-by': 'user' } } }
appsApi.readNamespacedDeployment.mockResolvedValue(userOwned)
coreApi.readNamespacedService.mockResolvedValue(userOwned)
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(userOwned)
rbacApi.readNamespacedRoleBinding.mockResolvedValue(userOwned)
rbacApi.readNamespacedRole.mockResolvedValue(userOwned)
coreApi.readNamespacedServiceAccount.mockResolvedValue(userOwned)
await reconciler.reconcile(makeHost({ name: 'chatllm' }))
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(rbacApi.deleteNamespacedRoleBinding).not.toHaveBeenCalled()
expect(rbacApi.deleteNamespacedRole).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedServiceAccount).not.toHaveBeenCalled()
})
it('preserves existing runtime on transient Host secret read error', async () => {
const err = new Error('api unavailable') as Error & { code?: number }
err.code = 500
coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
await reconciler.reconcile(makeHost({ name: 'chatllm' }))
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
})
})
describe('HostReconciler Kubernetes request deadline wiring', () => {
it('wraps every default finite Kubernetes client, including lazy CustomObjects', async () => {
const rawClients = Array.from({ length: 5 }, () => ({
probe: vi.fn(async (_request: unknown, _options?: unknown) => ({})),
}))
const pendingClients = [...rawClients]
const kubeConfig = {
makeApiClient: vi.fn(() => pendingClients.shift()),
} as unknown as k8s.KubeConfig
const reconciler = new HostReconciler(kubeConfig)
const internals = reconciler as unknown as {
appsApi: { probe(request: unknown): Promise<unknown> }
coreApi: { probe(request: unknown): Promise<unknown> }
networkingApi: { probe(request: unknown): Promise<unknown> }
rbacApi: { probe(request: unknown): Promise<unknown> }
customApi: { probe(request: unknown): Promise<unknown> }
}
for (const client of [
internals.appsApi,
internals.coreApi,
internals.networkingApi,
internals.rbacApi,
internals.customApi,
]) {
await client.probe({})
}
expect(kubeConfig.makeApiClient).toHaveBeenCalledTimes(5)
for (const rawClient of rawClients) {
expect(rawClient.probe).toHaveBeenCalledWith(
{},
expect.objectContaining({
middleware: [expect.objectContaining({ pre: expect.any(Function) })],
middlewareMergeStrategy: 'append',
})
)
}
})
})
function makeBasicReconciler() {
return new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
}
describe('HostReconciler fleet failure isolation', () => {
it('continues reconciling later Hosts and aggregates the failure', async () => {
const reconciler = makeBasicReconciler()
// Cleanup stays deferred (authority unwired), isolating reconcile behavior.
const firstFailure = new Error('first Host failed')
const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
if (host.name === 'host-a') throw firstFailure
})
let thrown: unknown
try {
await reconciler.fullReconcile([makeHost({ name: 'host-a' }), makeHost({ name: 'host-b' })])
} catch (error) {
thrown = error
}
expect(reconcile.mock.calls.map(([host]) => host.name)).toEqual(['host-a', 'host-b'])
expect(thrown).toBeInstanceOf(HostFleetReconcileError)
expect((thrown as HostFleetReconcileError).hostFailures).toEqual([firstFailure])
// Authority is unwired, so cleanup fail-closes (defers) rather than deleting.
expect((thrown as HostFleetReconcileError).cleanupFailures).toEqual([])
})
it('continues with Host B when Host A reaches its Kubernetes request deadline', async () => {
const reconciler = makeBasicReconciler()
const timeout = new HostK8sRequestTimeoutError('AppsV1Api.readNamespacedDeployment', 30_000)
const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
if (host.name === 'host-a') throw timeout
})
let thrown: unknown
try {
await reconciler.fullReconcile([makeHost({ name: 'host-a' }), makeHost({ name: 'host-b' })])
} catch (error) {
thrown = error
}
expect(reconcile.mock.calls.map(([host]) => host.name)).toEqual(['host-a', 'host-b'])
expect(thrown).toBeInstanceOf(HostFleetReconcileError)
expect((thrown as HostFleetReconcileError).hostFailures).toEqual([timeout])
})
it('aggregates candidate-discovery list failures under known authority', async () => {
const reconciler = makeBasicReconciler()
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(name => (name === 'host-a' ? makeHost({ name }) : undefined))
vi.spyOn(reconciler, 'reconcile').mockResolvedValue()
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
}
const inventoryFailure = new Error('Host Deployment inventory failed')
vi.spyOn(internals, 'listManagedHostDeployments').mockRejectedValue(inventoryFailure)
let thrown: unknown
try {
await reconciler.fullReconcile([makeHost({ name: 'host-a' })])
} catch (error) {
thrown = error
}
expect(thrown).toBeInstanceOf(HostFleetReconcileError)
expect((thrown as HostFleetReconcileError).hostFailures).toEqual([])
expect((thrown as HostFleetReconcileError).cleanupFailures).toContain(inventoryFailure)
})
it('preserves simultaneous Host and cleanup failures after every phase has run', async () => {
const reconciler = makeBasicReconciler()
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(name =>
name === 'host-a' || name === 'host-b' ? makeHost({ name }) : undefined
)
const hostFailure = new Error('Host deployment failed')
const cleanupFailure = new Error('Host inventory failed')
const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
if (host.name === 'host-a') throw hostFailure
})
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
}
vi.spyOn(internals, 'listManagedHostDeployments').mockRejectedValue(cleanupFailure)
let thrown: unknown
try {
await reconciler.fullReconcile([makeHost({ name: 'host-a' }), makeHost({ name: 'host-b' })])
} catch (error) {
thrown = error
}
expect(reconcile.mock.calls.map(([host]) => host.name)).toEqual(['host-a', 'host-b'])
expect(thrown).toBeInstanceOf(HostFleetReconcileError)
expect((thrown as HostFleetReconcileError).hostFailures).toEqual([hostFailure])
expect((thrown as HostFleetReconcileError).cleanupFailures).toContain(cleanupFailure)
})
it('deletes each confirmed orphan bundle and aggregates delete failures', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
customApi: { getNamespacedCustomObject: getObj } as unknown as k8s.CustomObjectsApi,
})
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(() => undefined)
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
const orphanFailure = new Error('first orphan cleanup failed')
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([
{
metadata: {
name: 'orphan-a',
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'orphan-a',
},
},
} as k8s.V1Deployment,
{
metadata: {
name: 'orphan-b',
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'orphan-b',
},
},
} as k8s.V1Deployment,
])
const del = vi
.spyOn(internals, 'deleteHostRuntimeResources')
.mockRejectedValueOnce(orphanFailure)
.mockResolvedValueOnce()
let thrown: unknown
try {
await reconciler.fullReconcile([])
} catch (error) {
thrown = error
}
expect(del.mock.calls.map(([name]) => name).sort()).toEqual(['orphan-a', 'orphan-b'])
expect(getObj).toHaveBeenCalledTimes(2)
expect(thrown).toBeInstanceOf(HostFleetReconcileError)
expect((thrown as HostFleetReconcileError).cleanupFailures).toContain(orphanFailure)
})
})
/**
* Current value of clerum_hcc_host_delete_cleanup_total for one `outcome`
* label. Read from the live HCC registry (metrics are NOT mocked here), so a
* skip that fails to record is caught rather than passing silently.
*/
async function readHostDeleteCleanupOutcome(outcome: string): Promise<number> {
const metric = registry.getSingleMetric('clerum_hcc_host_delete_cleanup_total')
if (!metric) throw new Error('clerum_hcc_host_delete_cleanup_total is not registered')
const snapshot = await metric.get()
const match = snapshot.values.find(entry => entry.labels.outcome === outcome)
return match?.value ?? 0
}
function makeCleanupReconciler(customApi: { getNamespacedCustomObject: ReturnType<typeof vi.fn> }) {
return new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
customApi: customApi as unknown as k8s.CustomObjectsApi,
})
}
describe('HostReconciler bounded fleet workers', () => {
it('admits at most the configured number of Host reconciles concurrently', async () => {
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
// Isolate reconcile concurrency from the cleanup phase.
vi.spyOn(
reconciler as unknown as {
collectHostCleanupFailures(...args: unknown[]): Promise<unknown[]>
},
'collectHostCleanupFailures'
).mockResolvedValue([])
let active = 0
let maxActive = 0
const gates: Array<() => void> = []
vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {
active += 1
maxActive = Math.max(maxActive, active)
await new Promise<void>(resolve => gates.push(resolve))
active -= 1
})
const hosts = ['h1', 'h2', 'h3', 'h4'].map(name => makeHost({ name }))
const pass = reconciler.fullReconcile(hosts)
// Only two workers may run at once; the pool must not admit h3/h4 yet.
await vi.waitFor(() => expect(gates.length).toBe(2))
expect(maxActive).toBe(2)
// Drain: each release lets the pool admit the next queued Host key.
for (let i = 0; i < hosts.length; i++) {
await vi.waitFor(() => expect(gates.length).toBeGreaterThan(0))
gates.shift()!()
}
await pass
expect(maxActive).toBe(2)
})
it('keeps urgent per-Host events outside the fleet worker budget', async () => {
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
vi.spyOn(
reconciler as unknown as {
collectHostCleanupFailures(...args: unknown[]): Promise<unknown[]>
},
'collectHostCleanupFailures'
).mockResolvedValue([])
let active = 0
let maxActive = 0
const gates = new Map<string, () => void>()
vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
active += 1
maxActive = Math.max(maxActive, active)
await new Promise<void>(resolve => gates.set(host.name, resolve))
active -= 1
})
const fleetHosts = ['f1', 'f2', 'f3', 'f4'].map(name => makeHost({ name }))
const pass = reconciler.fullReconcile(fleetHosts)
await vi.waitFor(() => expect(gates.size).toBe(2)) // fleet budget saturated
// An urgent event for an unrelated Host must still be admitted immediately,
// even though the fleet workers already occupy the whole budget.
const urgent = reconciler.reconcile(makeHost({ name: 'urgent' }), 'urgent')
await vi.waitFor(() => expect(gates.has('urgent')).toBe(true))
expect(maxActive).toBe(3) // 2 fleet + 1 urgent
// Drain repeatedly so the remaining fleet workers (f3/f4) are admitted and
// released as earlier ones finish.
const settled = Promise.all([pass, urgent])
for (let i = 0; i < 12; i++) {
for (const [name, resolve] of [...gates]) {
gates.delete(name)
resolve()
}
await Promise.resolve()
await Promise.resolve()
}
await settled
})
})
describe('HostReconciler lifecycle-only vs full pass cleanup boundary', () => {
it('reconcileHosts (lifecycle-only) runs NO orphan cleanup; fullReconcile does', async () => {
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
const internals = reconciler as unknown as {
collectHostReconcileFailures(...args: unknown[]): Promise<unknown[]>
collectHostCleanupFailures(...args: unknown[]): Promise<unknown[]>
}
const reconcileFailures = vi
.spyOn(internals, 'collectHostReconcileFailures')
.mockResolvedValue([])
const cleanup = vi.spyOn(internals, 'collectHostCleanupFailures').mockResolvedValue([])
// The lifecycle-only pass is what CommunicationChannel recovery and the
// heartbeat-driven fleet convergence dispatch through (k8sClient
// reconcileHosts()). It MUST NOT run destructive orphan cleanup: a
// lifecycle pass whose watch authority is stale could otherwise delete a
// just-created Host's children (Risk register: "Stale full pass deletes a
// new Host's resources" — critical). Only the authoritative full pass,
// which captures watch authority up front, is allowed to clean up.
await reconciler.reconcileHosts([makeHost({ name: 'h1' })])
expect(reconcileFailures).toHaveBeenCalledTimes(1)
expect(cleanup).not.toHaveBeenCalled()
// The authoritative full pass is the ONLY caller that runs orphan cleanup.
await reconciler.fullReconcile([makeHost({ name: 'h1' })])
expect(cleanup).toHaveBeenCalledTimes(1)
})
})
describe('HostReconciler bounded orphan cleanup (Addendum 2)', () => {
it('deletes orphan bundles through the bounded worker pool, never an unbounded fan-out', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(() => undefined) // every candidate is an orphan
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue(
['orphan-a', 'orphan-b', 'orphan-c', 'orphan-d'].map(
name =>
({
metadata: {
name,
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': name,
},
},
}) as k8s.V1Deployment
)
)
let active = 0
let maxActive = 0
const gates: Array<() => void> = []
vi.spyOn(internals, 'deleteHostRuntimeResources').mockImplementation(async () => {
active += 1
maxActive = Math.max(maxActive, active)
await new Promise<void>(resolve => gates.push(resolve))
active -= 1
})
const pass = reconciler.fullReconcile([])
// Bound = HCC_HOST_FULL_RECONCILE_CONCURRENCY (2): only two orphan bundles
// delete at once; the pool must not admit orphan-c/orphan-d yet.
await vi.waitFor(() => expect(gates.length).toBe(2))
expect(maxActive).toBe(2)
for (let i = 0; i < 4; i++) {
await vi.waitFor(() => expect(gates.length).toBeGreaterThan(0))
gates.shift()!()
}
await pass
expect(maxActive).toBe(2)
})
})
describe('HostReconciler bounded channel-reader revision fan-out (Addendum 2)', () => {
it('patches every affected Host through the bounded worker pool', async () => {
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
;(
reconciler as unknown as {
setFindCommunicationChannelsByCredentialsSecretName(fn: () => unknown[]): void
}
).setFindCommunicationChannelsByCredentialsSecretName(() =>
['h1', 'h2', 'h3', 'h4'].map(hostRef => ({ spec: { hostRef } }))
)
let active = 0
let maxActive = 0
const patched: string[] = []
const gates: Array<() => void> = []
vi.spyOn(
reconciler as unknown as {
patchChannelReaderRevisionAnnotation(name: string): Promise<void>
},
'patchChannelReaderRevisionAnnotation'
).mockImplementation(async (name: string) => {
patched.push(name)
active += 1
maxActive = Math.max(maxActive, active)
await new Promise<void>(resolve => gates.push(resolve))
active -= 1
})
const pass = reconciler.reconcileChannelReaderRevision('shared-secret', 'channels')
// Bound = 2: at most two affected Hosts patch concurrently.
await vi.waitFor(() => expect(gates.length).toBe(2))
expect(maxActive).toBe(2)
for (let i = 0; i < 4; i++) {
await vi.waitFor(() => expect(gates.length).toBeGreaterThan(0))
gates.shift()!()
}
await pass
expect(maxActive).toBe(2)
// Complete fan-out: every affected Host is patched exactly once.
expect(patched.sort()).toEqual(['h1', 'h2', 'h3', 'h4'])
})
})
describe('HostReconciler #827 cleanup hardening', () => {
it('suppresses the delete when the Host reappears in cache after the fresh read (F2 TOCTOU)', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 3 }))
// resolveCurrentHost reports "absent" during discovery and the authority
// gate, then "present" on the re-check INSIDE the per-Host serializer —
// modelling a same-name Host recreated in the admission window. The F2
// in-serializer re-check must suppress the destructive delete.
let calls = 0
reconciler.setResolveCurrentHost(name => {
calls += 1
return calls >= 3 ? makeHost({ name }) : undefined
})
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([
{
metadata: {
name: 'gone',
namespace: 'mcp-host',
labels: { 'clerum.io/managed-by': 'host-context-controller', 'clerum.io/host': 'gone' },
},
} as k8s.V1Deployment,
])
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
await reconciler.fullReconcile([])
expect(getObj).toHaveBeenCalledTimes(1) // the fresh read happened (said absent)
expect(del).not.toHaveBeenCalled() // but the in-serializer re-check suppressed the delete
})
it('discovers an orphan from a surviving owned resource when the Deployment is already gone (#827 item 4)', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(() => undefined) // "gone" absent from cache
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
// The Deployment sentinel is already gone...
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([])
// ...but a PVC leftover survives with ownership labels in the host namespace.
;(
reconciler as unknown as {
coreApi: { listNamespacedPersistentVolumeClaim: ReturnType<typeof vi.fn> }
}
).coreApi.listNamespacedPersistentVolumeClaim.mockResolvedValue({
items: [
{
metadata: {
name: 'gone-workspace',
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'gone',
},
},
},
],
})
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
await reconciler.fullReconcile([])
// The surviving PVC alone identified the orphan Host and drove its bundle delete.
expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'gone' }))
expect(del).toHaveBeenCalledWith('gone', 'mcp-host')
})
})
describe('HostReconciler delete serialization', () => {
it('serializes a delete behind an in-flight reconcile for the same Host', async () => {
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(createMockAppsApi()),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
const order: string[] = []
const reconcileGate = deferred()
vi.spyOn(
reconciler as unknown as { reconcileCore(...args: unknown[]): Promise<void> },
'reconcileCore'
).mockImplementation(async () => {
order.push('reconcile:start')
await reconcileGate.promise
order.push('reconcile:end')
})
vi.spyOn(
reconciler as unknown as { deleteHostRuntimeResources(...args: unknown[]): Promise<void> },
'deleteHostRuntimeResources'
).mockImplementation(async () => {
order.push('delete:start')
order.push('delete:end')
})
const reconcilePromise = reconciler.reconcile(makeHost({ name: 'chatllm' }))
const deletePromise = reconciler.reconcileDelete('chatllm', 'mcp-host')
await Promise.resolve()
await Promise.resolve()
// The delete must NOT begin while the same Host's reconcile is in flight.
expect(order).toEqual(['reconcile:start'])
reconcileGate.resolve()
await Promise.all([reconcilePromise, deletePromise])
expect(order).toEqual(['reconcile:start', 'reconcile:end', 'delete:start', 'delete:end'])
})
// Recovery-path TOCTOU parity with the F2 fence in collectHostCleanupFailures.
// The recovery delete (k8sClient.dispatchRecoveredHostDelete) diffs LIST-time
// names only; a same-name Host recreated AFTER that diff and admitted to the
// per-Host chain FIRST would otherwise have its freshly-reconciled bundle —
// workspace PVC, per-Host RBAC, runtime-token Secret, channel-reader
// resources, NetworkPolicies — wiped by the stale delete queued behind it.
it('suppresses the delete when skipIf reports a recreation inside the admission window', async () => {
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: vi.fn() })
const internals = reconciler as unknown as {
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
const before = await readHostDeleteCleanupOutcome('superseded')
// The fence is evaluated INSIDE the serializer, so it observes the cache as
// of admission — not as of enqueue. Here it reports "recreated".
let evaluatedInsideSerializer = false
await reconciler.reconcileDelete('recreated-in-window', 'mcp-host', {
skipIf: () => {
evaluatedInsideSerializer = true
return true
},
})
expect(evaluatedInsideSerializer).toBe(true)
expect(del).not.toHaveBeenCalled()
// The skip is observable, not silent, and reuses the watch path's vocabulary.
expect(await readHostDeleteCleanupOutcome('superseded')).toBe(before + 1)
})
it('still deletes the bundle when skipIf reports the Host really is gone', async () => {
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: vi.fn() })
const internals = reconciler as unknown as {
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
await reconciler.reconcileDelete('really-gone', 'mcp-host', { skipIf: () => false })
expect(del).toHaveBeenCalledWith('really-gone', 'mcp-host')
})
it('keeps the existing unconditional behavior for callers that pass no options', async () => {
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: vi.fn() })
const internals = reconciler as unknown as {
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
await reconciler.reconcileDelete('no-opts', 'mcp-host')
expect(del).toHaveBeenCalledWith('no-opts', 'mcp-host')
})
})
describe('HostReconciler destructiveCleanupAllowed property', () => {
it('permits deletion only with known authority, a stable generation, cache omission, and a confirmed 404', () => {
expect(
destructiveCleanupAllowed({
watchAuthorityKnown: true,
capturedWatchGeneration: 7,
currentWatchGeneration: 7,
currentCacheOmitsHost: true,
freshAuthoritativeRead: 'confirmed404',
})
).toBe(true)
})
it.each([
['authority unknown', { watchAuthorityKnown: false }],
['generation drifted', { currentWatchGeneration: 8 }],
['host still present in cache', { currentCacheOmitsHost: false }],
['fresh read present', { freshAuthoritativeRead: 'present' as const }],
['fresh read errored', { freshAuthoritativeRead: 'error' as const }],
])('refuses deletion when %s', (_label, override) => {
expect(
destructiveCleanupAllowed({
watchAuthorityKnown: true,
capturedWatchGeneration: 7,
currentWatchGeneration: 7,
currentCacheOmitsHost: true,
freshAuthoritativeRead: 'confirmed404',
...override,
})
).toBe(false)
})
})
describe('HostReconciler cleanup authority gate', () => {
function orphanDeployment(hostName: string) {
return {
metadata: {
name: hostName,
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': hostName,
},
},
} as k8s.V1Deployment
}
function wireCleanup(
reconciler: HostReconciler,
opts: {
authority: { known: boolean; generation: number }
currentAuthority?: { known: boolean; generation: number }
cache: Set<string>
candidates: k8s.V1Deployment[]
}
) {
let firstCall = true
reconciler.setHostWatchAuthority(() => {
if (firstCall) {
firstCall = false
return opts.authority
}
return opts.currentAuthority ?? opts.authority
})
reconciler.setResolveCurrentHost(name =>
opts.cache.has(name) ? makeHost({ name }) : undefined
)
const internals = reconciler as unknown as {
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue(opts.candidates)
return vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue()
}
it('deletes an orphan bundle after a fresh 404 while authority is known and stable', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
cache: new Set(),
candidates: [orphanDeployment('gone-host')],
})
await reconciler.fullReconcile([])
expect(getObj).toHaveBeenCalledTimes(1)
expect(del).toHaveBeenCalledWith('gone-host', 'mcp-host')
})
it('retains a candidate that is still present in the current cache', async () => {
const getObj = vi.fn()
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
cache: new Set(['live-host']),
candidates: [orphanDeployment('live-host')],
})
await reconciler.fullReconcile([])
expect(getObj).not.toHaveBeenCalled()
expect(del).not.toHaveBeenCalled()
})
it('defers all cleanup when watch authority is unknown', async () => {
const getObj = vi.fn()
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: false, generation: 3 },
cache: new Set(),
candidates: [orphanDeployment('gone-host')],
})
await reconciler.fullReconcile([])
expect(getObj).not.toHaveBeenCalled()
expect(del).not.toHaveBeenCalled()
})
it('defers all cleanup when the watch generation changed during the pass', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
currentAuthority: { known: true, generation: 4 },
cache: new Set(),
candidates: [orphanDeployment('gone-host')],
})
await reconciler.fullReconcile([])
expect(del).not.toHaveBeenCalled()
})
it('defers orphan deletion when authority changes before serializer admission', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const authority = { known: true, generation: 3 }
reconciler.setHostWatchAuthority(() => ({ ...authority }))
reconciler.setResolveCurrentHost(() => undefined)
const internals = reconciler as unknown as {
lifecycle: {
serializeByHost(name: string, task: () => Promise<void>): Promise<void>
}
listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
}
vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([
orphanDeployment('authority-race-host'),
])
const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue(undefined)
const blockerStarted = deferred()
const releaseBlocker = deferred()
const occupied = internals.lifecycle.serializeByHost('authority-race-host', async () => {
blockerStarted.resolve(undefined)
await releaseBlocker.promise
})
await blockerStarted.promise
const cleanup = reconciler.fullReconcile([])
await vi.waitFor(() => expect(getObj).toHaveBeenCalledOnce())
authority.known = false
authority.generation = 4
releaseBlocker.resolve(undefined)
await occupied
await cleanup
expect(del).not.toHaveBeenCalled()
})
it('defers an orphan whose fresh authoritative read fails', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 503 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
cache: new Set(),
candidates: [orphanDeployment('maybe-gone')],
})
await reconciler.fullReconcile([])
expect(getObj).toHaveBeenCalledTimes(1)
expect(del).not.toHaveBeenCalled()
})
it('retains an orphan whose fresh authoritative read shows it present', async () => {
const getObj = vi.fn().mockResolvedValue({ metadata: { name: 'reappeared' } })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
cache: new Set(),
candidates: [orphanDeployment('reappeared')],
})
await reconciler.fullReconcile([])
expect(getObj).toHaveBeenCalledTimes(1)
expect(del).not.toHaveBeenCalled()
})
it('never deletes a candidate whose owning Host cannot be derived from labels', async () => {
const getObj = vi.fn()
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
const del = wireCleanup(reconciler, {
authority: { known: true, generation: 3 },
cache: new Set(),
candidates: [
{
metadata: {
name: 'unlabeled',
namespace: 'mcp-host',
labels: { 'clerum.io/managed-by': 'host-context-controller' },
},
} as k8s.V1Deployment,
],
})
await reconciler.fullReconcile([])
expect(getObj).not.toHaveBeenCalled()
expect(del).not.toHaveBeenCalled()
})
})
describe('HostReconciler Host inventory mutation authority', () => {
function wireAuthoritativeHost(
reconciler: HostReconciler,
host: HostCRD,
authorityRef: { current: { known: boolean; generation: number } }
): void {
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostWatchAuthority(() => authorityRef.current)
}
function stubPreDeploymentReconcileEffects(reconciler: HostReconciler): void {
vi.spyOn(reconciler as any, 'validateHostSecret').mockResolvedValue({ ok: true })
vi.spyOn(reconciler as any, 'ensurePvc').mockResolvedValue(true)
vi.spyOn(reconciler as any, 'ensureService').mockResolvedValue(true)
for (const method of [
'ensureHostServiceAccount',
'ensureHostRole',
'ensureHostRoleBinding',
'ensureMcpHostIngressNetworkPolicy',
'ensureMcpHostGfsEgressNetworkPolicy',
'ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy',
'ensureRpcProxyMcpHostIngressNetworkPolicy',
'ensureRpcProxyHostEgressNetworkPolicy',
'ensureDesktopNetworkPolicy',
'ensureChannelReaderEgressNetworkPolicy',
'ensureWorkflowApprovalReaderHostEgressNetworkPolicy',
]) {
vi.spyOn(reconciler as any, method).mockResolvedValue('up_to_date')
}
}
function runtimeTokenScopeWithoutChannels(host: HostCRD): string {
return (
HostReconciler as unknown as {
runtimeTokenScopeHash(selected: HostCRD, hasChannelIngress: boolean): string
}
).runtimeTokenScopeHash(host, false)
}
it('rejects a queued reconcile when Host authority changes before admission', async () => {
const reconciler = makeBasicReconciler()
const host = makeHost({ name: 'authority-race' })
const authority = { current: { known: true, generation: 7 } }
wireAuthoritativeHost(reconciler, host, authority)
const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
const blockerStarted = deferred()
const releaseBlocker = deferred()
const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
blockerStarted.resolve(undefined)
await releaseBlocker.promise
}) as Promise<void>
await blockerStarted.promise
const pending = reconciler.reconcile(host)
authority.current = { known: false, generation: 8 }
releaseBlocker.resolve(undefined)
await occupied
await expect(pending).rejects.toThrow(/Host inventory authority/)
expect(reconcileCore).not.toHaveBeenCalled()
})
it('withdraws a queued fleet worker after Context authority retires and recovers', async () => {
const reconciler = makeBasicReconciler()
const hostA = makeHost({ name: 'context-authority-a' })
const hostB = makeHost({ name: 'context-authority-b' })
const hosts = new Map([
[hostA.name, hostA],
[hostB.name, hostB],
])
reconciler.setResolveCurrentHost(name => hosts.get(name))
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 7 }))
const mutationAuthority = {
current: {
known: true,
hostRevision: 7,
contextRevision: 11,
},
}
;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)
const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
vi.spyOn(reconciler as any, 'collectHostCleanupFailures').mockResolvedValue([])
const blockerStarted = deferred()
const releaseBlocker = deferred()
const occupied = (reconciler as any).lifecycle.serializeByHost(hostB.name, async () => {
blockerStarted.resolve(undefined)
await releaseBlocker.promise
}) as Promise<void>
await blockerStarted.promise
const fleet = reconciler.fullReconcile([hostA, hostB])
await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))
// Context desired state changed (its content revision advanced) while Host
// authority remained continuously known: a host-only lease would miss this
// transition and admit Host B with stale derived mounts. A pure watch
// reconnect with identical content would NOT advance the revision and is
// correctly admitted — that is the point of the content-identity fence.
// After #490 the fence still retires Host B; the fleet aggregator withdraws
// HostInventoryAuthorityUnavailableError instead of failing the pass.
mutationAuthority.current = {
known: true,
hostRevision: 7,
contextRevision: 13,
}
const before = await readLabeledCounter('clerum_hcc_host_fleet_benign_supersessions_total', {
error: 'HostInventoryAuthorityUnavailableError',
})
releaseBlocker.resolve(undefined)
await occupied
await expect(fleet).resolves.toBeUndefined()
expect(reconcileCore).toHaveBeenCalledTimes(1)
expect(reconcileCore).not.toHaveBeenCalledWith(hostB, expect.any(Function))
expect(
await readLabeledCounter('clerum_hcc_host_fleet_benign_supersessions_total', {
error: 'HostInventoryAuthorityUnavailableError',
})
).toBe(before + 1)
})
it('retires a Host reconcile when its Context is superseded within the same watch', async () => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'context-revision-race', contextRef: 'context-a' }),
uid: 'context-revision-race-uid',
generation: 5,
}
const contextV1 = { name: 'context-a', spec: { sharedFileSystems: [] } }
const contextV2 = {
name: 'context-a',
spec: {
sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
},
}
let selectedContext: unknown = contextV1
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
hostRevision: 7,
contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [selectedContext])
const secretReadStarted = deferred()
const releaseSecretRead = deferred()
vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
secretReadStarted.resolve(undefined)
await releaseSecretRead.promise
return { ok: true }
})
const ensureServiceAccount = vi
.spyOn(reconciler as any, 'ensureHostServiceAccount')
.mockResolvedValue(undefined)
// R1-M2: a benign mid-pass supersession must NOT be reported as a hard
// reconcile failure. The single reconcile() catch used to fire
// enqueueControllerError('reconcile_exception') → controller_error telemetry
// (status:'failed') AND an administrative 'failed' outcome to control-api for
// these retirements. Spy (observe-only) to prove the benign branch emits
// neither, while still re-throwing so callers treat it as a retire.
const enqueueControllerError = vi.spyOn(reconciler as any, 'enqueueControllerError')
const enqueueAdministrativeOutcome = vi.spyOn(reconciler as any, 'enqueueAdministrativeOutcome')
const pending = reconciler.reconcile(host)
await secretReadStarted.promise
selectedContext = contextV2
releaseSecretRead.resolve(undefined)
await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
expect(ensureServiceAccount).not.toHaveBeenCalled()
expect(enqueueControllerError).not.toHaveBeenCalled()
expect(enqueueAdministrativeOutcome).not.toHaveBeenCalledWith(
expect.anything(),
'failed',
expect.anything()
)
})
it('keeps a Host reconcile admitted when a watch replaces dependencies with deep-equal values', async () => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'deep-equal-dependency', contextRef: 'context-a' }),
uid: 'deep-equal-dependency-uid',
generation: 5,
}
let selectedContext: unknown = {
name: 'context-a',
spec: {
sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
},
}
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
hostRevision: 7,
contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [selectedContext])
const secretReadStarted = deferred()
const releaseSecretRead = deferred()
vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
secretReadStarted.resolve(undefined)
await releaseSecretRead.promise
return { ok: true }
})
const admitted = new Error('deep-equal dependency admitted')
const ensureServiceAccount = vi
.spyOn(reconciler as any, 'ensureHostServiceAccount')
.mockRejectedValue(admitted)
const pending = reconciler.reconcile(host)
await secretReadStarted.promise
selectedContext = structuredClone(selectedContext)
releaseSecretRead.resolve(undefined)
await expect(pending).rejects.toBe(admitted)
expect(ensureServiceAccount).toHaveBeenCalled()
})
it.each([
[
'referenced SharedFileSystem revision',
{ name: 'documents', spec: { storageClassName: 'fast-v1' } },
{ name: 'documents', spec: { storageClassName: 'fast-v2' } },
],
['referenced SharedFileSystem mountability', true, false],
[
'Host-scoped CommunicationChannel revision',
{ name: 'telegram', spec: { hostRef: 'dependency-race' } },
{
name: 'telegram',
spec: {
hostRef: 'dependency-race',
credentialsSecretRef: { name: 'telegram-v2' },
},
},
],
])('retires a Host reconcile when its %s changes', async (_label, original, replacement) => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'dependency-race', contextRef: 'context-a' }),
uid: 'dependency-race-uid',
generation: 5,
}
const context = { name: 'context-a', spec: { sharedFileSystems: [] } }
let selectedDependency: unknown = original
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
hostRevision: 7,
contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])
const secretReadStarted = deferred()
const releaseSecretRead = deferred()
vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
secretReadStarted.resolve(undefined)
await releaseSecretRead.promise
return { ok: true }
})
const ensureServiceAccount = vi
.spyOn(reconciler as any, 'ensureHostServiceAccount')
.mockResolvedValue(undefined)
const pending = reconciler.reconcile(host)
await secretReadStarted.promise
selectedDependency = replacement
releaseSecretRead.resolve(undefined)
await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
expect(ensureServiceAccount).not.toHaveBeenCalled()
})
it('does not delete runtime after a Secret rejection when Host authority changes in flight', async () => {
const reconciler = makeBasicReconciler()
const host = { ...makeHost({ name: 'secret-authority-race' }), uid: 'host-uid', generation: 3 }
const authority = { current: { known: true, generation: 7 } }
wireAuthoritativeHost(reconciler, host, authority)
const secretReadStarted = deferred()
const releaseSecretRead = deferred()
vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
secretReadStarted.resolve(undefined)
await releaseSecretRead.promise
return {
ok: false,
reason: 'SecretNotFound',
message: 'secret missing in stale spec',
}
})
const deleteRuntime = vi
.spyOn(reconciler as any, 'deleteHostRuntimeResources')
.mockResolvedValue(undefined)
const pending = reconciler.reconcile(host)
await secretReadStarted.promise
authority.current = { known: false, generation: 8 }
releaseSecretRead.resolve(undefined)
await expect(pending).rejects.toThrow(/Host inventory authority/)
expect(deleteRuntime).not.toHaveBeenCalled()
})
it.each([
[
'UID',
(host: HostCRD) => ({ ...host, uid: 'replacement-host-uid', generation: 1 }),
/Host identity/,
],
[
'spec generation',
(host: HostCRD) => ({
...host,
generation: (host.generation ?? 0) + 1,
spec: { ...host.spec, secretRef: 'replacement-secret' },
}),
/Host spec generation/,
],
] as const)(
'does not delete runtime after a Secret rejection when the Host %s changes in flight',
async (_, replaceCurrent, expectedError) => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'secret-identity-race' }),
uid: 'host-uid',
generation: 3,
}
let current: HostCRD = host
const authority = { current: { known: true, generation: 7 } }
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => authority.current)
const secretReadStarted = deferred()
const releaseSecretRead = deferred()
vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
secretReadStarted.resolve(undefined)
await releaseSecretRead.promise
return {
ok: false,
reason: 'SecretNotFound',
message: 'secret missing in stale spec',
}
})
const deleteRuntime = vi
.spyOn(reconciler as any, 'deleteHostRuntimeResources')
.mockResolvedValue(undefined)
const pending = reconciler.reconcile(host)
await secretReadStarted.promise
current = replaceCurrent(host)
releaseSecretRead.resolve(undefined)
await expect(pending).rejects.toThrow(expectedError)
expect(deleteRuntime).not.toHaveBeenCalled()
}
)
it('revalidates the Host spec after token provisioning and before Deployment mutation', async () => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'deployment-generation-race' }),
uid: 'host-uid',
generation: 3,
}
let current = host
const authority = { current: { known: true, generation: 11 } }
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => authority.current)
stubPreDeploymentReconcileEffects(reconciler)
const tokenProvisionStarted = deferred()
const releaseTokenProvision = deferred()
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {
tokenProvisionStarted.resolve(undefined)
await releaseTokenProvision.promise
return { revision: 'stale-revision', scopeHash: 'stale-scope' }
})
const ensureDeployment = vi.spyOn(reconciler as any, 'ensureDeployment').mockResolvedValue(true)
const pending = reconciler.reconcile(host)
await tokenProvisionStarted.promise
current = {
...host,
generation: 4,
spec: { ...host.spec, contextRef: 'replacement-context' },
}
releaseTokenProvision.resolve(undefined)
await expect(pending).rejects.toThrow(/Host spec generation/)
expect(ensureDeployment).not.toHaveBeenCalled()
})
it('revalidates the Host spec at the final Deployment create-state boundary', async () => {
const appsApi = createMockAppsApi()
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
const host = {
...makeHost({ name: 'deployment-create-race' }),
uid: 'host-uid',
generation: 3,
}
let current = host
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
stubPreDeploymentReconcileEffects(reconciler)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
revision: 'runtime-revision',
scopeHash: runtimeTokenScopeWithoutChannels(host),
})
let channelIngressReads = 0
vi.spyOn(reconciler as any, 'hasChannelIngress').mockImplementation(() => {
channelIngressReads += 1
if (channelIngressReads === 2) {
current = {
...host,
generation: 4,
spec: { ...host.spec, contextRef: 'replacement-context' },
}
}
return false
})
await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
expect(channelIngressReads).toBe(2)
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('revalidates the Host spec after a Deployment create conflict and before replace', async () => {
const appsApi = createMockAppsApi()
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
const host = {
...makeHost({ name: 'deployment-replace-race' }),
uid: 'host-uid',
generation: 3,
}
let current = host
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
stubPreDeploymentReconcileEffects(reconciler)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
revision: 'runtime-revision',
scopeHash: runtimeTokenScopeWithoutChannels(host),
})
vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {
current = {
...host,
generation: 4,
spec: { ...host.spec, contextRef: 'replacement-context' },
}
return {
metadata: {
name: host.name,
namespace: host.namespace,
uid: 'deployment-uid',
resourceVersion: 'rv-existing',
},
}
})
await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('revalidates the Host spec before retrying a conflicted Deployment replace', async () => {
const appsApi = createMockAppsApi()
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(createMockCoreApi()),
networkingApi: asNetworkingApi(createMockNetworkingApi()),
rbacApi: asRbacApi(createMockRbacApi()),
})
const host = {
...makeHost({ name: 'deployment-retry-race' }),
uid: 'host-uid',
generation: 3,
}
let current = host
let deploymentReads = 0
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
stubPreDeploymentReconcileEffects(reconciler)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
revision: 'runtime-revision',
scopeHash: runtimeTokenScopeWithoutChannels(host),
})
vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
appsApi.readNamespacedDeployment.mockImplementation(async () => {
deploymentReads += 1
if (deploymentReads === 2) {
current = {
...host,
generation: 4,
spec: { ...host.spec, contextRef: 'replacement-context' },
}
}
return {
metadata: {
name: host.name,
namespace: host.namespace,
uid: 'deployment-uid',
resourceVersion: `rv-existing-${deploymentReads}`,
},
}
})
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
const random = vi.spyOn(Math, 'random').mockReturnValue(0)
try {
await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
} finally {
random.mockRestore()
}
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
})
it('resolves the latest authoritative Host object at reconcile admission', async () => {
const reconciler = makeBasicReconciler()
const stale = { ...makeHost({ name: 'latest-host' }), generation: 1 }
const current = {
...stale,
generation: 2,
spec: { ...stale.spec, contextRef: 'current-context' },
}
const authority = { current: { known: true, generation: 7 } }
wireAuthoritativeHost(reconciler, current, authority)
const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
await reconciler.reconcile(stale)
expect(reconcileCore).toHaveBeenCalledOnce()
expect(reconcileCore).toHaveBeenCalledWith(
expect.objectContaining({
generation: 2,
spec: expect.objectContaining({ contextRef: 'current-context' }),
}),
expect.any(Function)
)
})
it('does not patch channel-reader after Host authority changes during revision resolution', async () => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'channel-authority-race' }),
uid: 'channel-authority-race-uid',
generation: 3,
}
const authority = { current: { known: true, generation: 9 } }
wireAuthoritativeHost(reconciler, host, authority)
const revisionStarted = deferred()
const releaseRevision = deferred<string>()
vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
async () => {
revisionStarted.resolve(undefined)
return releaseRevision.promise
}
)
const patchDeployment = vi
.spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
.mockResolvedValue(undefined)
const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
await revisionStarted.promise
authority.current = { known: false, generation: 10 }
releaseRevision.resolve('resolved-revision')
await pending
expect(patchDeployment).not.toHaveBeenCalled()
})
it.each([
['UID', (host: HostCRD) => ({ ...host, uid: 'replacement-channel-reader-uid' })],
[
'spec generation',
(host: HostCRD) => ({
...host,
generation: (host.generation ?? 0) + 1,
spec: { ...host.spec, contextRef: 'replacement-context' },
}),
],
] as const)(
'does not patch channel-reader when the Host %s changes during revision resolution',
async (_, replaceCurrent) => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'channel-host-race' }),
uid: 'channel-host-race-uid',
generation: 3,
}
let current: HostCRD = host
reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 9 }))
const revisionStarted = deferred()
const releaseRevision = deferred<string>()
vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
async () => {
revisionStarted.resolve(undefined)
return releaseRevision.promise
}
)
const patchDeployment = vi
.spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
.mockResolvedValue(undefined)
const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
await revisionStarted.promise
current = replaceCurrent(host)
releaseRevision.resolve('resolved-revision')
await pending
expect(patchDeployment).not.toHaveBeenCalled()
}
)
it('serializes a fresh channel-reader revision behind an older in-flight Host reconcile', async () => {
const reconciler = makeBasicReconciler()
const host = {
...makeHost({ name: 'channel-revision-order' }),
uid: 'channel-revision-order-uid',
generation: 4,
}
const authority = { current: { known: true, generation: 9 } }
wireAuthoritativeHost(reconciler, host, authority)
const staleReconcileStarted = deferred()
const releaseStaleReconcile = deferred()
const order: string[] = []
let liveRevision = ''
vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async () => {
order.push('reconcile-a:start')
staleReconcileStarted.resolve(undefined)
await releaseStaleReconcile.promise
liveRevision = 'revision-a'
order.push('reconcile-a:write')
})
vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue(
'revision-b'
)
const patchDeployment = vi
.spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
.mockImplementation(async (request: any) => {
const { body } = request
liveRevision = body.spec.template.metadata.annotations['clerum.io/credentials-revision']
order.push('revision-b:patch')
})
const staleReconcile = reconciler.reconcile(host)
await staleReconcileStarted.promise
const freshRevision = reconciler.patchChannelReaderRevisionAnnotation(host.name)
// Give B the opportunity to patch before A is released. Without a shared
// Host lane, B writes first and A deterministically overwrites it.
await Promise.resolve()
await Promise.resolve()
releaseStaleReconcile.resolve(undefined)
await Promise.all([staleReconcile, freshRevision])
expect(liveRevision).toBe('revision-b')
expect(patchDeployment).toHaveBeenCalledOnce()
expect(order).toEqual(['reconcile-a:start', 'reconcile-a:write', 'revision-b:patch'])
})
it('rejects a queued delete when Host authority changes before admission', async () => {
const reconciler = makeBasicReconciler()
const host = makeHost({ name: 'delete-authority-race' })
const authority = { current: { known: true, generation: 11 } }
reconciler.setResolveCurrentHost(() => undefined)
reconciler.setHostWatchAuthority(() => authority.current)
const deleteRuntime = vi
.spyOn(reconciler as any, 'deleteHostRuntimeResources')
.mockResolvedValue(undefined)
const blockerStarted = deferred()
const releaseBlocker = deferred()
const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
blockerStarted.resolve(undefined)
await releaseBlocker.promise
}) as Promise<void>
await blockerStarted.promise
const pending = reconciler.reconcileDelete(host.name, host.namespace)
authority.current = { known: false, generation: 12 }
releaseBlocker.resolve(undefined)
await occupied
await expect(pending).rejects.toThrow(/Host inventory authority/)
expect(deleteRuntime).not.toHaveBeenCalled()
})
})