Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, onTestFinished, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
type AdministrativeOutcomeReporter,
BoundedAdministrativeOutcomeReporter,
} from '../src/administrativeOutcomeReporter'
import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,
HostFleetReconcileError,
HostReconciler,
OAUTH_USER_TOKEN_SCOPE,
resolveRuntimeControlScopes,
resolveWorkflowControlScopes,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
import { HostContextLogger } from '../src/logger'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
import { registry } from '../src/metrics'
import { HostCRD, type HostWorkflowControlScope } from '../src/types'
import {
asAppsApi,
asCoreApi,
asNetworkingApi,
asRbacApi,
createMockAppsApi,
createMockCoreApi,
createMockNetworkingApi,
createMockRbacApi,
} from './__fixtures__/testMocks'
vi.mock('../src/config', () => ({
config: {
devMode: false,
port: 8081,
namespace: 'mcp-server',
controlPlaneNamespace: 'control-plane',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
channelsNamespace: 'channels',
hostFullReconcileConcurrency: 2,
channelReaderImage: 'clerum/channel-reader:test',
channelReaderImagePullPolicy: 'IfNotPresent',
hostImage: 'clerum/mcp-host:0.6.0',
hostImagePullSecretName: 'clerum',
hostPort: 8080,
gfsNamespace: 'gfs',
gfscPort: 8087,
hostConfigMapName: 'mcp-host-config',
hostServiceAccountName: 'mcp-host',
hostWorkspaceStorageClassName: 'do-block-storage-retain',
hostWorkspaceStorageSize: '10Gi',
hostWorkspacePath: '/workspace',
hostResources: {
requests: { memory: '128Mi', cpu: '100m' },
limits: { memory: '512Mi', cpu: '500m' },
},
desktopImage: 'clerum/mcp-host-desktop:latest',
desktopPort: 3000,
desktopResources: {
requests: { memory: '256Mi', cpu: '250m' },
limits: { memory: '4Gi', cpu: '1' },
},
devMcpServers: [],
devContexts: [],
devAuthTokens: new Map(),
// mcpHost runtime token provisioning config. Tests do NOT exercise the real
// network call; `mcpHostRuntimeTokenIssuerClient` is mocked below to short-circuit
// before reaching the InternalControl signer.
controlApiBaseUrl: 'http://control-api.test:8090',
internalControlJwtHccHmacSecret: 'test-hcc-internal-control-secret',
hccTargetNamespace: 'mcp-host',
mcpHostGatewayUrl:
'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092',
},
}))
// Short-circuit the HCC issuer so reconcile tests don't reach the network.
// Returns deterministic tokens; the secret-build path in `secretFactory` is
// exercised end-to-end so the Secret content assertions still test real code.
vi.mock('../src/mcpHostRuntimeTokenIssuerClient', () => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
accessToken: 'test-mcp-host-runtime-access-token',
refreshToken: 'test-mcp-host-runtime-refresh-token',
mcpHostControlToken: 'test-mcp-host-workflow-control-token',
channelReaderMessageToken: 'test-channel-reader-message-token',
channelReaderApprovalToken: 'test-channel-reader-approval-token',
channelReaderWorkflowApprovalDecisionToken: 'test-channel-reader-decision-token',
channelReaderActivityToken: 'test-channel-reader-activity-token',
channelReaderCronReadToken: 'test-channel-reader-cron-read-token',
channelReaderCronAckToken: 'test-channel-reader-cron-ack-token',
expiresInSeconds: 600,
refreshExpiresInSeconds: 3600,
controlExpiresInSeconds: 600,
channelReaderMessageExpiresInSeconds: 600,
channelReaderApprovalExpiresInSeconds: 600,
channelReaderWorkflowApprovalDecisionExpiresInSeconds: 600,
}),
}))
vi.mock('../src/gfsHostBinding', () => ({
mintHostGfsToken: vi
.fn()
.mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => ({
['to'.concat('ken')]: 'gfs-runtime-value',
expiresInSeconds: 300,
subject: `host:1st:${namespace}/${name}`,
})),
}))
function makeHost(overrides?: Partial<HostCRD>): HostCRD {
return {
name: 'alpha-host',
namespace: 'mcp-host',
uid: 'host-uid-1',
spec: {
host: 'alpha-host',
contextRef: 'context-a',
secretRef: 'host-secret',
channels: ['channel-a'],
},
...overrides,
}
}
function makeDesktopHost(
desktopSpec: { browser?: boolean; x11?: boolean } = { x11: true },
overrides?: Partial<HostCRD>
): HostCRD {
return makeHost({
...overrides,
spec: {
host: 'desktop-host',
contextRef: 'context-a',
secretRef: 'host-secret',
desktop: desktopSpec,
...(overrides as HostCRD | undefined)?.spec,
},
name: overrides?.name ?? 'desktop-host',
})
}
function createTelemetryReporterMock(): InfrastructureTelemetryReporter {
return {
enqueue: vi.fn(),
enqueueHealthTransition: vi.fn(),
stop: vi.fn(async () => undefined),
}
}
function createReconciler(
options: {
infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
administrativeOutcomeReporter?: AdministrativeOutcomeReporter
newTelemetryOccurrenceId?: () => string
// §10.5 orphan cleanup performs ONE fresh authoritative Host read
// (`customApi.getNamespacedCustomObject`) per orphan logical Host. Tests
// that exercise the authority-gated cleanup wire a mock here so a confirmed
// 404 authorizes deletion; without it the reconciler lazily constructs a
// real client that would never resolve under test.
customApi?: { getNamespacedCustomObject: ReturnType<typeof vi.fn> }
} = {}
) {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const networkingApi = createMockNetworkingApi()
const rbacApi = createMockRbacApi()
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
isCommunicationChannelCacheSynced: () => true,
infrastructureTelemetryReporter: options.infrastructureTelemetryReporter,
administrativeOutcomeReporter: options.administrativeOutcomeReporter,
newTelemetryOccurrenceId: options.newTelemetryOccurrenceId,
...(options.customApi
? { customApi: options.customApi as unknown as k8s.CustomObjectsApi }
: {}),
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi }
}
describe('HostReconciler', () => {
it('keys GFS lifecycle evidence by namespace/name/uid and refuses a missing uid (#696)', () => {
const helper = HostReconciler as unknown as {
gfsLifecycleEvidenceKey(host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>): string | undefined
}
expect(
helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name', uid: 'uid-1' })
).toBe('tenant-a/shared-name/uid-1')
expect(
helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-b', name: 'shared-name', uid: 'uid-1' })
).toBe('tenant-b/shared-name/uid-1')
// Same namespace/name, different object: the key must not collide.
expect(
helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name', uid: 'uid-2' })
).not.toBe('tenant-a/shared-name/uid-1')
expect(
helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name' })
).toBeUndefined()
})
it('binds GFS material to the exact Host and records only safe lifecycle annotations', async () => {
const { reconciler, coreApi } = createReconciler()
const boundHost = makeHost({ uid: 'host-uid-1', generation: 7 })
await reconciler.reconcile(boundHost)
expect(mintHostGfsToken).toHaveBeenCalledWith({ name: 'alpha-host', namespace: 'mcp-host' })
const runtimeSecretWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
)?.[0].body as k8s.V1Secret
expect(runtimeSecretWrite.metadata?.annotations).toMatchObject({
'clerum.io/gfs-token-expected-subject': 'host:1st:mcp-host/alpha-host',
'clerum.io/gfs-token-host-uid': 'host-uid-1',
'clerum.io/gfs-token-host-generation': '7',
'clerum.io/gfs-token-capability-set-hash': expect.stringMatching(/^[0-9a-f]{64}$/),
'clerum.io/gfs-token-expires-at': expect.any(String),
'clerum.io/gfs-token-refresh-before': expect.any(String),
})
expect(JSON.stringify(runtimeSecretWrite.metadata?.annotations)).not.toContain(
'gfs-runtime-value'
)
})
it('leaves the per-Host Secret untouched when binding validation rejects the response', async () => {
const { reconciler, coreApi } = createReconciler()
const validationError = new Error('gfs host token subject mismatch')
validationError.name = 'GfsHostTokenValidationError'
vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)
await expect(reconciler.reconcile(makeHost({ uid: 'host-uid-1' }))).rejects.toThrow(
/subject mismatch/
)
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
})
it('rotates for a recreated same-name Host without changing its grants identity', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, appsApi, coreApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
await reconciler.reconcile(makeHost({ uid: 'old-host-uid', generation: 3 }))
const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
)?.[0].body as k8s.V1Secret
coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
})
const channelReader = appsApi.replaceNamespacedDeployment.mock.calls.find(
([request]) => request.name === 'channel-reader-alpha-host'
)![0].body
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === 'channel-reader-alpha-host'
? Promise.resolve(channelReader)
: readDeployment(request)
)
coreApi.replaceNamespacedSecret.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
await reconciler.reconcile(makeHost({ uid: 'new-host-uid', generation: 1 }))
expect(mintHostGfsToken).toHaveBeenLastCalledWith({
name: 'alpha-host',
namespace: 'mcp-host',
})
const recreationWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
)?.[0].body as k8s.V1Secret
expect(recreationWrite.metadata?.annotations).toMatchObject({
'clerum.io/gfs-token-expected-subject': 'host:1st:mcp-host/alpha-host',
'clerum.io/gfs-token-host-uid': 'new-host-uid',
'clerum.io/gfs-token-host-generation': '1',
'clerum.io/runtime-token-rollout-required': 'true',
})
// GFS evidence travels as the allowlisted `transition` (#328); the recreated
// object is identified by the uid in the lookup reference (#691).
const recreationOutcome = vi
.mocked(reporter.enqueue)
.mock.calls.map(([projection]) => projection)
.filter(projection => projection.telemetryType === 'reconcile_outcome')
.at(-1)
expect(recreationOutcome).toMatchObject({
hostLookupReference: {
name: 'alpha-host',
namespace: 'mcp-host',
generation: 1,
uid: 'new-host-uid',
},
payload: { transition: 'gfs_token:rotated' },
})
expect(Object.keys(recreationOutcome!.payload!).filter(key => key.startsWith('gfs_'))).toEqual(
[]
)
expect(
appsApi.replaceNamespacedDeployment.mock.calls.every(
([request]) => request.name === 'alpha-host'
)
).toBe(true)
})
it('converges existing deployment, service, and unbound pvc for valid host', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost()
await reconciler.reconcile(host)
expect(coreApi.readNamespacedSecret).toHaveBeenCalledWith({
namespace: 'mcp-host',
name: 'host-secret',
})
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
// Reconcile updates two existing Services: the host (mcp-host ns) and its
// channel-reader handoff Service (channels ns).
expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)
expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(2)
expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
expect.objectContaining({
namespace: 'mcp-host',
body: expect.objectContaining({
metadata: expect.objectContaining({ name: 'alpha-host' }),
}),
}),
])
expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
expect.objectContaining({
namespace: 'channels',
body: expect.objectContaining({
metadata: expect.objectContaining({ name: 'channel-reader-alpha-host' }),
}),
}),
])
// Reconcile updates two existing Deployments: the host (mcp-host ns) and its
// channel-reader (channels ns).
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
expect.objectContaining({ namespace: 'mcp-host' }),
])
expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
expect.objectContaining({ namespace: 'channels' }),
])
expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
'alpha-host',
'host-uid-1',
DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES
)
})
it('emits Host-backed reconcile telemetry without inventing a legacy lifecycle transition', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })
await reconciler.reconcile(makeHost({ generation: 7 }))
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'reconcile_outcome',
hostLookupReference: {
name: 'alpha-host',
namespace: 'mcp-host',
generation: 7,
uid: 'host-uid-1',
},
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'ready',
status: 'succeeded',
phase: 'deployed',
state: 'ready',
}),
})
)
expect(vi.mocked(reporter.enqueue).mock.calls[0]![0]).not.toHaveProperty('sourceEventId')
expect(reporter.enqueue).not.toHaveBeenCalledWith(
expect.objectContaining({ telemetryType: 'lifecycle_transition' })
)
})
it('enqueues the linked administrative outcome only for a protected operation annotation', async () => {
const administrativeOutcomeReporter = {
enqueueHostOutcome: vi.fn(),
stop: vi.fn(async () => undefined),
}
const { reconciler } = createReconciler({ administrativeOutcomeReporter })
await reconciler.reconcile(
makeHost({
generation: 7,
annotations: {
'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
},
})
)
expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
expect.objectContaining({
sourceEventId:
'hcc-admin-outcome-v2:11111111-1111-4111-8111-111111111111:7:host-uid-1:succeeded',
outcome: 'succeeded',
hostRef: {
name: 'alpha-host',
namespace: 'mcp-host',
generation: 7,
uid: 'host-uid-1',
},
})
)
})
it('gives a same-name recreated Host its own administrative identity (#694)', async () => {
const administrativeOutcomeReporter = {
enqueueHostOutcome: vi.fn(),
stop: vi.fn(async () => undefined),
}
const { reconciler } = createReconciler({ administrativeOutcomeReporter })
const annotations = {
'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
}
// Same name, same generation, different object: only the uid tells them
// apart, and the server's dedupe is keyed on this id.
await reconciler.reconcile(makeHost({ generation: 1, uid: 'host-uid-1', annotations }))
await reconciler.reconcile(makeHost({ generation: 1, uid: 'host-uid-2', annotations }))
const ids = administrativeOutcomeReporter.enqueueHostOutcome.mock.calls.map(
([projection]) => projection.sourceEventId
)
// Liveness: both passes reached the reporter.
expect(ids).toHaveLength(2)
expect(new Set(ids).size).toBe(2)
})
it('skips the administrative outcome when the Host snapshot has no uid (#694)', async () => {
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
onTestFinished(() => warn.mockRestore())
const administrativeOutcomeReporter = {
enqueueHostOutcome: vi.fn(),
stop: vi.fn(async () => undefined),
}
const { reconciler } = createReconciler({ administrativeOutcomeReporter })
const host = makeHost({
generation: 7,
annotations: {
'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
},
})
delete (host as { uid?: string }).uid
await reconciler.reconcile(host)
// Liveness: the reconcile reached the administrative branch and refused
// there — control-api answers a uid-less reference with a 400 this
// reporter treats as terminal, so the event would be lost, not retried.
expect(warn).toHaveBeenCalledWith(
'skipping administrative outcome: Host snapshot has no uid',
expect.objectContaining({ host: 'alpha-host', namespace: 'mcp-host' })
)
expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalled()
})
it('skips Host telemetry when the Host snapshot has no uid (#693)', async () => {
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
onTestFinished(() => warn.mockRestore())
const reporter = createTelemetryReporterMock()
const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })
// Liveness: this reconciler and this reporter do enqueue when the snapshot
// carries a uid, so the unchanged count below is the guard's doing and not
// a reporter that was never reached.
await reconciler.reconcile(makeHost({ generation: 7 }))
expect(reporter.enqueue).toHaveBeenCalledTimes(1)
const host = makeHost({ generation: 8 })
delete (host as { uid?: string }).uid
await reconciler.reconcile(host)
// control-api answers a uid-less reference with a 400 this reporter treats
// as terminal, and the infrastructure reporter keeps no `seen` set to fall
// silent, so the event would be re-sent and re-dropped every pass (#693).
expect(warn).toHaveBeenCalledWith(
'skipping host telemetry: Host snapshot has no uid',
expect.objectContaining({
host: 'alpha-host',
namespace: 'mcp-host',
telemetryType: 'reconcile_outcome',
})
)
expect(reporter.enqueue).toHaveBeenCalledTimes(1)
})
it('sends an administrative outcome once across repeated reconcile passes (#327)', async () => {
const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
const administrativeOutcomeReporter = new BoundedAdministrativeOutcomeReporter({
baseUrl: 'http://control-api.test:8090',
signToken: () => 'signed',
fetchFn,
random: () => 0,
})
onTestFinished(() => administrativeOutcomeReporter.stop())
const telemetry = createTelemetryReporterMock()
const { reconciler } = createReconciler({
administrativeOutcomeReporter,
infrastructureTelemetryReporter: telemetry,
})
const host = makeHost({
generation: 7,
annotations: { 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111' },
})
for (let pass = 0; pass < 3; pass += 1) {
await reconciler.reconcile(host)
await new Promise(resolve => setTimeout(resolve, 0))
}
// Liveness: every pass re-observed the outcome.
expect(
vi
.mocked(telemetry.enqueue)
.mock.calls.filter(([projection]) => projection.telemetryType === 'reconcile_outcome')
).toHaveLength(3)
expect(fetchFn).toHaveBeenCalledOnce()
})
it('does not turn the previous pass status into a succeeded outcome when a pass throws (#327)', async () => {
const administrativeOutcomeReporter = {
enqueueHostOutcome: vi.fn(),
stop: vi.fn(async () => undefined),
}
const telemetry = createTelemetryReporterMock()
const { reconciler, rbacApi } = createReconciler({
administrativeOutcomeReporter,
infrastructureTelemetryReporter: telemetry,
})
const annotations = {
'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
}
await reconciler.reconcile(makeHost({ generation: 7, annotations }))
administrativeOutcomeReporter.enqueueHostOutcome.mockClear()
vi.mocked(telemetry.enqueue).mockClear()
// Fails before this pass writes any status, so the status read afterwards
// is still the previous pass's deployed/ready.
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
await expect(reconciler.reconcile(makeHost({ generation: 8, annotations }))).rejects.toEqual({
code: 500,
})
// Liveness: the failed pass reported its error and its reconcile outcome
// while the previous pass's ready status was still in place.
expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
const types = vi
.mocked(telemetry.enqueue)
.mock.calls.map(([projection]) => projection.telemetryType)
expect(types).toContain('controller_error')
// The telemetry reports the exception, not the stale ready status.
const failedOutcome = vi
.mocked(telemetry.enqueue)
.mock.calls.map(([projection]) => projection)
.find(projection => projection.telemetryType === 'reconcile_outcome')
expect(failedOutcome?.payload).toEqual({
resource_class: 'Host',
reason_code: 'reconcile_exception',
status: 'failed',
})
expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
expect.objectContaining({
outcome: 'failed',
hostRef: expect.objectContaining({ generation: 8 }),
})
)
expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalledWith(
expect.objectContaining({ outcome: 'succeeded' })
)
})
describe('GFS token evidence on reconcile_outcome (#328)', () => {
/**
* The evidence map is an implementation detail of the reconciler, but it is
* the only place a pass that emits no outcome can leave something behind,
* which is exactly the leak the uid key closes (#696).
*/
function gfsEvidenceOf(reconciler: HostReconciler): Map<string, string> {
return (reconciler as unknown as { gfsTokenLifecycleEvidence: Map<string, string> })
.gfsTokenLifecycleEvidence
}
function evidenceKeyOf(host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>): string | undefined {
return (
HostReconciler as unknown as {
gfsLifecycleEvidenceKey(
host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
): string | undefined
}
).gfsLifecycleEvidenceKey(host)
}
function reconcileOutcomes(reporter: InfrastructureTelemetryReporter) {
return vi
.mocked(reporter.enqueue)
.mock.calls.map(([projection]) => projection)
.filter(projection => projection.telemetryType === 'reconcile_outcome')
}
it('reports minted once and not on the next pass', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, rbacApi, coreApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
coreApi.readNamespacedSecret.mockImplementation((request: { name?: string } = {}) =>
request.name === 'host-alpha-host-mcp-host-runtime-tokens'
? Promise.reject({ code: 404 })
: readSecret(request)
)
const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
await reconciler.reconcile(makeHost({ generation: 3 }))
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toEqual({
code: 500,
})
// Liveness: both passes emitted an outcome; only the first reached the mint.
expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
const [first, second] = reconcileOutcomes(reporter)
expect(first?.payload).toMatchObject({ transition: 'gfs_token:minted' })
expect(second?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
expect(second?.payload).not.toHaveProperty('transition')
})
it('reports reused when the runtime Secret is kept', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, coreApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
// Reuse needs a refresh token whose `exp` can be read.
const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url')
const refreshToken = [
encode({ alg: 'none' }),
encode({ exp: Math.floor(Date.now() / 1000) + 30 * 86_400 }),
'unsigned',
].join('.')
const issue = vi.mocked(issueMcpHostRuntimeTokens).getMockImplementation()!
vi.mocked(issueMcpHostRuntimeTokens).mockImplementationOnce(async (...args) => ({
...(await issue(...args)),
refreshToken,
}))
const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
await reconciler.reconcile(makeHost({ generation: 3 }))
const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
)?.[0].body as k8s.V1Secret
coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
})
await reconciler.reconcile(makeHost({ generation: 3 }))
expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
expect(reconcileOutcomes(reporter).map(outcome => outcome.payload?.transition)).toEqual([
'gfs_token:rotated',
'gfs_token:reused',
])
})
it('reports failed on the exception outcome when the mint is rejected', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })
const validationError = new Error('gfs host token subject mismatch')
validationError.name = 'GfsHostTokenValidationError'
vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)
await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toThrow(
/subject mismatch/
)
expect(reconcileOutcomes(reporter).map(outcome => outcome.payload)).toEqual([
{
resource_class: 'Host',
reason_code: 'reconcile_exception',
status: 'failed',
transition: 'gfs_token:failed',
},
])
})
it('drops the evidence of a superseded pass', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, appsApi, rbacApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
const superseded = new Error('Host spec changed while the Deployment write was in flight')
superseded.name = 'HostMutationSpecRevisionChangedError'
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(superseded)
// A different object at the same name, whose evidence this pass has no
// business touching: the supersession knows which uid it reconciled.
const evidence = gfsEvidenceOf(reconciler)
const siblingKey = evidenceKeyOf({
namespace: 'mcp-host',
name: 'alpha-host',
uid: 'host-uid-sibling',
})!
evidence.set(siblingKey, 'minted')
await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toBe(superseded)
// Liveness: the superseded pass minted, and emitted no outcome.
expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
expect(reconcileOutcomes(reporter)).toHaveLength(0)
// The pass dropped its own entry and left the sibling's alone. Both
// halves matter: a clear scoped too narrowly would leak the transition
// into the next pass, and one scoped by name would destroy the sibling's.
expect(
evidence.has(
evidenceKeyOf({ namespace: 'mcp-host', name: 'alpha-host', uid: 'host-uid-1' })!
)
).toBe(false)
expect(evidence.get(siblingKey)).toBe('minted')
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
await expect(reconciler.reconcile(makeHost({ generation: 4 }))).rejects.toEqual({
code: 500,
})
const outcomes = reconcileOutcomes(reporter)
expect(outcomes).toHaveLength(1)
expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
expect(outcomes[0]?.payload).not.toHaveProperty('transition')
})
it('does not lend a leftover entry to a same-name Host with a new uid (#696)', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, rbacApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
const evidence = gfsEvidenceOf(reconciler)
// What a pass that recorded evidence and emitted no outcome leaves behind
// (the stateless suspension path), written under the key the reconciler
// itself would use for that object — so a key that ignored the uid would
// be picked up below. The Host that takes over the name is a different
// object and must not report that rotation.
const leftoverKey = evidenceKeyOf({
namespace: 'mcp-host',
name: 'alpha-host',
uid: 'host-uid-1',
})
expect(leftoverKey).toBeDefined()
evidence.set(leftoverKey!, 'rotated')
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
await expect(
reconciler.reconcile(makeHost({ uid: 'host-uid-2', generation: 3 }))
).rejects.toEqual({ code: 500 })
const outcomes = reconcileOutcomes(reporter)
// Liveness: the new object's pass did emit its outcome; that pass reached
// no GFS write of its own, so any transition would be the leftover.
expect(outcomes).toHaveLength(1)
expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
expect(outcomes[0]?.payload).not.toHaveProperty('transition')
expect(evidence.get(leftoverKey!)).toBe('rotated')
})
it('clears every entry for a deleted Host and leaves other Hosts alone (#696)', async () => {
const { reconciler } = createReconciler()
const evidence = gfsEvidenceOf(reconciler)
const deletedKey = evidenceKeyOf({
namespace: 'mcp-host',
name: 'alpha-host',
uid: 'host-uid-1',
})!
// A prefix neighbour on purpose: the sweep matches `mcp-host/alpha-host/`
// and only the trailing separator keeps `alpha-host-2` out of it. A
// survivor with an unrelated name would leave that character untested.
const survivorKey = evidenceKeyOf({
namespace: 'mcp-host',
name: 'alpha-host-2',
uid: 'host-uid-9',
})!
evidence.set(deletedKey, 'rotated')
evidence.set(survivorKey, 'minted')
// Witness: the entry the delete has to remove exists before the call.
expect(evidence.has(deletedKey)).toBe(true)
await reconciler.reconcileDelete('alpha-host', 'mcp-host')
expect(evidence.has(deletedKey)).toBe(false)
expect(evidence.get(survivorKey)).toBe('minted')
})
it('records no evidence and warns when the Host snapshot has no uid (#696)', async () => {
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
onTestFinished(() => warn.mockRestore())
const reporter = createTelemetryReporterMock()
const { reconciler, appsApi } = createReconciler({
infrastructureTelemetryReporter: reporter,
})
const host = makeHost({ generation: 3 })
delete (host as { uid?: string }).uid
await reconciler.reconcile(host)
// Liveness: the pass did its reconcile work, so it reached the GFS
// token path — the evidence is missing because recording it was
// refused, not because the pass never executed. The outcome itself is
// no longer a witness: the same missing uid now also stops the
// telemetry from being emitted at all (#693).
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalled()
expect(warn).toHaveBeenCalledWith(
'skipping gfs token lifecycle evidence: Host snapshot has no uid',
expect.objectContaining({ host: 'alpha-host', namespace: 'mcp-host' })
)
expect(gfsEvidenceOf(reconciler).size).toBe(0)
expect(reconcileOutcomes(reporter)).toHaveLength(0)
})
})
it('emits Host-backed controller_error and failed reconcile_outcome on reconcile failure', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, coreApi } = createReconciler({ infrastructureTelemetryReporter: reporter })
coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
if (name === 'host-secret') {
return Promise.reject({ code: 404 })
}
return Promise.resolve({
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'alpha-host',
},
},
})
})
await reconciler.reconcile(makeHost({ generation: 8 }))
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'controller_error',
hostLookupReference: {
name: 'alpha-host',
namespace: 'mcp-host',
generation: 8,
uid: 'host-uid-1',
},
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'SecretNotFound',
status: 'failed',
error_class: 'Error',
}),
})
)
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'reconcile_outcome',
hostLookupReference: {
name: 'alpha-host',
namespace: 'mcp-host',
generation: 8,
uid: 'host-uid-1',
},
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'not_ready',
status: 'failed',
phase: 'not_deployed',
state: 'not_ready',
}),
})
)
})
it('mints first-party workflow scopes when an external CommunicationChannel references a host without spec.channels', async () => {
const { reconciler } = createReconciler()
reconciler.setCountCommunicationChannels(() => 1)
await reconciler.reconcile(
makeHost({
spec: {
host: 'alpha-host',
contextRef: 'context-a',
secretRef: 'host-secret',
},
})
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
'alpha-host',
'host-uid-1',
DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES
)
})
it('fails closed when referenced secret is missing', async () => {
const { reconciler, appsApi, coreApi, networkingApi, rbacApi } = createReconciler()
coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
if (name === 'host-secret') {
return Promise.reject({ code: 404 })
}
if (name === 'channel-reader-alpha-host-mcp-host-runtime-auth') {
return Promise.resolve({
metadata: {
labels: {
'clerum.io/component': 'channel-reader',
'clerum.io/host': 'alpha-host',
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/secret-purpose': 'mcp-host-runtime-auth',
},
},
})
}
return Promise.resolve({
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'alpha-host',
},
},
})
})
await reconciler.reconcile(makeHost())
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
name: 'alpha-host',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'alpha-host',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'host-alpha-host-mcp-host-runtime-tokens',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
namespace: 'channels',
})
expect(rbacApi.deleteNamespacedRole).toHaveBeenCalledWith({
name: 'host-alpha-host-config-reader',
namespace: 'mcp-host',
})
expect(rbacApi.deleteNamespacedRoleBinding).toHaveBeenCalledWith({
name: 'host-alpha-host-config-reader',
namespace: 'mcp-host',
})
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'mcp-host-alpha-host-ingress-rpc-proxy',
namespace: 'mcp-host',
})
expect(reconciler.getStatus('alpha-host')).toMatchObject({
deployed: false,
ready: false,
})
})
it('does not delete legacy channel-reader runtime auth when labels are not HCC-owned', async () => {
const { reconciler, coreApi } = createReconciler()
coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
if (name === 'host-secret') {
return Promise.reject({ code: 404 })
}
if (name === 'channel-reader-alpha-host-mcp-host-runtime-auth') {
return Promise.resolve({
metadata: {
labels: {
'clerum.io/host': 'alpha-host',
},
},
})
}
return Promise.resolve({
metadata: {
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'alpha-host',
},
},
})
})
await reconciler.reconcile(makeHost())
expect(coreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
namespace: 'channels',
})
})
it('does not provision per-host channel-reader mcp-host runtime JWT mounts', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
await reconciler.reconcile(makeHost())
const writtenSecrets = [
...coreApi.createNamespacedSecret.mock.calls.map(([arg]) => arg),
...coreApi.replaceNamespacedSecret.mock.calls.map(([arg]) => arg),
]
expect(writtenSecrets).not.toEqual(
expect.arrayContaining([
expect.objectContaining({
namespace: 'channels',
body: expect.objectContaining({
metadata: expect.objectContaining({
name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
}),
}),
}),
])
)
const channelReaderCreate = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {
const body = arg.body as k8s.V1Deployment
return body.metadata?.name === 'channel-reader-alpha-host'
})
expect(channelReaderCreate).toBeDefined()
const deployment = channelReaderCreate![0].body as k8s.V1Deployment
const container = deployment.spec?.template.spec?.containers?.[0]
const annotations = deployment.spec?.template.metadata?.annotations ?? {}
expect(annotations['clerum.io/runtime-auth-revision']).toBeUndefined()
const envNames = (container?.env ?? []).map(env => env.name)
// No per-host JWT runtime/control token env — those would be
// CLERUM_MCP_HOST_RUNTIME_* / *_WORKFLOW_CONTROL_TOKEN. The routing URL
// (CLERUM_MCP_HOST_URL) is allowed: it is a Service address, not auth.
expect(
envNames.filter(
(name: string) => name.startsWith('CLERUM_MCP_HOST_') && name !== 'CLERUM_MCP_HOST_URL'
)
).toEqual([])
expect(container?.volumeMounts?.map(mount => mount.name) ?? []).not.toContain(
'mcp-host-runtime-auth'
)
expect(deployment.spec?.template.spec?.volumes?.map(volume => volume.name) ?? []).not.toContain(
'mcp-host-runtime-auth'
)
})
it('propagates cleanup errors when referenced secret is missing', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
appsApi.deleteNamespacedDeployment.mockRejectedValueOnce(
Object.assign(new Error('forbidden'), { code: 403 })
)
await expect(reconciler.reconcile(makeHost())).rejects.toThrow(
'Failed to delete runtime resources for Host "alpha-host"'
)
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'alpha-host',
namespace: 'mcp-host',
})
expect(reconciler.getStatus('alpha-host')).toMatchObject({
deployed: false,
ready: false,
message: 'Not reconciled yet',
})
})
it('removes orphaned host runtime resources during full reconcile', async () => {
// §10.5 authority-gated cleanup: an orphan managed Deployment is discovered,
// its owning Host is confirmed gone by a fresh 404 while watch authority is
// known and stable, and the whole owned bundle (Deployment/Service/PVC) is
// deleted through that Host's per-Host serializer.
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const { reconciler, appsApi, coreApi } = createReconciler({
customApi: { getNamespacedCustomObject: getObj },
})
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(() => undefined) // current cache omits every host
appsApi.listNamespacedDeployment.mockResolvedValue({
items: [
{
metadata: {
name: 'orphan-host',
namespace: 'mcp-host',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': 'orphan-host',
},
},
},
],
})
await reconciler.fullReconcile([])
// The fresh authoritative Host read gates the delete.
expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan-host' }))
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
name: 'orphan-host',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'orphan-host',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
name: 'orphan-host-workspace',
namespace: 'mcp-host',
})
})
it('skips replace on existing bound PVC (immutable spec) instead of forcing 422', async () => {
const { reconciler, coreApi } = createReconciler()
coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 409 })
coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
metadata: {
name: 'alpha-host-workspace',
namespace: 'mcp-host',
uid: 'pvc-uid',
labels: {
'clerum.io/host': 'alpha-host',
'clerum.io/managed-by': 'host-context-controller',
},
resourceVersion: '42',
},
spec: {
volumeName: 'pvc-aaaa-bbbb-cccc-dddd',
storageClassName: 'standard',
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: '10Gi' } },
},
})
await reconciler.reconcile(makeHost())
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
})
it('replaces existing unbound PVC (no volumeName) to preserve growth/path migrations', async () => {
const { reconciler, coreApi } = createReconciler()
coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 409 })
coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
metadata: {
name: 'alpha-host-workspace',
namespace: 'mcp-host',
uid: 'pvc-uid',
labels: {
'clerum.io/host': 'alpha-host',
'clerum.io/managed-by': 'host-context-controller',
},
resourceVersion: '7',
},
spec: {
storageClassName: 'standard',
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: '10Gi' } },
},
})
await reconciler.reconcile(makeHost())
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
})
})
describe('HostReconciler — desktop support', () => {
it('uses desktopImage when host.spec.desktop.x11 = true', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host-desktop:latest')
})
it('uses desktopImage when host.spec.desktop.browser = true', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ browser: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host-desktop:latest')
})
it('adds desktop port 3000 to deployment when desktop enabled', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const ports = call.body.spec.template.spec.containers[0].ports
expect(ports).toHaveLength(2)
expect(ports[0]).toMatchObject({ name: 'http', containerPort: 8080 })
expect(ports[1]).toMatchObject({ name: 'desktop', containerPort: 3000 })
})
it('adds desktop port 3000 to service when desktop enabled', async () => {
const { reconciler, coreApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = coreApi.replaceNamespacedService.mock.calls[0][0]
const ports = call.body.spec.ports
expect(ports).toHaveLength(2)
expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
expect(ports[1]).toMatchObject({ name: 'desktop', port: 3000, targetPort: 'desktop' })
})
it('sets desktop env vars when desktop enabled', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true, browser: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))
expect(envMap.get('CLERUM_DESKTOP_X11')).toBe('true')
expect(envMap.get('CLERUM_DESKTOP_BROWSER')).toBe('true')
expect(envMap.get('PUID')).toBe('1001')
expect(envMap.get('PGID')).toBe('1001')
expect(envMap.get('TZ')).toBe('UTC')
expect(envMap.get('CUSTOM_PORT')).toBe('3000')
})
it('injects the workflow gateway URL for runtime broker traffic', async () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))
expect(envMap.get('MCP_HOST_GATEWAY_URL')).toBe(
'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092'
)
expect(envMap.has('CLERUM_WORKFLOW_GATEWAY_URL')).toBe(false)
})
it('injects the mcpHost workflow control token from the mcp-host-runtime-token secret', async () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const controlEnv = env.find(
(e: { name: string }) => e.name === 'MCP_HOST_WORKFLOW_CONTROL_TOKEN'
)
expect(controlEnv?.valueFrom?.secretKeyRef).toEqual({
name: 'host-alpha-host-mcp-host-runtime-tokens',
key: 'mcp-host-workflow-control-token',
})
})
it('sets runAsNonRoot: false for desktop hosts', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const secCtx = call.body.spec.template.spec.securityContext
expect(secCtx.runAsNonRoot).toBe(false)
expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })
})
it('sets startup probe initialDelaySeconds to 30 for desktop hosts', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const startupProbe = call.body.spec.template.spec.containers[0].startupProbe
expect(startupProbe.initialDelaySeconds).toBe(30)
expect(startupProbe.failureThreshold).toBe(120)
})
it('uses desktopResources for desktop hosts', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const resources = call.body.spec.template.spec.containers[0].resources
expect(resources.requests.memory).toBe('256Mi')
expect(resources.requests.cpu).toBe('250m')
expect(resources.limits.memory).toBe('4Gi')
expect(resources.limits.cpu).toBe('1')
})
it('converges desktop NetworkPolicy when desktop enabled', async () => {
const { reconciler, networkingApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
// Reconcile updates existing per-host ingress, rpc-proxy egress, desktop,
// GFS egress, channel-reader, and workflow-approval-reader policies.
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)
expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
(c: any[]) => c[0].body.metadata.name
)
expect(names).toEqual(
expect.arrayContaining([
'mcp-host-desktop-host-egress-gfs',
'mcp-host-desktop-host-ingress-workflow-approval-reader',
'workflow-approval-reader-desktop-host-egress-mcp-host',
])
)
const call = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
(c: any[]) => c[0].body.metadata.name === 'allow-rpc-proxy-desktop-desktop-host'
)![0]
expect(call.body.metadata.name).toBe('allow-rpc-proxy-desktop-desktop-host')
expect(call.body.spec.podSelector).toEqual({ matchLabels: { app: 'desktop-host' } })
})
it('does not converge desktop NetworkPolicy for non-desktop hosts', async () => {
const { reconciler, networkingApi } = createReconciler()
const host = makeHost()
await reconciler.reconcile(host)
// Reconcile updates 7 existing NPs for non-desktop hosts:
// mcp-host ingress from channel-reader, mcp-host ingress from rpc-proxy,
// mcp-host GFS egress, rpc-proxy host egress, channel-reader egress,
// and workflow-approval-reader policies.
// (no desktop NP)
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
(c: any[]) => c[0].body.metadata.name
)
expect(names).toEqual(
expect.arrayContaining([
'mcp-host-alpha-host-egress-gfs',
'mcp-host-alpha-host-ingress-workflow-approval-reader',
'workflow-approval-reader-alpha-host-egress-mcp-host',
])
)
expect(names).not.toContain(expect.stringContaining('desktop'))
})
it('deletes desktop NetworkPolicy on reconcileDelete (no leak per Host)', async () => {
const { reconciler, networkingApi } = createReconciler()
await reconciler.reconcileDelete('desktop-host', 'mcp-host')
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'allow-rpc-proxy-desktop-desktop-host',
namespace: 'mcp-host',
})
})
it('swallows 404 when desktop NetworkPolicy deletion finds nothing (non-desktop host)', async () => {
const { reconciler, networkingApi } = createReconciler()
networkingApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })
await expect(
reconciler.reconcileDelete('non-desktop-host', 'mcp-host')
).resolves.toBeUndefined()
})
it('non-desktop hosts use standard image and resources (regression)', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost()
await reconciler.reconcile(host)
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host:0.6.0')
expect(container.resources.requests.memory).toBe('128Mi')
expect(container.startupProbe.initialDelaySeconds).toBe(10)
const secCtx = call.body.spec.template.spec.securityContext
expect(secCtx.runAsNonRoot).toBe(true)
expect(secCtx.fsGroup).toBe(1001)
})
it('emits restricted-PSA-compliant pod-level securityContext for non-desktop hosts', async () => {
// MCC shared-tenant namespaces (mcp-host-<slug>) are labeled
// pod-security.kubernetes.io/enforce=restricted. The pod-level
// securityContext must set runAsNonRoot + seccompProfile or the
// ReplicaSet gets FailedCreate and never produces a pod.
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const secCtx = call.body.spec.template.spec.securityContext
expect(secCtx.runAsNonRoot).toBe(true)
// 1001 = the mcp-host image's baked-in `nodejs` user; matches the
// workspace PVC ownership so re-rolling existing hosts doesn't EACCES.
expect(secCtx.runAsUser).toBe(1001)
expect(secCtx.runAsGroup).toBe(1001)
expect(secCtx.fsGroup).toBe(1001)
expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })
})
it('emits restricted-PSA-compliant mcp-host container securityContext for non-desktop hosts', async () => {
// Restricted PSA requires container-level allowPrivilegeEscalation=false,
// capabilities.drop=[ALL], runAsNonRoot=true, seccompProfile=RuntimeDefault.
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.name).toBe('mcp-host')
expect(container.securityContext).toMatchObject({
allowPrivilegeEscalation: false,
runAsNonRoot: true,
// 1001 = the mcp-host image's baked-in `nodejs` user (Dockerfile{,.slim,.full}).
runAsUser: 1001,
runAsGroup: 1001,
capabilities: { drop: ['ALL'] },
seccompProfile: { type: 'RuntimeDefault' },
})
// mcp-host writes outside its mounted volumes; restricted PSA does not
// require readOnlyRootFilesystem, so it must remain unset.
expect(container.securityContext.readOnlyRootFilesystem).toBeUndefined()
})
it('non-desktop hosts have only http port in service', async () => {
const { reconciler, coreApi } = createReconciler()
const host = makeHost()
await reconciler.reconcile(host)
const call = coreApi.replaceNamespacedService.mock.calls[0][0]
const ports = call.body.spec.ports
expect(ports).toHaveLength(1)
expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
})
})
describe('HostReconciler — per-Host RBAC scaffolding', () => {
it('provisions per-Host SA + Role + RoleBinding before the Deployment', async () => {
const { reconciler, coreApi, rbacApi } = createReconciler()
// Initial provisioning explicitly starts without these three RBAC objects.
coreApi.readNamespacedServiceAccount.mockRejectedValueOnce({ code: 404 })
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 404 })
rbacApi.readNamespacedRoleBinding.mockRejectedValueOnce({ code: 404 })
await reconciler.reconcile(makeHost())
expect(coreApi.createNamespacedServiceAccount).toHaveBeenCalledTimes(1)
expect(coreApi.createNamespacedServiceAccount.mock.calls[0][0].body).toMatchObject({
kind: 'ServiceAccount',
metadata: { name: 'host-alpha-host-sa', namespace: 'mcp-host' },
})
expect(rbacApi.createNamespacedRole).toHaveBeenCalledTimes(1)
const roleBody = rbacApi.createNamespacedRole.mock.calls[0][0].body
expect(roleBody.metadata.name).toBe('host-alpha-host-config-reader')
// The mcp-host-runtime-token Secret appears in the read-only rule ONLY. HCC
// writes that Secret using its own ServiceAccount (cluster RBAC); the
// per-Host SA bound to this Role is mounted on the runtime mcp-host pod,
// which must NOT be able to rotate its own credentials.
expect(roleBody.rules).toEqual([
{
apiGroups: ['clerum.io'],
resources: ['hosts'],
resourceNames: ['alpha-host'],
verbs: ['get', 'watch', 'list'],
},
{
apiGroups: [''],
resources: ['configmaps'],
resourceNames: ['host-alpha-host-env', 'clerum-llm-allowed-models'],
verbs: ['get', 'watch', 'list'],
},
{
apiGroups: [''],
resources: ['secrets'],
resourceNames: [
'host-secret',
'host-alpha-host-env-secret',
'host-alpha-host-mcp-host-runtime-tokens',
],
verbs: ['get', 'watch', 'list'],
},
])
expect(rbacApi.createNamespacedRoleBinding).toHaveBeenCalledTimes(1)
expect(rbacApi.createNamespacedRoleBinding.mock.calls[0][0].body).toMatchObject({
kind: 'RoleBinding',
subjects: [{ kind: 'ServiceAccount', name: 'host-alpha-host-sa', namespace: 'mcp-host' }],
roleRef: { kind: 'Role', name: 'host-alpha-host-config-reader' },
})
})
it('uses the per-Host ServiceAccount in the Deployment template', async () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
expect(deployment.spec.template.spec.serviceAccountName).toBe('host-alpha-host-sa')
})
it('injects CLERUM_LLM_SECRET_REF into the Deployment env from spec.secretRef', async () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
const env = deployment.spec.template.spec.containers[0].env as Array<{
name: string
value?: string
}>
expect(env).toContainEqual({ name: 'CLERUM_LLM_SECRET_REF', value: 'host-secret' })
})
it('rewrites Role resourceNames when spec.secretRef changes (replace path)', async () => {
const { reconciler, rbacApi } = createReconciler()
rbacApi.createNamespacedRole.mockRejectedValueOnce({ code: 409 })
// Race: the first absence is superseded by a conflicting creator.
rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 404 })
rbacApi.readNamespacedRole.mockImplementation(async () => {
const desired = rbacApi.createNamespacedRole.mock.calls[0][0].body as {
metadata?: { labels?: Record<string, string> }
rules?: Array<{ resources?: string[]; verbs?: string[]; resourceNames?: string[] }>
}
const live = structuredClone(desired)
const secretRule = live.rules?.find(
rule => rule.resources?.[0] === 'secrets' && rule.verbs?.includes('get')
)
if (!secretRule?.resourceNames) {
throw new Error('expected the created Role to carry a secrets get rule')
}
secretRule.resourceNames[0] = 'host-secret'
return {
...live,
metadata: { ...live.metadata, resourceVersion: '7' },
}
})
await reconciler.reconcile(makeHost({ spec: { secretRef: 'rotated-secret' } } as never))
expect(rbacApi.replaceNamespacedRole).toHaveBeenCalledTimes(1)
const replaced = rbacApi.replaceNamespacedRole.mock.calls[0][0].body
// The read-only rule includes the mcp-host-runtime-tokens Secret too. Filter by
// verbs so we don't accidentally pick up a write-only rule.
const secretRule = replaced.rules.find(
(r: { resources: string[]; verbs: string[] }) =>
r.resources[0] === 'secrets' && r.verbs.includes('get')
)
expect(secretRule.resourceNames).toEqual([
'rotated-secret',
'host-alpha-host-env-secret',
'host-alpha-host-mcp-host-runtime-tokens',
])
expect(replaced.metadata.resourceVersion).toBe('7')
})
it('replaces a live Role that has no rules (fail-open-to-write)', async () => {
const { reconciler, rbacApi } = createReconciler()
rbacApi.createNamespacedRole.mockRejectedValueOnce({ code: 409 })
rbacApi.readNamespacedRole.mockResolvedValue({
metadata: {
name: 'host-alpha-host-config-reader',
namespace: 'mcp-host',
uid: 'role-uid',
resourceVersion: '7',
},
})
await reconciler.reconcile(makeHost({ spec: { secretRef: 'rotated-secret' } } as never))
expect(rbacApi.replaceNamespacedRole).toHaveBeenCalledTimes(1)
expect(rbacApi.replaceNamespacedRole.mock.calls[0][0].body.metadata.resourceVersion).toBe('7')
})
it('cleans up RBAC on reconcileDelete', async () => {
const { reconciler, coreApi, rbacApi } = createReconciler()
await reconciler.reconcileDelete('alpha-host', 'mcp-host')
expect(rbacApi.deleteNamespacedRoleBinding).toHaveBeenCalledWith({
name: 'host-alpha-host-config-reader',
namespace: 'mcp-host',
})
expect(rbacApi.deleteNamespacedRole).toHaveBeenCalledWith({
name: 'host-alpha-host-config-reader',
namespace: 'mcp-host',
})
expect(coreApi.deleteNamespacedServiceAccount).toHaveBeenCalledWith({
name: 'host-alpha-host-sa',
namespace: 'mcp-host',
})
})
it('treats existing RBAC as success when create returns 409 (idempotent)', async () => {
const { reconciler, coreApi, rbacApi } = createReconciler()
coreApi.createNamespacedServiceAccount.mockRejectedValueOnce({ code: 409 })
rbacApi.createNamespacedRoleBinding.mockRejectedValueOnce({ code: 409 })
await expect(reconciler.reconcile(makeHost())).resolves.toBeUndefined()
expect(reconciler.getStatus('alpha-host').deployed).toBe(true)
})
})
describe('buildChannelReaderDeployment', () => {
it('returns Deployment with name channel-reader-<host>', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
expect(dep.metadata?.name).toBe('channel-reader-alpha-host')
expect(dep.metadata?.namespace).toBe('channels')
})
it('sets management labels', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost({ name: 'agent-a' }), '')
// app.kubernetes.io/name MUST be per-Host (channel-reader-<host>) to
// avoid collision with the static Deployment's selector which matches
// supersets (see comment in buildChannelReaderDeployment).
expect(dep.metadata?.labels).toMatchObject({
app: 'channel-reader',
'app.kubernetes.io/name': 'channel-reader-agent-a',
'clerum.io/host': 'agent-a',
'clerum.io/managed-by': 'host-context-controller',
})
expect(dep.spec?.template?.metadata?.labels).toMatchObject({
app: 'channel-reader',
'app.kubernetes.io/name': 'channel-reader-agent-a',
'clerum.io/host': 'agent-a',
})
})
it('sets CLERUM_HOST_REF env to the host name', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost({ name: 'agent-a' }), '')
const env = dep.spec!.template!.spec!.containers[0].env!
expect(env).toContainEqual({ name: 'CLERUM_HOST_REF', value: 'agent-a' })
})
it('injects CLERUM_NAMESPACE via downward API fieldRef metadata.namespace', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const env = dep.spec!.template!.spec!.containers[0].env!
expect(env).toContainEqual({
name: 'CLERUM_NAMESPACE',
valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
})
})
it('does not inject direct control-api credentials into channel-reader', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const env = dep.spec!.template!.spec!.containers[0].env!
const envFrom = dep.spec!.template!.spec!.containers[0].envFrom ?? []
const rendered = JSON.stringify({ env, envFrom })
expect(rendered).not.toContain('CLERUM_CONTROL_API_SERVICE_TOKEN')
expect(rendered).not.toContain('CONTROL_API_SERVICE_TOKEN')
expect(rendered).not.toContain('channel-reader-internal-tokens')
})
it('injects CLERUM_MCP_HOST_URL pointing at the slug-scoped mcp-host Service in host.namespace', () => {
const { reconciler } = createReconciler()
// mcp-host Service is named host.name and lives in host.namespace
// (mcp-server here / mcp-host-<slug> in MCC), NOT the channels namespace.
const dep = (reconciler as any).buildChannelReaderDeployment(
makeHost({ name: 'agent-a', namespace: 'mcp-host-agent-a' }),
''
)
const env = dep.spec!.template!.spec!.containers[0].env!
expect(env).toContainEqual({
name: 'CLERUM_MCP_HOST_URL',
value: 'http://agent-a.mcp-host-agent-a.svc.cluster.local:8080',
})
})
it('mounts envFrom on configMapRef clerum-channel-reader-config', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const envFrom = dep.spec!.template!.spec!.containers[0].envFrom!
expect(envFrom).toContainEqual({ configMapRef: { name: 'clerum-channel-reader-config' } })
})
it('sets clerum.io/credentials-revision annotation when revision is provided', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), 'abc123')
expect(dep.spec?.template?.metadata?.annotations).toMatchObject({
'clerum.io/credentials-revision': 'abc123',
})
})
it('omits credentials-revision annotation when revision is empty (initial create)', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const ann = dep.spec?.template?.metadata?.annotations ?? {}
expect(ann['clerum.io/credentials-revision']).toBeUndefined()
})
it('runs as non-root with read-only filesystem', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const sc = dep.spec!.template!.spec!.containers[0].securityContext!
expect(sc).toMatchObject({
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
runAsNonRoot: true,
})
})
it('emits restricted-PSA-compliant container securityContext (drop ALL + seccompProfile)', () => {
// MCC shared-tenant namespaces (channels-<slug>) are labeled
// pod-security.kubernetes.io/enforce=restricted. capabilities.drop=[ALL]
// and seccompProfile=RuntimeDefault are required alongside the existing
// allowPrivilegeEscalation/runAsNonRoot fields.
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const sc = dep.spec!.template!.spec!.containers[0].securityContext!
expect(sc).toMatchObject({
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
runAsNonRoot: true,
runAsUser: 1000,
runAsGroup: 1000,
capabilities: { drop: ['ALL'] },
seccompProfile: { type: 'RuntimeDefault' },
})
})
it('uses imagePullPolicy from config', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
expect(dep.spec!.template!.spec!.containers[0].imagePullPolicy).toBe('IfNotPresent')
})
it('keeps internal notification auth separate from mcp-host runtime access', () => {
// Channel-reader keeps its internal notification token path separate and no
// longer receives per-host JWT files for direct mcp-host runtime calls.
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
const envNames = (dep.spec?.template?.spec?.containers?.[0]?.env ?? []).map(
(e: { name: string }) => e.name
)
expect(envNames).not.toContain('CLERUM_CHANNEL_READER_BOOTSTRAP_TOKEN')
expect(envNames).not.toContain('CLERUM_CONTROL_API_URL')
// No per-host JWT runtime/control token env. The routing URL
// (CLERUM_MCP_HOST_URL) is allowed: it is a Service address, not auth.
expect(
envNames.filter(
(name: string) => name.startsWith('CLERUM_MCP_HOST_') && name !== 'CLERUM_MCP_HOST_URL'
)
).toEqual([])
})
it('sets replicas to 0 when countCommunicationChannels returns 0 (default)', () => {
const { reconciler } = createReconciler()
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
expect(dep.spec.replicas).toBe(0)
})
it('sets replicas to 1 when countCommunicationChannels returns 1+', () => {
const { reconciler } = createReconciler()
reconciler.setCountCommunicationChannels(() => 1)
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
expect(dep.spec.replicas).toBe(1)
})
it('passes host.name to countCommunicationChannels (not a global sum)', () => {
// Regression guard: if the count was accidentally summed across hosts,
// alpha-host would see 5 instead of 3. The replica is still 1 either
// way, but the per-host count must be passed through faithfully so
// future metrics (clerum_channel_reader_replicas{host=}) remain
// host-scoped.
const { reconciler } = createReconciler()
const captured: string[] = []
reconciler.setCountCommunicationChannels(host => {
captured.push(host)
return host === 'alpha-host' ? 3 : 5
})
;(reconciler as any).buildChannelReaderDeployment(makeHost(), '')
expect(captured).toEqual(['alpha-host'])
})
})
describe('reconcileChannelReaderDeployment', () => {
it('creates Deployment when absent (createNamespacedDeployment succeeds)', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockResolvedValue({})
appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({
namespace: 'channels',
body: expect.objectContaining({
metadata: expect.objectContaining({ name: 'channel-reader-a', namespace: 'channels' }),
}),
})
})
it('replaces a present Deployment with drift without creating', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue({
metadata: { name: 'channel-reader-a', namespace: 'channels', resourceVersion: '42' },
})
appsApi.replaceNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledWith({
name: 'channel-reader-a',
namespace: 'channels',
body: expect.objectContaining({
metadata: expect.objectContaining({ name: 'channel-reader-a', resourceVersion: '42' }),
}),
})
})
it('does not replace a converged channel-reader Deployment with server defaults', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost({ name: 'a' })
const existing = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
const deploymentSpec = existing.spec
if (!deploymentSpec?.template.spec) throw new Error('expected channel-reader PodSpec')
const podSpec = deploymentSpec.template.spec
existing.metadata = {
...existing.metadata,
resourceVersion: '42',
uid: 'channel-reader-uid',
generation: 7,
creationTimestamp: new Date('2026-07-10T00:00:00Z'),
}
existing.status = { readyReplicas: 1, availableReplicas: 1 }
existing.spec = {
...deploymentSpec,
progressDeadlineSeconds: 600,
revisionHistoryLimit: 10,
strategy: {
type: 'RollingUpdate',
rollingUpdate: { maxSurge: '25%', maxUnavailable: '25%' },
},
template: {
...deploymentSpec.template,
metadata: {
...deploymentSpec.template.metadata,
annotations: { 'kubectl.kubernetes.io/restartedAt': '2026-07-10T00:00:00Z' },
},
spec: {
...podSpec,
securityContext: {},
dnsPolicy: 'ClusterFirst',
restartPolicy: 'Always',
schedulerName: 'default-scheduler',
serviceAccount: podSpec.serviceAccountName,
terminationGracePeriodSeconds: 30,
containers: podSpec.containers.map(container => ({
...container,
terminationMessagePath: '/dev/termination-log',
terminationMessagePolicy: 'File',
env: container.env?.map(env =>
env.valueFrom?.fieldRef
? {
...env,
valueFrom: {
...env.valueFrom,
fieldRef: { ...env.valueFrom.fieldRef, apiVersion: 'v1' },
},
}
: env
),
})),
},
},
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(host)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('replaces a channel-reader Deployment with a non-default strategy', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost({ name: 'a' })
const existing = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
existing.metadata = { ...existing.metadata, resourceVersion: '42' }
existing.spec!.strategy = { type: 'Recreate' }
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
appsApi.replaceNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(host)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('clears a stale controller-owned revision while preserving operational annotations', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost({ name: 'a' })
const existing = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
existing.metadata = { ...existing.metadata, resourceVersion: '42' }
existing.spec!.template.metadata = {
...existing.spec!.template.metadata,
annotations: {
'clerum.io/credentials-revision': 'stale-revision',
'kubectl.kubernetes.io/restartedAt': '2026-07-10T00:00:00Z',
},
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
appsApi.replaceNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(host)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
.body as k8s.V1Deployment
expect(
replacement.spec?.template.metadata?.annotations?.['clerum.io/credentials-revision']
).toBeUndefined()
expect(
replacement.spec?.template.metadata?.annotations?.['kubectl.kubernetes.io/restartedAt']
).toBe('2026-07-10T00:00:00Z')
})
it('stops retrying when a fresh channel-reader Deployment converges after a conflict', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost({ name: 'a' })
const stale = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
stale.metadata = { ...stale.metadata, resourceVersion: '42' }
stale.spec!.template.spec!.containers[0].image = 'clerum/channel-reader:stale'
const converged = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
converged.metadata = { ...converged.metadata, resourceVersion: '43' }
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(converged)
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(host)
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('does not overwrite a channel-reader whose ownership changes during a retry', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost({ name: 'a' })
const stale = structuredClone(
(reconciler as any).buildChannelReaderDeployment(host, '')
) as k8s.V1Deployment
stale.metadata = { ...stale.metadata, resourceVersion: '42' }
stale.spec!.template.spec!.containers[0].image = 'clerum/channel-reader:stale'
const foreign = structuredClone(stale)
foreign.metadata = {
...foreign.metadata,
resourceVersion: '43',
labels: { ...foreign.metadata?.labels, 'clerum.io/host': 'other-host' },
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(foreign)
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await expect((reconciler as any).reconcileChannelReaderDeployment(host)).rejects.toThrow(
/ownership changed to host "other-host"/
)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('reads existing Secret revision and writes annotation on initial create', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
appsApi.createNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockResolvedValue({
data: { 'telegram-bot-token': Buffer.from('tok-1').toString('base64') },
})
// Seed the CC cache so computeChannelReaderRevisionForHost finds a Secret ref
reconciler.setFindCommunicationChannelsByHostRef(() => [
{
name: 'cc-a',
namespace: 'channels',
spec: { hostRef: 'a', credentialsSecretRef: { name: 'channel-reader-a-credentials' } },
},
])
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
const ann = call.body.spec.template.metadata.annotations
expect(ann['clerum.io/credentials-revision']).toMatch(/^[0-9a-f]{64}$/)
})
it('skips reconcile if existing Deployment is owned by a different host', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue({
metadata: {
name: 'channel-reader-a',
namespace: 'channels',
labels: { 'clerum.io/host': 'OTHER-HOST' },
},
})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('surfaces a transient failure when read-after-409 races with deletion (404)', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockRejectedValue({ code: 404 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await expect(
(reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
).rejects.toMatchObject({ code: 404 })
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
})
describe('reconcile / reconcileDelete with channel-reader', () => {
it('reconcile(host) calls reconcileChannelReaderDeployment', async () => {
const { reconciler } = createReconciler()
const spy = vi
.spyOn(reconciler as any, 'reconcileChannelReaderDeployment')
.mockResolvedValue(undefined)
await reconciler.reconcile(makeHost({ name: 'a' }))
expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }), expect.any(Function))
})
it('reconcileDelete(name, ns) deletes channel-reader-<name> Deployment in channels ns', async () => {
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockResolvedValue({})
await reconciler.reconcileDelete('a', 'mcp-host')
expect(appsApi.deleteNamespacedDeployment.mock.calls).toContainEqual([
{ name: 'channel-reader-a', namespace: 'channels' },
])
})
it('reconcileDelete tolerates 404 on missing channel-reader Deployment', async () => {
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockRejectedValue({ code: 404 })
await expect(reconciler.reconcileDelete('a', 'mcp-host')).resolves.toBeUndefined()
})
it('records channel-reader failure and rejects so the caller can retry convergence', async () => {
const { reconciler } = createReconciler()
vi.spyOn(reconciler as any, 'reconcileChannelReaderDeployment').mockRejectedValue(
new Error('boom')
)
await expect(reconciler.reconcile(makeHost({ name: 'a' }))).rejects.toThrow(
'Failed to converge channel-reader resources for Host "a"'
)
expect(reconciler.getStatus('a')).toMatchObject({
deployed: true,
ready: true,
message: expect.stringContaining('channel-reader: boom'),
})
})
})
describe('orphan sweep on fullReconcile (channel-reader)', () => {
it('deletes channel-reader Deployments without a matching Host', async () => {
// §10.5: an orphan is DISCOVERED via its channel-reader Deployment candidate,
// its owning Host is confirmed gone by a fresh 404 under known/stable watch
// authority, and that Host's owned bundle (which cascades to the channels-ns
// channel-reader Deployment) is deleted. The live Host "a" is retained.
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const { reconciler, appsApi } = createReconciler({
customApi: { getNamespacedCustomObject: getObj },
})
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(name => (name === 'a' ? makeHost({ name }) : undefined))
// listNamespacedDeployment is called twice during fullReconcile:
// 1) mcp-host ns (with labelSelector) for the existing host orphan sweep
// 2) channels ns (no labelSelector) for the channel-reader sweep
// Mock based on namespace so we don't pollute the host-side sweep.
appsApi.listNamespacedDeployment.mockImplementation(({ namespace }: { namespace: string }) => {
if (namespace === 'channels') {
return Promise.resolve({
items: [
{
metadata: {
name: 'channel-reader-a',
namespace: 'channels',
labels: {
app: 'channel-reader',
'clerum.io/host': 'a',
'clerum.io/managed-by': 'host-context-controller',
},
},
},
{
metadata: {
name: 'channel-reader-orphan',
namespace: 'channels',
labels: {
app: 'channel-reader',
'clerum.io/host': 'orphan',
'clerum.io/managed-by': 'host-context-controller',
},
},
},
{ metadata: { name: 'unrelated-thing', namespace: 'channels', labels: {} } },
],
})
}
return Promise.resolve({ items: [] })
})
appsApi.deleteNamespacedDeployment.mockResolvedValue({})
// Desired list is empty: "a" is live purely by current-cache presence
// (setResolveCurrentHost above), which is what §10.5 retention keys on.
await reconciler.fullReconcile([])
// The orphan ("orphan") got a fresh authoritative read; the live host ("a")
// stayed in cache and was never read/deleted.
expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan' }))
expect(getObj).not.toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }))
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
name: 'channel-reader-orphan',
namespace: 'channels',
})
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
name: 'channel-reader-a',
namespace: 'channels',
})
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
name: 'unrelated-thing',
namespace: 'channels',
})
})
it('deletes the owning Host bundle for an orphan discovered via a channel-reader Secret', async () => {
// The old per-name Secret sweep became discover-owner + delete-the-Host-bundle
// (§10.5): a managed channel-reader Secret whose owning Host is gone is a
// DISCOVERY signal. Its owner ("orphan") is confirmed absent by a fresh 404
// under known/stable authority, and that Host's entire owned bundle is
// deleted through its per-Host serializer. The live host "a" is retained.
// (The bundle deletes the Host's own managed Secrets — e.g.
// `channel-reader-orphan-mcp-host-runtime-auth` — not an arbitrary
// discovered name, so we assert at the bundle boundary.)
const getObj = vi.fn().mockRejectedValue({ code: 404 })
const { reconciler, coreApi } = createReconciler({
customApi: { getNamespacedCustomObject: getObj },
})
reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
reconciler.setResolveCurrentHost(name => (name === 'a' ? makeHost({ name }) : undefined))
coreApi.listNamespacedSecret.mockResolvedValue({
items: [
{
metadata: {
name: 'channel-reader-a-credentials',
namespace: 'channels',
labels: {
'clerum.io/component': 'channel-reader',
'clerum.io/host': 'a',
'clerum.io/managed-by': 'host-context-controller',
},
},
},
{
metadata: {
name: 'channel-reader-orphan-credentials',
namespace: 'channels',
labels: {
'clerum.io/component': 'channel-reader',
'clerum.io/host': 'orphan',
'clerum.io/managed-by': 'host-context-controller',
},
},
},
],
})
const deleteBundle = vi
.spyOn(reconciler as any, 'deleteHostRuntimeResources')
.mockResolvedValue(undefined)
// Desired list is empty: "a" is live purely by current-cache presence
// (setResolveCurrentHost above), which is what §10.5 retention keys on.
await reconciler.fullReconcile([])
expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan' }))
expect(deleteBundle).toHaveBeenCalledWith('orphan', 'mcp-host')
expect(deleteBundle).not.toHaveBeenCalledWith('a', 'mcp-host')
})
it('defers the orphan bundle and deletes nothing when watch authority is unknown', async () => {
// §10.5 gate proven on purpose: unknown watch authority short-circuits the
// whole cleanup BEFORE any candidate is gathered or any fresh read is made,
// so nothing is deleted even though an orphan channel-reader Deployment is
// present. This is the accidental safety the removed global tail used to
// provide, now made explicit.
const getObj = vi.fn()
const { reconciler, appsApi } = createReconciler({
customApi: { getNamespacedCustomObject: getObj },
})
reconciler.setHostWatchAuthority(() => ({ known: false, generation: 1 }))
reconciler.setResolveCurrentHost(() => undefined)
appsApi.listNamespacedDeployment.mockImplementation(({ namespace }: { namespace: string }) =>
namespace === 'channels'
? Promise.resolve({
items: [
{
metadata: {
name: 'channel-reader-orphan',
namespace: 'channels',
labels: {
app: 'channel-reader',
'clerum.io/host': 'orphan',
'clerum.io/managed-by': 'host-context-controller',
},
},
},
],
})
: Promise.resolve({ items: [] })
)
const deleteBundle = vi
.spyOn(reconciler as any, 'deleteHostRuntimeResources')
.mockResolvedValue(undefined)
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
await reconciler.fullReconcile([])
expect(getObj).not.toHaveBeenCalled()
expect(deleteBundle).not.toHaveBeenCalled()
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
name: 'channel-reader-orphan',
namespace: 'channels',
})
expect(warn).toHaveBeenCalledWith('Deferring orphan cleanup', { reason: 'authority_unknown' })
warn.mockRestore()
})
it('does NOT delete Secrets missing the clerum.io/managed-by label', async () => {
const { reconciler, coreApi } = createReconciler()
coreApi.listNamespacedSecret.mockResolvedValue({
items: [
{
metadata: {
name: 'hand-rolled-channel-secret',
namespace: 'channels',
labels: {
'clerum.io/component': 'channel-reader',
'clerum.io/host': 'unknown-host',
// intentionally NO managed-by
},
},
},
],
})
coreApi.deleteNamespacedSecret.mockResolvedValue({})
await reconciler.fullReconcile([])
expect(coreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
name: 'hand-rolled-channel-secret',
namespace: 'channels',
})
})
})
describe('sweepLegacyStaticChannelReader', () => {
// Issue #273: HCC startup sweep that retires the static
// `clerum-channel-reader` Deployment so per-Host pods don't collide
// with it on Telegram getUpdates long-poll (409 Conflict).
// Empirical reproduction in #273 comment.
it('deletes the static clerum-channel-reader Deployment when it exists', async () => {
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockResolvedValue({})
await reconciler.sweepLegacyStaticChannelReader()
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(1)
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
name: 'clerum-channel-reader',
namespace: 'channels',
})
})
it('swallows 404 (already gone — steady state)', async () => {
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockRejectedValue(
Object.assign(new Error('not found'), { code: 404 })
)
// Must not throw — 404 is the steady-state after the legacy retirement.
await expect(reconciler.sweepLegacyStaticChannelReader()).resolves.toBeUndefined()
expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(1)
})
it('logs but does not throw on non-404 errors (e.g. 403 RBAC)', async () => {
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockRejectedValue(
Object.assign(new Error('forbidden'), { code: 403 })
)
const errorSpy = vi.spyOn(HostContextLogger.prototype, 'error').mockImplementation(() => {})
// Reviewer-style assertion: startup MUST continue even if the sweep
// fails — per-Host reconciles are still required.
await expect(reconciler.sweepLegacyStaticChannelReader()).resolves.toBeUndefined()
expect(errorSpy).toHaveBeenCalledWith(
expect.stringContaining('legacy-sweep failed'),
expect.objectContaining({ err: expect.anything() })
)
errorSpy.mockRestore()
})
it('targets the channels namespace (not mcp-host) and the exact name', async () => {
// Regression guard: if someone refactors namespace config, the sweep
// must not accidentally target a Deployment named `clerum-channel-reader`
// in a different namespace.
const { reconciler, appsApi } = createReconciler()
appsApi.deleteNamespacedDeployment.mockResolvedValue({})
await reconciler.sweepLegacyStaticChannelReader()
const call = appsApi.deleteNamespacedDeployment.mock.calls[0][0] as {
name: string
namespace: string
}
expect(call.name).toBe('clerum-channel-reader')
expect(call.namespace).toBe('channels')
})
})
describe('setCountCommunicationChannels', () => {
it('defaults to a function returning 0 when not set', () => {
const { reconciler } = createReconciler()
expect((reconciler as any).countCommunicationChannels('any-host')).toBe(0)
})
it('uses the injected callback to count CCs for a host', () => {
const { reconciler } = createReconciler()
const counts = new Map([
['alpha-host', 3],
['beta-host', 0],
])
reconciler.setCountCommunicationChannels(host => counts.get(host) ?? 0)
expect((reconciler as any).countCommunicationChannels('alpha-host')).toBe(3)
expect((reconciler as any).countCommunicationChannels('beta-host')).toBe(0)
expect((reconciler as any).countCommunicationChannels('unknown-host')).toBe(0)
})
})
describe('resolveWorkflowControlScopes (F25)', () => {
const DEFAULT: HostWorkflowControlScope[] = [
'workflow:list',
'workflow:read',
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
]
it('exports the canonical first-party scope set for explicit declarations', () => {
expect(DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES).toEqual(DEFAULT)
})
it('does not grant workflow-control scopes when workflowControl is absent without channel ingress', () => {
expect(resolveWorkflowControlScopes(undefined)).toEqual([])
})
it('defaults first-party workflow-control scopes when a channel-bearing Host omits workflowControl', () => {
expect(resolveWorkflowControlScopes(undefined, { hasChannelIngress: true })).toEqual(DEFAULT)
})
it('does not grant workflow-control scopes when the block is present but scopes is omitted', () => {
expect(resolveWorkflowControlScopes({})).toEqual([])
})
it('honors an explicit empty scopes list as an intentional opt-out (does NOT default)', () => {
expect(resolveWorkflowControlScopes({ scopes: [] })).toEqual([])
})
it('honors an explicit scopes list exactly as declared', () => {
expect(resolveWorkflowControlScopes({ scopes: ['workflow:list', 'workflow:read'] })).toEqual([
'workflow:list',
'workflow:read',
])
})
it('returns a fresh array (callers may sort/mutate without corrupting the default)', () => {
const a = resolveWorkflowControlScopes({ scopes: DEFAULT })
a.sort()
expect(DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES).toEqual(DEFAULT)
})
})
// ── B2: CC cache-sync gate on replica decision ─────────────────────────────
describe('buildChannelReaderDeployment — B2 CC cache-sync gate', () => {
it('sets replicas to 1 when cache is synced and CC count is 1', () => {
// When the CC cache has completed its initial list and the host has 1+ CCs,
// the Deployment should be created with replicas=1.
const { reconciler } = createReconciler()
reconciler.setCountCommunicationChannels(() => 1)
reconciler.setIsCommunicationChannelCacheSynced(() => true)
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '', undefined)
expect(dep.spec.replicas).toBe(1)
})
it('preserves replicas=1 when cache is NOT synced and an existing Deployment has replicas=1', () => {
// When the CC cache is still empty (startup / load failure) AND we know the
// existing Deployment had replicas=1, we MUST NOT scale to 0 — the channel-reader
// is live and scaling it down would drop inbound messages.
const { reconciler } = createReconciler()
// ccCacheSyncedFn returns false (unsynced)
reconciler.setIsCommunicationChannelCacheSynced(() => false)
// countCommunicationChannels returns 0 (cache empty)
reconciler.setCountCommunicationChannels(() => 0)
const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '', 1)
expect(dep.spec.replicas).toBe(1)
})
})
describe('reconcileChannelReaderDeployment — B2: preserve replicas on unsynced cache (409 path)', () => {
it('keeps replicas=1 on the replace body when cache is unsynced and existing Deployment has replicas=1', async () => {
// Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced).
// The existing live Deployment has replicas=1 (channel-reader is running).
// reconcileChannelReaderDeployment must NOT scale it to 0.
const { reconciler, appsApi, coreApi } = createReconciler()
// Force the 409 → read-existing → replace path
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue({
metadata: {
name: 'channel-reader-a',
namespace: 'channels',
resourceVersion: '77',
labels: { 'clerum.io/host': 'a' },
},
spec: { replicas: 1 },
})
appsApi.replaceNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
// CC cache is unsynced; count returns 0
reconciler.setIsCommunicationChannelCacheSynced(() => false)
reconciler.setCountCommunicationChannels(() => 0)
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
// The replace call body must carry replicas=1, not 0
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
const replaceBody = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
.body as k8s.V1Deployment
expect(replaceBody.spec?.replicas).toBe(1)
})
})
async function readFleetBenignSupersessions(errorName: string): Promise<number> {
const metric = registry.getSingleMetric('clerum_hcc_host_fleet_benign_supersessions_total')
if (!metric) throw new Error('clerum_hcc_host_fleet_benign_supersessions_total is not registered')
const snapshot = await metric.get()
return snapshot.values.find(entry => entry.labels.error === errorName)?.value ?? 0
}
describe('collectHostReconcileFailures benign supersession (#490)', () => {
function nameEquivalent(name: string, message: string): Error {
const error = new Error(message)
error.name = name
return error
}
it.each([
'HostInventoryAuthorityUnavailableError',
'HostMutationIdentityChangedError',
'HostMutationSpecRevisionChangedError',
'HostMutationDependencyChangedError',
] as const)(
'resolves reconcileHosts when every fleet error is name-equivalent %s',
async errorName => {
const { reconciler } = createReconciler()
const withdrawn = nameEquivalent(
errorName,
`Host "${errorName}" superseded before Host "alpha-host" reconcile admission`
)
vi.spyOn(reconciler, 'reconcile').mockRejectedValue(withdrawn)
const errorSpy = vi.spyOn(HostContextLogger.prototype, 'error').mockImplementation(() => {})
const before = await readFleetBenignSupersessions(errorName)
await expect(reconciler.reconcileHosts([makeHost()])).resolves.toBeUndefined()
expect(await readFleetBenignSupersessions(errorName)).toBe(before + 1)
expect(
errorSpy.mock.calls.some(call => String(call[0]).includes('Fleet reconcile failed'))
).toBe(false)
errorSpy.mockRestore()
}
)
it('still rejects a lookalike error name that is not in the benign set', async () => {
const { reconciler } = createReconciler()
vi.spyOn(reconciler, 'reconcile').mockRejectedValue(
nameEquivalent('HostMutationSpecRevisionChanged', 'almost the withdrawn name')
)
await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(
HostFleetReconcileError
)
})
it('still rejects HostFleetReconcileError for a non-benign fleet error', async () => {
const { reconciler } = createReconciler()
vi.spyOn(reconciler, 'reconcile').mockRejectedValue(new Error('boom'))
await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(
HostFleetReconcileError
)
})
it('still rejects when a mixed pass has at least one genuine host failure', async () => {
const { reconciler } = createReconciler()
vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
if (host.name === 'alpha-host') {
throw nameEquivalent(
'HostMutationSpecRevisionChangedError',
'Host spec generation changed before Host "alpha-host" reconcile admission'
)
}
throw new Error('boom')
})
await expect(
reconciler.reconcileHosts([makeHost(), makeHost({ name: 'beta-host' })])
).rejects.toMatchObject({
name: 'HostFleetReconcileError',
hostFailures: [expect.objectContaining({ message: 'boom' })],
})
})
})
describe('HostReconciler oauth:user-token runtime scope provisioning', () => {
// Extract the runtime-token scope arg from the LAST issuer call, sorted for
// stable comparison. This is the REAL derive → issuance payload: the mock
// captures whatever `resolveWorkflowControlScopesForHost` (the production
// derive) actually produced — no hand-minted JWT.
function lastIssuedScopes(): string[] {
const calls = vi.mocked(issueMcpHostRuntimeTokens).mock.calls
const last = calls[calls.length - 1]
return [...(last?.[2] ?? [])].sort()
}
// Access the private static scope-hash used for drift detection.
const scopeHashOf = (
HostReconciler as unknown as {
runtimeTokenScopeHash(
host: HostCRD,
hasChannelIngress?: boolean,
frontsOAuthServer?: boolean,
projection?: { derivedScopes?: string[]; driftHashInput?: string },
grokProjection?: { derivedScopes?: string[]; driftHashInput?: string }
): string
}
).runtimeTokenScopeHash
it('keeps the pre-Grok scope hash when Grok has no derived scopes', () => {
const host = makeHost()
const codex = { derivedScopes: ['llm:codex:execute'], driftHashInput: '{"k":1}' }
expect(scopeHashOf(host, false, false, codex, { derivedScopes: [] })).toBe(
scopeHashOf(host, false, false, codex)
)
expect(
scopeHashOf(host, false, false, codex, { derivedScopes: ['llm:grok:execute'] })
).not.toBe(scopeHashOf(host, false, false, codex))
})
it('requests oauth:user-token through the real issuance payload when the Host fronts an enabled oauth mcp-server', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const { reconciler } = createReconciler()
// Host CRD carries NO oauth scope — the value is derive-only. HCC learns
// the Host fronts an oauth server exclusively from this injected probe.
reconciler.setHostFrontsOAuthServer(async () => true)
await reconciler.reconcile(makeHost())
// Goes through the REAL derive (resolveWorkflowControlScopesForHost →
// resolveRuntimeControlScopes). Without the derive change this scope is
// absent and the assertion fails.
expect(lastIssuedScopes()).toEqual(
[...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES, OAUTH_USER_TOKEN_SCOPE].sort()
)
expect(lastIssuedScopes()).toContain('oauth:user-token')
})
it('does NOT request oauth:user-token when the Host fronts no oauth mcp-server', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const { reconciler } = createReconciler()
reconciler.setHostFrontsOAuthServer(async () => false)
await reconciler.reconcile(makeHost())
expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
expect(lastIssuedScopes()).not.toContain('oauth:user-token')
})
it('defaults to NO oauth:user-token when the probe is unwired', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const { reconciler } = createReconciler()
// No setHostFrontsOAuthServer — production default is fail-closed false.
await reconciler.reconcile(makeHost())
expect(lastIssuedScopes()).not.toContain('oauth:user-token')
})
it('fails closed (no oauth:user-token) when the oauth-server probe throws', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')
try {
const { reconciler } = createReconciler()
reconciler.setHostFrontsOAuthServer(async () => {
throw new Error('context read failed')
})
await reconciler.reconcile(makeHost())
expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
expect(lastIssuedScopes()).not.toContain('oauth:user-token')
// A new Host has no retained grant to preserve, so the unknown answer
// fails closed at issuance, and says so.
const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))
expect(retryWarn?.[1]).toMatchObject({ attempts: 3, err: 'context read failed' })
} finally {
warnSpy.mockRestore()
}
})
it('toggling oauth-server presence changes the runtime-token scope hash (drift re-issues)', () => {
const host = makeHost()
const withoutOAuth = scopeHashOf(host, true, false)
const withOAuth = scopeHashOf(host, true, true)
expect(withOAuth).not.toBe(withoutOAuth)
})
it('mint scope set matches the scope-hash input (no mint/hash divergence)', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const { reconciler } = createReconciler()
reconciler.setHostFrontsOAuthServer(async () => true)
const host = makeHost()
await reconciler.reconcile(host)
// The scopes minted into the token (issuer payload) must equal the exact
// set the drift hash is computed over — both flow through
// resolveRuntimeControlScopes, so a hash miss would re-issue forever.
const hashInput = [
...resolveRuntimeControlScopes(host.spec.workflowControl, {
hasChannelIngress: true,
frontsOAuthServer: true,
}),
].sort()
expect(lastIssuedScopes()).toEqual(hashInput)
expect(hashInput).toContain('oauth:user-token')
})
})
describe('HostReconciler.observeFrontsOAuthServer transient-blip retry (R4-M9)', () => {
// The oauth-server probe is exercised through the REAL reconciler built by
// createReconciler() and the REAL setHostFrontsOAuthServer setter — no
// hand-minted value. We assert on the private observation directly (rather
// than the full reconcile) on purpose: the deployment drift guard
// (ensureCurrentRuntimeTokenScope) re-reads the observation mid-reconcile, so
// a probe that throws once and then returns `true` would be MASKED at the
// public issuance observable. The observation IS the value whose flip drives
// the churn this fixes; what an unobserved answer means is decided by its
// callers and covered at the reconcile level.
const observeFronts = (
reconciler: HostReconciler,
host: HostCRD
): Promise<
| { observed: true; frontsOAuthServer: boolean }
| { observed: false; attempts: number; error: unknown }
> =>
(
reconciler as unknown as {
observeFrontsOAuthServer(
host: HostCRD
): Promise<
| { observed: true; frontsOAuthServer: boolean }
| { observed: false; attempts: number; error: unknown }
>
}
).observeFrontsOAuthServer(host)
it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {
// Regression case (T3): without the bounded retry the throw is reported as
// unobserved and this assertion fails. With the retry the second attempt
// returns the authoritative `true`.
const { reconciler } = createReconciler()
let calls = 0
reconciler.setHostFrontsOAuthServer(async () => {
calls += 1
if (calls === 1) throw new Error('transient apiserver read blip')
return true
})
await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
observed: true,
frontsOAuthServer: true,
})
expect(calls).toBe(2)
})
it('does NOT retry an authoritative false: probe returns false without throwing → false, called once', async () => {
// Control: a real scope change RETURNS false; the retry must not fire, or a
// genuine "no longer oauth" would be smeared into a stale `true`.
const { reconciler } = createReconciler()
let calls = 0
reconciler.setHostFrontsOAuthServer(async () => {
calls += 1
return false
})
await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
observed: true,
frontsOAuthServer: false,
})
expect(calls).toBe(1)
})
it('reports an unobserved answer after exhausting retries instead of collapsing it to false', async () => {
const { reconciler } = createReconciler()
let calls = 0
const outage = new Error('sustained apiserver outage')
reconciler.setHostFrontsOAuthServer(async () => {
calls += 1
throw outage
})
await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
observed: false,
attempts: 3,
error: outage,
})
// Retried up to the bounded ceiling (3 total attempts).
expect(calls).toBe(3)
})
})