Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { isDeepStrictEqual } from 'node:util'
import { asApiserverDeployment } from '../src/__tests__/asApiserverDeployment'
import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeReporter'
import { config } from '../src/config'
import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
type EffectiveHostLifecycle,
HostReconciler,
OAUTH_USER_TOKEN_SCOPE,
type ResolvedSfsMount,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
import { HostContextLogger } from '../src/logger'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
import {
MCP_HOST_GFS_TOKEN_SECRET_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
} from '../src/secretFactory'
import { HostCRD, HostCrdStatus } from '../src/types'
import {
type MockAppsApi,
type MockCustomApi,
asAppsApi,
asCoreApi,
asCustomApi,
asNetworkingApi,
asRbacApi,
createMockAppsApi,
createMockCoreApi,
createMockCustomApi,
createMockNetworkingApi,
createMockRbacApi,
} from './__fixtures__/testMocks'
vi.mock('../src/config', () => ({
config: {
devMode: false,
port: 8081,
namespace: 'mcp-server',
controlPlaneNamespace: 'control-plane',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
channelsNamespace: 'channels',
channelReaderImage: 'clerum/channel-reader:test',
channelReaderImagePullPolicy: 'IfNotPresent',
hostImage: 'clerum/mcp-host:0.6.0',
hostImagePullPolicy: 'Always',
hostImagePullSecretName: 'clerum',
hostPort: 8080,
gfsNamespace: 'gfs',
gfscPort: 8087,
hostConfigMapName: 'mcp-host-config',
hostServiceAccountName: 'mcp-host',
hostWorkspaceStorageClassName: 'do-block-storage-retain',
hostWorkspaceStorageSize: '10Gi',
hostWorkspacePath: '/workspace',
workspaceLayoutMerge: true,
hostResources: {
requests: { memory: '128Mi', cpu: '100m' },
limits: { memory: '512Mi', cpu: '500m' },
},
desktopImage: 'clerum/mcp-host-desktop:latest',
desktopPort: 3000,
desktopResources: {
requests: { memory: '256Mi', cpu: '250m' },
limits: { memory: '4Gi', cpu: '1000m' },
},
devMcpServers: [],
devContexts: [],
devAuthTokens: new Map(),
controlApiBaseUrl: 'http://control-api.test:8090',
internalControlJwtHccHmacSecret: 'test-hcc-internal-control-secret',
hccTargetNamespace: 'mcp-host',
mcpHostGatewayUrl:
'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092',
},
}))
// Short-circuit the HCC issuer so reconcile tests don't reach the network.
vi.mock('../src/mcpHostRuntimeTokenIssuerClient', () => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
accessToken: 'test-mcp-host-runtime-access-token',
refreshToken: 'test-mcp-host-runtime-refresh-token',
mcpHostControlToken: 'test-mcp-host-workflow-control-token',
channelReaderMessageToken: 'test-channel-reader-message-token',
channelReaderApprovalToken: 'test-channel-reader-approval-token',
channelReaderWorkflowApprovalDecisionToken: 'test-channel-reader-decision-token',
channelReaderActivityToken: 'test-channel-reader-activity-token',
channelReaderCronReadToken: 'test-channel-reader-cron-read-token',
channelReaderCronAckToken: 'test-channel-reader-cron-ack-token',
expiresInSeconds: 600,
refreshExpiresInSeconds: 3600,
controlExpiresInSeconds: 600,
channelReaderMessageExpiresInSeconds: 600,
channelReaderApprovalExpiresInSeconds: 600,
channelReaderWorkflowApprovalDecisionExpiresInSeconds: 600,
}),
}))
vi.mock('../src/gfsHostBinding', () => ({
mintHostGfsToken: vi.fn(async (namespace: string, name: string) => ({
['to' + 'ken']: 'gfs-runtime-value',
expiresInSeconds: 600,
subject: `host:1st:${namespace}/${name}`,
})),
}))
function makeHost(overrides?: Partial<HostCRD>): HostCRD {
return {
name: 'alpha-host',
namespace: 'mcp-host',
uid: 'alpha-host-uid',
spec: {
host: 'alpha-host',
contextRef: 'context-a',
secretRef: 'host-secret',
},
...overrides,
}
}
function makeStatelessHost(
overrides: { name?: string; spec?: Partial<HostCRD['spec']>; status?: HostCrdStatus } = {}
): HostCRD {
const name = overrides.name ?? 'stateless-host'
return {
name,
namespace: 'mcp-host',
uid: `${name}-uid`,
spec: {
host: name,
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
...overrides.spec,
},
...(overrides.status ? { status: overrides.status } : {}),
}
}
function suspendedStatus(wakeHandledGeneration = 0): HostCrdStatus {
return { lifecycle: { state: 'suspended', wakeHandledGeneration } }
}
function createTelemetryReporterMock(): InfrastructureTelemetryReporter {
return {
enqueue: vi.fn(),
enqueueHealthTransition: vi.fn(),
stop: vi.fn(async () => undefined),
}
}
function hostApiObject(host: HostCRD) {
return {
metadata: {
name: host.name,
namespace: host.namespace,
uid: host.uid,
resourceVersion: host.resourceVersion ?? '42',
annotations: host.annotations,
},
spec: host.spec,
status: host.status,
}
}
function createReconciler(deps?: {
countCommunicationChannels?: (hostName: string) => number
isCommunicationChannelCacheSynced?: () => boolean
resolveContextMounts?: (host: HostCRD) => Promise<ResolvedSfsMount[]>
infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
administrativeOutcomeReporter?: AdministrativeOutcomeReporter
}) {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const networkingApi = createMockNetworkingApi()
const rbacApi = createMockRbacApi()
const customApi = createMockCustomApi()
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
customApi: asCustomApi(customApi),
now: () => new Date('2026-07-03T00:00:00.000Z'),
...deps,
// Lifecycle tests model a fully initialized watcher unless a case is
// explicitly exercising the fail-closed cache-startup behavior.
isCommunicationChannelCacheSynced: deps?.isCommunicationChannelCacheSynced ?? (() => true),
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
}
function runtimeTokenProvision(host: HostCRD, hasChannelIngress = false) {
const internals = HostReconciler as unknown as {
runtimeTokenScopeHash(host: HostCRD, hasChannelIngress: boolean): string
}
return {
revision: 'runtime-revision',
scopeHash: internals.runtimeTokenScopeHash(host, hasChannelIngress),
}
}
/**
* A hand-built credential record with the identity, revision and refresh-window
* annotations only. It deliberately omits the bootstrap-state, rollout marker
* and scope hash the producer always writes, so it models a legacy record. It
* is kept for the held-runtime identity guards and the GFS renewal window,
* which read none of those. A test whose decision depends on the bootstrap or
* scope annotations uses `mintedRuntimeCredentialRecord` instead.
*/
function runtimeCredentialRecord(
host: HostCRD,
options: {
gfsRefreshBefore?: string
frontsOAuthServer?: string
hasChannelIngress?: string
hostUid?: string
managedByHost?: boolean
} = {}
) {
const data = {
[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]: Buffer.from('access-value').toString('base64'),
[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]: Buffer.from('refresh-value').toString('base64'),
[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]: Buffer.from('control-value').toString('base64'),
[MCP_HOST_GFS_TOKEN_SECRET_KEY]: Buffer.from('gfs-value').toString('base64'),
}
const revision = (
HostReconciler as unknown as {
runtimeTokenSecretRevision(data: Record<string, string>): string
}
).runtimeTokenSecretRevision(data)
return {
metadata: {
name: `host-${host.name}-mcp-host-runtime-tokens`,
namespace: host.namespace,
resourceVersion: '88',
labels:
options.managedByHost === false
? { 'clerum.io/managed-by': 'someone-else' }
: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': host.name,
},
annotations: {
'clerum.io/gfs-token-host-uid': options.hostUid ?? host.uid ?? '',
'clerum.io/runtime-token-secret-revision': revision,
'clerum.io/runtime-token-has-channel-ingress': options.hasChannelIngress ?? 'false',
'clerum.io/runtime-token-fronts-oauth-server': options.frontsOAuthServer ?? 'false',
'clerum.io/gfs-token-refresh-before':
options.gfsRefreshBefore ?? '2999-01-01T00:00:00.000Z',
},
},
data,
} as unknown as k8s.V1Secret
}
function trustedRuntimeDeployment(
reconciler: HostReconciler,
host: HostCRD,
options: { readyReplicas?: number; replicas?: number } = {}
) {
const deployment = reconciler.buildDeployment(host)
deployment.metadata = {
...deployment.metadata,
uid: `deployment-${host.name}`,
resourceVersion: '91',
generation: 1,
labels: {
...(deployment.metadata?.labels ?? {}),
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': host.name,
},
annotations: {
...(deployment.metadata?.annotations ?? {}),
'clerum.io/host-uid': host.uid ?? '',
},
}
if (deployment.spec) {
deployment.spec.replicas = options.replicas ?? 1
}
deployment.status = {
observedGeneration: deployment.metadata.generation,
replicas: deployment.spec?.replicas ?? 1,
updatedReplicas: deployment.spec?.replicas ?? 1,
readyReplicas: options.readyReplicas ?? 1,
}
return deployment
}
function markPersistedRuntimeTrusted(
deployment: k8s.V1Deployment,
host: HostCRD,
readyReplicas = 1
): k8s.V1Deployment {
deployment.metadata = {
...deployment.metadata,
generation: deployment.metadata?.generation ?? 1,
labels: {
...(deployment.metadata?.labels ?? {}),
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': host.name,
},
annotations: {
...(deployment.metadata?.annotations ?? {}),
'clerum.io/host-uid': host.uid ?? '',
},
}
deployment.status = { readyReplicas }
return deployment
}
function encodedRuntimeRefreshMaterial(expiresAtMs: number): string {
const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url')
return [
encode({ alg: 'none', typ: 'JWT' }),
encode({ exp: Math.floor(expiresAtMs / 1000) }),
'test-signature',
].join('.')
}
const PRODUCER_RUNTIME_ANNOTATIONS = [
'clerum.io/runtime-token-bootstrap-state',
'clerum.io/runtime-token-rollout-required',
'clerum.io/runtime-token-scope-hash',
] as const
/** Fails loud when a credential fixture lacks an annotation the producer always writes. */
function assertProducerRuntimeAnnotations(record: k8s.V1Secret): k8s.V1Secret {
const annotations = record.metadata?.annotations ?? {}
const missing = PRODUCER_RUNTIME_ANNOTATIONS.filter(key => !(key in annotations))
if (missing.length > 0) {
throw new Error(`runtime credential fixture lacks producer annotations: ${missing.join(', ')}`)
}
return record
}
/**
* The runtime credential record as HCC persists it, produced by one real
* `ensureMcpHostRuntimeTokenSecret` mint pass on a dedicated producer
* reconciler, so the caller's API mocks record none of the producer's calls.
* The written `stringData` is stored as base64 `data`, the way the apiserver
* returns it. With `bootstrap: 'consumed'` (the default) a second real pass
* against a Ready Deployment running the minted revision consumes the
* bootstrap, which is what a running pod's record looks like. `annotations`
* overrides only what a test drives explicitly.
*/
async function mintedRuntimeCredentialRecord(
host: HostCRD,
options: {
frontsOAuthServer?: boolean
hasChannelIngress?: boolean
bootstrap?: 'fresh' | 'consumed'
annotations?: Record<string, string>
} = {}
): Promise<k8s.V1Secret> {
const {
reconciler: producer,
appsApi,
coreApi,
} = createReconciler({
countCommunicationChannels: () => (options.hasChannelIngress ? 1 : 0),
})
producer.setHostFrontsOAuthServer(async () => options.frontsOAuthServer ?? false)
const issue = vi.mocked(issueMcpHostRuntimeTokens)
const issueDefaults = issue.getMockImplementation()
if (!issueDefaults) throw new Error('issueMcpHostRuntimeTokens mock has no implementation')
// A decodable refresh token, so a later reuse decision can reach `current`.
issue.mockImplementationOnce(async (...args) => ({
...(await issueDefaults(...args)),
refreshToken: encodedRuntimeRefreshMaterial(Date.now() + 3_600_000),
}))
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (producer as any).ensureMcpHostRuntimeTokenSecret(host)
const created = coreApi.createNamespacedSecret.mock.calls.at(-1)?.[0].body as
k8s.V1Secret | undefined
if (!created?.stringData) throw new Error('producer pass wrote no runtime credential Secret')
let record: k8s.V1Secret = {
...created,
metadata: { ...created.metadata, resourceVersion: '88' },
data: Object.fromEntries(
Object.entries(created.stringData).map(([key, value]) => [
key,
Buffer.from(value).toString('base64'),
])
),
}
delete record.stringData
if (options.bootstrap !== 'fresh') {
const revision = record.metadata?.annotations?.['clerum.io/runtime-token-secret-revision']
if (!revision) throw new Error('producer pass wrote no runtime credential revision')
const running = trustedRuntimeDeployment(producer, host)
running.spec!.template!.metadata!.annotations = {
...running.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': revision,
}
appsApi.readNamespacedDeployment.mockResolvedValue(running)
coreApi.readNamespacedSecret.mockResolvedValue(record)
await (producer as any).ensureMcpHostRuntimeTokenSecret(host)
const consumed = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as
k8s.V1Secret | undefined
if (
consumed?.metadata?.annotations?.['clerum.io/runtime-token-bootstrap-state'] !== 'consumed'
) {
throw new Error('producer pass did not consume the runtime credential bootstrap')
}
record = consumed
}
record.metadata = {
...record.metadata,
annotations: { ...(record.metadata?.annotations ?? {}), ...options.annotations },
}
return assertProducerRuntimeAnnotations(record)
}
function withReadableRuntimeRefreshMaterial(record: k8s.V1Secret): k8s.V1Secret {
const material =
(
HostReconciler as unknown as {
runtimeTokenSecretData(record: k8s.V1Secret): Record<string, string> | null
}
).runtimeTokenSecretData(record) ?? {}
return {
...record,
data: {
...material,
[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]: Buffer.from(
encodedRuntimeRefreshMaterial(Date.now() + 3_600_000)
).toString('base64'),
},
}
}
/** Model persisted API state so a loss-of-authority test cannot invent a template. */
function persistHostDeployment(appsApi: MockAppsApi, host: HostCRD, initial: k8s.V1Deployment) {
let live = structuredClone(initial)
live.metadata = {
...live.metadata,
uid: live.metadata?.uid ?? 'deployment-uid',
resourceVersion: live.metadata?.resourceVersion ?? '73',
}
const read = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === host.name ? Promise.resolve(structuredClone(live)) : read(request)
)
const replace = appsApi.replaceNamespacedDeployment.getMockImplementation()!
appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
if (request.name !== host.name) return replace(request)
expect(request.body.metadata.resourceVersion).toBe(live.metadata!.resourceVersion)
live = structuredClone(request.body)
return live
})
return () => live
}
/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */
function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {
const calls = [
...appsApi.createNamespacedDeployment.mock.calls,
...appsApi.replaceNamespacedDeployment.mock.calls,
]
const call = calls.find(
([arg]) => (arg as { body?: k8s.V1Deployment })?.body?.metadata?.name === name
)
if (!call) {
throw new Error(`No Deployment create/replace call found for "${name}"`)
}
return (call[0] as { body: k8s.V1Deployment }).body
}
function containerEnv(dep: k8s.V1Deployment): k8s.V1EnvVar[] {
const env = dep.spec?.template?.spec?.containers?.[0]?.env
if (!env) {
throw new Error('Deployment has no mcp-host container env')
}
return env
}
function envValue(dep: k8s.V1Deployment, name: string): string {
const entry = containerEnv(dep).find(e => e.name === name)
if (!entry || entry.value === undefined) {
throw new Error(`env var "${name}" not found on the mcp-host container`)
}
return entry.value
}
/**
* Every /status value written through the CustomObjects API, reconstructed
* from the JSON Patch ops. The lifecycle writers use three shapes:
* - full-status seed: `[{ add /status }]` (heartbeat cores + fresh-Host seed)
* - targeted sub-object: `[{ add /status/lifecycle }, { add /status/conditions }]`
* (writeLifecycleStatusToCluster, D2 — never clobbers un-computed fields)
* either optionally preceded by an `add /metadata/resourceVersion` precondition
* op (D3). This helper strips the precondition op and merges the remaining ops
* into a single HostCrdStatus so assertions stay shape-agnostic.
*/
function lifecycleStatusWrites(customApi: MockCustomApi): HostCrdStatus[] {
return customApi.patchNamespacedCustomObjectStatus.mock.calls.map(([arg]) => {
const body = (arg as { body: Array<{ op: string; path: string; value: unknown }> }).body
if (!Array.isArray(body)) {
throw new Error(`Unexpected status patch body: ${JSON.stringify(body)}`)
}
const ops = body.filter(op => op.path !== '/metadata/resourceVersion')
let status: HostCrdStatus = {}
for (const op of ops) {
if (op.path === '/status') {
status = op.value as HostCrdStatus
} else if (op.path === '/status/lifecycle') {
status = { ...status, lifecycle: op.value as HostCrdStatus['lifecycle'] }
} else if (op.path === '/status/conditions') {
status = { ...status, conditions: op.value as HostCrdStatus['conditions'] }
} else {
throw new Error(`Unexpected status patch op path: ${JSON.stringify(op)}`)
}
}
return status
})
}
function rejectedCondition(status: HostCrdStatus) {
const cond = status.conditions?.find(c => c.type === 'StatelessEnableRejected')
if (!cond) {
throw new Error('StatelessEnableRejected condition missing from status write')
}
return cond
}
describe('HostReconciler resource rollout bootstrap', () => {
const initialResources = structuredClone(config.hostResources)
const initialDesktopResources = structuredClone(config.desktopResources)
const initialRefreshBefore = config.mcpHostBootstrapRefreshBeforeSec
const issue = vi.mocked(issueMcpHostRuntimeTokens)
const initialIssue = issue.getMockImplementation()!
const gfsMint = vi.mocked(mintHostGfsToken)
const initialGfsMint = gfsMint.getMockImplementation()!
const BOOTSTRAP = 'clerum.io/runtime-token-bootstrap-state'
const SECRET_REVISION = 'clerum.io/runtime-token-secret-revision'
const APPLIED_REVISION = 'clerum.io/runtime-token-revision'
const ROLLOUT = 'clerum.io/runtime-token-rollout-required'
const fixtures: HostReconciler[] = []
afterEach(() => {
config.hostResources = structuredClone(initialResources)
config.desktopResources = structuredClone(initialDesktopResources)
config.mcpHostBootstrapRefreshBeforeSec = initialRefreshBefore
issue.mockImplementation(initialIssue)
gfsMint.mockImplementation(initialGfsMint)
vi.useRealTimers()
// Full reconciles schedule readiness polls; keep their timers inside this fixture.
for (const reconciler of fixtures.splice(0)) {
for (const timer of (reconciler as any).readinessTimers.values()) clearTimeout(timer)
}
})
async function runtime(
options: {
desktop?: boolean
stateless?: boolean
draining?: boolean
consumed?: boolean
oauth?: boolean
canonicalResources?: boolean
} = {}
) {
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(new Date('2026-07-03T00:00:00.000Z'))
// A finite producer-derived renewal window exercises maintenance before expiry.
config.mcpHostBootstrapRefreshBeforeSec = 90
const host = makeHost({
generation: 1,
...(options.stateless
? makeStatelessHost({
status: {
lifecycle: {
state: options.draining ? 'draining' : 'active',
wakeHandledGeneration: 0,
},
},
})
: {}),
...(options.desktop ? { spec: { ...makeHost().spec, desktop: { browser: true } } } : {}),
})
let cacheSynced = true
const apis = createReconciler({ isCommunicationChannelCacheSynced: () => cacheSynced })
const { reconciler, appsApi, coreApi, customApi } = apis
fixtures.push(reconciler)
reconciler.setHostFrontsOAuthServer(async () => options.oauth ?? false)
const readHost = customApi.getNamespacedCustomObject.getMockImplementation()!
customApi.getNamespacedCustomObject.mockImplementation(async request =>
request.plural === 'hosts'
? {
...hostApiObject(host),
metadata: { ...hostApiObject(host).metadata, generation: host.generation },
}
: readHost(request)
)
let issuance = 0
let gfsIssuance = 0
const events: string[] = []
// Only the issuer boundary is simulated. Distinct synthetic emissions and
// decodable future refresh expiries exercise the real persistence/reuse path.
issue.mockImplementation(async (...args) => {
events.push('issue')
const emission = ++issuance
return {
...(await initialIssue(...args)),
accessToken: `synthetic-access-${emission}`,
refreshToken: encodedRuntimeRefreshMaterial(Date.now() + 3_600_000 + emission * 1000),
mcpHostControlToken: `synthetic-control-${emission}`,
}
})
// Independent synthetic GFS emissions expose content-hash changes without
// changing the runtime access/refresh/control material.
gfsMint.mockImplementation(async (...args) => ({
...(await initialGfsMint(...args)),
token: `synthetic-gfs-${++gfsIssuance}`,
}))
let secret: k8s.V1Secret | null = null
let deployment: k8s.V1Deployment | null = null
let secretVersion = 0
let deploymentVersion = 0
let templateChanges = 0
let replacementFailures: number[] = []
let secretConflict = false
const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
const writeSecret = (body: k8s.V1Secret): k8s.V1Secret => {
secret = structuredClone(body)
secret.data = {
...secret.data,
...Object.fromEntries(
Object.entries(secret.stringData ?? {}).map(([key, value]) => [
key,
Buffer.from(value).toString('base64'),
])
),
}
delete secret.stringData
secret.metadata = {
...secret.metadata,
uid: 'runtime-secret-uid',
resourceVersion: String(++secretVersion),
}
events.push('persist-secret')
return structuredClone(secret)
}
coreApi.readNamespacedSecret.mockImplementation(async request => {
if (request.name !== `host-${host.name}-mcp-host-runtime-tokens`) return readSecret(request)
if (!secret) throw { code: 404 }
return structuredClone(secret)
})
const createSecret = coreApi.createNamespacedSecret.getMockImplementation()!
coreApi.createNamespacedSecret.mockImplementation(async request => {
const { body } = request
if (body.metadata?.name !== `host-${host.name}-mcp-host-runtime-tokens`)
return createSecret(request)
if (secret) throw { code: 409 }
return writeSecret(body)
})
const replaceSecret = coreApi.replaceNamespacedSecret.getMockImplementation()!
coreApi.replaceNamespacedSecret.mockImplementation(async request => {
const { body } = request
if (request.name !== `host-${host.name}-mcp-host-runtime-tokens`)
return replaceSecret(request)
if (!secret || body.metadata?.resourceVersion !== secret.metadata?.resourceVersion)
throw { code: 409 }
if (secretConflict) {
secretConflict = false
// An unrelated writer won CAS. The next read must see its new version.
secret.metadata!.resourceVersion = String(++secretVersion)
throw { code: 409 }
}
return writeSecret(body)
})
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
const persistDeployment = (body: k8s.V1Deployment) => {
const result = asApiserverDeployment(body)
// The recorded-default overlay does not merge Date objects. A real SDK
// read preserves these timestamps on a replica-only replacement.
for (const [incoming, persisted] of [
[body.metadata, result.metadata],
[body.spec?.template.metadata, result.spec?.template.metadata],
]) {
if (incoming?.creationTimestamp instanceof Date && persisted) {
persisted.creationTimestamp = structuredClone(incoming.creationTimestamp)
}
}
if (options.canonicalResources) {
// Closed equivalences from apimachinery Quantity parse/serialization,
// independent of the production guard's implementation.
for (const container of result.spec!.template.spec!.containers) {
for (const kind of ['requests', 'limits'] as const) {
const values = container.resources?.[kind]
if (values?.cpu === '1000m') values.cpu = '1'
if (values?.cpu === '0.1') values.cpu = '100m'
if (values?.cpu === '0.025') values.cpu = '25m'
if (values?.memory === '1024Mi') values.memory = '1Gi'
}
}
}
return result
}
appsApi.readNamespacedDeployment.mockImplementation(async request => {
if (request.name !== host.name) return readDeployment(request)
if (!deployment) throw { code: 404 }
return structuredClone(deployment)
})
const createDeployment = appsApi.createNamespacedDeployment.getMockImplementation()!
appsApi.createNamespacedDeployment.mockImplementation(async request => {
const { body } = request
if (body.metadata?.name !== host.name) return createDeployment(request)
if (deployment) throw { code: 409 }
deployment = persistDeployment(body)
deployment.metadata = {
...deployment.metadata,
uid: 'runtime-deployment-uid',
resourceVersion: String(++deploymentVersion),
generation: 1,
}
events.push('create-deployment')
return structuredClone(deployment)
})
const replaceDeployment = appsApi.replaceNamespacedDeployment.getMockImplementation()!
appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
const { body } = request
if (request.name !== host.name) return replaceDeployment(request)
if (!deployment || body.metadata?.resourceVersion !== deployment.metadata?.resourceVersion)
throw { code: 409 }
const failure = replacementFailures.shift()
if (failure) {
if (failure === 409) deployment.metadata!.resourceVersion = String(++deploymentVersion)
events.push(`replace-${failure}`)
throw { code: failure }
}
const previous = deployment
deployment = persistDeployment(body)
deployment.metadata = {
...deployment.metadata,
uid: previous.metadata!.uid,
resourceVersion: String(++deploymentVersion),
generation:
previous.metadata!.generation! +
(isDeepStrictEqual(previous.spec, deployment.spec) ? 0 : 1),
}
if (!isDeepStrictEqual(previous.spec!.template, deployment.spec!.template)) templateChanges++
// Kubernetes status belongs to the prior generation until advance() is called.
deployment.status = structuredClone(previous.status)
events.push('replace-deployment')
return structuredClone(deployment)
})
const advance = (status: Partial<k8s.V1DeploymentStatus> = {}) => {
if (!deployment) throw new Error('No persisted Deployment')
const replicas = deployment.spec!.replicas ?? 1
deployment.status = {
observedGeneration: deployment.metadata!.generation,
replicas,
updatedReplicas: replicas,
readyReplicas: replicas,
availableReplicas: replicas,
unavailableReplicas: 0,
...status,
}
deployment.metadata!.resourceVersion = String(++deploymentVersion)
}
await reconciler.reconcile(host)
advance()
if (options.consumed !== false) {
await reconciler.reconcile(host)
expect(secret!.metadata!.annotations![BOOTSTRAP]).toBe('consumed')
}
events.length = 0
templateChanges = 0
issue.mockClear()
return {
...apis,
host,
events,
advance,
secret: () => secret!,
deployment: () => deployment!,
reconcile: () => reconciler.reconcile(host),
failReplacements: (...codes: number[]) => {
replacementFailures = codes
},
conflictSecret: () => {
secretConflict = true
},
unbindLegacyPreparation: () => {
delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-applied-revision']
secret!.metadata!.resourceVersion = String(++secretVersion)
},
setCacheSynced: (value: boolean) => {
cacheSynced = value
},
templateChanges: () => templateChanges,
recreateDeployment: () => {
deployment!.metadata!.uid = 'recreated-runtime-deployment-uid'
deployment!.metadata!.resourceVersion = String(++deploymentVersion)
deployment!.metadata!.generation = 1
},
}
}
it.each([
['CPU request', 'requests', 'cpu', '25m'],
['memory request', 'requests', 'memory', '192Mi'],
['CPU limit', 'limits', 'cpu', '750m'],
['memory limit', 'limits', 'memory', '768Mi'],
] as const)(
'persists usable bootstrap before replacing a Ready runtime for a %s change',
async (_label, kind, resource, value) => {
const fixture = await runtime()
const before = fixture.secret().metadata!.annotations![SECRET_REVISION]
config.hostResources[kind][resource] = value
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(issue).toHaveBeenCalledOnce()
const revision = fixture.secret().metadata!.annotations![SECRET_REVISION]
expect(revision).not.toBe(before)
expect(fixture.secret().metadata!.annotations).toMatchObject({
[BOOTSTRAP]: 'fresh',
[ROLLOUT]: 'true',
})
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
revision
)
expect(
fixture.deployment().spec!.template.spec!.containers[0].resources![kind]![resource]
).toBe(value)
expect(fixture.deployment().metadata).toMatchObject({
uid: 'runtime-deployment-uid',
generation: 2,
})
await fixture.reconcile() // Ready from generation 1 must not acknowledge generation 2.
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })
await fixture.reconcile() // A starting rollout must not issue or replace again.
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations).toMatchObject({
[BOOTSTRAP]: 'consumed',
[ROLLOUT]: 'false',
})
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.events.filter(event => event === 'replace-deployment')).toHaveLength(1)
expect(fixture.secret().metadata!.annotations![SECRET_REVISION]).toBe(revision)
}
)
it.each([409, 503])('reuses persisted preparation after a Deployment %s failure', async code => {
const fixture = await runtime()
config.hostResources.requests.cpu = '25m'
fixture.failReplacements(...(code === 409 ? [409, 409, 409] : [503]))
await expect(fixture.reconcile()).rejects.toMatchObject({ code })
const revision = fixture.secret().metadata!.annotations![SECRET_REVISION]
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
expect(fixture.deployment().metadata!.generation).toBe(1)
await fixture.reconcile()
await fixture.reconcile()
fixture.advance()
await fixture.reconcile()
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.secret().metadata!.annotations![SECRET_REVISION]).toBe(revision)
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
expect(fixture.events.filter(event => event === 'replace-deployment')).toHaveLength(1)
})
it('rereads CAS after a Secret conflict and persists before any Deployment replacement', async () => {
const fixture = await runtime()
config.hostResources.requests.cpu = '25m'
fixture.conflictSecret()
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'issue', 'persist-secret', 'replace-deployment'])
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
fixture.secret().metadata!.annotations![SECRET_REVISION]
)
})
it.each([
{ observedGeneration: 0 },
{ updatedReplicas: 0 },
{ replicas: 2, readyReplicas: 2 },
{ readyReplicas: 0, availableReplicas: 0 },
])('does not consume fresh bootstrap before current-generation convergence: %j', async status => {
const fixture = await runtime({ consumed: false })
fixture.advance(status)
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
expect(issue).not.toHaveBeenCalled()
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('keeps an API-defaulted Ready no-op stable, including metadata-only resource policy reordering', async () => {
const fixture = await runtime()
config.hostResources = {
limits: { cpu: initialResources.limits.cpu, memory: initialResources.limits.memory },
requests: { cpu: initialResources.requests.cpu, memory: initialResources.requests.memory },
}
await fixture.reconcile()
await fixture.reconcile()
expect(fixture.events).toEqual([])
expect(issue).not.toHaveBeenCalled()
})
it('leaves an unchanged Desktop runtime stable when Host resources change', async () => {
const fixture = await runtime({ desktop: true })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual([])
expect(issue).not.toHaveBeenCalled()
expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(
initialDesktopResources
)
})
it('prepares bootstrap for a change to the selected Desktop resource policy', async () => {
const fixture = await runtime({ desktop: true })
config.desktopResources.limits.memory = '5Gi'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(
config.desktopResources
)
})
it('preserves a held template despite a resource policy change', async () => {
const fixture = await runtime({ stateless: true })
fixture.setCacheSynced(false)
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual([])
expect(issue).not.toHaveBeenCalled()
expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(
initialResources
)
})
it('does not consume a resource rollout through GFS-only renewal with old-generation Ready status', async () => {
const fixture = await runtime({ stateless: true })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
fixture.setCacheSynced(false)
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
expect(issue).toHaveBeenCalledOnce()
expect(fixture.events.filter(event => event === 'replace-deployment')).toHaveLength(1)
})
it('retains consumed-bootstrap NotReady recovery', async () => {
const fixture = await runtime()
fixture.advance({ readyReplicas: 0, availableReplicas: 0 })
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
})
it.each(['Desktop CPU', 'Host memory'])(
'keeps equivalent canonical API quantities stable: %s',
async kind => {
const desktop = kind === 'Desktop CPU'
if (!desktop) config.hostResources.requests.memory = '1024Mi'
const fixture = await runtime({ desktop, canonicalResources: true, consumed: false })
for (let pass = 0; pass < 3; pass++) {
await fixture.reconcile()
fixture.advance()
}
expect(issue).not.toHaveBeenCalled()
expect(fixture.templateChanges()).toBe(0)
expect(fixture.deployment().metadata!.generation).toBe(1)
}
)
it('keeps the reviewed decimal CPU no-op stable across four canonicalized reconciles', async () => {
config.hostResources.requests.cpu = '0.1'
const fixture = await runtime({ canonicalResources: true, consumed: false })
const revision = fixture.secret().metadata!.annotations![SECRET_REVISION]
for (let pass = 0; pass < 4; pass++) {
await fixture.reconcile()
fixture.advance()
}
expect(issue).not.toHaveBeenCalled()
expect(fixture.templateChanges()).toBe(0)
expect(fixture.deployment().metadata!.generation).toBe(1)
expect(fixture.secret().metadata!.annotations![SECRET_REVISION]).toBe(revision)
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'100m'
)
})
it('rolls the reviewed decimal CPU transition once and settles after four canonicalized reconciles', async () => {
const fixture = await runtime({ canonicalResources: true })
config.hostResources.requests.cpu = '0.025'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
const appliedGeneration = fixture.deployment().metadata!.generation
expect(appliedGeneration).toBe(2)
for (let pass = 0; pass < 4; pass++) {
fixture.advance()
await fixture.reconcile()
}
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
expect(fixture.deployment().metadata!.generation).toBe(appliedGeneration)
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
prepared
)
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'25m'
)
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it.each([false, undefined] as const)(
'blocks an active resource change with an unprepared provision (%s), then resumes with persisted preparation',
async proof => {
const fixture = await runtime()
const template = structuredClone(fixture.deployment().spec!.template)
const generation = fixture.deployment().metadata!.generation
const credentialData = { ...fixture.secret().data }
config.hostResources.requests.cpu = '25m'
// Model a provisioning-contract regression at its boundary. The real
// Deployment mutation guard must reject the missing delivery proof.
const provision = vi
.spyOn(fixture.reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue({
revision: template.metadata!.annotations![APPLIED_REVISION],
scopeHash: fixture.secret().metadata!.annotations!['clerum.io/runtime-token-scope-hash'],
deploymentUid: fixture.deployment().metadata!.uid,
freshBootstrapPrepared: proof,
})
try {
await fixture.reconcile()
expect(issue).not.toHaveBeenCalled()
expect(fixture.events).toEqual([])
expect(fixture.secret().data).toEqual(credentialData)
expect(fixture.deployment().spec!.template).toEqual(template)
expect(fixture.deployment().metadata!.generation).toBe(generation)
expect(fixture.reconciler.getStatus(fixture.host.name)).toMatchObject({
deployed: true,
ready: false,
message: 'Waiting for a prepared runtime bootstrap before changing resources',
})
} finally {
provision.mockRestore()
}
// The same intended change must proceed once real provisioning supplies
// persisted material, so refusal cannot hide a permanent convergence block.
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'25m'
)
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
fixture.secret().metadata!.annotations![SECRET_REVISION]
)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
expect(issue).toHaveBeenCalledOnce()
}
)
it('defers resource preparation when retained OAuth authority is temporarily unobserved', async () => {
const fixture = await runtime({ oauth: true })
config.hostResources.requests.cpu = '25m'
fixture.reconciler.setHostFrontsOAuthServer(async () => {
throw new Error('simulated OAuth observation unavailable')
})
await fixture.reconcile()
expect(issue).not.toHaveBeenCalled()
expect(fixture.events).toEqual([])
expect(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-fronts-oauth-server']
).toBe('true')
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'100m'
)
fixture.reconciler.setHostFrontsOAuthServer(async () => true)
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(issue.mock.calls[0][2]).toContain(OAUTH_USER_TOKEN_SCOPE)
expect(fixture.templateChanges()).toBe(1)
})
it.each([
['Host', false],
['Desktop', true],
] as const)(
'does not reuse a routine renewal for a new %s resource rollout',
async (_kind, desktop) => {
const fixture = await runtime({ desktop })
const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]
vi.setSystemTime(
Date.parse(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-refresh-before']
) + 1
)
await fixture.reconcile()
const renewed = fixture.secret().metadata!.annotations![SECRET_REVISION]
expect(renewed).not.toBe(applied)
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
expect(fixture.secret().metadata!.annotations![ROLLOUT]).toBe('false')
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
applied
)
expect(fixture.templateChanges()).toBe(0)
expect(issue).toHaveBeenCalledOnce()
// A routine renewal may be consumed after a container restart without an
// applied revision change. Its fresh record is not rollout preparation.
fixture.events.length = 0
const resources = desktop ? config.desktopResources : config.hostResources
resources.requests.cpu = '25m'
await fixture.reconcile()
const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
expect(issue).toHaveBeenCalledTimes(2)
expect(prepared).not.toBe(renewed)
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
prepared
)
expect(fixture.secret().metadata!.annotations![ROLLOUT]).toBe('true')
// The new preparation survives NotReady startup; it is not renewed on
// every reconcile, and is acknowledged only after convergence.
fixture.advance({ readyReplicas: 0, availableReplicas: 0, unavailableReplicas: 1 })
for (let pass = 0; pass < 3; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledTimes(2)
expect(fixture.templateChanges()).toBe(1)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
expect(issue).toHaveBeenCalledTimes(2)
}
)
it('reuses an intrinsic renewal persisted before a failed resource replacement', async () => {
const fixture = await runtime()
vi.setSystemTime(
Date.parse(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-refresh-before']
) + 1
)
config.hostResources.requests.cpu = '25m'
fixture.failReplacements(503)
await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })
const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
prepared
)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('binds and delivers legacy unbound preparation even when its rollout marker is already true', async () => {
const fixture = await runtime()
config.hostResources.requests.cpu = '25m'
fixture.failReplacements(503)
await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })
// The previously shipped preparation did not bind its target until acknowledgement.
fixture.unbindLegacyPreparation()
fixture.events.length = 0
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.events).toEqual(['persist-secret', 'replace-deployment'])
expect(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
).toBe(fixture.deployment().metadata!.uid)
for (let pass = 0; pass < 3; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('applies and acknowledges the preparation when intrinsic renewal and resources coincide', async () => {
const fixture = await runtime()
vi.setSystemTime(
Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-refresh-before']) + 1
)
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
prepared
)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('renews held GFS while strict bootstrap acknowledgement remains pending', async () => {
const fixture = await runtime({ stateless: true, consumed: false })
const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]
const runtimeData = { ...fixture.secret().data }
fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
fixture.setCacheSynced(false)
gfsMint.mockClear()
vi.setSystemTime(Date.now() + 550_000)
await fixture.reconcile()
await fixture.reconcile()
vi.setSystemTime(Date.now() + 100_000)
await fixture.reconcile()
expect(gfsMint).toHaveBeenCalledOnce()
expect(issue).not.toHaveBeenCalled()
expect(
Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-expires-at'])
).toBeGreaterThan(Date.now())
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
for (const key of [
MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
]) {
expect(fixture.secret().data![key]).toBe(runtimeData[key])
}
expect(fixture.secret().data![MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(
runtimeData[MCP_HOST_GFS_TOKEN_SECRET_KEY]
)
fixture.setCacheSynced(true)
await fixture.reconcile()
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
applied
)
expect(fixture.templateChanges()).toBe(0)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('recovers once after an observed partial Ready bootstrap subsequently loses Ready', async () => {
const fixture = await runtime({ consumed: false })
fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
fixture.advance({ readyReplicas: 0, availableReplicas: 0, updatedReplicas: 0 })
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
})
it('does not mistake stale-generation Ready for possible bootstrap use during slow startup', async () => {
const fixture = await runtime({ consumed: false })
fixture.advance({ observedGeneration: 0, updatedReplicas: 0, readyReplicas: 1 })
await fixture.reconcile()
fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })
for (let pass = 0; pass < 5; pass++) await fixture.reconcile()
expect(issue).not.toHaveBeenCalled()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
expect(fixture.templateChanges()).toBe(0)
})
it('prepares for a changed Deployment UID observed after credential persistence', async () => {
const fixture = await runtime()
config.hostResources.requests.cpu = '25m'
const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!
let reads = 0
fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {
if (request.name === fixture.host.name && ++reads === 2) fixture.recreateDeployment()
return read(request)
})
await fixture.reconcile()
expect(issue).toHaveBeenCalledTimes(2)
expect(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
).toBe('recreated-runtime-deployment-uid')
expect(fixture.templateChanges()).toBe(1)
await fixture.reconcile()
expect(issue).toHaveBeenCalledTimes(2)
})
it('does not reuse a consumed record bound to a previous Deployment UID', async () => {
const fixture = await runtime()
fixture.recreateDeployment()
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(
fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
).toBe('recreated-runtime-deployment-uid')
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
})
it('replans resource preparation against drift observed at the mutation read with the same UID', async () => {
const fixture = await runtime()
const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!
let reads = 0
fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {
if (request.name === fixture.host.name && ++reads === 2) {
fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'
fixture.deployment().metadata!.generation!++
fixture.advance()
}
return read(request)
})
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(issue).toHaveBeenCalledOnce()
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'100m'
)
})
it('prepares a consumed-bootstrap resource transition without acknowledging a partial Ready runtime', async () => {
const fixture = await runtime()
fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
for (let pass = 0; pass < 3; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('prepares an immediate resource repair for a fresh applied bootstrap without Ready replicas', async () => {
const fixture = await runtime({ consumed: false })
fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
fixture.secret().metadata!.annotations![SECRET_REVISION]
)
for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
fixture.advance()
await fixture.reconcile()
expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
})
it('recovers a NotReady consumed bootstrap when its resources also drift', async () => {
const fixture = await runtime()
fixture.advance({ readyReplicas: 0, availableReplicas: 0 })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(issue).toHaveBeenCalledOnce()
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'25m'
)
for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
})
it('replans same-UID resource drift introduced by a conflicting Deployment writer', async () => {
config.desktopResources.requests.cpu = '1000m'
const fixture = await runtime({ desktop: true, canonicalResources: true })
const replace = fixture.appsApi.replaceNamespacedDeployment.getMockImplementation()!
let changed = false
fixture.appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
if (request.name === fixture.host.name && !changed) {
changed = true
fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'
fixture.deployment().metadata!.generation!++
fixture.advance()
// The external writer's new resourceVersion makes this first PUT conflict.
fixture.events.push('replace-409')
}
return replace(request)
})
await fixture.reconcile()
expect(fixture.events).toEqual(['replace-409', 'issue', 'persist-secret', 'replace-deployment'])
expect(issue).toHaveBeenCalledOnce()
expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
'1'
)
expect(fixture.templateChanges()).toBe(1)
})
it('defers resource transitions while the owned stateless runtime is draining', async () => {
const fixture = await runtime({ stateless: true, draining: true })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(issue).not.toHaveBeenCalled()
expect(fixture.events).toEqual([])
expect(fixture.templateChanges()).toBe(0)
expect(fixture.deployment().spec!.replicas).toBe(1)
})
it('delivers intrinsic bootstrap recovery while draining and preserves the applied resources', async () => {
const fixture = await runtime({ stateless: true, draining: true })
const resources = structuredClone(
fixture.deployment().spec!.template.spec!.containers[0].resources
)
fixture.advance({ readyReplicas: 0, availableReplicas: 0 })
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
fixture.secret().metadata!.annotations![SECRET_REVISION]
)
expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(resources)
for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
expect(fixture.templateChanges()).toBe(1)
})
it('suspends without applying resource drift or resurrecting the owned stateless runtime', async () => {
const fixture = await runtime({ stateless: true })
const template = structuredClone(fixture.deployment().spec!.template)
const runtimeData = { ...fixture.secret().data }
fixture.host.status = suspendedStatus()
config.hostResources.requests.cpu = '25m'
await fixture.reconcile()
expect(fixture.deployment().spec!.replicas).toBe(0)
expect(fixture.deployment().spec!.template).toEqual(template)
expect(fixture.secret().data).toEqual(runtimeData)
expect(issue).not.toHaveBeenCalled()
expect(fixture.events).toEqual(['replace-deployment'])
await fixture.reconcile()
expect(fixture.deployment().spec!.replicas).toBe(0)
expect(fixture.events).toEqual(['replace-deployment'])
expect(issue).not.toHaveBeenCalled()
})
it('keeps consumed runtime credentials and the applied revision stable after held GFS renewal', async () => {
const fixture = await runtime({ stateless: true })
const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]
const before = { ...fixture.secret().data }
fixture.setCacheSynced(false)
gfsMint.mockClear()
vi.setSystemTime(Date.now() + 550_000)
await fixture.reconcile()
expect(gfsMint).toHaveBeenCalledOnce()
expect(issue).not.toHaveBeenCalled()
for (const key of [
MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
])
expect(fixture.secret().data![key]).toBe(before[key])
fixture.setCacheSynced(true)
await fixture.reconcile()
expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
applied
)
expect(fixture.templateChanges()).toBe(0)
expect(issue).not.toHaveBeenCalled()
})
})
describe('HostReconciler stateless lifecycle — buildDeployment replicas', () => {
it('pins replicas=1 and the Recreate strategy for a non-stateless host', () => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(makeHost())
expect(dep.spec?.replicas).toBe(1)
expect(dep.spec?.strategy).toEqual({ type: 'Recreate' })
expect(dep.spec?.template.spec?.priorityClassName).toBeUndefined()
})
it('derives replicas=0 for stateless+suspended from the CRD status', () => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))
expect(dep.spec?.replicas).toBe(0)
})
it('forces stateless+suspended to replicas=1 when CommunicationChannels exist by default', () => {
const { reconciler } = createReconciler({
countCommunicationChannels: () => 1,
})
const dep = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))
expect(dep.spec?.replicas).toBe(1)
})
it('preserves stateless+suspended replicas while the channel cache is unsynced', () => {
const { reconciler } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
const dep = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))
expect(dep.spec?.replicas).toBe(0)
expect(dep.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
})
it('derives replicas=1 for stateless+active and stateless+draining', () => {
const { reconciler } = createReconciler()
const active = reconciler.buildDeployment(makeStatelessHost())
expect(active.spec?.replicas).toBe(1)
const draining = reconciler.buildDeployment(
makeStatelessHost({ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 1 } } })
)
expect(draining.spec?.replicas).toBe(1)
})
it('assigns the interactive priority class to every effective stateless Host', () => {
const { reconciler } = createReconciler()
const active = reconciler.buildDeployment(makeStatelessHost())
const suspended = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))
expect(active.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
expect(suspended.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
})
it('derives replicas=0 from an explicit suspended lifecycle argument', () => {
const { reconciler } = createReconciler()
const lifecycle: EffectiveHostLifecycle = { stateless: true, state: 'suspended' }
const dep = reconciler.buildDeployment(makeStatelessHost(), [], '', lifecycle)
expect(dep.spec?.replicas).toBe(0)
})
})
describe('HostReconciler ensureDeployment — idempotent replacement', () => {
function withKubernetesProbeDefaults(probe: k8s.V1Probe | undefined): k8s.V1Probe | undefined {
if (!probe) return probe
const persisted = {
...probe,
successThreshold: 1,
httpGet: probe.httpGet ? { ...probe.httpGet, scheme: 'HTTP' } : probe.httpGet,
}
if (persisted.initialDelaySeconds === 0) delete persisted.initialDelaySeconds
return persisted
}
function existingDeployment(
reconciler: HostReconciler,
host: HostCRD,
runtimeTokenRevision: string,
lifecycle?: EffectiveHostLifecycle
): k8s.V1Deployment {
const deployment = structuredClone(
reconciler.buildDeployment(host, [], runtimeTokenRevision, lifecycle)
)
const deploymentSpec = deployment.spec
if (!deploymentSpec) throw new Error('expected Host Deployment spec')
const podSpec = deploymentSpec.template.spec
if (!podSpec) throw new Error('expected Host Deployment PodSpec')
deployment.metadata = {
...deployment.metadata,
resourceVersion: '42',
uid: 'deployment-uid',
generation: 7,
creationTimestamp: new Date('2026-07-10T00:00:00Z'),
annotations: {
...deployment.metadata?.annotations,
'deployment.kubernetes.io/revision': '7',
},
}
deployment.status = { readyReplicas: 1, availableReplicas: 1 }
deployment.spec = {
...deploymentSpec,
progressDeadlineSeconds: 600,
revisionHistoryLimit: 10,
template: {
...deploymentSpec.template,
metadata: {
...deploymentSpec.template.metadata,
annotations: {
...deploymentSpec.template.metadata?.annotations,
'kubectl.kubernetes.io/restartedAt': '2026-07-10T00:00:00Z',
},
},
spec: {
...podSpec,
dnsPolicy: 'ClusterFirst',
restartPolicy: 'Always',
schedulerName: 'default-scheduler',
serviceAccount: `host-${host.name}-sa`,
terminationGracePeriodSeconds: 30,
containers: podSpec.containers.map(container => ({
...container,
terminationMessagePath: '/dev/termination-log',
terminationMessagePolicy: 'File',
startupProbe: withKubernetesProbeDefaults(container.startupProbe),
livenessProbe: withKubernetesProbeDefaults(container.livenessProbe),
readinessProbe: withKubernetesProbeDefaults(container.readinessProbe),
env: container.env?.map(env =>
env.valueFrom?.fieldRef
? {
...env,
valueFrom: {
...env.valueFrom,
fieldRef: { ...env.valueFrom.fieldRef, apiVersion: 'v1' },
},
}
: env
),
})),
volumes: (podSpec.volumes ?? []).map(volume =>
volume.secret ? { ...volume, secret: { ...volume.secret, defaultMode: 420 } } : volume
),
},
},
}
return deployment
}
it('does not replace a converged stateful Deployment when Kubernetes only adds defaults', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(
existingDeployment(reconciler, host, 'revision-a')
)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledOnce()
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
namespace: 'mcp-host',
name: 'chatllm',
})
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('does not replace a converged stateless Deployment when Kubernetes omits zero probe delays', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeStatelessHost()
const active: EffectiveHostLifecycle = { stateless: true, state: 'active' }
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(
existingDeployment(reconciler, host, 'revision-a', active)
)
await (reconciler as any).ensureDeployment(host, [], 'revision-a', active)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('replaces an existing Deployment when an HCC-owned nested field is stale', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const existing = existingDeployment(reconciler, host, 'revision-a')
const container = existing.spec?.template.spec?.containers?.[0]
if (!container) throw new Error('expected mcp-host container')
container.securityContext = {
...container.securityContext,
readOnlyRootFilesystem: true,
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('replaces an existing Deployment when a defaultable Deployment field is non-default', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const existing = existingDeployment(reconciler, host, 'revision-a')
existing.spec!.progressDeadlineSeconds = 30
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('replaces an existing Deployment when a defaultable Pod field is non-default', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const existing = existingDeployment(reconciler, host, 'revision-a')
existing.spec!.template.spec!.terminationGracePeriodSeconds = 120
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('treats an unrecognized admission mutation as drift', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const existing = existingDeployment(reconciler, host, 'revision-a')
existing.spec!.template.spec!.tolerations = [
{ key: 'admission.example.io/injected', operator: 'Exists', effect: 'NoSchedule' },
]
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it.each([
[
'readiness successThreshold',
(deployment: k8s.V1Deployment) => {
deployment.spec!.template.spec!.containers![0].readinessProbe!.successThreshold = 2
},
],
[
'readiness HTTP scheme',
(deployment: k8s.V1Deployment) => {
deployment.spec!.template.spec!.containers![0].readinessProbe!.httpGet!.scheme = 'HTTPS'
},
],
[
'fieldRef API version',
(deployment: k8s.V1Deployment) => {
const namespaceEnv = deployment.spec!.template.spec!.containers![0].env!.find(
env => env.name === 'CLERUM_NAMESPACE'
)
namespaceEnv!.valueFrom!.fieldRef!.apiVersion = 'v2'
},
],
[
'Secret defaultMode',
(deployment: k8s.V1Deployment) => {
const runtimeVolume = deployment.spec!.template.spec!.volumes!.find(
volume => volume.name === 'mcp-host-runtime-tokens'
)
runtimeVolume!.secret!.defaultMode = 384
},
],
[
'Secret optional flag',
(deployment: k8s.V1Deployment) => {
const runtimeVolume = deployment.spec!.template.spec!.volumes!.find(
volume => volume.name === 'mcp-host-runtime-tokens'
)
runtimeVolume!.secret!.optional = true
},
],
])('replaces an existing Deployment when %s is non-default', async (_case, mutate) => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const existing = existingDeployment(reconciler, host, 'revision-a')
mutate(existing)
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(existing)
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('stops retrying when a fresh read converges after a replace conflict', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
const stale = existingDeployment(reconciler, host, 'revision-a')
const container = stale.spec?.template.spec?.containers?.[0]
if (!container) throw new Error('expected mcp-host container')
container.securityContext = {
...container.securityContext,
readOnlyRootFilesystem: true,
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment
.mockResolvedValueOnce(stale)
.mockResolvedValueOnce(existingDeployment(reconciler, host, 'revision-a'))
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
it('replaces an existing Deployment when the runtime token revision changes', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeHost({ name: 'chatllm' })
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(
existingDeployment(reconciler, host, 'revision-a')
)
await (reconciler as any).ensureDeployment(host, [], 'revision-b')
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
.body as k8s.V1Deployment
expect(replacement.metadata?.resourceVersion).toBe('42')
expect(
replacement.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']
).toBe('revision-b')
})
it('replaces an existing Deployment when the stateless lifecycle target changes', async () => {
const { reconciler, appsApi } = createReconciler()
const host = makeStatelessHost()
const active: EffectiveHostLifecycle = { stateless: true, state: 'active' }
const suspended: EffectiveHostLifecycle = { stateless: true, state: 'suspended' }
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue(
existingDeployment(reconciler, host, 'revision-a', active)
)
await (reconciler as any).ensureDeployment(host, [], 'revision-a', suspended)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
.body as k8s.V1Deployment
expect(replacement.metadata?.resourceVersion).toBe('42')
expect(replacement.spec?.replicas).toBe(0)
expect(
replacement.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']
).toBe('revision-a')
})
it('replaces a running stateful Host with the stateless template when lifecycle is enabled', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const stateful = makeHost({ name: 'transition-host' })
await reconciler.reconcile(stateful)
const runningStateful = structuredClone(hostDeploymentBody(appsApi, 'transition-host'))
runningStateful.metadata = { ...runningStateful.metadata, resourceVersion: '42' }
runningStateful.status = { readyReplicas: 1, availableReplicas: 1 }
expect(runningStateful.spec?.replicas).toBe(1)
expect(containerEnv(runningStateful).map(entry => entry.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
appsApi.replaceNamespacedDeployment.mockClear()
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === stateful.name ? Promise.resolve(runningStateful) : readDeployment(request)
)
const stateless = makeStatelessHost({ name: 'transition-host' })
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: { name: stateless.name, namespace: stateless.namespace, uid: stateless.uid },
spec: stateless.spec,
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
await reconciler.reconcile(stateless)
const hostReplacements = appsApi.replaceNamespacedDeployment.mock.calls.filter(
([request]) => request.name === stateful.name
)
expect(hostReplacements).toHaveLength(1)
const replacement = hostReplacements[0][0].body as k8s.V1Deployment
// Witness: the flip really changed the lifecycle-specific template.
expect(replacement.spec?.replicas).toBe(1)
expect(replacement.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
expect(envValue(replacement, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
// Storage is identical across the flip: mounts, storage env and the init.
const storage = (dep: k8s.V1Deployment) => ({
mounts: dep.spec?.template.spec?.containers?.[0]?.volumeMounts,
store: envValue(dep, 'CLERUM_SESSION_STORE'),
dbDir: envValue(dep, 'CLERUM_SESSION_DB_DIR'),
inits: dep.spec?.template.spec?.initContainers,
})
expect(JSON.stringify(storage(replacement))).toBe(JSON.stringify(storage(runningStateful)))
expect(storage(replacement).dbDir).toBe('/workspace')
expect(storage(replacement).inits?.map(container => container.name)).toEqual([
'workspace-layout-merge',
])
})
})
describe('HostReconciler stateless lifecycle — env injection', () => {
it('creates the stateless template directly for a new Host', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const stateless = makeStatelessHost({ name: 'new-stateless-host' })
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: { name: stateless.name, namespace: stateless.namespace, uid: stateless.uid },
spec: stateless.spec,
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
// Model a genuinely new principal Deployment, then expose its created state.
let principalCreated = false
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request => {
if (request.name === stateless.name && !principalCreated) return Promise.reject({ code: 404 })
return readDeployment(request)
})
appsApi.createNamespacedDeployment.mockImplementation(async request => {
if (request.body.metadata?.name === stateless.name) principalCreated = true
return request.body
})
await reconciler.reconcile(stateless)
const creates = appsApi.createNamespacedDeployment.mock.calls.filter(
([arg]) => (arg as { body?: k8s.V1Deployment }).body?.metadata?.name === 'new-stateless-host'
)
const replacements = appsApi.replaceNamespacedDeployment.mock.calls.filter(
([arg]) => (arg as { body?: k8s.V1Deployment }).body?.metadata?.name === 'new-stateless-host'
)
expect(creates).toHaveLength(1)
expect(replacements).toHaveLength(0)
const created = hostDeploymentBody(appsApi, 'new-stateless-host')
expect(created.spec?.replicas).toBe(1)
expect(created.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
expect(created.spec?.template.spec?.initContainers?.map(container => container.name)).toEqual([
'workspace-layout-merge',
])
expect(envValue(created, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(created.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({
name: 'workspace',
mountPath: '/workspace',
})
})
it('injects the stateless env and the root storage env when stateless is enabled', () => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(makeStatelessHost())
expect(envValue(dep, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
})
it('injects the root storage env but not the stateless env when stateless is off', () => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(makeHost())
expect(containerEnv(dep).map(e => e.name)).not.toContain('CLERUM_STATELESS_LIFECYCLE')
// Witness: the same container carries the storage env every Host gets.
expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
})
})
describe('HostReconciler stateless lifecycle — rejection matrix', () => {
it('preserves a rejected stateful runtime across cold channel LIST failure and recovery', async () => {
const host = makeStatelessHost()
const previous = createReconciler({ countCommunicationChannels: () => 1 })
previous.customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
vi.spyOn(previous.reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
runtimeTokenProvision(host, true)
)
await previous.reconciler.reconcile(host)
host.status = lifecycleStatusWrites(previous.customApi).at(-1)
expect(rejectedCondition(host.status!).reason).toBe('ActiveCommunicationChannels')
const applied = hostDeploymentBody(previous.appsApi, host.name)
applied.metadata = { ...applied.metadata, uid: 'deployment-uid', resourceVersion: '73' }
const baseline = JSON.stringify(applied.spec!.template)
expect(containerEnv(applied).some(entry => entry.name === 'CLERUM_STATELESS_LIFECYCLE')).toBe(
false
)
let synced = false
let channels = 0
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => channels,
isCommunicationChannelCacheSynced: () => synced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const live = persistHostDeployment(appsApi, host, applied)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
runtimeTokenProvision(host, true)
)
await reconciler.reconcile(host)
expect(JSON.stringify(live().spec!.template)).toBe(baseline)
expect(live().spec!.replicas).toBe(1)
expect(
appsApi.replaceNamespacedDeployment.mock.calls.filter(([r]) => r.name === host.name)
).toHaveLength(0)
host.status = lifecycleStatusWrites(customApi).at(-1)
synced = true
channels = 1
await reconciler.reconcile(host)
expect(JSON.stringify(live().spec!.template)).toBe(baseline)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'ActiveCommunicationChannels'
)
})
it('does not use a stale positive channel cache to change an applied stateless template', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => false,
})
const read = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === host.name ? Promise.reject({ code: 404 }) : read(request)
)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
applied.metadata = { ...applied.metadata, uid: 'deployment-uid', resourceVersion: '74' }
const live = persistHostDeployment(appsApi, host, applied)
await reconciler.reconcile(host)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
})
it('defers creating a missing Deployment until the channel inventory is authoritative', async () => {
let synced = false
const host = makeStatelessHost()
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => synced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const read = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === host.name ? Promise.reject({ code: 404 }) : read(request)
)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
await reconciler.reconcile(host)
expect(
appsApi.createNamespacedDeployment.mock.calls.filter(
([r]) => r.body.metadata.name === host.name
)
).toHaveLength(0)
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })
synced = true
await reconciler.reconcile(host)
expect(
appsApi.createNamespacedDeployment.mock.calls.filter(
([r]) => r.body.metadata.name === host.name
)
).toHaveLength(1)
})
it('preserves the applied suspended replica count during cache loss', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
const current = structuredClone(applied)
current.metadata = { ...current.metadata, uid: 'deployment-uid', resourceVersion: '41' }
const baseline = JSON.stringify(current.spec!.template)
appsApi.readNamespacedDeployment.mockImplementation(async () => structuredClone(current))
await reconciler.reconcile(host)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(current.spec!.replicas).toBe(0)
expect(JSON.stringify(applied.spec!.template)).toBe(baseline)
expect(current.spec!.template).toEqual(applied.spec!.template)
})
it.each([
'foreign owner',
'missing UID',
'missing resourceVersion',
'prior Host UID',
'unannotated legacy',
])('does not scale an unverified Deployment during cache loss: %s', async invalid => {
const host = makeStatelessHost({ status: suspendedStatus() })
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
applied.metadata = { ...applied.metadata, uid: 'deployment-uid', resourceVersion: '73' }
if (invalid === 'foreign owner') applied.metadata.labels!['clerum.io/host'] = 'another-host'
if (invalid === 'missing UID') delete applied.metadata.uid
if (invalid === 'missing resourceVersion') delete applied.metadata.resourceVersion
if (invalid === 'prior Host UID') {
applied.metadata.annotations = {
...applied.metadata.annotations,
'clerum.io/host-uid': 'prior-host-uid',
}
}
if (invalid === 'unannotated legacy')
delete applied.metadata.annotations?.['clerum.io/host-uid']
appsApi.readNamespacedDeployment.mockResolvedValue(applied)
await expect(reconciler.reconcile(host)).rejects.toThrow('unverified Deployment')
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
})
it('never scales an unannotated legacy Deployment with an existing UID and resourceVersion', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
const { reconciler, appsApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
const legacy = reconciler.buildDeployment(host)
delete legacy.metadata?.annotations?.['clerum.io/host-uid']
legacy.metadata = {
...legacy.metadata,
uid: 'deployment-uid',
resourceVersion: '73',
}
appsApi.readNamespacedDeployment.mockResolvedValue(legacy)
// Derive the held lifecycle from the Host itself so the fixture cannot
// contradict the durable suspended status.
const heldLifecycle = reconciler.getEffectiveLifecycle(host)
expect(heldLifecycle).toEqual({
stateless: true,
state: 'suspended',
suspensionBlocked: true,
})
await expect(
(reconciler as any).ensureDeployment(host, [], undefined, {
...heldLifecycle,
allowScaleUpDuringHold: true,
})
).rejects.toThrow('unverified Deployment')
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
namespace: host.namespace,
name: host.name,
})
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
})
it('throws held-path PVC and Service errors before any credential or Deployment mutation', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
const ingress = vi
.spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
.mockResolvedValue('up_to_date')
coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))
coreApi.readNamespacedService.mockRejectedValueOnce(new Error('Service read failed'))
const rejection = await reconciler.reconcile(host).then(
() => undefined,
(error: unknown) => error
)
expect(rejection).toBeInstanceOf(AggregateError)
expect((rejection as AggregateError).errors.map(error => (error as Error).message)).toEqual([
'PVC read failed',
'Service read failed',
])
// Liveness witness: the pass reached the resource reads and the policy
// phase before rejecting.
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
namespace: host.namespace,
name: `${host.name}-workspace`,
})
expect(coreApi.readNamespacedService).toHaveBeenCalledWith({
namespace: host.namespace,
name: host.name,
})
expect(ingress).toHaveBeenCalledWith(host)
expect(provision).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(live().spec!.template).toEqual(templateBefore)
})
it('rethrows a single held-path PVC error unwrapped and never enters the hold', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
const ingress = vi
.spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
.mockResolvedValue('up_to_date')
const ready = vi.spyOn(reconciler as any, 'checkDeploymentReady')
coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))
const rejection = await reconciler.reconcile(host).then(
() => undefined,
(error: unknown) => error
)
expect(rejection).toBeInstanceOf(Error)
expect(rejection).not.toBeInstanceOf(AggregateError)
expect((rejection as Error).message).toBe('PVC read failed')
// Liveness witness: the pass read the PVC and Service and ran the policy
// phase, the last step before the throw. The hold path (which would mint
// credentials for this held wake and poll readiness) never ran.
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
namespace: host.namespace,
name: `${host.name}-workspace`,
})
expect(coreApi.readNamespacedService).toHaveBeenCalledWith({
namespace: host.namespace,
name: host.name,
})
expect(ingress).toHaveBeenCalledWith(host)
expect(provision).not.toHaveBeenCalled()
expect(ready).not.toHaveBeenCalled()
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(live().spec!.template).toEqual(templateBefore)
})
it('waits for a verified applied runtime when a wake is requested and the live Deployment has no applied-runtime annotation', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
// A legacy Deployment: owned by the Host and at zero replicas, but its pod
// template carries no runtime-token-revision annotation.
const legacy = reconciler.buildDeployment(host)
legacy.spec!.replicas = 0
delete legacy.spec!.template!.metadata!.annotations?.['clerum.io/runtime-token-revision']
markPersistedRuntimeTrusted(legacy, host, 0)
const live = persistHostDeployment(appsApi, host, legacy)
const templateBefore = structuredClone(live().spec!.template)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
await reconciler.reconcile(host)
expect(reconciler.getStatus(host.name)).toEqual({
deployed: true,
ready: false,
message:
'Waiting for a verified applied runtime before waking during CommunicationChannel inventory loss',
})
// Liveness witness: the held pass read the live Deployment before deciding.
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
namespace: host.namespace,
name: host.name,
})
expect(provision).not.toHaveBeenCalled()
expect(poll).not.toHaveBeenCalled()
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(live().spec!.template).toEqual(templateBefore)
})
it('converges the held runtime boundary and reports policy failures without polling', async () => {
// An active, not-Ready held runtime would be polled if the boundary were
// complete, so only the policy failure keeps the poll from running.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
const templateBefore = structuredClone(live().spec!.template)
const replicasBefore = live().spec!.replicas
const poll = vi.spyOn(reconciler as any, 'pollReadiness')
const ingress = vi
.spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
.mockRejectedValueOnce(new Error('ingress policy failed'))
const gfsEgress = vi
.spyOn(reconciler as any, 'ensureMcpHostGfsEgressNetworkPolicy')
.mockResolvedValue('up_to_date')
const codexEgress = vi
.spyOn(reconciler as any, 'reconcileMcpHostCodexProxyEgressNetworkPolicy')
.mockResolvedValue('up_to_date')
await reconciler.reconcile(host)
expect(ingress).toHaveBeenCalledOnce()
expect(ingress).toHaveBeenCalledWith(host)
expect(gfsEgress).toHaveBeenCalledOnce()
expect(codexEgress).toHaveBeenCalledOnce()
expect(live().spec!.replicas).toBe(replicasBefore)
expect(live().spec!.template).toEqual(templateBefore)
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
message: expect.stringContaining('mcp-host ingress: ingress policy failed'),
})
expect(reconciler.getStatus(host.name)?.message).not.toContain('Host PVC')
expect(reconciler.getStatus(host.name)?.message).not.toContain('Host Service')
expect(reconciler.getStatus(host.name)?.message).not.toContain('GFS egress')
expect(reconciler.getStatus(host.name)?.message).not.toContain('Codex proxy egress')
// Liveness witness: the hold reached its readiness check and found the
// runtime not Ready, the state in which a complete boundary is polled.
expect(readySpy).toHaveBeenCalledWith(host.name, host.namespace)
await expect(readySpy.mock.results[0].value).resolves.toBe(false)
expect(poll).not.toHaveBeenCalled()
})
it('polls readiness after a held wake scales the Host up from zero', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
runtimeTokenProvision(host)
)
const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
await reconciler.reconcile(host)
expect(live().spec!.replicas).toBe(1)
expect(reconciler.getStatus(host.name)).toEqual({
deployed: true,
ready: false,
message: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
})
expect(poll).toHaveBeenCalledOnce()
expect(poll).toHaveBeenCalledWith(host.name, host.namespace)
})
it.each([
{
runtime: 'Ready',
readyReplicas: 1,
ready: true,
message: 'CommunicationChannel inventory unavailable; preserving applied runtime',
},
{
runtime: 'not Ready',
readyReplicas: 0,
ready: false,
message: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
},
])(
'holds a held wake on a suspended Host whose applied runtime still runs ($runtime)',
async ({ readyReplicas, ready, message }) => {
// The suspension reached the Host status but its scale-down never reached
// the Deployment, so the applied runtime still runs at one replica.
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 1
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, readyReplicas)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
await reconciler.reconcile(host)
// Liveness witness: the held pass reached its readiness check.
expect(readySpy).toHaveBeenCalledWith(host.name, host.namespace)
await expect(readySpy.mock.results[0].value).resolves.toBe(readyReplicas > 0)
expect(reconciler.getStatus(host.name)).toEqual({ deployed: true, ready, message })
// A running runtime renews only its GFS credential; it never mints for
// the wake, and a not-Ready one is polled instead of renewed.
expect(provision.mock.calls).toEqual(
readyReplicas > 0 ? [[host, { targetSuspended: false, refreshGfsOnly: true }]] : []
)
expect(poll.mock.calls).toEqual(readyReplicas > 0 ? [] : [[host.name, host.namespace]])
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(templateBefore)
}
)
it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi } = createReconciler()
appsApi.readNamespacedDeployment
.mockRejectedValueOnce({ code: 404 })
.mockRejectedValueOnce({ code: 404 })
appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
await expect((reconciler as any).ensureDeployment(host, [], 'runtime-revision')).resolves.toBe(
false
)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
it('refreshes only the GFS token for a running held Host during cache loss', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
markPersistedRuntimeTrusted(applied, host)
const live = persistHostDeployment(appsApi, host, applied)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
expect(provision).toHaveBeenCalledOnce()
expect(provision).toHaveBeenCalledWith(host, {
refreshGfsOnly: true,
targetSuspended: false,
})
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
})
it('fails closed when held credential renewal cannot verify the runtime identity', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
markPersistedRuntimeTrusted(applied, host)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
// The record was minted for a previous Host incarnation, so the real
// refreshGfsOnly provision refuses to renew it and returns null.
coreApi.readNamespacedSecret.mockResolvedValue(
runtimeCredentialRecord(host, { hostUid: 'prior-host-uid' }) as any
)
const runtimeSecretName = `host-${host.name}-mcp-host-runtime-tokens`
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
await reconciler.reconcile(host)
// Liveness witness: the held renewal read the runtime credential record.
expect(coreApi.readNamespacedSecret).toHaveBeenCalledWith({
name: runtimeSecretName,
namespace: host.namespace,
})
expect(reconciler.getStatus(host.name)).toEqual({
deployed: true,
ready: false,
message: 'Held runtime credential renewal is not safely available',
})
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
expect(
coreApi.replaceNamespacedSecret.mock.calls.filter(
([request]) => (request as { name?: string }).name === runtimeSecretName
)
).toHaveLength(0)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(templateBefore)
})
it('keeps a suspended Host at zero replicas during cache loss without a pending wake', async () => {
const host = makeStatelessHost({ status: suspendedStatus(3) })
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
applied.spec!.replicas = 0
const live = persistHostDeployment(appsApi, host, applied)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
expect(provision).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle).toMatchObject({
state: 'suspended',
wakeHandledGeneration: 3,
})
})
it('keeps a preserved suspended wake target at zero replicas during cache loss without a pending wake', async () => {
// This Deployment IS a preserved wake target (applied runtime revision at
// zero replicas), so only the missing wake keeps the hold from minting
// bootstrap credentials and scaling it up.
const host = makeStatelessHost({ status: suspendedStatus() })
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
const ensureDeployment = vi.spyOn(reconciler as any, 'ensureDeployment')
const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
await reconciler.reconcile(host)
// Liveness witness: the pass read the live Deployment and applied the hold
// with the suspended, cache-loss lifecycle.
expect(
appsApi.readNamespacedDeployment.mock.calls.filter(([request]) => request.name === host.name)
.length
).toBeGreaterThan(0)
expect(ensureDeployment).toHaveBeenCalledOnce()
expect(ensureDeployment.mock.calls[0][3]).toMatchObject({
state: 'suspended',
suspensionBlocked: true,
allowScaleUpDuringHold: false,
})
await expect(ensureDeployment.mock.results[0].value).resolves.toBe(true)
expect(reconciler.getStatus(host.name)).toEqual({
deployed: true,
ready: false,
message: 'CommunicationChannel inventory unavailable; preserving suspended Host replicas',
})
expect(provision).not.toHaveBeenCalled()
expect(poll).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(live().spec!.template).toEqual(templateBefore)
})
it('does not poll readiness when the held Deployment disappears before it is applied', async () => {
// An active, not-Ready held runtime with a complete boundary is polled, so
// only the unapplied Deployment keeps the poll from running.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
markPersistedRuntimeTrusted(applied, host, 0)
persistHostDeployment(appsApi, host, applied)
const persistedRead = appsApi.readNamespacedDeployment.getMockImplementation()!
let deleted = false
appsApi.readNamespacedDeployment.mockImplementation(request =>
deleted && request.name === host.name ? Promise.reject({ code: 404 }) : persistedRead(request)
)
const realEnsureDeployment = (reconciler as any).ensureDeployment.bind(reconciler)
const ensureDeployment = vi
.spyOn(reconciler as any, 'ensureDeployment')
.mockImplementation(async (...args: unknown[]) => {
// The Deployment is deleted after the hold read it and before it is applied.
deleted = true
return realEnsureDeployment(...args)
})
const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
await reconciler.reconcile(host)
// Liveness witness: the hold reached ensureDeployment, whose re-read found
// the Deployment gone and left it unapplied.
expect(ensureDeployment).toHaveBeenCalledOnce()
await expect(ensureDeployment.mock.results[0].value).resolves.toBe(false)
expect(reconciler.getStatus(host.name)).toEqual({
deployed: false,
ready: false,
message: 'Waiting for CommunicationChannel inventory before creating runtime',
})
expect(
appsApi.createNamespacedDeployment.mock.calls.filter(
([request]) => request.body.metadata.name === host.name
)
).toHaveLength(0)
expect(readySpy).not.toHaveBeenCalled()
expect(poll).not.toHaveBeenCalled()
})
it('uses the fenced replica-only update for a pending wake during cache loss', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
expect(provision).toHaveBeenCalledOnce()
expect(provision).toHaveBeenCalledWith(host, {
forceFreshForWake: true,
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
expect(provision.mock.invocationCallOrder[0]).toBeLessThan(
appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
)
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
})
it('does not apply a channel rejection after that inventory loses authority mid-reconcile', async () => {
let synced = true
const host = makeStatelessHost()
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => synced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
const live = persistHostDeployment(appsApi, host, applied)
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
synced = false
return readPolicy(request)
})
await reconciler.reconcile(host)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
})
it('preserves the template when authority is lost during the final asynchronous scope lookup', async () => {
let synced = true
let lookup = 0
const host = makeStatelessHost()
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => synced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
const live = persistHostDeployment(appsApi, host, applied)
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
runtimeTokenProvision(host)
)
// The final lookup loses channel authority and its McpServer read fails too:
// neither an unobservable OAuth answer nor the lost authority may commit
// the template.
reconciler.setHostFrontsOAuthServer(async () => {
if (++lookup === 1) return false
synced = false
throw new Error('mcp-server watch retired')
})
await reconciler.reconcile(host)
// One observed lookup, then the final lookup's three attempts.
expect(lookup).toBe(4)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
})
it('preserves an independently confirmed desktop rejection while the channel cache is unknown', async () => {
const host = makeStatelessHost({ spec: { desktop: { browser: true } } })
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
coreApi.readNamespacedSecret.mockResolvedValue({
metadata: {
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/host': host.name,
},
annotations: {
'clerum.io/gfs-token-host-uid': host.uid,
'clerum.io/runtime-token-has-channel-ingress': 'true',
'clerum.io/runtime-token-fronts-oauth-server': 'false',
},
},
data: {},
} as any)
await reconciler.reconcile(host)
const condition = rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!)
expect(condition.reason).toBe('DesktopEnabled')
expect(condition.status).toBe('True')
expect(
containerEnv(hostDeploymentBody(appsApi, host.name)).some(
entry => entry.name === 'CLERUM_STATELESS_LIFECYCLE'
)
).toBe(false)
})
it('defers a confirmed desktop rejection when no trusted scope observation exists', async () => {
const host = makeStatelessHost({ spec: { desktop: { browser: true } } })
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => false,
})
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === host.name ? Promise.reject({ code: 404 }) : readDeployment(request)
)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
await reconciler.reconcile(host)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'DesktopEnabled'
)
expect(
[
...appsApi.createNamespacedDeployment.mock.calls,
...appsApi.replaceNamespacedDeployment.mock.calls,
].filter(
([request]) => (request as { body?: k8s.V1Deployment }).body?.metadata?.name === host.name
)
).toHaveLength(0)
expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })
})
it('does not let a failed live probe narrow a retained OAuth grant before its renewal window', async () => {
const host = makeStatelessHost()
const { reconciler, coreApi, appsApi } = createReconciler({
isCommunicationChannelCacheSynced: () => true,
})
const record = await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })
coreApi.readNamespacedSecret.mockResolvedValue(record)
const running = trustedRuntimeDeployment(reconciler, host)
running.spec!.template.metadata!.annotations = {
...running.spec!.template.metadata!.annotations,
'clerum.io/runtime-token-revision':
record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
}
appsApi.readNamespacedDeployment.mockResolvedValue(running)
const oauthResolver = vi.fn(async (): Promise<boolean> => {
throw new Error('oauth lookup unavailable')
})
reconciler.setHostFrontsOAuthServer(oauthResolver)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
// Liveness witness: the record was read and the live probe ran all its attempts.
expect(coreApi.readNamespacedSecret).toHaveBeenCalled()
expect(oauthResolver).toHaveBeenCalledTimes(3)
expect(provision).toBeNull()
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
})
it('uses the retained channel observation but a live OAuth observation while channel authority is unavailable', async () => {
const host = makeStatelessHost()
const countChannels = vi.fn(() => 0)
const { reconciler, coreApi } = createReconciler({
countCommunicationChannels: countChannels,
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(
await mintedRuntimeCredentialRecord(host, {
frontsOAuthServer: false,
hasChannelIngress: true,
})
)
const oauthResolver = vi.fn(async () => true)
reconciler.setHostFrontsOAuthServer(oauthResolver)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
expect(provision).not.toBeNull()
expect(oauthResolver).toHaveBeenCalledWith(host)
expect(countChannels).not.toHaveBeenCalled()
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).toContain(
OAUTH_USER_TOKEN_SCOPE
)
const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
expect(write.metadata?.annotations).toMatchObject({
'clerum.io/runtime-token-fronts-oauth-server': 'true',
'clerum.io/runtime-token-has-channel-ingress': 'true',
})
})
describe('McpServer OAuth observation unavailable while channel authority is synced', () => {
/**
* A Host whose Ready pod runs a consumed credential that grants
* oauth:user-token, over stateful Secret and Deployment mocks.
*/
async function oauthGrantedRuntime(annotations: Record<string, string> = {}) {
const host = makeHost()
const { reconciler, appsApi, coreApi } = createReconciler()
let record = await mintedRuntimeCredentialRecord(host, {
frontsOAuthServer: true,
annotations,
})
const secretName = record.metadata!.name!
const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
coreApi.readNamespacedSecret.mockImplementation(request =>
request.name === secretName ? Promise.resolve(structuredClone(record)) : readSecret(request)
)
coreApi.replaceNamespacedSecret.mockImplementation(async request => {
if (request.name === secretName) record = structuredClone(request.body as k8s.V1Secret)
return request.body
})
let live = trustedRuntimeDeployment(reconciler, host)
live.spec!.template!.metadata!.annotations = {
...live.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision':
record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
}
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request =>
request.name === host.name
? Promise.resolve(structuredClone(live))
: readDeployment(request)
)
const replaceDeployment = appsApi.replaceNamespacedDeployment.getMockImplementation()!
appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
if (request.name !== host.name) return replaceDeployment(request)
// The replaced pod stays Ready, as a running runtime does.
live = { ...structuredClone(request.body), status: { readyReplicas: 1 } }
return live
})
const hostDeploymentWrites = () =>
[
...appsApi.createNamespacedDeployment.mock.calls,
...appsApi.replaceNamespacedDeployment.mock.calls,
...appsApi.patchNamespacedDeployment.mock.calls,
].filter(([request]) => (request as { name?: string }).name === host.name)
return {
host,
reconciler,
appsApi,
coreApi,
record: () => record,
hostDeploymentWrites,
}
}
it('keeps oauth:user-token and does not roll the Deployment when the McpServer probe fails between relists', async () => {
const { host, reconciler, appsApi, record, hostDeploymentWrites } =
await oauthGrantedRuntime()
const oauthResolver = vi.fn(async (): Promise<boolean> => true)
reconciler.setHostFrontsOAuthServer(oauthResolver)
const issue = vi.mocked(issueMcpHostRuntimeTokens)
issue.mockClear()
for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)
// The steady state is stable: no runtime mint while the grant is observed.
expect(issue).not.toHaveBeenCalled()
oauthResolver.mockImplementation(async () => {
throw new Error('mcp-server watch retired')
})
oauthResolver.mockClear()
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
appsApi.patchNamespacedDeployment.mockClear()
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
try {
await reconciler.reconcile(host)
expect(issue).not.toHaveBeenCalled()
expect(hostDeploymentWrites()).toHaveLength(0)
expect(record().metadata!.annotations).toMatchObject({
'clerum.io/runtime-token-fronts-oauth-server': 'true',
'clerum.io/runtime-token-rollout-required': 'false',
})
// Liveness witness: the live probe ran all its attempts and the reconcile
// deferred with the reason it could not decide.
expect(oauthResolver).toHaveBeenCalledTimes(3)
expect(warn).toHaveBeenCalledWith(
'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',
{
host: host.name,
namespace: host.namespace,
observation: 'frontsOAuthServer',
attempts: 3,
err: 'mcp-server watch retired',
}
)
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
message: 'Waiting for authoritative scope observation',
})
} finally {
warn.mockRestore()
}
})
it('lets the Deployment drift guard keep an observed grant when its own McpServer read fails', async () => {
const { host, reconciler, record } = await oauthGrantedRuntime()
const oauthResolver = vi.fn(async (): Promise<boolean> => true)
reconciler.setHostFrontsOAuthServer(oauthResolver)
const issue = vi.mocked(issueMcpHostRuntimeTokens)
issue.mockClear()
for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)
expect(issue).not.toHaveBeenCalled()
const steadyStatus = structuredClone(reconciler.getStatus(host.name))
// Issuance observes the grant; every later read (the drift guard's) fails.
oauthResolver.mockClear()
oauthResolver.mockImplementation(async () => {
if (oauthResolver.mock.calls.length === 1) return true
throw new Error('mcp-server watch retired')
})
await reconciler.reconcile(host)
// Liveness witness: the drift guard read after issuance and exhausted its retries.
expect(oauthResolver.mock.calls.length).toBeGreaterThanOrEqual(4)
expect(issue).not.toHaveBeenCalled()
expect(reconciler.getStatus(host.name)).toEqual(steadyStatus)
expect(record().metadata!.annotations).toMatchObject({
'clerum.io/runtime-token-fronts-oauth-server': 'true',
'clerum.io/runtime-token-rollout-required': 'false',
})
})
it('falls back to a fail-closed mint when the OAuth observation is unavailable inside the renewal window', async () => {
const { host, reconciler, record } = await oauthGrantedRuntime({
'clerum.io/runtime-token-refresh-before': '2000-01-01T00:00:00.000Z',
})
const oauthResolver = vi.fn(async (): Promise<boolean> => {
throw new Error('mcp-server watch retired')
})
reconciler.setHostFrontsOAuthServer(oauthResolver)
const issue = vi.mocked(issueMcpHostRuntimeTokens)
issue.mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
// Liveness witness: the live probe ran all its attempts before the mint.
expect(oauthResolver).toHaveBeenCalledTimes(3)
expect(provision).not.toBeNull()
expect(issue).toHaveBeenCalledOnce()
expect(issue.mock.calls[0]?.[2]).not.toContain(OAUTH_USER_TOKEN_SCOPE)
expect(record().metadata!.annotations).toMatchObject({
'clerum.io/runtime-token-fronts-oauth-server': 'false',
'clerum.io/runtime-token-rollout-required': 'true',
})
})
})
describe('held wake OAuth scope during channel cache loss', () => {
function heldZeroReplicaWake(options: {
cacheSynced: boolean
retainedFrontsOAuth: 'true' | 'false'
resolveFrontsOAuth: () => Promise<boolean>
}) {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => options.cacheSynced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const runtimeRecord = runtimeCredentialRecord(host, {
frontsOAuthServer: options.retainedFrontsOAuth,
})
const runtimeRecordReads: string[] = []
coreApi.readNamespacedSecret.mockImplementation(({ name }) => {
if (name?.includes('runtime-tokens')) {
runtimeRecordReads.push(name)
return Promise.resolve(runtimeRecord)
}
return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} } as any)
})
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const oauthResolver = vi.fn(options.resolveFrontsOAuth)
reconciler.setHostFrontsOAuthServer(oauthResolver)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
return { host, reconciler, coreApi, live, oauthResolver, runtimeRecordReads }
}
function runtimeTokenWrite(coreApi: ReturnType<typeof createMockCoreApi>): k8s.V1Secret {
const write = coreApi.replaceNamespacedSecret.mock.calls.find(([request]) =>
request.name?.includes('runtime-tokens')
)?.[0].body as k8s.V1Secret | undefined
if (!write) throw new Error('No runtime token Secret write recorded')
return write
}
it.each([false, true])(
'mints without oauth:user-token when the OAuth mcp-server was removed (cache synced: %s)',
async cacheSynced => {
const { host, reconciler, coreApi, live, oauthResolver } = heldZeroReplicaWake({
cacheSynced,
retainedFrontsOAuth: 'true',
resolveFrontsOAuth: async () => false,
})
await reconciler.reconcile(host)
expect(oauthResolver).toHaveBeenCalledWith(host)
expect(vi.mocked(issueMcpHostRuntimeTokens)).toHaveBeenCalledOnce()
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls[0]?.[2]).not.toContain(
OAUTH_USER_TOKEN_SCOPE
)
expect(
runtimeTokenWrite(coreApi).metadata?.annotations?.[
'clerum.io/runtime-token-fronts-oauth-server'
]
).toBe('false')
expect(live().spec!.replicas).toBe(1)
}
)
it('skips a held wake mint when the OAuth read fails and the retained scope would grant it', async () => {
const { host, reconciler, coreApi, live, oauthResolver, runtimeRecordReads } =
heldZeroReplicaWake({
cacheSynced: false,
retainedFrontsOAuth: 'true',
resolveFrontsOAuth: async () => {
throw new Error('McpServer read unavailable')
},
})
const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
try {
await reconciler.reconcile(host)
expect(warn).toHaveBeenCalledWith(
'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
{
host: host.name,
namespace: host.namespace,
observation: 'frontsOAuthServer',
attempts: 3,
err: 'McpServer read unavailable',
}
)
} finally {
warn.mockRestore()
}
expect(runtimeRecordReads.length).toBeGreaterThan(0)
expect(oauthResolver).toHaveBeenCalledTimes(3)
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
expect(
coreApi.replaceNamespacedSecret.mock.calls.filter(([request]) =>
request.name?.includes('runtime-tokens')
)
).toHaveLength(0)
expect(live().spec!.replicas).toBe(0)
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
message: 'Waiting for authoritative scope observation',
})
})
it('uses a retained non-granting OAuth observation when the OAuth read fails', async () => {
const { host, reconciler, coreApi, live, oauthResolver } = heldZeroReplicaWake({
cacheSynced: false,
retainedFrontsOAuth: 'false',
resolveFrontsOAuth: async () => {
throw new Error('McpServer read unavailable')
},
})
await reconciler.reconcile(host)
expect(oauthResolver).toHaveBeenCalledTimes(3)
expect(vi.mocked(issueMcpHostRuntimeTokens)).toHaveBeenCalledOnce()
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls[0]?.[2]).not.toContain(
OAUTH_USER_TOKEN_SCOPE
)
expect(
runtimeTokenWrite(coreApi).metadata?.annotations?.[
'clerum.io/runtime-token-fronts-oauth-server'
]
).toBe('false')
expect(live().spec!.replicas).toBe(1)
})
})
it('does not renew held-runtime GFS before its refresh window', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(runtimeCredentialRecord(host))
appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
expect(provision).not.toBeNull()
expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
})
it('renews only held-runtime GFS at the refresh boundary', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
const record = runtimeCredentialRecord(host, {
gfsRefreshBefore: '2000-01-01T00:00:00.000Z',
})
coreApi.readNamespacedSecret.mockResolvedValue(record)
const deployment = trustedRuntimeDeployment(reconciler, host)
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()
expect(provision.revision).toBe('applied-runtime-revision')
const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]).toBe(
record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]
)
expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]).toBe(
record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]
)
expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]).toBe(
record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]
)
expect(write.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(
record.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]
)
})
it('never falls back to full issuance from a closed held-runtime GFS renewal', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
expect(provision).toBeNull()
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
})
it('does not treat a requested but unready held runtime as ready for GFS renewal', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(
runtimeCredentialRecord(host, { gfsRefreshBefore: '2000-01-01T00:00:00.000Z' })
)
appsApi.readNamespacedDeployment.mockResolvedValue(
trustedRuntimeDeployment(reconciler, host, { readyReplicas: 0 })
)
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
expect(provision).toBeNull()
expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
})
it('does not reassign a prior Host UID during held-runtime GFS renewal', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(
runtimeCredentialRecord(host, { hostUid: 'prior-host-uid' })
)
appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
expect(provision).toBeNull()
expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
})
describe('held-runtime GFS renewal identity guards', () => {
const dueForRenewal = { gfsRefreshBefore: '2000-01-01T00:00:00.000Z' }
/** Runs one refreshGfsOnly provision against a due record and a Ready Deployment. */
async function renewHeldGfs(
scenario: {
host?: HostCRD
record?: (host: HostCRD) => k8s.V1Secret
deployment?: (reconciler: HostReconciler, host: HostCRD) => k8s.V1Deployment | null
} = {}
) {
const host = scenario.host ?? makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(
scenario.record ? scenario.record(host) : runtimeCredentialRecord(host, dueForRenewal)
)
const deployment = scenario.deployment
? scenario.deployment(reconciler, host)
: trustedRuntimeDeployment(reconciler, host)
if (deployment) {
appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
} else {
appsApi.readNamespacedDeployment.mockRejectedValue({ code: 404 })
}
vi.mocked(mintHostGfsToken).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
refreshGfsOnly: true,
})
return {
provision,
mints: vi.mocked(mintHostGfsToken).mock.calls.length,
recordReads: coreApi.readNamespacedSecret.mock.calls.length,
deploymentReads: appsApi.readNamespacedDeployment.mock.calls.length,
recordWrites: coreApi.replaceNamespacedSecret.mock.calls.length,
}
}
/** The unmodified scenario mints, so every refusal below is a real refusal. */
async function expectControlRenews() {
const control = await renewHeldGfs()
expect(control.provision).not.toBeNull()
expect(control.mints).toBe(1)
expect(control.recordWrites).toBe(1)
}
async function expectRefusedAfterReads(scenario: Parameters<typeof renewHeldGfs>[0]) {
await expectControlRenews()
const refused = await renewHeldGfs(scenario)
// Liveness witness: the credential record and the Deployment were both read
// before the guard refused, so the guarded path was entered.
expect(refused.recordReads).toBeGreaterThan(0)
expect(refused.deploymentReads).toBeGreaterThan(0)
expect(refused.provision).toBeNull()
expect(refused.mints).toBe(0)
expect(refused.recordWrites).toBe(0)
}
it('refuses to renew an incomplete credential record', async () => {
await expectRefusedAfterReads({
record: host => {
const record = runtimeCredentialRecord(host, dueForRenewal)
delete record.data![MCP_HOST_GFS_TOKEN_SECRET_KEY]
return record
},
})
})
it.each([
[
'managed by another controller',
(host: HostCRD) =>
runtimeCredentialRecord(host, { ...dueForRenewal, managedByHost: false }),
],
[
'labelled for another Host',
(host: HostCRD) => {
const record = runtimeCredentialRecord(host, dueForRenewal)
record.metadata!.labels!['clerum.io/host'] = 'other-host'
return record
},
],
])('refuses to renew a credential record %s', async (_label, record) => {
await expectRefusedAfterReads({ record })
})
it.each<[string, (deployment: k8s.V1Deployment) => void]>([
[
'labelled for another Host',
deployment => {
deployment.metadata!.labels!['clerum.io/host'] = 'other-host'
},
],
[
'managed by another controller',
deployment => {
deployment.metadata!.labels!['clerum.io/managed-by'] = 'someone-else'
},
],
[
'named after another Deployment',
deployment => {
deployment.metadata!.name = 'other-host'
},
],
[
'created for a prior Host incarnation',
deployment => {
deployment.metadata!.annotations!['clerum.io/host-uid'] = 'prior-host-uid'
},
],
[
'being deleted',
deployment => {
deployment.metadata!.deletionTimestamp = new Date('2026-07-02T00:00:00.000Z')
},
],
[
'without a uid',
deployment => {
delete deployment.metadata!.uid
},
],
[
'without a resourceVersion',
deployment => {
delete deployment.metadata!.resourceVersion
},
],
[
'without a pod template spec',
deployment => {
delete deployment.spec!.template!.spec
},
],
])('refuses to renew when the Deployment is %s', async (_label, corrupt) => {
await expectRefusedAfterReads({
deployment: (reconciler, host) => {
const deployment = trustedRuntimeDeployment(reconciler, host)
corrupt(deployment)
return deployment
},
})
})
it('refuses to renew when the Deployment is absent', async () => {
await expectRefusedAfterReads({ deployment: () => null })
})
it.each<[string, (deployment: k8s.V1Deployment) => void]>([
[
'scaled to zero replicas',
deployment => {
deployment.spec!.replicas = 0
},
],
[
'without a replica count',
deployment => {
delete deployment.spec!.replicas
},
],
])('refuses to renew when the Ready-status Deployment is %s', async (_label, corrupt) => {
await expectRefusedAfterReads({
deployment: (reconciler, host) => {
// readyReplicas stays 1 so only the replica-count guard can refuse.
const deployment = trustedRuntimeDeployment(reconciler, host, { readyReplicas: 1 })
corrupt(deployment)
return deployment
},
})
})
describe('with an unconsumed wake bootstrap', () => {
const APPLIED_REVISION = 'applied-runtime-revision'
/** A due record whose fresh bootstrap is bound to the given Deployment UID. */
const freshBootstrapBoundTo = (boundUid: string) => (host: HostCRD) => {
const record = runtimeCredentialRecord(host, dueForRenewal)
Object.assign(record.metadata!.annotations!, {
'clerum.io/runtime-token-bootstrap-state': 'fresh',
'clerum.io/runtime-token-bootstrap-deployment-uid': boundUid,
'clerum.io/runtime-token-bootstrap-applied-revision': APPLIED_REVISION,
})
return record
}
const appliedDeployment = (reconciler: HostReconciler, host: HostCRD) => {
const deployment = trustedRuntimeDeployment(reconciler, host)
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': APPLIED_REVISION,
}
return deployment
}
it('acknowledges a bound bootstrap and renews due GFS in one CAS write', async () => {
const host = makeStatelessHost()
const bound = await renewHeldGfs({
host,
record: freshBootstrapBoundTo(`deployment-${host.name}`),
deployment: appliedDeployment,
})
// Liveness witness: the record was read and the consumed marker was written.
expect(bound.recordReads).toBeGreaterThan(0)
expect(bound.recordWrites).toBe(1)
expect(bound.provision.revision).toBe(APPLIED_REVISION)
expect(bound.mints).toBe(1)
})
it('renews once the bootstrap is bound to a different Deployment UID', async () => {
const host = makeStatelessHost()
const bound = await renewHeldGfs({
host,
record: freshBootstrapBoundTo(`deployment-${host.name}`),
deployment: appliedDeployment,
})
expect(bound.mints).toBe(1)
expect(bound.recordWrites).toBe(1)
const replaced = await renewHeldGfs({
host,
record: freshBootstrapBoundTo('deployment-replaced-uid'),
deployment: appliedDeployment,
})
expect(replaced.mints).toBe(1)
expect(replaced.recordWrites).toBe(1)
})
})
})
it('consumes a fresh bootstrap on GFS-only refresh when the Ready Deployment runs its revision', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
try {
const t0 = Date.parse('2026-07-03T00:00:00.000Z')
vi.setSystemTime(t0)
let cacheSynced = true
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
})
reconciler.setHostFrontsOAuthServer(async () => false)
// The real producer mints a fresh bootstrap and the pod boots Ready on it,
// with channel authority lost before the resync that would consume it.
let record = await mintedRuntimeCredentialRecord(host, { bootstrap: 'fresh' })
const deployedRevision =
record.metadata!.annotations!['clerum.io/runtime-token-secret-revision']
expect(record.metadata!.annotations!['clerum.io/runtime-token-bootstrap-state']).toBe('fresh')
const deployment = trustedRuntimeDeployment(reconciler, host)
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': deployedRevision,
}
appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
coreApi.readNamespacedSecret.mockImplementation(async () => structuredClone(record))
coreApi.replaceNamespacedSecret.mockImplementation(async request => {
const body = request.body as k8s.V1Secret
// Two writes on one resourceVersion would be a 409 from the apiserver.
expect(body.metadata?.resourceVersion).toBe(record.metadata!.resourceVersion)
record = structuredClone({
...body,
metadata: {
...body.metadata,
resourceVersion: String(Number(body.metadata!.resourceVersion) + 1),
},
})
return record
})
vi.mocked(mintHostGfsToken).mockClear()
// A renewed GFS credential differs from the one the pod booted with, so
// the record's revision moves away from the deployed one.
vi.mocked(mintHostGfsToken).mockImplementationOnce(async ({ name, namespace }) => ({
// The literal key type keeps the mock assignable to GfsHostToken.
[('to' + 'ken') as 'token']: 'gfs-renewed-value',
expiresInSeconds: 600,
subject: `host:1st:${namespace}/${name}`,
}))
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
cacheSynced = false
vi.setSystemTime(t0 + 601_000)
await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, { refreshGfsOnly: true })
vi.setSystemTime(t0 + 601_500)
await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, { refreshGfsOnly: true })
expect(record.metadata!.annotations!['clerum.io/runtime-token-bootstrap-state']).toBe(
'consumed'
)
cacheSynced = true
vi.setSystemTime(t0 + 602_000)
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
// The independent renewal and strict acknowledgement share one CAS write.
expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()
expect(coreApi.replaceNamespacedSecret.mock.calls.length).toBe(1)
expect(record.metadata!.annotations!['clerum.io/runtime-token-rollout-required']).not.toBe(
'true'
)
// The Ready pod already rotated the refresh token it booted with, so moving
// the Deployment to another revision without a runtime mint would hand the
// new pod a revoked token.
if (provision.revision !== deployedRevision) {
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length).toBeGreaterThan(0)
}
expect(provision.revision).toBe(deployedRevision)
} finally {
vi.useRealTimers()
}
})
it('keeps a Ready held runtime deployed when its GFS renewal fails, without retry backoff', async () => {
// An administrative intent on the Host: the pass that keeps the runtime
// Ready must report exactly one outcome for it.
const host: HostCRD = {
...makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
}),
generation: 4,
annotations: {
'clerum.io/administrative-intent-id': '22222222-2222-4222-8222-222222222222',
},
}
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const administrativeOutcomeReporter = {
enqueueHostOutcome: vi.fn(),
stop: vi.fn(async () => undefined),
}
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
infrastructureTelemetryReporter,
administrativeOutcomeReporter,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
reconciler.setHostFrontsOAuthServer(async () => false)
// A running pod's consumed record whose GFS credential is due for renewal.
const record = await mintedRuntimeCredentialRecord(host, {
annotations: { 'clerum.io/gfs-token-refresh-before': '2000-01-01T00:00:00.000Z' },
})
const runtimeSecretName = record.metadata!.name!
const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
coreApi.readNamespacedSecret.mockImplementation(request =>
request.name === runtimeSecretName
? Promise.resolve(structuredClone(record))
: readSecret(request)
)
const applied = trustedRuntimeDeployment(reconciler, host)
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision':
record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
}
const live = persistHostDeployment(appsApi, host, applied)
const gfsMint = vi.mocked(mintHostGfsToken)
const defaultGfsMint = gfsMint.getMockImplementation()!
gfsMint.mockClear()
gfsMint.mockImplementation(async () => {
throw new Error('gfs token endpoint unavailable')
})
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
try {
const started = performance.now()
await reconciler.reconcile(host)
const elapsedMs = performance.now() - started
// Liveness witness: the hold attempted the GFS renewal exactly once.
expect(gfsMint).toHaveBeenCalledOnce()
expect(elapsedMs).toBeLessThan(1000)
expect(reconciler.getStatus(host.name)).toEqual({
deployed: true,
ready: true,
message: 'Held runtime credential renewal failed; runtime kept',
})
const controllerErrors = vi
.mocked(infrastructureTelemetryReporter.enqueue)
.mock.calls.filter(([event]) => event.telemetryType === 'controller_error')
expect(controllerErrors).toHaveLength(1)
expect(controllerErrors[0]![0].payload).toMatchObject({
reason_code: 'RuntimeCredentialRenewalFailed',
status: 'failed',
})
// The renewal failure is telemetry; the intent's one outcome is the Ready
// runtime, never a `failed` next to a `succeeded` for the same generation.
expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledOnce()
expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
expect.objectContaining({
sourceEventId:
'hcc-admin-outcome-v2:22222222-2222-4222-8222-222222222222:4:stateless-host-uid:succeeded',
outcome: 'succeeded',
})
)
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
} finally {
gfsMint.mockImplementation(defaultGfsMint)
}
})
it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
for (let pass = 1; pass <= 3; pass++) {
await reconciler.reconcile(host)
// One credential mint per pass, made before that pass's replica-only update.
expect(provision).toHaveBeenCalledTimes(pass)
expect(provision).toHaveBeenLastCalledWith(host, {
forceFreshForWake: true,
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
const passReplace = appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
expect(provision.mock.invocationCallOrder.at(-1)!).toBeLessThan(passReplace)
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
// Scale the held Deployment back to zero so the next pass recovers it again.
live().spec!.replicas = 0
live().status = { readyReplicas: 0 }
}
})
it('does not roll a held wake bootstrap when the watch recovers before Ready', async () => {
let cacheSynced = false
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
let runtimeRecord = runtimeCredentialRecord(host)
coreApi.readNamespacedSecret.mockImplementation(({ name }) =>
name?.includes('runtime-tokens')
? Promise.resolve(runtimeRecord)
: Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} } as any)
)
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const issueRuntimeMaterial = vi.mocked(issueMcpHostRuntimeTokens)
issueRuntimeMaterial.mockClear()
await reconciler.reconcile(host)
expect(issueRuntimeMaterial).toHaveBeenCalledOnce()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template!.metadata!.annotations?.['clerum.io/runtime-token-revision']).toBe(
'applied-runtime-revision'
)
const wakeWrite = coreApi.replaceNamespacedSecret.mock.calls.find(([request]) =>
request.name?.includes('runtime-tokens')
)?.[0].body as k8s.V1Secret
expect(wakeWrite.metadata?.annotations).toMatchObject({
'clerum.io/runtime-token-bootstrap-state': 'fresh',
'clerum.io/runtime-token-rollout-required': 'false',
'clerum.io/runtime-token-bootstrap-deployment-uid': 'deployment-uid',
'clerum.io/runtime-token-bootstrap-applied-revision': 'applied-runtime-revision',
})
runtimeRecord = withReadableRuntimeRefreshMaterial(wakeWrite)
issueRuntimeMaterial.mockClear()
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
cacheSynced = true
await reconciler.reconcile(host)
expect(issueRuntimeMaterial).not.toHaveBeenCalled()
expect(
appsApi.replaceNamespacedDeployment.mock.calls.filter(
([request]) => request.body?.metadata?.name === host.name
)
).toHaveLength(0)
live().status = {
observedGeneration: live().metadata!.generation,
replicas: 1,
updatedReplicas: 1,
readyReplicas: 1,
}
coreApi.replaceNamespacedSecret.mockClear()
await reconciler.reconcile(host)
expect(issueRuntimeMaterial).not.toHaveBeenCalled()
expect(
appsApi.replaceNamespacedDeployment.mock.calls.filter(
([request]) => request.body?.metadata?.name === host.name
)
).toHaveLength(0)
const consumedWrite = coreApi.replaceNamespacedSecret.mock.calls.find(([request]) =>
request.name?.includes('runtime-tokens')
)?.[0].body as k8s.V1Secret
expect(consumedWrite.metadata?.annotations?.['clerum.io/runtime-token-bootstrap-state']).toBe(
'consumed'
)
runtimeRecord = consumedWrite
live().spec!.replicas = 0
live().status = { readyReplicas: 0 }
issueRuntimeMaterial.mockClear()
await reconciler.reconcile(host)
expect(issueRuntimeMaterial).toHaveBeenCalledOnce()
expect(live().spec!.replicas).toBe(1)
})
it('marks a newly created preserved wake bootstrap for scale-only delivery', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler()
const deployment = trustedRuntimeDeployment(reconciler, host, {
replicas: 0,
readyReplicas: 0,
})
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
const issueRuntimeMaterial = vi.mocked(issueMcpHostRuntimeTokens)
issueRuntimeMaterial.mockClear()
const first = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
forceFreshForWake: true,
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
expect(first.revision).toBe('applied-runtime-revision')
const created = coreApi.createNamespacedSecret.mock.calls.find(([request]) =>
request.body?.metadata?.name?.includes('runtime-tokens')
)?.[0].body as k8s.V1Secret
expect(created.metadata?.annotations).toMatchObject({
'clerum.io/runtime-token-bootstrap-state': 'fresh',
'clerum.io/runtime-token-rollout-required': 'false',
'clerum.io/runtime-token-bootstrap-applied-revision': 'applied-runtime-revision',
})
const readableCreated = withReadableRuntimeRefreshMaterial(created)
coreApi.readNamespacedSecret.mockResolvedValue(readableCreated)
deployment.spec!.replicas = 1
issueRuntimeMaterial.mockClear()
const second = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
expect(issueRuntimeMaterial).not.toHaveBeenCalled()
expect(second.revision).toBe('applied-runtime-revision')
})
describe('wake bootstrap binding to the applied Deployment', () => {
const BOOTSTRAP_UID = 'clerum.io/runtime-token-bootstrap-deployment-uid'
const BOOTSTRAP_REVISION = 'clerum.io/runtime-token-bootstrap-applied-revision'
const APPLIED_REVISION = 'applied-runtime-revision'
/**
* Mints a wake bootstrap for a zero-replica Deployment through the real
* producer, then models the wake: the same Deployment at one replica that is
* not Ready yet, with the persisted record bound to it.
*/
async function boundWakeBootstrap() {
const host = makeStatelessHost()
const { reconciler, appsApi, coreApi } = createReconciler()
const deployment = trustedRuntimeDeployment(reconciler, host, {
replicas: 0,
readyReplicas: 0,
})
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': APPLIED_REVISION,
}
appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
const issueRuntimeMaterial = vi.mocked(issueMcpHostRuntimeTokens)
issueRuntimeMaterial.mockClear()
await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
forceFreshForWake: true,
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
const created = coreApi.createNamespacedSecret.mock.calls.find(([request]) =>
request.body?.metadata?.name?.includes('runtime-tokens')
)?.[0].body as k8s.V1Secret
expect(created.metadata?.annotations).toMatchObject({
'clerum.io/runtime-token-bootstrap-state': 'fresh',
[BOOTSTRAP_UID]: deployment.metadata!.uid,
[BOOTSTRAP_REVISION]: APPLIED_REVISION,
})
const record = withReadableRuntimeRefreshMaterial(created)
coreApi.readNamespacedSecret.mockResolvedValue(record)
deployment.spec!.replicas = 1
/** One follow-up provision on the woken Deployment, with the counters it moved. */
const provisionOnWokenDeployment = async () => {
issueRuntimeMaterial.mockClear()
coreApi.replaceNamespacedSecret.mockClear()
const recordReadsBefore = coreApi.readNamespacedSecret.mock.calls.length
const deploymentReadsBefore = appsApi.readNamespacedDeployment.mock.calls.length
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
return {
provision,
mints: issueRuntimeMaterial.mock.calls.length,
recordReads: coreApi.readNamespacedSecret.mock.calls.length - recordReadsBefore,
deploymentReads:
appsApi.readNamespacedDeployment.mock.calls.length - deploymentReadsBefore,
writes: coreApi.replaceNamespacedSecret.mock.calls.map(([request]) => request.body),
}
}
return { deployment, record, provisionOnWokenDeployment }
}
it('reuses a wake bootstrap bound to the applied Deployment UID and revision', async () => {
const { provisionOnWokenDeployment } = await boundWakeBootstrap()
const reused = await provisionOnWokenDeployment()
// Liveness witness: the record and the Deployment were read and the reuse
// path returned the applied revision it binds.
expect(reused.recordReads).toBeGreaterThan(0)
expect(reused.deploymentReads).toBeGreaterThan(0)
expect(reused.provision.revision).toBe(APPLIED_REVISION)
expect(reused.mints).toBe(0)
expect(reused.writes).toHaveLength(0)
})
it.each<[string, (deployment: k8s.V1Deployment) => void]>([
[
'Deployment UID',
deployment => {
deployment.metadata!.uid = 'deployment-replaced-uid'
},
],
[
'applied runtime revision',
deployment => {
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'other-applied-revision',
}
},
],
])('mints again once the bound %s changes', async (_label, changeDeployment) => {
const { deployment, provisionOnWokenDeployment } = await boundWakeBootstrap()
const reused = await provisionOnWokenDeployment()
expect(reused.recordReads).toBeGreaterThan(0)
expect(reused.mints).toBe(0)
changeDeployment(deployment)
const reissued = await provisionOnWokenDeployment()
expect(reissued.mints).toBe(1)
expect(reissued.writes).toHaveLength(1)
expect((reissued.writes[0] as k8s.V1Secret).metadata?.annotations).toMatchObject({
'clerum.io/runtime-token-bootstrap-state': 'fresh',
})
})
it.each<[string, (deployment: k8s.V1Deployment, record: k8s.V1Secret) => void]>([
[
'empty Deployment UID',
(deployment, record) => {
deployment.metadata!.uid = ''
record.metadata!.annotations![BOOTSTRAP_UID] = ''
},
],
[
'empty applied runtime revision',
(deployment, record) => {
deployment.spec!.template!.metadata!.annotations = {
...deployment.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': '',
}
record.metadata!.annotations![BOOTSTRAP_REVISION] = ''
},
],
])('does not treat an %s on both sides as a binding', async (_label, empty) => {
const { deployment, record, provisionOnWokenDeployment } = await boundWakeBootstrap()
const reused = await provisionOnWokenDeployment()
expect(reused.recordReads).toBeGreaterThan(0)
expect(reused.mints).toBe(0)
empty(deployment, record)
const reissued = await provisionOnWokenDeployment()
expect(reissued.mints).toBe(1)
expect(reissued.writes).toHaveLength(1)
})
})
it('keeps an active held Deployment at zero when credential minting is unavailable', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.replicas = 0
applied.spec!.template!.metadata!.annotations = {
...applied.spec!.template!.metadata!.annotations,
'clerum.io/runtime-token-revision': 'applied-runtime-revision',
}
markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
appsApi.replaceNamespacedDeployment.mockClear()
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(null)
await reconciler.reconcile(host)
expect(provision).toHaveBeenCalledOnce()
expect(provision).toHaveBeenCalledWith(host, {
forceFreshForWake: true,
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(0)
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
message: 'Waiting for authoritative scope observation',
})
})
it('reconciles an active Host twice through cache loss without changing its session path or template', async () => {
let cacheSynced = true
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
let serverHost = host
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(serverHost))
await reconciler.reconcile(host)
const live = persistHostDeployment(appsApi, host, hostDeploymentBody(appsApi, host.name))
const baseline = structuredClone(live().spec?.template)
expect(envValue(live(), 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
for (let cycle = 1; cycle <= 2; cycle++) {
cacheSynced = false
// Exercise the fresh-read race as well as the cached active state: a
// suspend may have landed after the initial Host watch event.
serverHost = makeStatelessHost({ status: suspendedStatus(2 + cycle) })
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
await reconciler.reconcile(host)
const deployment = live()
expect(deployment.spec?.replicas).toBe(1)
expect(deployment.spec?.template).toEqual(baseline)
expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
expect(reconciler.getEffectiveLifecycle(host)).toMatchObject({
stateless: true,
suspensionBlocked: true,
})
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('suspended')
expect(
appsApi.replaceNamespacedDeployment.mock.calls.filter(([r]) => r.name === host.name)
).toHaveLength(0)
cacheSynced = true
serverHost = host
await reconciler.reconcile(host)
expect(live().spec?.template).toEqual(baseline)
}
})
it('preserves a suspended Host while the CommunicationChannel cache is unsynced', async () => {
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
const host = makeStatelessHost({ status: suspendedStatus() })
const live = persistHostDeployment(
appsApi,
host,
createReconciler().reconciler.buildDeployment(host)
)
await reconciler.reconcile(host)
const deployment = live()
expect(deployment.spec?.replicas).toBe(0)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle?.state).toBe('suspended')
const condition = rejectedCondition(writes[0])
expect(condition.status).toBe('False')
expect(condition.reason).toContain('CommunicationChannelCacheUnsynced')
})
it('preserves the stateless template when the channel cache becomes unsynced during reconciliation', async () => {
let cacheSynced = true
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
})
const host = makeStatelessHost({ status: suspendedStatus(4) })
const applied = reconciler.buildDeployment(host)
markPersistedRuntimeTrusted(applied, host)
const live = persistHostDeployment(appsApi, host, applied)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
cacheSynced = false
return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).not.toHaveBeenCalled()
const deployment = live()
expect(deployment.spec?.replicas).toBe(0)
expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(2)
expect(writes[0].lifecycle?.state).toBe('suspended')
expect(writes[1].lifecycle).toMatchObject({
state: 'suspended',
wakeHandledGeneration: 4,
})
expect(rejectedCondition(writes[1]).reason).toBe('CommunicationChannelCacheUnsynced')
expect(rejectedCondition(writes[1]).status).toBe('False')
})
it('fails closed when a channel starts referencing the Host during reconciliation', async () => {
let channelCount = 0
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeStatelessHost({ status: suspendedStatus(5) })
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
return readPolicy(request)
})
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
const deployment = hostDeploymentBody(appsApi, host.name)
expect(deployment.spec?.replicas).toBe(1)
expect(containerEnv(deployment).map(entry => entry.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(2)
expect(writes[1].lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 5,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
expect(rejectedCondition(writes[1])).toMatchObject({
status: 'True',
reason: 'ActiveCommunicationChannels',
})
})
it('issues bootstrap material once with final scopes for an already-active late channel', async () => {
let channelCount = 0
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
return readPolicy(request)
})
const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)
const mintGfs = vi.mocked(mintHostGfsToken)
issueTokens.mockClear()
mintGfs.mockClear()
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(issueTokens).toHaveBeenCalledOnce()
expect(mintGfs).toHaveBeenCalledOnce()
expect(mintGfs).toHaveBeenCalledWith({ name: host.name, namespace: host.namespace })
expect(issueTokens).toHaveBeenCalledWith(
host.name,
host.uid,
expect.arrayContaining([
'workflow:list',
'workflow:read',
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
])
)
const deployment = hostDeploymentBody(appsApi, host.name)
expect(deployment.spec?.replicas).toBe(1)
expect(containerEnv(deployment).map(entry => entry.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
})
it('reissues channel-aware bootstrap material when a channel appears during provisioning', async () => {
let channelCount = 0
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeStatelessHost({ status: suspendedStatus(6) })
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)
const defaultIssueTokens = issueTokens.getMockImplementation()
issueTokens.mockClear()
issueTokens.mockImplementationOnce(async (hostName, hostUid, scopes) => {
const tokens = await defaultIssueTokens!(hostName, hostUid, scopes)
channelCount = 1
return tokens
})
await reconciler.reconcile(host)
expect(issueTokens).toHaveBeenCalledTimes(2)
expect(issueTokens.mock.calls[0][1]).toBe(host.uid)
expect(issueTokens.mock.calls[0][2]).not.toContain('workflow:trigger')
expect(issueTokens.mock.calls[1][2]).toContain('workflow:trigger')
const deployment = hostDeploymentBody(appsApi, host.name)
expect(deployment.spec?.replicas).toBe(1)
expect(containerEnv(deployment).map(entry => entry.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(2)
expect(writes[1].lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 6,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
})
it('reissues channel-aware bootstrap material for a stateful Host when a channel appears during provisioning', async () => {
let channelCount = 0
const { reconciler, appsApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeHost()
const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)
const defaultIssueTokens = issueTokens.getMockImplementation()
issueTokens.mockClear()
issueTokens.mockImplementationOnce(async (hostName, hostUid, scopes) => {
const tokens = await defaultIssueTokens!(hostName, hostUid, scopes)
channelCount = 1
return tokens
})
await reconciler.reconcile(host)
expect(issueTokens).toHaveBeenCalledTimes(2)
expect(issueTokens.mock.calls[0][1]).toBe(host.uid)
expect(issueTokens.mock.calls[0][2]).not.toContain('workflow:trigger')
expect(issueTokens.mock.calls[1][2]).toContain('workflow:trigger')
expect(
containerEnv(hostDeploymentBody(appsApi, host.name)).map(entry => entry.name)
).not.toContain('CLERUM_STATELESS_LIFECYCLE')
})
it('propagates runtime token provisioning failure so lifecycle convergence can retry', async () => {
const { reconciler } = createReconciler()
const failure = new Error('runtime token issuer unavailable')
vi.spyOn(reconciler as any, 'ensureMcpHostRuntimeTokenSecret').mockRejectedValue(failure)
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
await expect(reconciler.reconcile(makeStatelessHost())).rejects.toBe(failure)
expect(reconciler.getStatus('stateless-host')).toEqual(
expect.objectContaining({
deployed: false,
ready: false,
message: 'mcpHost runtime token provisioning failed',
})
)
} finally {
errorSpy.mockRestore()
}
})
it('propagates the principal Deployment mutation failure so lifecycle convergence can retry', async () => {
const { reconciler, appsApi } = createReconciler()
const failure = Object.assign(new Error('Deployment API unavailable'), { code: 503 })
appsApi.replaceNamespacedDeployment.mockImplementation(
async ({ body }: { body: k8s.V1Deployment }) => {
if (body.metadata?.name === 'stateless-host') throw failure
return {}
}
)
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
await expect(reconciler.reconcile(makeStatelessHost())).rejects.toBe(failure)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
} finally {
errorSpy.mockRestore()
}
})
it('rechecks channel policy when the Deployment existence read observes a late channel', async () => {
let channelCount = 0
let hostDeploymentReads = 0
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeStatelessHost({ status: suspendedStatus(7) })
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)
issueTokens.mockClear()
appsApi.createNamespacedDeployment.mockImplementation(
async ({ body }: { body: k8s.V1Deployment }) => {
if (body.metadata?.name === host.name) {
throw Object.assign(new Error('Deployment already exists'), { code: 409 })
}
return {}
}
)
appsApi.readNamespacedDeployment.mockImplementation(async ({ name }: { name: string }) => {
if (name === host.name) {
hostDeploymentReads += 1
if (hostDeploymentReads >= 2) channelCount = 1
return {
metadata: { name, namespace: host.namespace, resourceVersion: '9' },
spec: { replicas: 0 },
status: { readyReplicas: 1 },
}
}
return {
metadata: { name, namespace: 'channels', resourceVersion: '4' },
spec: { replicas: 1 },
status: { readyReplicas: 1 },
}
})
await reconciler.reconcile(host)
expect(issueTokens).toHaveBeenCalledTimes(2)
expect(issueTokens.mock.calls[0][1]).toBe(host.uid)
expect(issueTokens.mock.calls[0][2]).not.toContain('workflow:trigger')
expect(issueTokens.mock.calls[1][2]).toContain('workflow:trigger')
const hostReplace = appsApi.replaceNamespacedDeployment.mock.calls.find(
([request]) => request.name === host.name
)?.[0].body as k8s.V1Deployment | undefined
expect(hostReplace?.spec?.replicas).toBe(1)
expect(containerEnv(hostReplace!).map(entry => entry.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(2)
expect(writes[1].lifecycle?.reason).toBe(
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle'
)
})
it('does not repeat bootstrap provisioning when channel ingress already existed in spec', async () => {
let channelCount = 0
const { reconciler, customApi, networkingApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
const host = makeStatelessHost({
spec: { channels: ['existing-channel'] },
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host, true))
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).toHaveBeenCalledOnce()
})
it('refreshes only the GFS token during active cache loss without changing replicas', async () => {
let cacheSynced = true
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
})
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
const applied = reconciler.buildDeployment(host)
markPersistedRuntimeTrusted(applied, host)
const live = persistHostDeployment(appsApi, host, applied)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
cacheSynced = false
return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).toHaveBeenCalledOnce()
expect(provision).toHaveBeenCalledWith(host, {
refreshGfsOnly: true,
targetSuspended: false,
})
const deployment = live()
expect(deployment.spec?.replicas).toBe(1)
expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
})
it('rejects stateless by default when CommunicationChannels reference the host', async () => {
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 2,
})
await reconciler.reconcile(makeStatelessHost({ status: suspendedStatus() }))
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
expect(hostDeploymentBody(appsApi, 'channel-reader-stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle?.state).toBe('active')
expect(writes[0].lifecycle?.reason).toContain('CommunicationChannel')
const cond = rejectedCondition(writes[0])
expect(cond.status).toBe('True')
expect(cond.reason).toContain('ActiveCommunicationChannels')
})
it('returns to the normal stateless lifecycle after the last channel is removed', async () => {
let channelCount = 1
let serverHost = makeStatelessHost({ status: suspendedStatus(2) })
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => channelCount,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(serverHost))
await reconciler.reconcile(serverHost)
expect(hostDeploymentBody(appsApi, serverHost.name).spec?.replicas).toBe(1)
channelCount = 0
serverHost = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
await reconciler.reconcile(serverHost)
const accepted = hostDeploymentBody(appsApi, serverHost.name)
expect(accepted.spec?.replicas).toBe(1)
expect(envValue(accepted, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!)).toMatchObject({
status: 'False',
reason: 'StatelessEnabled',
})
serverHost = makeStatelessHost({ status: suspendedStatus(2) })
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
await reconciler.reconcile(serverHost)
expect(hostDeploymentBody(appsApi, serverHost.name).spec?.replicas).toBe(0)
})
it('surfaces a failed channel-reader scale-down and converges on retry', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 0,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const channelReaderName = `channel-reader-${host.name}`
const alreadyExists = Object.assign(new Error('Deployment already exists'), { code: 409 })
const scaleDownFailure = Object.assign(new Error('Deployment replace unavailable'), {
code: 503,
})
let channelReaderReplicas = 1
let failScaleDown = true
appsApi.createNamespacedDeployment.mockImplementation(
async ({ body }: { body: k8s.V1Deployment }) => {
if (body.metadata?.name === channelReaderName) throw alreadyExists
return {}
}
)
appsApi.readNamespacedDeployment.mockImplementation(async ({ name }: { name: string }) => {
if (name === channelReaderName) {
return {
metadata: {
name,
namespace: 'channels',
resourceVersion: '7',
labels: {
app: 'channel-reader',
'clerum.io/host': host.name,
'clerum.io/managed-by': 'host-context-controller',
},
},
spec: { replicas: channelReaderReplicas },
status: { readyReplicas: channelReaderReplicas },
}
}
return {
metadata: {
name,
namespace: host.namespace,
resourceVersion: '3',
labels: {
'clerum.io/host': host.name,
'clerum.io/managed-by': 'host-context-controller',
},
},
spec: { replicas: 1 },
status: { readyReplicas: 1 },
}
})
appsApi.replaceNamespacedDeployment.mockImplementation(
async ({ name, body }: { name: string; body: k8s.V1Deployment }) => {
if (name !== channelReaderName) return {}
if (failScaleDown) throw scaleDownFailure
channelReaderReplicas = body.spec?.replicas ?? 0
return {}
}
)
await expect(reconciler.reconcile(host)).rejects.toThrow(
`Failed to converge channel-reader resources for Host "${host.name}"`
)
expect(channelReaderReplicas).toBe(1)
expect(reconciler.getStatus(host.name).channelReader).toMatchObject({
expected: false,
ready: false,
message: expect.stringContaining('Deployment replace unavailable'),
})
failScaleDown = false
await expect(reconciler.reconcile(host)).resolves.toBeUndefined()
expect(channelReaderReplicas).toBe(0)
expect(reconciler.getStatus(host.name).channelReader).toMatchObject({
expected: false,
ready: false,
message: 'Scaled to 0 (no CommunicationChannels)',
})
})
it('retries when a channel-reader disappears after a create conflict', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const channelReaderName = `channel-reader-${host.name}`
const conflict = Object.assign(new Error('Deployment already exists'), { code: 409 })
const disappeared = Object.assign(new Error('Deployment disappeared'), { code: 404 })
let injectConflictRace = true
let channelReaderExists = false
appsApi.createNamespacedDeployment.mockImplementation(
async ({ body }: { body: k8s.V1Deployment }) => {
if (body.metadata?.name !== channelReaderName) return {}
if (injectConflictRace) throw conflict
channelReaderExists = true
return {}
}
)
appsApi.readNamespacedDeployment.mockImplementation(async ({ name }: { name: string }) => {
if (name === channelReaderName) {
if (!channelReaderExists) throw disappeared
return {
metadata: {
name,
namespace: 'channels',
resourceVersion: '9',
labels: {
app: 'channel-reader',
'clerum.io/host': host.name,
'clerum.io/managed-by': 'host-context-controller',
},
},
spec: { replicas: 1 },
status: { readyReplicas: 1 },
}
}
return {
metadata: {
name,
namespace: host.namespace,
resourceVersion: '4',
labels: {
'clerum.io/host': host.name,
'clerum.io/managed-by': 'host-context-controller',
},
},
spec: { replicas: 1 },
status: { readyReplicas: 1 },
}
})
await expect(reconciler.reconcile(host)).rejects.toThrow(
`Failed to converge channel-reader resources for Host "${host.name}"`
)
expect(reconciler.getStatus(host.name).channelReader).toMatchObject({
expected: true,
ready: false,
message: expect.stringContaining('Deployment disappeared'),
})
injectConflictRace = false
await expect(reconciler.reconcile(host)).resolves.toBeUndefined()
expect(channelReaderExists).toBe(true)
expect(reconciler.getStatus(host.name).channelReader).toMatchObject({
expected: true,
ready: true,
message: 'Running',
})
})
it('rejects when spec.desktop is present (and never suspends)', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
await reconciler.reconcile(
makeStatelessHost({ spec: { desktop: { x11: true } }, status: suspendedStatus() })
)
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle?.state).toBe('active')
const cond = rejectedCondition(writes[0])
expect(cond.status).toBe('True')
expect(cond.reason).toContain('DesktopEnabled')
})
it('rejects when 2+ SFS wfc pods are provably on different nodes', async () => {
const mounts: ResolvedSfsMount[] = [
{ name: 'sfs-a', namespace: 'mcp-host', pvcName: 'sfs-a-pvc', mountPath: '/mnt/sfs-a' },
{ name: 'sfs-b', namespace: 'mcp-host', pvcName: 'sfs-b-pvc', mountPath: '/mnt/sfs-b' },
]
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
resolveContextMounts: async () => mounts,
})
coreApi.listNamespacedPod.mockImplementation(
({ labelSelector }: { labelSelector?: string } = {}) =>
Promise.resolve({
items: [{ spec: { nodeName: labelSelector?.includes('sfs-a') ? 'node-1' : 'node-2' } }],
})
)
await reconciler.reconcile(makeStatelessHost({ status: suspendedStatus() }))
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle?.state).toBe('active')
expect(writes[0].lifecycle?.reason).toContain('node-1')
const cond = rejectedCondition(writes[0])
expect(cond.status).toBe('True')
expect(cond.reason).toContain('SfsColocationUnsatisfiable')
})
it('accepts 2+ SFS when all wfc pods share one node (suspension preserved)', async () => {
const mounts: ResolvedSfsMount[] = [
{ name: 'sfs-a', namespace: 'mcp-host', pvcName: 'sfs-a-pvc', mountPath: '/mnt/sfs-a' },
{ name: 'sfs-b', namespace: 'mcp-host', pvcName: 'sfs-b-pvc', mountPath: '/mnt/sfs-b' },
]
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
resolveContextMounts: async () => mounts,
})
coreApi.listNamespacedPod.mockResolvedValue({ items: [{ spec: { nodeName: 'node-1' } }] })
await reconciler.reconcile(makeStatelessHost({ status: suspendedStatus(4) }))
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({ state: 'suspended', wakeHandledGeneration: 4 })
expect(rejectedCondition(writes[0]).status).toBe('False')
})
it('hard-rejects on the FIRST stateless reconcile when a channel already exists (reverse arrival order)', async () => {
// Addendum 6 (order independence): the operator scenario is a STATEFUL Host
// with an active channel whose spec is then flipped to stateless:true. The
// decision is state-derived, not arrival-order-dependent — the confirmed
// channel must hard-reject from the very first stateless-requesting
// reconcile; stateless must never be transiently enabled.
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
// Prior stateful projection: the Host ran active with the channel and is
// now flipped to stateless:true. The AP-1 fresh read agrees with the cached
// snapshot — this scenario has no concurrent heartbeat transition, and the
// shared mock's default fresh object is a SUSPENDED host, which would
// otherwise inject an unrelated echo.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: {
name: host.name,
namespace: host.namespace,
uid: host.uid,
resourceVersion: '42',
},
spec: host.spec,
status: host.status,
})
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
// Never transiently enabled: EVERY status write carries the rejection and
// stays active. (This tree has no status.lifecycle.effectiveMode field, so
// the "never transiently stateless" invariant is asserted through the
// durable rejection condition + state, which is what it projects here.)
expect(writes.length).toBeGreaterThan(0)
for (const write of writes) {
expect(write.lifecycle?.state).toBe('active')
expect(rejectedCondition(write)).toMatchObject({
status: 'True',
reason: 'ActiveCommunicationChannels',
})
}
// The status reason is present immediately, naming the count + recovery.
expect(writes.at(-1)!.lifecycle?.reason).toBe(
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle'
)
// Stateful active runtime (no lifecycle env).
expect(hostDeploymentBody(appsApi, host.name).spec?.replicas).toBe(1)
expect(containerEnv(hostDeploymentBody(appsApi, host.name)).map(e => e.name)).not.toContain(
'CLERUM_STATELESS_LIFECYCLE'
)
})
it('surfaces the disassociation recovery action in the StatelessEnableRejected condition message', async () => {
// Operator-visibility contract (control-ui renders condition.message
// verbatim): the recovery action MUST live in the condition message, not
// only in lifecycle.reason.
const { reconciler, customApi } = createReconciler({
countCommunicationChannels: () => 3,
})
await reconciler.reconcile(makeStatelessHost())
const condition = rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!)
expect(condition.status).toBe('True')
expect(condition.reason).toBe('ActiveCommunicationChannels')
expect(condition.message).toBe(
'3 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle'
)
})
})
describe('HostReconciler stateless lifecycle — kill-switches', () => {
it('default channel policy kill-switches stateless to active + condition', async () => {
const { reconciler, appsApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
await reconciler.reconcile(makeStatelessHost({ status: suspendedStatus(3) }))
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 3,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
expect(rejectedCondition(writes[0]).status).toBe('True')
})
it('kill-switch: stateless:false returns a suspended host to active + replicas 1', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
await reconciler.reconcile(
makeStatelessHost({ spec: { lifecycle: { stateless: false } }, status: suspendedStatus(5) })
)
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 5 })
const cond = rejectedCondition(writes[0])
expect(cond.status).toBe('False')
expect(cond.reason).toBe('StatelessDisabled')
})
it('kill-switch: removing spec.lifecycle returns a suspended host to active', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const host = makeHost({ name: 'stateless-host', status: suspendedStatus(2) })
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: {
name: host.name,
namespace: host.namespace,
uid: host.uid,
resourceVersion: '42',
},
spec: host.spec,
status: host.status,
})
await reconciler.reconcile(host)
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 2 })
})
})
describe('HostReconciler stateless lifecycle — status write idempotence', () => {
it('writes status once for an unchanged assessment across reconciles', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
const host = makeStatelessHost()
await reconciler.reconcile(host)
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
await reconciler.reconcile(host)
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(infrastructureTelemetryReporter.enqueue).toHaveBeenCalledTimes(3)
expect(
vi
.mocked(infrastructureTelemetryReporter.enqueue)
.mock.calls.filter(([event]) => event.telemetryType === 'lifecycle_transition')
).toHaveLength(1)
})
it('binds the committed health transition to the Host uid (#691)', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
await reconciler.reconcile({ ...makeStatelessHost(), generation: 3 })
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledTimes(1)
expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledWith(
expect.objectContaining({
hostLookupReference: {
name: 'stateless-host',
namespace: 'mcp-host',
generation: 3,
uid: 'stateless-host-uid',
},
payload: { transition: 'lifecycle:suspended', state: 'suspended' },
})
)
})
it('never commits a lifecycle status for a Host snapshot without a uid (#693)', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
const withUid: HostCRD = { ...makeStatelessHost(), generation: 3 }
const withoutUid: HostCRD = { ...makeStatelessHost(), generation: 3 }
delete (withoutUid as { uid?: string }).uid
// Liveness: the identical fixture, differing only in the uid, does commit
// and does emit. The absence below is the missing uid, not a fixture that
// never had anything to commit.
await reconciler.reconcile(withUid)
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledTimes(1)
await reconciler.reconcile(withoutUid)
// The health-transition emitter carries a uid guard the compiler demands,
// but it is unreachable through reconcile: the commit that invokes it does
// not happen without a uid, so no uid-less reference can reach control-api
// by this route and earn the terminal 400 (#693).
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledTimes(1)
})
it('skips the write when the observed status already matches', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
const host = makeStatelessHost({
status: {
lifecycle: { state: 'active', wakeHandledGeneration: 0 },
conditions: [
{
type: 'StatelessEnableRejected',
status: 'False',
reason: 'StatelessEnabled',
message: 'stateless lifecycle is enabled',
lastTransitionTime: '2026-01-01T00:00:00.000Z',
},
{
type: 'StatelessPullPolicyRejected',
status: 'False',
reason: 'StatelessPullPolicyAccepted',
message: 'stateless imagePullPolicy resolves to Always',
lastTransitionTime: '2026-01-01T00:00:00.000Z',
},
],
},
})
// FIX 1 replicas guard: state the server-side truth explicitly — the
// fresh read must agree with the observed (converged) active status.
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: { name: 'stateless-host', namespace: 'mcp-host', uid: 'stateless-host-uid' },
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
await reconciler.reconcile(host)
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(infrastructureTelemetryReporter.enqueue).not.toHaveBeenCalledWith(
expect.objectContaining({ telemetryType: 'lifecycle_transition' })
)
})
it('never writes status for a host that has not opted into the lifecycle', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
await reconciler.reconcile(makeHost())
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(infrastructureTelemetryReporter.enqueue).not.toHaveBeenCalledWith(
expect.objectContaining({ telemetryType: 'lifecycle_transition' })
)
})
it('does not report a lifecycle transition when the status write fails', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
customApi.patchNamespacedCustomObjectStatus.mockRejectedValue(new Error('write failed'))
await reconciler.reconcile(makeStatelessHost())
expect(infrastructureTelemetryReporter.enqueue).not.toHaveBeenCalledWith(
expect.objectContaining({ telemetryType: 'lifecycle_transition' })
)
})
})
describe('HostReconciler stateless lifecycle — AP-1 fresh-read status writer', () => {
function freshHostRead(lifecycle: {
state: 'active' | 'draining' | 'suspended'
wakeHandledGeneration: number
reason?: string
}) {
return {
metadata: {
name: 'stateless-host',
namespace: 'mcp-host',
uid: 'stateless-host-uid',
resourceVersion: '42',
},
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: { lifecycle },
}
}
it('preserves a fresher heartbeat suspend over a stale accepted assessment (the 8th costume)', async () => {
const { reconciler, customApi } = createReconciler()
// Cached watch-cache snapshot: the reconcile assessment is derived from
// THIS (draining, gen 1) and has no conditions[], so the accepted-path
// status writer fires on the conditions transition.
const host = makeStatelessHost({
status: { lifecycle: { state: 'draining', wakeHandledGeneration: 1 } },
})
// Server-side truth: a heartbeat suspend landed AFTER the snapshot but
// BEFORE the writer's fresh read (higher generation). The precondition
// passes trivially, so a verbatim `assessment.lifecycle` write would
// resurrect the Host to draining/gen-1 with no wake. The AP-1 fix must
// re-source state + monotonic generation from fresh.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'suspended', wakeHandledGeneration: 5 })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.length).toBeGreaterThanOrEqual(1)
// Reverting the fix (writing assessment.lifecycle verbatim) yields
// { state: 'draining', wakeHandledGeneration: 1 } here and fails.
expect(writes.at(-1)?.lifecycle).toEqual({ state: 'suspended', wakeHandledGeneration: 5 })
})
it('never regresses wakeHandledGeneration below the fresh value on an accepted echo', async () => {
const { reconciler, customApi } = createReconciler()
// Cached snapshot active/gen 2 (no conditions → writer fires); fresh is
// active but at a higher generation a concurrent wake already handled.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'active', wakeHandledGeneration: 9 })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 9 })
})
it('kill-switch STILL forces active over a suspended fresh read (no over-correction)', async () => {
const { reconciler, customApi } = createReconciler()
// Kill-switch: stateless disabled on a currently-suspended Host. The
// assessment INTENDS state=active as an operator-visible kill-switch; that
// override must win even though the fresh read is still suspended.
const host = makeStatelessHost({
spec: { lifecycle: { stateless: false } },
status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 4 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'suspended', wakeHandledGeneration: 4 })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 4 })
})
it('rejection STILL forces active + message over a suspended fresh read (no over-correction)', async () => {
const { reconciler, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
// Rejection (an active CommunicationChannel references the Host) on a
// currently-suspended Host: the assessment forces active + the rejection
// message as reason. That override must win over the suspended fresh read.
const host = makeStatelessHost({
status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 3 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'suspended', wakeHandledGeneration: 3 })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 3,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
})
it('reject-while-active carries the rejection message onto the fresh active state (not dropped)', async () => {
const { reconciler, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
// A rejection fires while the Host is ALREADY active: assessment.state
// ('active') === cachedState ('active'), so the plain state-diff
// discriminator routes it to the ECHO branch, which would re-source the
// reason via isHeartbeatManagedLifecycleReason and DROP the rejection
// message (not heartbeat-managed). The reject-branch discriminator must
// recognise condition.status='True' as an INTENDED reason override and
// stamp the rejection message onto the fresh state instead.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 2 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'active', wakeHandledGeneration: 2 })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 2,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
})
it('confirmed incompatibility restores active state after a concurrent suspend', async () => {
const { reconciler, customApi } = createReconciler({
countCommunicationChannels: () => 1,
})
// A confirmed channel conflict has the existing always-on policy. If a
// heartbeat suspended the Host during the fresh read, reconcile wakes it.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 3 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({ state: 'suspended', wakeHandledGeneration: 5, reason: 'idle' })
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({
state: 'active',
wakeHandledGeneration: 5,
reason:
'1 CommunicationChannel(s) reference this Host; disassociate them to enable the requested stateless lifecycle',
})
})
it('pure accepted-path echo (no rejection) still preserves ONLY heartbeat-managed reasons', async () => {
const { reconciler, customApi } = createReconciler()
// Accepted (no rejection): condition.status='False'. The echo branch must
// keep the pre-FIX behaviour — state from fresh, and reason preserved only
// when heartbeat-managed. A non-heartbeat-managed fresh reason is dropped.
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
customApi.getNamespacedCustomObject.mockResolvedValue(
freshHostRead({
state: 'active',
wakeHandledGeneration: 0,
reason: 'some stale non-heartbeat reason',
})
)
await reconciler.reconcile(host)
const writes = lifecycleStatusWrites(customApi)
// Reason dropped (not heartbeat-managed, no rejection intending an override).
expect(writes.at(-1)?.lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 0 })
})
})
describe('HostReconciler storage layout — one shape for every lifecycle', () => {
const kinds = [
{ kind: 'stateful', host: () => makeHost(), workspacePath: '/workspace' },
{ kind: 'stateless', host: () => makeStatelessHost(), workspacePath: '/workspace' },
{
kind: 'desktop',
host: () => makeHost({ spec: { ...makeHost().spec, desktop: { browser: true } } }),
workspacePath: '/config/workspace',
},
]
it.each(kinds)(
'renders the root mount, storage env and merge init for a $kind Host',
({ host, workspacePath }) => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(host())
const podSpec = dep.spec?.template?.spec
const main = podSpec?.containers?.[0]
expect(main?.volumeMounts?.filter(mount => mount.name === 'workspace')).toEqual([
{ name: 'workspace', mountPath: workspacePath },
])
expect(JSON.stringify(dep)).not.toContain('subPath')
expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe(workspacePath)
expect(dep.spec?.strategy).toEqual({ type: 'Recreate' })
const inits = podSpec?.initContainers ?? []
expect(inits.map(init => init.name)).toEqual(['workspace-layout-merge'])
const [merge] = inits
expect(merge.image).toBe(main?.image)
expect(merge.imagePullPolicy).toBe(main?.imagePullPolicy)
expect(merge.securityContext).toEqual(main?.securityContext)
expect(merge.securityContext).toMatchObject({ runAsUser: 1001, runAsNonRoot: true })
expect(merge.volumeMounts).toEqual([{ name: 'workspace', mountPath: '/mnt/workspace-root' }])
expect(merge.command?.[2]).toContain('workspace layout merge refused')
// A Pod's scheduling request counts its largest init container; dev
// requests 25m CPU for a Host, so the merge stays at or below 25m/128Mi.
expect(merge.resources?.requests).toEqual({ cpu: '10m', memory: '32Mi' })
const names = [...(main?.env ?? []), ...(merge.env ?? [])].map(env => env.name)
// Witness: the env list is populated with the storage env asserted above.
expect(names).toContain('CLERUM_SESSION_DB_DIR')
for (const withdrawn of [
'CLERUM_STORAGE_LAYOUT',
'CLERUM_STORAGE_STORE_ID',
'CLERUM_STORAGE_GENESIS',
'CLERUM_DB_BARRIER_MODE',
]) {
expect(names).not.toContain(withdrawn)
}
}
)
it('changes only lifecycle fields when a Host flips lifecycle', () => {
const { reconciler } = createReconciler()
const stateful = reconciler.buildDeployment(makeHost())
const stateless = reconciler.buildDeployment(
makeHost({ spec: { ...makeHost().spec, lifecycle: { stateless: true } } })
)
// Witness: the lifecycle-specific template really differs.
expect(envValue(stateless, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(containerEnv(stateful).map(env => env.name)).not.toContain('CLERUM_STATELESS_LIFECYCLE')
expect(stateless.spec?.template?.spec?.priorityClassName).toBe('clerum-interactive-host')
expect(stateful.spec?.template?.spec?.priorityClassName).toBeUndefined()
const storage = (dep: k8s.V1Deployment) =>
JSON.stringify({
mounts: dep.spec?.template?.spec?.containers?.[0]?.volumeMounts,
volumes: dep.spec?.template?.spec?.volumes,
store: envValue(dep, 'CLERUM_SESSION_STORE'),
dbDir: envValue(dep, 'CLERUM_SESSION_DB_DIR'),
inits: dep.spec?.template?.spec?.initContainers,
strategy: dep.spec?.strategy,
})
expect(storage(stateless)).toBe(storage(stateful))
})
it('renders the same manifest for an explicit non-stateless lifecycle argument', () => {
const { reconciler } = createReconciler()
const dep = reconciler.buildDeployment(makeHost())
const explicit = reconciler.buildDeployment(makeHost(), [], '', {
stateless: false,
state: 'active',
})
expect(explicit).toEqual(dep)
expect(dep.spec?.template?.spec?.initContainers?.map(init => init.name)).toEqual([
'workspace-layout-merge',
])
})
describe('with CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false', () => {
afterEach(() => {
config.workspaceLayoutMerge = true
})
it.each(kinds)(
'renders no merge init and keeps the root layout for a $kind Host',
({ host, workspacePath }) => {
const { reconciler } = createReconciler()
// Witness: with the flag on, the same Host renders the merge init.
expect(
reconciler.buildDeployment(host()).spec?.template?.spec?.initContainers?.map(i => i.name)
).toEqual(['workspace-layout-merge'])
config.workspaceLayoutMerge = false
const dep = reconciler.buildDeployment(host())
expect(dep.spec?.template?.spec?.initContainers).toBeUndefined()
expect(dep.spec?.template?.spec?.containers?.[0]?.volumeMounts).toContainEqual({
name: 'workspace',
mountPath: workspacePath,
})
expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe(workspacePath)
expect(JSON.stringify(dep)).not.toContain('subPath')
}
)
})
})
describe('HostReconciler storage layout — upgrade from a v0.11.0 Deployment', () => {
// The fixtures are the Deployments v0.11.0's HostReconciler renders for these
// Hosts with this file's mocks (dumped from the tag, not written by hand).
// v0.10.0 renders the same template: statelessDeployment.ts is identical.
const legacy = (file: string): k8s.V1Deployment =>
JSON.parse(readFileSync(join(__dirname, '__fixtures__', file), 'utf8'))
const cases = [
{
state: 'active',
host: () => makeStatelessHost({ name: 'upgrade-active' }),
fixture: 'v0.11.0-stateless-active-deployment.json',
replicas: 1,
},
{
state: 'suspended',
host: () => makeStatelessHost({ name: 'upgrade-suspended', status: suspendedStatus(7) }),
fixture: 'v0.11.0-stateless-suspended-deployment.json',
replicas: 0,
},
]
it.each(cases)(
'replaces the split template of a $state stateless Host once',
async ({ host: makeUpgradeHost, fixture, replicas }) => {
const host = makeUpgradeHost()
const previous = legacy(fixture)
// Witness: the fixture carries the v0.11.0 split layout.
expect(previous.spec?.template.spec?.initContainers?.map(init => init.name)).toEqual([
'workspace-layout-init',
])
expect(previous.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({
name: 'workspace',
mountPath: '/var/lib/clerum/state',
subPath: 'state',
})
expect(previous.spec?.replicas).toBe(replicas)
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue({
...hostApiObject(host),
status: host.status ?? { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
const live = persistHostDeployment(appsApi, host, previous)
await reconciler.reconcile(host)
const replacements = () =>
appsApi.replaceNamespacedDeployment.mock.calls.filter(
([request]) => request.name === host.name
)
expect(replacements()).toHaveLength(1)
const replaced = live()
expect(replaced.spec?.template.spec?.initContainers?.map(init => init.name)).toEqual([
'workspace-layout-merge',
])
expect(JSON.stringify(replaced)).not.toContain('subPath')
expect(replaced.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({
name: 'workspace',
mountPath: '/workspace',
})
expect(envValue(replaced, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')
expect(replaced.spec?.strategy).toEqual({ type: 'Recreate' })
expect(replaced.spec?.replicas).toBe(replicas)
expect(envValue(replaced, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
// Witness: the replaced template is up to date, so the next reconcile
// leaves it alone.
await reconciler.reconcile(host)
expect(replacements()).toHaveLength(1)
}
)
})
describe('HostReconciler stateless lifecycle — suspension durability', () => {
it('keeps a suspended host at replicas=0 across full reconciles (no resurrection)', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const host = makeStatelessHost({ status: suspendedStatus(7) })
await reconciler.reconcile(host)
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({ state: 'suspended', wakeHandledGeneration: 7 })
expect(reconciler.getStatus('stateless-host').message).toContain('Suspended')
// Simulate the periodic resync: a second reconcile of the same cached CRD
// must not scale the Deployment back up (the resurrection bug).
appsApi.createNamespacedDeployment.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
await reconciler.reconcile(host)
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
// And the unchanged status is not rewritten.
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
})
})
describe('HostReconciler readiness poll generation fence', () => {
afterEach(() => {
vi.useRealTimers()
})
async function pollOnce(options: { supersede: boolean }) {
vi.useFakeTimers()
const { reconciler } = createReconciler()
const internals = reconciler as unknown as {
advanceReconcileGeneration(name: string): number
setStatus(name: string, status: { deployed: boolean; ready: boolean; message: string }): void
pollReadiness(
name: string,
namespace: string,
intervalMs?: number,
maxAttempts?: number
): void
}
const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady').mockResolvedValue(true)
internals.advanceReconcileGeneration('stateless-host')
internals.setStatus('stateless-host', {
deployed: true,
ready: false,
message: 'Deployed, waiting for readiness',
})
internals.pollReadiness('stateless-host', 'tenant-ns', 10, 2)
if (options.supersede) {
// A later reconcile pass owns the status now; the older poll must not
// overwrite its verdict with a stale 'Running'.
internals.advanceReconcileGeneration('stateless-host')
internals.setStatus('stateless-host', {
deployed: true,
ready: false,
message: 'Degraded: newer reconcile verdict',
})
}
await vi.advanceTimersByTimeAsync(10)
return { reconciler, readySpy }
}
it('reports Running when no newer reconcile superseded the poll', async () => {
const { reconciler, readySpy } = await pollOnce({ supersede: false })
expect(readySpy).toHaveBeenCalledTimes(1)
expect(reconciler.getStatus('stateless-host')).toEqual({
deployed: true,
ready: true,
message: 'Running',
})
})
it('keeps the newer reconcile verdict when the generation advanced during the poll', async () => {
const { reconciler, readySpy } = await pollOnce({ supersede: true })
// Liveness witness: the poll ran and observed a ready Deployment.
expect(readySpy).toHaveBeenCalledTimes(1)
expect(readySpy).toHaveBeenCalledWith('stateless-host', 'tenant-ns')
expect(reconciler.getStatus('stateless-host')).toEqual({
deployed: true,
ready: false,
message: 'Degraded: newer reconcile verdict',
})
})
})
describe('HostReconciler stateless lifecycle — probe tuning (Stage 6, W7)', () => {
it('gives the stateless pod an aggressive startup probe preserving the 310s allowance', () => {
const { reconciler } = createReconciler()
const container =
reconciler.buildDeployment(makeStatelessHost()).spec?.template?.spec?.containers?.[0]
expect(container?.startupProbe).toEqual({
httpGet: { path: '/v1/runtime/live', port: 'http' },
initialDelaySeconds: 0,
periodSeconds: 2,
timeoutSeconds: 2,
// 0 + 2s × 155 = 310s — same ceiling as the legacy 10 + 5s × 60.
failureThreshold: 155,
})
expect(container?.readinessProbe?.initialDelaySeconds).toBe(0)
expect(container?.readinessProbe?.periodSeconds).toBe(10)
})
it('keeps the non-stateless probes byte-identical to the legacy shape', () => {
const { reconciler } = createReconciler()
const container = reconciler.buildDeployment(makeHost()).spec?.template?.spec?.containers?.[0]
expect(container?.startupProbe).toEqual({
httpGet: { path: '/v1/runtime/live', port: 'http' },
initialDelaySeconds: 10,
periodSeconds: 5,
timeoutSeconds: 2,
failureThreshold: 60,
})
expect(container?.readinessProbe).toEqual({
httpGet: { path: '/v1/runtime/health', port: 'http' },
initialDelaySeconds: 20,
periodSeconds: 10,
timeoutSeconds: 2,
failureThreshold: 6,
})
})
})
describe('HostReconciler stateless lifecycle — guarded image pull policy (Stage 6, W5)', () => {
afterEach(() => {
config.statelessImagePullPolicy = ''
config.hostImage = 'clerum/mcp-host:0.6.0'
vi.restoreAllMocks()
})
function podContainers(dep: k8s.V1Deployment) {
const container = dep.spec?.template?.spec?.containers?.[0]
const init = dep.spec?.template?.spec?.initContainers?.[0]
if (!container) {
throw new Error('Deployment has no mcp-host container')
}
return { container, init }
}
it('applies IfNotPresent for an immutable sha-<gitsha> tag', () => {
config.statelessImagePullPolicy = 'IfNotPresent'
config.hostImage = 'clerum/mcp-host:sha-abc1234'
const { reconciler } = createReconciler()
const { container, init } = podContainers(reconciler.buildDeployment(makeStatelessHost()))
expect(container.imagePullPolicy).toBe('IfNotPresent')
expect(init?.imagePullPolicy).toBe('IfNotPresent')
})
it('applies IfNotPresent for a digest-pinned reference', () => {
config.statelessImagePullPolicy = 'IfNotPresent'
config.hostImage = `clerum/mcp-host@sha256:${'a'.repeat(64)}`
const { reconciler } = createReconciler()
const { container, init } = podContainers(reconciler.buildDeployment(makeStatelessHost()))
expect(container.imagePullPolicy).toBe('IfNotPresent')
expect(init?.imagePullPolicy).toBe('IfNotPresent')
})
it('IfNotPresent + mutable tag: policy KEPT (pod stays pullable) + advisory condition + warn', async () => {
// The image-skew guard must never override IfNotPresent to an unpullable
// Always for a node-local image (regression: T2 minikube ImagePullBackOff).
const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => {})
config.statelessImagePullPolicy = 'IfNotPresent' // hostImage stays 0.6.0 (mutable)
const { reconciler, appsApi, customApi } = createReconciler()
await reconciler.reconcile(makeStatelessHost())
const dep = hostDeploymentBody(appsApi, 'stateless-host')
const { container, init } = podContainers(dep)
// Policy is preserved — NOT forced to Always (which a node-local image
// cannot pull). This is the whole point of the guard being advisory.
expect(container.imagePullPolicy).toBe('IfNotPresent')
expect(init?.imagePullPolicy).toBe('IfNotPresent')
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
const cond = writes[0].conditions?.find(c => c.type === 'StatelessPullPolicyRejected')
if (!cond) {
throw new Error('StatelessPullPolicyRejected condition missing from status write')
}
// Operator-visible advisory (the stale-cached-image-on-wake risk stands).
expect(cond.status).toBe('True')
expect(cond.reason).toBe('MutableImageReference')
expect(cond.message).toContain('clerum/mcp-host:0.6.0')
expect(cond.message).not.toContain('Always is enforced')
const advisoryLogs = warnSpy.mock.calls.filter(args =>
String(args[0]).includes('serves old code on wake')
)
expect(advisoryLogs).toHaveLength(1)
expect(advisoryLogs[0][1]).toEqual(
expect.objectContaining({
imagePullPolicy: 'IfNotPresent',
image: expect.stringContaining('clerum/mcp-host:0.6.0'),
})
)
// A second reconcile of the same image does not repeat the advisory.
await reconciler.reconcile(makeStatelessHost())
expect(
warnSpy.mock.calls.filter(args => String(args[0]).includes('serves old code on wake'))
).toHaveLength(1)
})
it('inherits the global policy when the override is unset', () => {
// statelessImagePullPolicy stays '' (the default) → global 'Always'.
const { reconciler } = createReconciler()
const { container, init } = podContainers(reconciler.buildDeployment(makeStatelessHost()))
expect(container.imagePullPolicy).toBe('Always')
expect(init?.imagePullPolicy).toBe('Always')
})
it('keeps the non-stateless pod on the global policy even when the override is set', () => {
config.statelessImagePullPolicy = 'IfNotPresent'
config.hostImage = 'clerum/mcp-host:sha-abc1234'
const { reconciler } = createReconciler()
const { container, init } = podContainers(reconciler.buildDeployment(makeHost()))
expect(container.imagePullPolicy).toBe('Always')
// The merge init follows the main container's policy on every Host.
expect(init?.name).toBe('workspace-layout-merge')
expect(init?.imagePullPolicy).toBe('Always')
})
})