Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { HostReconciler } from '../src/hostReconciler'
import { HostCRD, HostCrdStatus, HostLifecycleState } from '../src/types'
import {
type MockAppsApi,
type MockCustomApi,
asAppsApi,
asCoreApi,
asCustomApi,
asNetworkingApi,
asRbacApi,
createMockAppsApi,
createMockCoreApi,
createMockCustomApi,
createMockNetworkingApi,
createMockRbacApi,
} from './__fixtures__/testMocks'
vi.mock('../src/config', () => ({
config: {
devMode: false,
port: 8081,
namespace: 'mcp-server',
controlPlaneNamespace: 'control-plane',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
channelsNamespace: 'channels',
channelReaderImage: 'clerum/channel-reader:test',
channelReaderImagePullPolicy: 'IfNotPresent',
hostImage: 'clerum/mcp-host:0.6.0',
hostImagePullPolicy: 'Always',
hostImagePullSecretName: 'clerum',
hostPort: 8080,
gfsNamespace: 'gfs',
gfscPort: 8087,
hostConfigMapName: 'mcp-host-config',
hostServiceAccountName: 'mcp-host',
hostWorkspaceStorageClassName: 'do-block-storage-retain',
hostWorkspaceStorageSize: '10Gi',
hostWorkspacePath: '/workspace',
hostResources: {
requests: { memory: '128Mi', cpu: '100m' },
limits: { memory: '512Mi', cpu: '500m' },
},
desktopImage: 'clerum/mcp-host-desktop:latest',
desktopPort: 3000,
desktopResources: {
requests: { memory: '256Mi', cpu: '250m' },
limits: { memory: '4Gi', cpu: '1000m' },
},
devMcpServers: [],
devContexts: [],
devAuthTokens: new Map(),
controlApiBaseUrl: 'http://control-api.test:8090',
internalControlJwtHccHmacSecret: 'test-hcc-internal-control-secret',
hccTargetNamespace: 'mcp-host',
mcpHostGatewayUrl:
'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092',
},
}))
// Short-circuit the HCC issuer so reconcile tests don't reach the network.
vi.mock('../src/mcpHostRuntimeTokenIssuerClient', () => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
accessToken: 'test-mcp-host-runtime-access-token',
refreshToken: 'test-mcp-host-runtime-refresh-token',
mcpHostControlToken: 'test-mcp-host-workflow-control-token',
channelReaderMessageToken: 'test-channel-reader-message-token',
channelReaderApprovalToken: 'test-channel-reader-approval-token',
channelReaderWorkflowApprovalDecisionToken: 'test-channel-reader-decision-token',
channelReaderActivityToken: 'test-channel-reader-activity-token',
channelReaderCronReadToken: 'test-channel-reader-cron-read-token',
channelReaderCronAckToken: 'test-channel-reader-cron-ack-token',
expiresInSeconds: 600,
refreshExpiresInSeconds: 3600,
controlExpiresInSeconds: 600,
channelReaderMessageExpiresInSeconds: 600,
channelReaderApprovalExpiresInSeconds: 600,
channelReaderWorkflowApprovalDecisionExpiresInSeconds: 600,
}),
}))
vi.mock('../src/gfsHostBinding', () => ({
mintHostGfsToken: vi.fn(async (namespace: string, name: string) => ({
['to' + 'ken']: 'gfs-runtime-value',
expiresInSeconds: 600,
subject: `host:1st:${namespace}/${name}`,
})),
}))
const WAKE_ANNOTATION = 'clerum.io/wake-requested'
function runtimeTokenProvision(host: HostCRD) {
const internals = HostReconciler as unknown as {
runtimeTokenScopeHash(host: HostCRD, hasChannelIngress: boolean): string
}
return {
revision: 'runtime-revision',
scopeHash: internals.runtimeTokenScopeHash(host, false),
}
}
/** Converged accepted-stateless condition, exactly as assessLifecycle writes it. */
function statelessEnabledCondition() {
return {
type: 'StatelessEnableRejected',
status: 'False' as const,
reason: 'StatelessEnabled',
message: 'stateless lifecycle is enabled',
lastTransitionTime: '2026-01-01T00:00:00.000Z',
}
}
/** Converged accepted pull-policy condition, exactly as assessLifecycle writes it. */
function statelessPullPolicyAcceptedCondition() {
return {
type: 'StatelessPullPolicyRejected',
status: 'False' as const,
reason: 'StatelessPullPolicyAccepted',
message: 'stateless imagePullPolicy resolves to Always',
lastTransitionTime: '2026-01-01T00:00:00.000Z',
}
}
/**
* Stateless Host in a given durable lifecycle state with an optional
* wake-requested annotation. The status carries the converged
* StatelessEnableRejected=False condition so the heavy reconcile body's
* status write is a no-op unless the wake actually changes something.
*/
function makeWakeHost(
opts: {
wakeRequested?: string
state?: HostLifecycleState
wakeHandledGeneration?: number
} = {}
): HostCRD {
const status: HostCrdStatus = {
lifecycle: {
state: opts.state ?? 'suspended',
wakeHandledGeneration: opts.wakeHandledGeneration ?? 0,
},
conditions: [statelessEnabledCondition(), statelessPullPolicyAcceptedCondition()],
}
return {
name: 'stateless-host',
namespace: 'mcp-host',
...(opts.wakeRequested !== undefined
? { annotations: { [WAKE_ANNOTATION]: opts.wakeRequested } }
: {}),
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status,
}
}
function createReconciler() {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const networkingApi = createMockNetworkingApi()
const rbacApi = createMockRbacApi()
const customApi = createMockCustomApi()
customApi.getNamespacedCustomObject.mockResolvedValue({
metadata: { name: 'stateless-host', namespace: 'mcp-host' },
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: { lifecycle: { state: 'suspended', wakeHandledGeneration: 0 }, conditions: [] },
})
const reconciler = new HostReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
customApi: asCustomApi(customApi),
now: () => new Date('2026-07-03T00:00:00.000Z'),
// Wake cases model the steady state after the watcher completed its
// CommunicationChannel initial list. Cache-startup fail-closed behavior
// is covered in hostReconciler.lifecycle.test.ts.
isCommunicationChannelCacheSynced: () => true,
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
}
/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */
function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {
const calls = [
...appsApi.createNamespacedDeployment.mock.calls,
...appsApi.replaceNamespacedDeployment.mock.calls,
]
const call = calls.find(
([arg]) => (arg as { body?: k8s.V1Deployment })?.body?.metadata?.name === name
)
if (!call) {
throw new Error(`No Deployment create/replace call found for "${name}"`)
}
return (call[0] as { body: k8s.V1Deployment }).body
}
/** Every replicas value sent in a create/replace of the mcp-host Deployment. */
function allHostDeploymentReplicas(appsApi: MockAppsApi, name: string): Array<number | undefined> {
return [
...appsApi.createNamespacedDeployment.mock.calls,
...appsApi.replaceNamespacedDeployment.mock.calls,
]
.map(([arg]) => (arg as { body?: k8s.V1Deployment })?.body)
.filter(body => body?.metadata?.name === name)
.map(body => body?.spec?.replicas)
}
/**
* Every /status value written through the CustomObjects API, reconstructed
* from the JSON Patch ops. The lifecycle writers use three shapes:
* - full-status seed: `[{ add /status }]` (heartbeat cores + fresh-Host seed)
* - targeted sub-object: `[{ add /status/lifecycle }, { add /status/conditions }]`
* (writeLifecycleStatusToCluster, D2 — never clobbers un-computed fields)
* either optionally preceded by an `add /metadata/resourceVersion` precondition
* op (D3). This helper strips the precondition op and merges the remaining ops
* into a single HostCrdStatus so assertions stay shape-agnostic.
*/
function lifecycleStatusWrites(customApi: MockCustomApi): HostCrdStatus[] {
return customApi.patchNamespacedCustomObjectStatus.mock.calls.map(([arg]) => {
const body = (arg as { body: Array<{ op: string; path: string; value: unknown }> }).body
if (!Array.isArray(body)) {
throw new Error(`Unexpected status patch body: ${JSON.stringify(body)}`)
}
const ops = body.filter(op => op.path !== '/metadata/resourceVersion')
let status: HostCrdStatus = {}
for (const op of ops) {
if (op.path === '/status') {
status = op.value as HostCrdStatus
} else if (op.path === '/status/lifecycle') {
status = { ...status, lifecycle: op.value as HostCrdStatus['lifecycle'] }
} else if (op.path === '/status/conditions') {
status = { ...status, conditions: op.value as HostCrdStatus['conditions'] }
} else {
throw new Error(`Unexpected status patch op path: ${JSON.stringify(op)}`)
}
}
return status
})
}
/**
* API-shaped fresh-read response mirroring makeWakeHost: the AP-1 hardened
* fast-path re-decides the wake from a FRESH GET at the commit point, so
* tests must serve the server-side truth through
* customApi.getNamespacedCustomObject. Carries a resourceVersion so the D3
* precondition op is exercised, and the converged conditions so the heavy
* body's status write stays a no-op unless a test overrides them.
*/
function freshWakeHostResponse(
opts: {
wakeRequested?: string
state?: HostLifecycleState
wakeHandledGeneration?: number
reason?: string
} = {}
) {
return {
metadata: {
name: 'stateless-host',
namespace: 'mcp-host',
resourceVersion: 'rv-1',
...(opts.wakeRequested !== undefined
? { annotations: { [WAKE_ANNOTATION]: opts.wakeRequested } }
: {}),
},
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: {
lifecycle: {
state: opts.state ?? 'suspended',
wakeHandledGeneration: opts.wakeHandledGeneration ?? 0,
...(opts.reason !== undefined ? { reason: opts.reason } : {}),
},
conditions: [statelessEnabledCondition(), statelessPullPolicyAcceptedCondition()],
},
}
}
/** A K8s 409 Conflict error as getErrorCode reads it (.code). */
function conflict409(): Error {
return Object.assign(new Error('the object has been modified'), { code: 409 })
}
describe('HostReconciler wake fast-path — wake × state transition matrix', () => {
it('wake in suspended flips the status to active, records the generation, and scales to 1', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
// Server-side truth for the fast-path's AP-1 commit-point read.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
const writes = lifecycleStatusWrites(customApi)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 2 })
// The fast-path's minimal scale patch targets the existing Deployment.
expect(appsApi.patchNamespacedDeployment).toHaveBeenCalledTimes(1)
const [scaleArg] = appsApi.patchNamespacedDeployment.mock.calls[0]
expect(scaleArg).toMatchObject({
name: 'stateless-host',
namespace: 'mcp-host',
body: { spec: { replicas: 1 } },
})
// The heavy body then converges the full Deployment on replicas=1 too.
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
// Exactly one durable write: the heavy body's status write is a
// converged no-op after the fast-path flip.
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
})
it('falls back to a deployment update when the minimal scale patch is forbidden', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
appsApi.patchNamespacedDeployment.mockRejectedValueOnce(
Object.assign(new Error('forbidden'), { code: 403 })
)
appsApi.readNamespacedDeployment.mockResolvedValueOnce({
metadata: { resourceVersion: 'rv-1' },
spec: {
replicas: 0,
selector: { matchLabels: { app: 'stateless-host' } },
template: {
metadata: { labels: { app: 'stateless-host' } },
spec: { containers: [] },
},
},
})
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
const replacement = appsApi.replaceNamespacedDeployment.mock.calls.find(
([arg]) => arg.name === 'stateless-host' && arg.namespace === 'mcp-host'
)?.[0]
expect(replacement?.body.spec?.replicas).toBe(1)
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('used deployment update fallback')
)
expect(
logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.some(line => line.includes('phase=replicas_patched'))
).toBe(true)
} finally {
logSpy.mockRestore()
warnSpy.mockRestore()
}
})
it('wake in draining cancels the drain: status active, NO scale call issued', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '4', state: 'draining', wakeHandledGeneration: 1 })
)
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '4', state: 'draining', wakeHandledGeneration: 1 })
)
const writes = lifecycleStatusWrites(customApi)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 4 })
// Replicas were never dropped while draining — no scale patch.
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
})
it('drained-pre-scale guard: a wake pending after a failed fast-path write aborts replicas=0', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const internals = reconciler as unknown as {
ensureMcpHostRuntimeTokenSecret: (
host: HostCRD,
options?: { forceFreshForWake?: boolean; targetSuspended?: boolean }
) => Promise<{ revision: string; scopeHash: string }>
}
const ensureBootstrap = vi
.spyOn(internals, 'ensureMcpHostRuntimeTokenSecret')
.mockImplementation(async host => runtimeTokenProvision(host))
// Simulate the wake landing in the drained-pre-scale window: the
// fast-path durable flip fails, so the assessment still observes
// suspended when the scale-down would derive from it.
customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(
new Error('simulated status write conflict')
)
// M2/AP-1: the fast-path's commit-point read AND the drained-pre-scale
// guard both re-check the wake against a FRESH GET (not the cache). The
// cluster holds the woken-pending state throughout (the fast-path status
// flip failed), so every fresh read must surface wake-requested=3 over
// handled=0 for the guard to abort replicas=0.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '3', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '3', state: 'suspended' }))
// The fast-path never scaled (its durable flip failed first) ...
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
// ... and the pending replicas:0 was aborted: every Deployment apply
// carried replicas=1, so the scale-down never reached the cluster.
const replicas = allHostDeploymentReplicas(appsApi, 'stateless-host')
expect(replicas.length).toBeGreaterThan(0)
expect(replicas.every(r => r === 1)).toBe(true)
expect(ensureBootstrap).toHaveBeenCalledWith(expect.anything(), {
forceFreshForWake: false,
targetSuspended: false,
prepareForResourceChange: true,
})
// The guard re-ran the durable wake transition.
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toMatchObject({
state: 'active',
wakeHandledGeneration: 3,
})
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
})
it('uses the fresh guard wake when the cached Host has not observed it yet', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const internals = reconciler as unknown as {
ensureMcpHostRuntimeTokenSecret: (
host: HostCRD,
options?: { forceFreshForWake?: boolean; targetSuspended?: boolean }
) => Promise<{ revision: string; scopeHash: string }>
}
const ensureBootstrap = vi
.spyOn(internals, 'ensureMcpHostRuntimeTokenSecret')
.mockImplementation(async host => runtimeTokenProvision(host))
// The cached Host does not carry the wake annotation, so the fast-path
// returns before its own fresh read. The later scale guard sees the live
// wake and must make bootstrap selection target the active state.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '3', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ state: 'suspended', wakeHandledGeneration: 0 }))
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
expect(ensureBootstrap).toHaveBeenCalledWith(expect.anything(), {
forceFreshForWake: false,
targetSuspended: false,
prepareForResourceChange: true,
})
})
it('wake in active records the generation only: no scale, no writes beyond the generation', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '3', state: 'active', wakeHandledGeneration: 1 })
)
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '3', state: 'active', wakeHandledGeneration: 1 })
)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 3 })
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(1)
})
it('repeated events with the same generation transition once; the redelivery is a no-op', async () => {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(appsApi.patchNamespacedDeployment).toHaveBeenCalledTimes(1)
// The redelivered event serves the woken status with the same annotation
// (the API reflects the fast-path's own /status write).
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '2', state: 'active', wakeHandledGeneration: 2 })
)
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(appsApi.patchNamespacedDeployment).toHaveBeenCalledTimes(1)
})
})
describe('HostReconciler wake fast-path — durability and restart behavior', () => {
it('HCC restart amnesia: wakeHandledGeneration == annotation does not re-wake the host', async () => {
// A fresh reconciler instance simulates the restart: no in-memory wake
// state survives, only the CRD annotation + status.
const { reconciler, appsApi, customApi } = createReconciler()
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '5', state: 'suspended', wakeHandledGeneration: 5 })
)
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
})
it('Deployment recreated while suspended stays at replicas=0 when the wake is already handled', async () => {
const { reconciler, appsApi } = createReconciler()
// The principal Deployment is absent until this reconcile creates it.
let principalCreated = false
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
appsApi.readNamespacedDeployment.mockImplementation(request => {
if (request.name === 'stateless-host' && !principalCreated)
return Promise.reject({ code: 404 })
return readDeployment(request)
})
appsApi.createNamespacedDeployment.mockImplementation(async request => {
if (request.body.metadata?.name === 'stateless-host') principalCreated = true
return request.body
})
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '2', state: 'suspended', wakeHandledGeneration: 2 })
)
const created = appsApi.createNamespacedDeployment.mock.calls
.map(([arg]) => (arg as { body?: k8s.V1Deployment })?.body)
.find(body => body?.metadata?.name === 'stateless-host')
if (!created) {
throw new Error('expected the mcp-host Deployment to be created')
}
expect(created.spec?.replicas).toBe(0)
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
})
it('fast-path acts before any readiness or pod-listing API is touched', async () => {
const { reconciler, appsApi, coreApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
const statusOrder = customApi.patchNamespacedCustomObjectStatus.mock.invocationCallOrder[0]
const scaleOrder = appsApi.patchNamespacedDeployment.mock.invocationCallOrder[0]
// Durable flip first, then the scale patch.
expect(statusOrder).toBeLessThan(scaleOrder)
// Every readiness/waiting API call (Deployment reads, pod listings)
// happens strictly AFTER the fast-path completed — the fast path never
// awaits pod readiness.
const readinessOrders = [
...appsApi.readNamespacedDeployment.mock.invocationCallOrder,
...coreApi.listNamespacedPod.mock.invocationCallOrder,
]
expect(readinessOrders.length).toBeGreaterThan(0)
for (const order of readinessOrders) {
expect(order).toBeGreaterThan(scaleOrder)
}
})
it('malformed annotation means no wake intent: logged loudly once, never throws', async () => {
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
await expect(
reconciler.reconcile(makeWakeHost({ wakeRequested: 'not-a-number', state: 'suspended' }))
).resolves.toBeUndefined()
// A second delivery of the same malformed value must not repeat the log.
await reconciler.reconcile(
makeWakeHost({ wakeRequested: 'not-a-number', state: 'suspended' })
)
// No wake happened: the host stays suspended at replicas=0.
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(hostDeploymentBody(appsApi, 'stateless-host').spec?.replicas).toBe(0)
const malformedLogs = errorSpy.mock.calls.filter(args =>
String(args[0]).includes('Malformed clerum.io/wake-requested')
)
expect(malformedLogs).toHaveLength(1)
expect(JSON.parse(String(malformedLogs[0][0])).msg).toContain('"not-a-number"')
} finally {
errorSpy.mockRestore()
}
})
})
describe('HostReconciler wake fast-path — structured wake timestamps (Stage 6, W2)', () => {
it('emits wake_observed, status_flipped, replicas_patched with the same generation', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
const wakeLines = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.startsWith('[StatelessWake]'))
expect(wakeLines).toEqual([
expect.stringMatching(
/^\[StatelessWake\] host=stateless-host generation=2 phase=wake_observed ts=\d+$/
),
expect.stringMatching(
/^\[StatelessWake\] host=stateless-host generation=2 phase=status_flipped ts=\d+$/
),
expect.stringMatching(
/^\[StatelessWake\] host=stateless-host generation=2 phase=replicas_patched ts=\d+$/
),
])
} finally {
logSpy.mockRestore()
}
})
it('emits no replicas_patched (and no scale metric) for a cancel-drain wake', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '4', state: 'draining', wakeHandledGeneration: 1 })
)
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '4', state: 'draining', wakeHandledGeneration: 1 })
)
const lines = logSpy.mock.calls.map(args => JSON.parse(String(args[0])).msg as string)
const phases = lines
.filter(line => line.startsWith('[StatelessWake]'))
.map(line => /phase=(\w+)/.exec(line)?.[1])
expect(phases).toEqual(['wake_observed', 'status_flipped'])
expect(lines.some(line => line.startsWith('[StatelessMetric]'))).toBe(false)
} finally {
logSpy.mockRestore()
}
})
})
describe('HostReconciler — scale-transition counter (Stage 6 metric)', () => {
it('increments per host in both directions and emits suspended_applied on the way down', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, customApi } = createReconciler()
const host = makeWakeHost({ wakeRequested: '2', state: 'suspended' })
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
// suspended → wake → replicas=1 (direction=up)
await reconciler.reconcile(host)
// active → heartbeat suspension → replicas=0 (direction=down).
// Server-side truth for the suspend core's fresh-read guard: the
// durable draining write already landed with the woken generation.
customApi.getNamespacedCustomObject
.mockResolvedValueOnce({
metadata: { name: 'stateless-host', namespace: 'mcp-host' },
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
})
// After the suspend PATCH lands the server is suspended — the FIX 1
// replicas guard re-reads fresh before deriving replicas=0.
.mockResolvedValue({
metadata: { name: 'stateless-host', namespace: 'mcp-host' },
spec: {
host: 'stateless-host',
contextRef: 'context-a',
secretRef: 'host-secret',
lifecycle: { stateless: true },
},
status: {
lifecycle: { state: 'suspended', wakeHandledGeneration: 2, reason: 'idle' },
},
})
await reconciler.suspendHostFromHeartbeat(host, 'idle', 2)
const metricLines = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.startsWith('[StatelessMetric]'))
expect(metricLines).toEqual([
'[StatelessMetric] scale_transition host=stateless-host direction=up total=1',
'[StatelessMetric] scale_transition host=stateless-host direction=down total=2',
])
const suspendLines = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.startsWith('[StatelessSuspend]'))
expect(suspendLines).toEqual([
expect.stringMatching(
/^\[StatelessSuspend\] host=stateless-host phase=suspended_applied ts=\d+$/
),
])
} finally {
logSpy.mockRestore()
}
})
})
describe('HostReconciler wake fast-path — AP-1 fresh re-decide at the commit point', () => {
it('skips when the cached snapshot shows a pending wake but FRESH already handled it: no write, no duplicate phase logs', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
// A sibling (or a prior fast-path pass over another snapshot) already
// handled generation 3: the server-side truth is active with
// handled == requested.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '3', state: 'active', wakeHandledGeneration: 3 })
)
// Stale cached event payload: suspended with the wake still pending.
await reconciler.reconcile(makeWakeHost({ wakeRequested: '3', state: 'suspended' }))
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
const lines = logSpy.mock.calls.map(args => JSON.parse(String(args[0])).msg as string)
expect(lines.filter(line => line.startsWith('[StatelessWake]'))).toEqual([])
expect(lines.filter(line => line.includes('Wake fast-path for'))).toEqual([])
} finally {
logSpy.mockRestore()
}
})
it('never resurrects stale cached conditions/reason: the write is a targeted /status/lifecycle op', async () => {
const { reconciler, customApi } = createReconciler()
// FRESH carries the converged conditions + heartbeat reason a hardened
// sibling wrote AFTER the cached snapshot was captured.
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({
wakeRequested: '2',
state: 'suspended',
reason: 'sibling-written-reason',
})
)
// CACHED snapshot: stale conditions and a stale lifecycle reason that
// pre-date the sibling's write.
const host = makeWakeHost({ wakeRequested: '2', state: 'suspended' })
host.status = {
lifecycle: { state: 'suspended', wakeHandledGeneration: 0, reason: 'stale-cached-reason' },
conditions: [
{
type: 'StatelessEnableRejected',
status: 'True',
reason: 'StaleCachedCondition',
message: 'stale cached condition',
lastTransitionTime: '2020-01-01T00:00:00.000Z',
},
],
}
await reconciler.reconcile(host)
// The fast-path emits ONLY the targeted lifecycle op (plus the D3
// resourceVersion precondition): the whole-/status spread that would
// resurrect the stale cached conditions over the sibling's fresher ones
// is gone, and the stale cached reason is dropped, not written.
const [firstPatch] = customApi.patchNamespacedCustomObjectStatus.mock.calls
const body = (firstPatch[0] as { body: Array<{ op: string; path: string; value: unknown }> })
.body
expect(body.filter(op => op.path !== '/metadata/resourceVersion')).toEqual([
{
op: 'add',
path: '/status/lifecycle',
value: { state: 'active', wakeHandledGeneration: 2 },
},
])
expect(body.some(op => op.path === '/status' || op.path === '/status/conditions')).toBe(false)
})
it('normal wake: fresh requested > handled flips to active with the fresh generation, each phase once', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '5', state: 'suspended' })
)
await reconciler.reconcile(makeWakeHost({ wakeRequested: '5', state: 'suspended' }))
const writes = lifecycleStatusWrites(customApi)
expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 5 })
expect(appsApi.patchNamespacedDeployment).toHaveBeenCalledTimes(1)
const phases = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.startsWith('[StatelessWake]'))
.map(line => /phase=(\w+)/.exec(line)?.[1])
expect(phases).toEqual(['wake_observed', 'status_flipped', 'replicas_patched'])
} finally {
logSpy.mockRestore()
}
})
it('409 conflict: re-reads fresh and retries, then completes the wake (D3 idiom)', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' })
)
customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict409())
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
// Attempt 1 (409) + attempt 2 (success); the heavy body's status write
// is a converged no-op after the flip.
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
// The retry re-READ fresh before re-building (initial read + 409 re-read).
expect(customApi.getNamespacedCustomObject.mock.calls.length).toBeGreaterThanOrEqual(2)
const writes = lifecycleStatusWrites(customApi)
expect(writes.at(-1)?.lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 2 })
expect(appsApi.patchNamespacedDeployment).toHaveBeenCalledTimes(1)
const flipped = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.includes('phase=status_flipped'))
expect(flipped).toHaveLength(1)
} finally {
logSpy.mockRestore()
}
})
it('409 conflict then FRESH shows the wake handled: the retry skips instead of double-writing', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
const { reconciler, appsApi, customApi } = createReconciler()
customApi.getNamespacedCustomObject
.mockResolvedValueOnce(freshWakeHostResponse({ wakeRequested: '2', state: 'suspended' }))
// The racing writer that caused the 409 handled the wake itself.
.mockResolvedValue(
freshWakeHostResponse({ wakeRequested: '2', state: 'active', wakeHandledGeneration: 2 })
)
customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict409())
await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended' }))
// Only the 409'd attempt reached the API — the re-decided build skipped.
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
const flipped = logSpy.mock.calls
.map(args => JSON.parse(String(args[0])).msg as string)
.filter(line => line.includes('phase=status_flipped'))
expect(flipped).toEqual([])
} finally {
logSpy.mockRestore()
}
})
})