Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): fail closed on a stale McpServer cache and hold a running suspended runtime

Alfredo Lopez6df1be439c78
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index a1411f78d..cadfc56e6 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.366",
+ "version": "0.1.367",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.366",
+ "version": "0.1.367",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 53372b194..db79a39c8 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.366",
+ "version": "0.1.367",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index c4d0a6b98..d296655ec 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -5137,17 +5137,25 @@ export class HostReconciler {
// A legacy Deployment without a runtime-token-revision annotation is not
// a preserved wake target, so it stays at zero replicas until channel
// authority returns and the normal path rewrites its template.
- const preservedWakeTarget =
+ const appliedHostRuntime =
HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
- (liveDeployment?.spec?.replicas ?? 1) === 0 &&
HostReconciler.deploymentRuntimeTokenRevision(liveDeployment) !== ''
+ const liveReplicas = liveDeployment?.spec?.replicas ?? 1
+ const preservedWakeTarget = appliedHostRuntime && liveReplicas === 0
+ // A suspension recorded in status whose scale-down never reached the
+ // Deployment leaves the applied runtime running. A wake on it needs no
+ // mint and no replica change: it is held like any running runtime.
+ const wakeOnRunningRuntime = wakeRequested && appliedHostRuntime && liveReplicas > 0
const runtimeIsReady =
!!liveDeployment &&
HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
HostReconciler.deploymentReady(liveDeployment)
const scaleUpForRecovery = lifecycle.effective.state === 'active' && preservedWakeTarget
const allowScaleUp = preservedWakeTarget && (wakeRequested || scaleUpForRecovery)
- if (!liveDeployment || ((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget)) {
+ if (
+ !liveDeployment ||
+ ((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget && !wakeOnRunningRuntime)
+ ) {
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
@@ -5161,7 +5169,10 @@ export class HostReconciler {
// A pod starting from zero will consume bootstrap credentials. Mint and
// persist them before the replica-only update; a failed mint aborts here.
provisioningRequired = true
- } else if (lifecycle.effective.state === 'active' && runtimeIsReady) {
+ } else if (
+ (lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&
+ runtimeIsReady
+ ) {
const provision = await this.provisionRuntimeTokenRevision(host, {
targetSuspended: false,
refreshGfsOnly: true,
@@ -5203,11 +5214,13 @@ export class HostReconciler {
revalidateHostMutationBoundary
)
revalidateHostMutationBoundary()
- if (lifecycle.effective.state !== 'suspended' || allowScaleUp) {
+ const runtimeExpectedToRun =
+ lifecycle.effective.state !== 'suspended' || allowScaleUp || wakeOnRunningRuntime
+ if (runtimeExpectedToRun) {
this.lifecycle.markHostNotSuspended(host.name)
}
const deploymentReady =
- lifecycle.effective.state !== 'suspended' &&
+ (lifecycle.effective.state !== 'suspended' || wakeOnRunningRuntime) &&
applied &&
(await this.checkDeploymentReady(host.name, host.namespace))
revalidateHostMutationBoundary()
@@ -5232,7 +5245,7 @@ export class HostReconciler {
? 'Waiting for CommunicationChannel inventory before creating runtime'
: failures.length > 0
? `degraded — Host runtime boundary incomplete while preserving applied runtime (${failures.join('; ')})`
- : lifecycle.effective.state === 'suspended' && !allowScaleUp
+ : !runtimeExpectedToRun
? 'CommunicationChannel inventory unavailable; preserving suspended Host replicas'
: deploymentReady
? 'CommunicationChannel inventory unavailable; preserving applied runtime'
@@ -5241,12 +5254,7 @@ export class HostReconciler {
// Mirror the normal path: a runtime expected to run (including a held
// wake from zero) is polled until Ready; a degraded boundary is not,
// because the poll would overwrite the degraded verdict with "Running".
- if (
- applied &&
- !deploymentReady &&
- failures.length === 0 &&
- (lifecycle.effective.state !== 'suspended' || allowScaleUp)
- ) {
+ if (applied && !deploymentReady && failures.length === 0 && runtimeExpectedToRun) {
this.pollReadiness(host.name, host.namespace)
}
}
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index a7b2ce7e2..b47602de8 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -10769,8 +10769,10 @@ describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)'
})
/** Drive the REAL getServerInfosByContext: populate the server cache and answer
- * the Context read the producer makes with an allow-list of those servers. */
+ * the Context read the producer makes with an allow-list of those servers.
+ * The cache is an observation only while its LIST -> WATCH pair is live. */
const withServers = (watcher: McpServerWatcher, servers: any[]): void => {
+ markMcpServerInventoryAuthoritative(watcher)
for (const s of servers) (watcher as any).servers.set(s.name, s)
mocks.getNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) =>
plural === 'contexts'
@@ -10815,6 +10817,60 @@ describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)'
await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)
watcher.stop()
})
+
+ /** The resolver McpServerWatcher wires into HostReconciler at construction. */
+ const wiredResolver = (watcher: McpServerWatcher): ((host: HostCRD) => Promise<boolean>) => {
+ const calls = (watcher.getHostReconciler() as any).setHostFrontsOAuthServer.mock.calls
+ expect(calls).toHaveLength(1)
+ return calls[0][0]
+ }
+
+ it('does not serve a retired McpServer cache that would grant the OAuth scope', async () => {
+ const watcher = new McpServerWatcher()
+ withServers(watcher, [serverCRD({ name: 'notion', auth: { type: 'oauth' }, enabled: true })])
+ const resolve = wiredResolver(watcher)
+ // Liveness witness: while the inventory is authoritative, this same cache
+ // and wired resolver grant the scope.
+ await expect(resolve(hostFor('ctx'))).resolves.toBe(true)
+
+ // A McpServer watch outage retires the inventory but keeps its last cache,
+ // which still holds the enabled OAuth server an admin may have disabled.
+ expect((watcher as any).retireMcpServerWatch()).toBe(true)
+ expect((watcher as any).servers.get('notion')?.spec.enabled).toBe(true)
+
+ await expect(resolve(hostFor('ctx'))).rejects.toThrow(
+ 'McpServer inventory is not authoritative; OAuth scope is unobserved'
+ )
+ watcher.stop()
+ })
+
+ it('does not serve a McpServer cache that lost authority during the Context read', async () => {
+ const watcher = new McpServerWatcher()
+ withServers(watcher, [serverCRD({ name: 'notion', auth: { type: 'oauth' }, enabled: true })])
+ const contextRead = deferred<unknown>()
+ const contextReads: string[] = []
+ mocks.getNamespacedCustomObject.mockImplementation(
+ async ({ plural, name }: { plural: string; name: string }) => {
+ contextReads.push(`${plural}/${name}`)
+ return contextRead.promise
+ }
+ )
+
+ const pending = wiredResolver(watcher)(hostFor('ctx'))
+ await flushMicrotasks()
+ // Liveness witness: the observation reached the live Context read.
+ expect(contextReads).toEqual(['contexts/ctx'])
+ expect((watcher as any).retireMcpServerWatch()).toBe(true)
+ contextRead.resolve({
+ metadata: { name: 'ctx', namespace: 'mcp-server' },
+ spec: { contextId: 'ctx', mcpServers: ['notion'] },
+ })
+
+ await expect(pending).rejects.toThrow(
+ 'McpServer inventory lost authority during the OAuth scope read'
+ )
+ watcher.stop()
+ })
})
describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-close regime)', () => {
// Regression for the clerum-dev livelock. PR #205 coupled /ready to a safety
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index b758206e5..7c07e1335 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -2519,9 +2519,22 @@ export class McpServerWatcher implements McpServerProvider {
* already filters to enabled + allowed servers) that mcp-host discovery uses,
* so HCC does not need a second cross-CRD read path. HostReconciler wraps this
* call in a fail-closed guard, so a thrown read there yields no oauth scope.
+ *
+ * A retired McpServer watch keeps its last `servers` cache, so an answer read
+ * from it is only an observation while the same LIST -> WATCH generation stays
+ * authoritative across the whole read. Otherwise this throws: a server an
+ * admin disabled during the outage must not re-grant the scope from the stale
+ * cache, and HostReconciler treats the throw as an unobserved read.
*/
private async hostFrontsOAuthServer(host: HostCRD): Promise<boolean> {
+ const watchGeneration = this.mcpWatchGeneration
+ if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {
+ throw new Error('McpServer inventory is not authoritative; OAuth scope is unobserved')
+ }
const servers = await this.getServerInfosByContext(host.spec.contextRef)
+ if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {
+ throw new Error('McpServer inventory lost authority during the OAuth scope read')
+ }
return servers.some(server => server.enabled && server.auth?.type === 'oauth')
}
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index ce3c386fa..f96b8057e 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1297,6 +1297,62 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(poll).toHaveBeenCalledWith(host.name, host.namespace)
})
+ it.each([
+ {
+ runtime: 'Ready',
+ readyReplicas: 1,
+ ready: true,
+ message: 'CommunicationChannel inventory unavailable; preserving applied runtime',
+ },
+ {
+ runtime: 'not Ready',
+ readyReplicas: 0,
+ ready: false,
+ message: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
+ },
+ ])(
+ 'holds a held wake on a suspended Host whose applied runtime still runs ($runtime)',
+ async ({ readyReplicas, ready, message }) => {
+ // The suspension reached the Host status but its scale-down never reached
+ // the Deployment, so the applied runtime still runs at one replica.
+ const host = makeStatelessHost({ status: suspendedStatus() })
+ host.annotations = { 'clerum.io/wake-requested': '1' }
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 1
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, readyReplicas)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const templateBefore = structuredClone(live().spec!.template)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
+
+ await reconciler.reconcile(host)
+
+ // Liveness witness: the held pass reached its readiness check.
+ expect(readySpy).toHaveBeenCalledWith(host.name, host.namespace)
+ await expect(readySpy.mock.results[0].value).resolves.toBe(readyReplicas > 0)
+ expect(reconciler.getStatus(host.name)).toEqual({ deployed: true, ready, message })
+ // A running runtime renews only its GFS credential; it never mints for
+ // the wake, and a not-Ready one is polled instead of renewed.
+ expect(provision.mock.calls).toEqual(
+ readyReplicas > 0 ? [[host, { targetSuspended: false, refreshGfsOnly: true }]] : []
+ )
+ expect(poll.mock.calls).toEqual(readyReplicas > 0 ? [] : [[host.name, host.namespace]])
+ expect(live().spec!.replicas).toBe(1)
+ expect(live().spec!.template).toEqual(templateBefore)
+ }
+ )
+
it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {
const host = makeStatelessHost()
const { reconciler, appsApi } = createReconciler()
@@ -1407,6 +1463,105 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
})
})
+ it('keeps a preserved suspended wake target at zero replicas during cache loss without a pending wake', async () => {
+ // This Deployment IS a preserved wake target (applied runtime revision at
+ // zero replicas), so only the missing wake keeps the hold from minting
+ // bootstrap credentials and scaling it up.
+ const host = makeStatelessHost({ status: suspendedStatus() })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const templateBefore = structuredClone(live().spec!.template)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+ const ensureDeployment = vi.spyOn(reconciler as any, 'ensureDeployment')
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
+
+ await reconciler.reconcile(host)
+
+ // Liveness witness: the pass read the live Deployment and applied the hold
+ // with the suspended, cache-loss lifecycle.
+ expect(
+ appsApi.readNamespacedDeployment.mock.calls.filter(([request]) => request.name === host.name)
+ .length
+ ).toBeGreaterThan(0)
+ expect(ensureDeployment).toHaveBeenCalledOnce()
+ expect(ensureDeployment.mock.calls[0][3]).toMatchObject({
+ state: 'suspended',
+ suspensionBlocked: true,
+ allowScaleUpDuringHold: false,
+ })
+ await expect(ensureDeployment.mock.results[0].value).resolves.toBe(true)
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: true,
+ ready: false,
+ message: 'CommunicationChannel inventory unavailable; preserving suspended Host replicas',
+ })
+ expect(provision).not.toHaveBeenCalled()
+ expect(poll).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(0)
+ expect(live().spec!.template).toEqual(templateBefore)
+ })
+
+ it('does not poll readiness when the held Deployment disappears before it is applied', async () => {
+ // An active, not-Ready held runtime with a complete boundary is polled, so
+ // only the unapplied Deployment keeps the poll from running.
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ markPersistedRuntimeTrusted(applied, host, 0)
+ persistHostDeployment(appsApi, host, applied)
+ const persistedRead = appsApi.readNamespacedDeployment.getMockImplementation()!
+ let deleted = false
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ deleted && request.name === host.name ? Promise.reject({ code: 404 }) : persistedRead(request)
+ )
+ const realEnsureDeployment = (reconciler as any).ensureDeployment.bind(reconciler)
+ const ensureDeployment = vi
+ .spyOn(reconciler as any, 'ensureDeployment')
+ .mockImplementation(async (...args: unknown[]) => {
+ // The Deployment is deleted after the hold read it and before it is applied.
+ deleted = true
+ return realEnsureDeployment(...args)
+ })
+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
+
+ await reconciler.reconcile(host)
+
+ // Liveness witness: the hold reached ensureDeployment, whose re-read found
+ // the Deployment gone and left it unapplied.
+ expect(ensureDeployment).toHaveBeenCalledOnce()
+ await expect(ensureDeployment.mock.results[0].value).resolves.toBe(false)
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: false,
+ ready: false,
+ message: 'Waiting for CommunicationChannel inventory before creating runtime',
+ })
+ expect(
+ appsApi.createNamespacedDeployment.mock.calls.filter(
+ ([request]) => request.body.metadata.name === host.name
+ )
+ ).toHaveLength(0)
+ expect(readySpy).not.toHaveBeenCalled()
+ expect(poll).not.toHaveBeenCalled()
+ })
+
it('uses the fenced replica-only update for a pending wake during cache loss', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
@@ -1686,8 +1841,22 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
throw new Error('McpServer read unavailable')
},
})
-
- await reconciler.reconcile(host)
+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
+ try {
+ await reconciler.reconcile(host)
+ expect(warn).toHaveBeenCalledWith(
+ 'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ observation: 'frontsOAuthServer',
+ attempts: 3,
+ err: 'McpServer read unavailable',
+ }
+ )
+ } finally {
+ warn.mockRestore()
+ }
expect(runtimeRecordReads.length).toBeGreaterThan(0)
expect(oauthResolver).toHaveBeenCalledTimes(3)
diff --git a/host-context-controller/test/statelessLifecycleTracker.test.ts b/host-context-controller/test/statelessLifecycleTracker.test.ts
index ec9e1494c..01b6cfaa2 100644
--- a/host-context-controller/test/statelessLifecycleTracker.test.ts
+++ b/host-context-controller/test/statelessLifecycleTracker.test.ts
@@ -236,6 +236,51 @@ describe('StatelessLifecycleTracker — D8 idle rule', () => {
expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
})
+ it('repeats the suspension-blocked fresh check after the tracker is stopped', async () => {
+ const port = makePort()
+ port.getEffectiveLifecycle.mockReturnValue({
+ stateless: true,
+ state: 'active',
+ suspensionBlocked: true,
+ })
+ const tracker = makeTracker({ port, host: makeHost() })
+
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ // The epoch's single fresh check ran, and the second blocked beat skipped it.
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(2)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
+
+ tracker.stop()
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(3)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
+ expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
+ })
+
+ it('reports a cancel-drain with nothing to revert as settled', async () => {
+ const port = makePort()
+ const tracker = makeTracker({ port })
+ const cancelDrain = (
+ tracker as unknown as {
+ cancelDrainOnEvidence(hostRef: string, host: HostCRD, bypass?: boolean): Promise<boolean>
+ }
+ ).cancelDrainOnEvidence.bind(tracker)
+ const active = makeHost()
+ const suspended = makeHost({ lifecycle: { state: 'suspended', wakeHandledGeneration: 0 } })
+ const draining = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
+
+ // Cached active or suspended evidence needs no revert: settled, no write.
+ await expect(cancelDrain(active.name, active)).resolves.toBe(true)
+ await expect(cancelDrain(suspended.name, suspended)).resolves.toBe(true)
+ expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()
+
+ // Liveness witness: cached draining evidence does reach the revert write.
+ await expect(cancelDrain(draining.name, draining)).resolves.toBe(true)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledOnce()
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(draining)
+ })
+
it('retries a failed suspension-blocked fresh check and still reverts cached draining evidence', async () => {
const port = makePort()
port.getEffectiveLifecycle.mockReturnValue({