Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): fail closed on a stale McpServer cache and hold a running suspended runtime
Alfredo Lopez6df1be439c78
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex a1411f78d..cadfc56e6 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.366",+ "version": "0.1.367","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.366",+ "version": "0.1.367","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 53372b194..db79a39c8 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.366",+ "version": "0.1.367","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex c4d0a6b98..d296655ec 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -5137,17 +5137,25 @@ export class HostReconciler {// A legacy Deployment without a runtime-token-revision annotation is not// a preserved wake target, so it stays at zero replicas until channel// authority returns and the normal path rewrites its template.- const preservedWakeTarget =+ const appliedHostRuntime =HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&- (liveDeployment?.spec?.replicas ?? 1) === 0 &&HostReconciler.deploymentRuntimeTokenRevision(liveDeployment) !== ''+ const liveReplicas = liveDeployment?.spec?.replicas ?? 1+ const preservedWakeTarget = appliedHostRuntime && liveReplicas === 0+ // A suspension recorded in status whose scale-down never reached the+ // Deployment leaves the applied runtime running. A wake on it needs no+ // mint and no replica change: it is held like any running runtime.+ const wakeOnRunningRuntime = wakeRequested && appliedHostRuntime && liveReplicas > 0const runtimeIsReady =!!liveDeployment &&HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&HostReconciler.deploymentReady(liveDeployment)const scaleUpForRecovery = lifecycle.effective.state === 'active' && preservedWakeTargetconst allowScaleUp = preservedWakeTarget && (wakeRequested || scaleUpForRecovery)- if (!liveDeployment || ((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget)) {+ if (+ !liveDeployment ||+ ((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget && !wakeOnRunningRuntime)+ ) {this.setStatus(host.name, {deployed: !!liveDeployment,ready: false,@@ -5161,7 +5169,10 @@ export class HostReconciler {// A pod starting from zero will consume bootstrap credentials. Mint and// persist them before the replica-only update; a failed mint aborts here.provisioningRequired = true- } else if (lifecycle.effective.state === 'active' && runtimeIsReady) {+ } else if (+ (lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&+ runtimeIsReady+ ) {const provision = await this.provisionRuntimeTokenRevision(host, {targetSuspended: false,refreshGfsOnly: true,@@ -5203,11 +5214,13 @@ export class HostReconciler {revalidateHostMutationBoundary)revalidateHostMutationBoundary()- if (lifecycle.effective.state !== 'suspended' || allowScaleUp) {+ const runtimeExpectedToRun =+ lifecycle.effective.state !== 'suspended' || allowScaleUp || wakeOnRunningRuntime+ if (runtimeExpectedToRun) {this.lifecycle.markHostNotSuspended(host.name)}const deploymentReady =- lifecycle.effective.state !== 'suspended' &&+ (lifecycle.effective.state !== 'suspended' || wakeOnRunningRuntime) &&applied &&(await this.checkDeploymentReady(host.name, host.namespace))revalidateHostMutationBoundary()@@ -5232,7 +5245,7 @@ export class HostReconciler {? 'Waiting for CommunicationChannel inventory before creating runtime': failures.length > 0? `degraded — Host runtime boundary incomplete while preserving applied runtime (${failures.join('; ')})`- : lifecycle.effective.state === 'suspended' && !allowScaleUp+ : !runtimeExpectedToRun? 'CommunicationChannel inventory unavailable; preserving suspended Host replicas': deploymentReady? 'CommunicationChannel inventory unavailable; preserving applied runtime'@@ -5241,12 +5254,7 @@ export class HostReconciler {// Mirror the normal path: a runtime expected to run (including a held// wake from zero) is polled until Ready; a degraded boundary is not,// because the poll would overwrite the degraded verdict with "Running".- if (- applied &&- !deploymentReady &&- failures.length === 0 &&- (lifecycle.effective.state !== 'suspended' || allowScaleUp)- ) {+ if (applied && !deploymentReady && failures.length === 0 && runtimeExpectedToRun) {this.pollReadiness(host.name, host.namespace)}}diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex a7b2ce7e2..b47602de8 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -10769,8 +10769,10 @@ describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)'})/** Drive the REAL getServerInfosByContext: populate the server cache and answer- * the Context read the producer makes with an allow-list of those servers. */+ * the Context read the producer makes with an allow-list of those servers.+ * The cache is an observation only while its LIST -> WATCH pair is live. */const withServers = (watcher: McpServerWatcher, servers: any[]): void => {+ markMcpServerInventoryAuthoritative(watcher)for (const s of servers) (watcher as any).servers.set(s.name, s)mocks.getNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) =>plural === 'contexts'@@ -10815,6 +10817,60 @@ describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)'await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)watcher.stop()})++ /** The resolver McpServerWatcher wires into HostReconciler at construction. */+ const wiredResolver = (watcher: McpServerWatcher): ((host: HostCRD) => Promise<boolean>) => {+ const calls = (watcher.getHostReconciler() as any).setHostFrontsOAuthServer.mock.calls+ expect(calls).toHaveLength(1)+ return calls[0][0]+ }++ it('does not serve a retired McpServer cache that would grant the OAuth scope', async () => {+ const watcher = new McpServerWatcher()+ withServers(watcher, [serverCRD({ name: 'notion', auth: { type: 'oauth' }, enabled: true })])+ const resolve = wiredResolver(watcher)+ // Liveness witness: while the inventory is authoritative, this same cache+ // and wired resolver grant the scope.+ await expect(resolve(hostFor('ctx'))).resolves.toBe(true)++ // A McpServer watch outage retires the inventory but keeps its last cache,+ // which still holds the enabled OAuth server an admin may have disabled.+ expect((watcher as any).retireMcpServerWatch()).toBe(true)+ expect((watcher as any).servers.get('notion')?.spec.enabled).toBe(true)++ await expect(resolve(hostFor('ctx'))).rejects.toThrow(+ 'McpServer inventory is not authoritative; OAuth scope is unobserved'+ )+ watcher.stop()+ })++ it('does not serve a McpServer cache that lost authority during the Context read', async () => {+ const watcher = new McpServerWatcher()+ withServers(watcher, [serverCRD({ name: 'notion', auth: { type: 'oauth' }, enabled: true })])+ const contextRead = deferred<unknown>()+ const contextReads: string[] = []+ mocks.getNamespacedCustomObject.mockImplementation(+ async ({ plural, name }: { plural: string; name: string }) => {+ contextReads.push(`${plural}/${name}`)+ return contextRead.promise+ }+ )++ const pending = wiredResolver(watcher)(hostFor('ctx'))+ await flushMicrotasks()+ // Liveness witness: the observation reached the live Context read.+ expect(contextReads).toEqual(['contexts/ctx'])+ expect((watcher as any).retireMcpServerWatch()).toBe(true)+ contextRead.resolve({+ metadata: { name: 'ctx', namespace: 'mcp-server' },+ spec: { contextId: 'ctx', mcpServers: ['notion'] },+ })++ await expect(pending).rejects.toThrow(+ 'McpServer inventory lost authority during the OAuth scope read'+ )+ watcher.stop()+ })})describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-close regime)', () => {// Regression for the clerum-dev livelock. PR #205 coupled /ready to a safetydiff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex b758206e5..7c07e1335 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -2519,9 +2519,22 @@ export class McpServerWatcher implements McpServerProvider {* already filters to enabled + allowed servers) that mcp-host discovery uses,* so HCC does not need a second cross-CRD read path. HostReconciler wraps this* call in a fail-closed guard, so a thrown read there yields no oauth scope.+ *+ * A retired McpServer watch keeps its last `servers` cache, so an answer read+ * from it is only an observation while the same LIST -> WATCH generation stays+ * authoritative across the whole read. Otherwise this throws: a server an+ * admin disabled during the outage must not re-grant the scope from the stale+ * cache, and HostReconciler treats the throw as an unobserved read.*/private async hostFrontsOAuthServer(host: HostCRD): Promise<boolean> {+ const watchGeneration = this.mcpWatchGeneration+ if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {+ throw new Error('McpServer inventory is not authoritative; OAuth scope is unobserved')+ }const servers = await this.getServerInfosByContext(host.spec.contextRef)+ if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {+ throw new Error('McpServer inventory lost authority during the OAuth scope read')+ }return servers.some(server => server.enabled && server.auth?.type === 'oauth')}diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex ce3c386fa..f96b8057e 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1297,6 +1297,62 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(poll).toHaveBeenCalledWith(host.name, host.namespace)})+ it.each([+ {+ runtime: 'Ready',+ readyReplicas: 1,+ ready: true,+ message: 'CommunicationChannel inventory unavailable; preserving applied runtime',+ },+ {+ runtime: 'not Ready',+ readyReplicas: 0,+ ready: false,+ message: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',+ },+ ])(+ 'holds a held wake on a suspended Host whose applied runtime still runs ($runtime)',+ async ({ readyReplicas, ready, message }) => {+ // The suspension reached the Host status but its scale-down never reached+ // the Deployment, so the applied runtime still runs at one replica.+ const host = makeStatelessHost({ status: suspendedStatus() })+ host.annotations = { 'clerum.io/wake-requested': '1' }+ const { reconciler, appsApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ const applied = reconciler.buildDeployment(host)+ applied.spec!.replicas = 1+ applied.spec!.template!.metadata!.annotations = {+ ...applied.spec!.template!.metadata!.annotations,+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',+ }+ markPersistedRuntimeTrusted(applied, host, readyReplicas)+ const live = persistHostDeployment(appsApi, host, applied)+ const templateBefore = structuredClone(live().spec!.template)+ const provision = vi+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')+ .mockResolvedValue(runtimeTokenProvision(host))+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)++ await reconciler.reconcile(host)++ // Liveness witness: the held pass reached its readiness check.+ expect(readySpy).toHaveBeenCalledWith(host.name, host.namespace)+ await expect(readySpy.mock.results[0].value).resolves.toBe(readyReplicas > 0)+ expect(reconciler.getStatus(host.name)).toEqual({ deployed: true, ready, message })+ // A running runtime renews only its GFS credential; it never mints for+ // the wake, and a not-Ready one is polled instead of renewed.+ expect(provision.mock.calls).toEqual(+ readyReplicas > 0 ? [[host, { targetSuspended: false, refreshGfsOnly: true }]] : []+ )+ expect(poll.mock.calls).toEqual(readyReplicas > 0 ? [] : [[host.name, host.namespace]])+ expect(live().spec!.replicas).toBe(1)+ expect(live().spec!.template).toEqual(templateBefore)+ }+ )+it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {const host = makeStatelessHost()const { reconciler, appsApi } = createReconciler()@@ -1407,6 +1463,105 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {})})+ it('keeps a preserved suspended wake target at zero replicas during cache loss without a pending wake', async () => {+ // This Deployment IS a preserved wake target (applied runtime revision at+ // zero replicas), so only the missing wake keeps the hold from minting+ // bootstrap credentials and scaling it up.+ const host = makeStatelessHost({ status: suspendedStatus() })+ const { reconciler, appsApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ const applied = reconciler.buildDeployment(host)+ applied.spec!.replicas = 0+ applied.spec!.template!.metadata!.annotations = {+ ...applied.spec!.template!.metadata!.annotations,+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',+ }+ markPersistedRuntimeTrusted(applied, host, 0)+ const live = persistHostDeployment(appsApi, host, applied)+ const templateBefore = structuredClone(live().spec!.template)+ const provision = vi+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')+ .mockResolvedValue(runtimeTokenProvision(host))+ const ensureDeployment = vi.spyOn(reconciler as any, 'ensureDeployment')+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)++ await reconciler.reconcile(host)++ // Liveness witness: the pass read the live Deployment and applied the hold+ // with the suspended, cache-loss lifecycle.+ expect(+ appsApi.readNamespacedDeployment.mock.calls.filter(([request]) => request.name === host.name)+ .length+ ).toBeGreaterThan(0)+ expect(ensureDeployment).toHaveBeenCalledOnce()+ expect(ensureDeployment.mock.calls[0][3]).toMatchObject({+ state: 'suspended',+ suspensionBlocked: true,+ allowScaleUpDuringHold: false,+ })+ await expect(ensureDeployment.mock.results[0].value).resolves.toBe(true)+ expect(reconciler.getStatus(host.name)).toEqual({+ deployed: true,+ ready: false,+ message: 'CommunicationChannel inventory unavailable; preserving suspended Host replicas',+ })+ expect(provision).not.toHaveBeenCalled()+ expect(poll).not.toHaveBeenCalled()+ expect(live().spec!.replicas).toBe(0)+ expect(live().spec!.template).toEqual(templateBefore)+ })++ it('does not poll readiness when the held Deployment disappears before it is applied', async () => {+ // An active, not-Ready held runtime with a complete boundary is polled, so+ // only the unapplied Deployment keeps the poll from running.+ const host = makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },+ })+ const { reconciler, appsApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ const applied = reconciler.buildDeployment(host)+ markPersistedRuntimeTrusted(applied, host, 0)+ persistHostDeployment(appsApi, host, applied)+ const persistedRead = appsApi.readNamespacedDeployment.getMockImplementation()!+ let deleted = false+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ deleted && request.name === host.name ? Promise.reject({ code: 404 }) : persistedRead(request)+ )+ const realEnsureDeployment = (reconciler as any).ensureDeployment.bind(reconciler)+ const ensureDeployment = vi+ .spyOn(reconciler as any, 'ensureDeployment')+ .mockImplementation(async (...args: unknown[]) => {+ // The Deployment is deleted after the hold read it and before it is applied.+ deleted = true+ return realEnsureDeployment(...args)+ })+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)++ await reconciler.reconcile(host)++ // Liveness witness: the hold reached ensureDeployment, whose re-read found+ // the Deployment gone and left it unapplied.+ expect(ensureDeployment).toHaveBeenCalledOnce()+ await expect(ensureDeployment.mock.results[0].value).resolves.toBe(false)+ expect(reconciler.getStatus(host.name)).toEqual({+ deployed: false,+ ready: false,+ message: 'Waiting for CommunicationChannel inventory before creating runtime',+ })+ expect(+ appsApi.createNamespacedDeployment.mock.calls.filter(+ ([request]) => request.body.metadata.name === host.name+ )+ ).toHaveLength(0)+ expect(readySpy).not.toHaveBeenCalled()+ expect(poll).not.toHaveBeenCalled()+ })+it('uses the fenced replica-only update for a pending wake during cache loss', async () => {const host = makeStatelessHost({ status: suspendedStatus() })host.annotations = { 'clerum.io/wake-requested': '1' }@@ -1686,8 +1841,22 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {throw new Error('McpServer read unavailable')},})-- await reconciler.reconcile(host)+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)+ try {+ await reconciler.reconcile(host)+ expect(warn).toHaveBeenCalledWith(+ 'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',+ {+ host: host.name,+ namespace: host.namespace,+ observation: 'frontsOAuthServer',+ attempts: 3,+ err: 'McpServer read unavailable',+ }+ )+ } finally {+ warn.mockRestore()+ }expect(runtimeRecordReads.length).toBeGreaterThan(0)expect(oauthResolver).toHaveBeenCalledTimes(3)diff --git a/host-context-controller/test/statelessLifecycleTracker.test.ts b/host-context-controller/test/statelessLifecycleTracker.test.tsindex ec9e1494c..01b6cfaa2 100644--- a/host-context-controller/test/statelessLifecycleTracker.test.ts+++ b/host-context-controller/test/statelessLifecycleTracker.test.ts@@ -236,6 +236,51 @@ describe('StatelessLifecycleTracker — D8 idle rule', () => {expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()})+ it('repeats the suspension-blocked fresh check after the tracker is stopped', async () => {+ const port = makePort()+ port.getEffectiveLifecycle.mockReturnValue({+ stateless: true,+ state: 'active',+ suspensionBlocked: true,+ })+ const tracker = makeTracker({ port, host: makeHost() })++ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })+ // The epoch's single fresh check ran, and the second blocked beat skipped it.+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(2)+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)++ tracker.stop()+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(3)+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)+ expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()+ })++ it('reports a cancel-drain with nothing to revert as settled', async () => {+ const port = makePort()+ const tracker = makeTracker({ port })+ const cancelDrain = (+ tracker as unknown as {+ cancelDrainOnEvidence(hostRef: string, host: HostCRD, bypass?: boolean): Promise<boolean>+ }+ ).cancelDrainOnEvidence.bind(tracker)+ const active = makeHost()+ const suspended = makeHost({ lifecycle: { state: 'suspended', wakeHandledGeneration: 0 } })+ const draining = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })++ // Cached active or suspended evidence needs no revert: settled, no write.+ await expect(cancelDrain(active.name, active)).resolves.toBe(true)+ await expect(cancelDrain(suspended.name, suspended)).resolves.toBe(true)+ expect(port.markHostActiveFromHeartbeat).not.toHaveBeenCalled()++ // Liveness witness: cached draining evidence does reach the revert write.+ await expect(cancelDrain(draining.name, draining)).resolves.toBe(true)+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledOnce()+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledWith(draining)+ })+it('retries a failed suspension-blocked fresh check and still reverts cached draining evidence', async () => {const port = makePort()port.getEffectiveLifecycle.mockReturnValue({