Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): converge policies with published runtime eligibility

Alfredo Lopez06355c987fdf
diff --git a/Makefile b/Makefile
index 4eea8240c..49fc7e38d 100644
--- a/Makefile
+++ b/Makefile
@@ -671,6 +671,13 @@ minikube-t2: ## Full orchestrator: T0, Real PostgreSQL T1, then exact-head T2
.PHONY: minikube-t2-np08-hcc-authorization
minikube-t2-np08-hcc-authorization: minikube-t2 ## Run canonical T2 including the required deployed NP-08 Host-to-HCC authorization journey
+# Dedicated #604 certification: the bounded health lane runs the real
+# policy lifecycle, business invocation and three watch reconnections.
+.PHONY: minikube-t2-hcc-networkpolicy-lifecycle
+minikube-t2-hcc-networkpolicy-lifecycle:
+ @T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \
+ T2_HEALTHCHECK_TIMEOUT_SECONDS=900 $(MAKE) minikube-t2
+
.PHONY: minikube-t2-runtime
minikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile
@T2_RUN_T0=false T2_RUN_T1=false \
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 20238072a..331343bce 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.310",
+ "version": "0.1.311",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.310",
+ "version": "0.1.311",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index b409f572e..bc676b887 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.310",
+ "version": "0.1.311",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 65eea1818..23c963224 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -3587,6 +3587,7 @@ describe('McpServerWatcher startup', () => {
}
)
await (watcher as any).startContextWatch('context-policy-rv')
+ ;(watcher as any).contextCacheSynced = true
const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)
await oldStarted.promise
@@ -6118,7 +6119,11 @@ describe('McpServerWatcher external egress retries', () => {
it('retries failed external egress reconciliation from an ADDED event', async () => {
const watcher = new McpServerWatcher()
+ // This test isolates the per-server retry; fleet convergence is covered
+ // separately and would independently retry the same failed egress.
+ vi.spyOn(watcher as any, 'runInitialNetworkPolicyConvergence').mockResolvedValue(undefined)
markMcpServerInventoryAuthoritative(watcher)
+ ;(watcher as any).contextCacheSynced = true
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
const bindingReconciler = (watcher as any).bindingReconciler
@@ -10934,6 +10939,97 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)
})
+ it('recovers policy verdict edges from re-LIST without invalidating on unrelated status (#604)', () => {
+ const watcher = new McpServerWatcher()
+ const state = watcher as any
+ const server = {
+ name: 'policy-server',
+ namespace: 'mcp-server',
+ uid: 'uid-policy',
+ generation: 1,
+ spec: {
+ image: 'fixture:v1',
+ contextRef: 'ctx',
+ transport: { type: 'streamableHttp', port: 3000 },
+ envSecret: { name: 'fixture-env', keys: [] },
+ },
+ status: {
+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],
+ },
+ }
+ state.installMcpServerSnapshot({ servers: [server], resourceVersion: '1' })
+ const initialRevision = state.mcpServerDesiredRevision
+ const recovered = {
+ ...server,
+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },
+ }
+ state.installMcpServerSnapshot({ servers: [recovered], resourceVersion: '2' })
+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 1)
+ state.installMcpServerSnapshot({ servers: [recovered], resourceVersion: '3' })
+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 1)
+ state.installMcpServerSnapshot({ servers: [server], resourceVersion: '4' })
+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 2)
+ })
+
+ it('propagates only an effective WATCH verdict edge into policy work (#604)', async () => {
+ const watcher = new McpServerWatcher()
+ const state = watcher as any
+ state.mcpServerCacheSynced = true
+ state.contextCacheSynced = true
+ const server = {
+ name: 'policy-server',
+ namespace: 'mcp-server',
+ uid: 'uid-policy',
+ generation: 1,
+ spec: {
+ image: 'fixture:v1',
+ contextRef: 'ctx',
+ transport: { type: 'streamableHttp', port: 3000 },
+ envSecret: { name: 'fixture-env', keys: [] },
+ },
+ status: {
+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],
+ },
+ }
+ state.servers.set(server.name, server)
+ const full = vi.spyOn(state, 'runInitialNetworkPolicyConvergence').mockResolvedValue(undefined)
+ const retry = vi
+ .spyOn(state.externalEgressCoordinator, 'scheduleRetry')
+ .mockImplementation(() => {})
+ const scoped = vi.spyOn(state, 'reconcileContextsForMcpServer').mockResolvedValue(undefined)
+ const callback = state.getMcpServerWatchCallback()
+ const event = (status: typeof server.status) => ({
+ metadata: {
+ name: server.name,
+ namespace: server.namespace,
+ uid: server.uid,
+ generation: server.generation,
+ },
+ spec: server.spec,
+ status,
+ })
+ await callback(
+ 'MODIFIED',
+ event({ conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] })
+ )
+ expect(full).toHaveBeenCalledTimes(1)
+ expect(scoped).toHaveBeenCalledTimes(1)
+ expect(retry).toHaveBeenCalledTimes(1)
+ for (let repeat = 0; repeat < 3; repeat++) {
+ await callback(
+ 'MODIFIED',
+ event({ conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] })
+ )
+ }
+ expect(scoped).toHaveBeenCalledTimes(1)
+ expect(retry).toHaveBeenCalledTimes(1)
+ await callback('MODIFIED', event(server.status))
+ expect(full).toHaveBeenCalledTimes(2)
+ expect(scoped).toHaveBeenCalledTimes(2)
+ expect(retry).toHaveBeenCalledTimes(2)
+ await watcher.stop()
+ })
+
it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {
const watcher = new McpServerWatcher()
const hostA = {
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 216143451..bef2ed366 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -41,6 +41,7 @@ import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'
+import { hccLogger } from './logger'
import type {
AuthorityContext,
AuthorityHost,
@@ -52,7 +53,9 @@ import type {
import {
confirmAuthoritativeMcpServerAbsence,
isMcpServerStatusOnlyUpdate,
+ runtimeDesired,
sameMcpServerDesiredRevision,
+ sameMcpServerPolicyRevision,
} from './mcpServerSafety'
import {
contextMetadataOnlyEventsTotal,
@@ -845,7 +848,7 @@ export class McpServerWatcher implements McpServerProvider {
listServers: () => [...this.servers.values()],
getCurrentServer: name => this.servers.get(name),
inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,
- sameDesiredRevision: sameMcpServerDesiredRevision,
+ sameDesiredRevision: sameMcpServerPolicyRevision,
enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),
mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),
replay: (type, server, retry) =>
@@ -1406,7 +1409,7 @@ export class McpServerWatcher implements McpServerProvider {
if (snapshot.servers.length !== this.servers.size) return true
for (const server of snapshot.servers) {
const previous = this.servers.get(server.name)
- if (previous === undefined || !sameMcpServerDesiredRevision(previous, server)) {
+ if (previous === undefined || !sameMcpServerPolicyRevision(previous, server)) {
return true
}
}
@@ -3919,7 +3922,7 @@ export class McpServerWatcher implements McpServerProvider {
let desiredStateChanged = false
if (type === 'ADDED' || type === 'MODIFIED') {
desiredStateChanged =
- previous === undefined || !sameMcpServerDesiredRevision(previous, server)
+ previous === undefined || !sameMcpServerPolicyRevision(previous, server)
this.servers.set(server.name, server)
} else if (type === 'DELETED') {
if (!previous?.uid || !server.uid || previous.uid === server.uid) {
@@ -3931,11 +3934,25 @@ export class McpServerWatcher implements McpServerProvider {
void this.runInitialNetworkPolicyConvergence()
}
- // HCC writes McpServer status during reconciliation. Those writes emit
- // MODIFIED events but do not change the desired runtime or policy state.
- // Suppress that self-induced work while still publishing the fresh status
- // through discovery. Any UID/spec/annotation/label change remains live.
+ // Most status writes do not change desired state. A runtime eligibility
+ // edge does change policy intent: propagate it without re-entering the
+ // runtime owner merely because it published status. Discovery still
+ // receives every update; UID/spec/annotation/label changes remain live.
if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {
+ if (runtimeDesired(previous) !== runtimeDesired(server)) {
+ // The revision bump above retires old policy work and requests a
+ // certified safety pass; replay also retries per-server additions.
+ this.externalEgressCoordinator.scheduleRetry('MODIFIED', server)
+ try {
+ await this.reconcileContextsForMcpServer(server, watchGeneration)
+ } catch (err) {
+ // The certified fleet pass and the server replay remain scheduled.
+ hccLogger.error('Runtime verdict policy propagation failed', {
+ serverName: server.name,
+ err,
+ })
+ }
+ }
this.changeCallback?.()
return
}
@@ -3950,6 +3967,42 @@ export class McpServerWatcher implements McpServerProvider {
}
}
+ private async reconcileContextsForMcpServer(
+ server: McpServerCRD,
+ watchGeneration: number
+ ): Promise<void> {
+ const contextGeneration = this.contextWatchGeneration
+ const isCurrent = (): boolean => {
+ const current = this.servers.get(server.name)
+ return (
+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
+ current !== undefined &&
+ sameMcpServerPolicyRevision(server, current) &&
+ this.hasContextInventoryAuthority(contextGeneration)
+ )
+ }
+ for (const selected of this.contexts.values()) {
+ if (!isCurrent()) return
+ if (!selected.spec.mcpServers?.includes(server.name)) continue
+ const contextId = selected.spec.contextId
+ await this.enqueueContextReconciliation(contextId, async () => {
+ if (!isCurrent()) return
+ const current = this.contexts.get(selected.name)
+ if (
+ current?.spec.contextId !== contextId ||
+ !current.spec.mcpServers?.includes(server.name)
+ )
+ return
+ const completed = await this.netPolReconciler.reconcileContext(current, {
+ isCurrent: () => isCurrent() && this.contexts.get(current.name) === current,
+ })
+ if (completed === false && isCurrent()) {
+ this.scheduleExternalEgressRetry('MODIFIED', server)
+ }
+ })
+ }
+ }
+
private async reconcileMcpServerWatchEvent(
type: ExternalEgressWatchEventType,
server: McpServerCRD,
@@ -3973,11 +4026,6 @@ export class McpServerWatcher implements McpServerProvider {
}
const deleteAllowed = () =>
this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)
- const contextInventoryGeneration = this.contextWatchGeneration
- const contextPolicyInventoryIsCurrent = (): boolean =>
- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
- this.hasContextInventoryAuthority(contextInventoryGeneration)
-
// External egress is part of the workload's pre-start contract. Reconcile
// it before HCC creates or updates any managed runtime Deployment so a
// stdio MCP with egressBindings cannot start before ExternalEgressReady.
@@ -4077,30 +4125,7 @@ export class McpServerWatcher implements McpServerProvider {
if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
server = this.servers.get(server.name)!
}
- // Re-reconcile cached Contexts after the server cache is populated.
- for (const selectedContext of this.contexts.values()) {
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- if (!selectedContext.spec.mcpServers?.includes(server.name)) continue
- const selectedContextId = selectedContext.spec.contextId
- console.log(
- `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`
- )
- await this.enqueueContextReconciliation(selectedContextId, async () => {
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- const currentContext = this.contexts.get(selectedContext.name)
- if (
- currentContext?.spec.contextId !== selectedContextId ||
- !currentContext.spec.mcpServers?.includes(server.name)
- ) {
- return
- }
- await this.netPolReconciler.reconcileContext(currentContext, {
- isCurrent: () =>
- contextPolicyInventoryIsCurrent() &&
- this.contexts.get(currentContext.name) === currentContext,
- })
- })
- }
+ await this.reconcileContextsForMcpServer(server, watchGeneration)
} else if (type === 'DELETED') {
const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })
diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.ts
index 32a0329a0..08821c206 100644
--- a/host-context-controller/src/mcpServerSafety.test.ts
+++ b/host-context-controller/src/mcpServerSafety.test.ts
@@ -2,7 +2,9 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest'
import {
confirmAuthoritativeMcpServerAbsence,
isMcpServerStatusOnlyUpdate,
+ runtimeDesired,
sameMcpServerDesiredRevision,
+ sameMcpServerPolicyRevision,
} from './mcpServerSafety'
import type { McpServerCRD } from './types'
@@ -312,3 +314,57 @@ describe('confirmAuthoritativeMcpServerAbsence', () => {
expect(resolveCurrent).toHaveBeenCalledTimes(2)
})
})
+
+describe('runtimeDesired policy contract (#604)', () => {
+ const reasons = ['SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied', 'ReadError']
+ function withVerdict(reason: string, managed = true): McpServerCRD {
+ const server = makeServer()
+ server.spec = {
+ ...server.spec,
+ managed,
+ envSecret: { name: 'fixture-env', keys: [{ secretKey: 'required', envVar: 'VALUE' }] },
+ }
+ server.status = { conditions: [{ type: 'SecretResolved', status: 'False', reason }] }
+ return server
+ }
+
+ it.each(reasons)('preserves ownership semantics for %s', reason => {
+ const managed = withVerdict(reason)
+ const wrc = withVerdict(reason, false)
+ expect(runtimeDesired(managed)).toBe(reason === 'ReadError')
+ expect(runtimeDesired(wrc)).toBe(true)
+ for (const server of [managed, wrc]) {
+ server.spec.enabled = false
+ expect(runtimeDesired(server)).toBe(false)
+ }
+ })
+
+ it('recovers when the reference is removed despite a stale negative condition', () => {
+ const server = withVerdict('SecretNotFound')
+ expect(runtimeDesired(server)).toBe(false)
+ delete server.spec.envSecret
+ expect(runtimeDesired(server)).toBe(true)
+ })
+
+ it('retires policy effects, but not runtime effects, on a published verdict edge', () => {
+ const previous = withVerdict('SecretNotFound')
+ const current = withVerdict('SecretFound')
+ expect(sameMcpServerDesiredRevision(previous, current)).toBe(true)
+ expect(sameMcpServerPolicyRevision(previous, current)).toBe(false)
+ expect(sameMcpServerPolicyRevision(current, previous)).toBe(false)
+ expect(sameMcpServerPolicyRevision(previous, withVerdict('SecretMissingKey'))).toBe(true)
+ expect(
+ sameMcpServerPolicyRevision(
+ withVerdict('SecretNotFound', false),
+ withVerdict('SecretFound', false)
+ )
+ ).toBe(true)
+ })
+
+ it('does not require status for an otherwise desired server', () => {
+ expect(runtimeDesired(makeServer())).toBe(true)
+ const server = withVerdict('SecretNotFound')
+ server.status = undefined
+ expect(runtimeDesired(server)).toBe(true)
+ })
+})
diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.ts
index 5b15bda16..bcdde5a58 100644
--- a/host-context-controller/src/mcpServerSafety.ts
+++ b/host-context-controller/src/mcpServerSafety.ts
@@ -91,3 +91,33 @@ export async function confirmAuthoritativeMcpServerAbsence({
return inventoryAuthoritative() && !resolveCurrent()
}
+
+/** The runtime owner is the only source of credential validation. */
+export function shouldFailClosedForSecretFailure(reason: string | undefined): boolean {
+ return (
+ reason === 'SecretNotFound' || reason === 'SecretMissingKey' || reason === 'SecretAccessDenied'
+ )
+}
+
+/**
+ * HCC owns policies for both runtime owners, but its Secret verdict only
+ * controls HCC-managed runtime. WRC permits optional keys HCC cannot judge.
+ * A removed reference makes an old SecretResolved condition irrelevant.
+ */
+export function runtimeDesired(server: McpServerCRD): boolean {
+ if (server.spec.enabled === false) return false
+ if (server.spec.managed === false || !server.spec.envSecret) return true
+ const condition = server.status?.conditions?.find(item => item.type === 'SecretResolved')
+ return !shouldFailClosedForSecretFailure(condition?.reason)
+}
+
+/** Policy effects also depend on the effective published runtime verdict. */
+export function sameMcpServerPolicyRevision(
+ expected: McpServerCRD,
+ current: McpServerCRD
+): boolean {
+ return (
+ sameMcpServerDesiredRevision(expected, current) &&
+ runtimeDesired(expected) === runtimeDesired(current)
+ )
+}
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 3dc6a0ea8..385abb2f2 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -4,6 +4,13 @@ import type { RecordWithTtl } from 'node:dns'
import * as dns from 'node:dns/promises'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
+import {
+ asAppsApi,
+ asCoreApi,
+ asCustomApi,
+ createMockAppsApi,
+ createMockCoreApi,
+} from '../test/__fixtures__/testMocks'
import { asApiserverNetworkPolicy } from './__tests__/asApiserverNetworkPolicy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL, POLICY_TYPE_LABEL } from './constants'
@@ -18,6 +25,7 @@ import {
PUBLIC_EGRESS_EXCEPT_CIDRS,
sameContextDesiredRevision,
} from './networkPolicyReconciler'
+import { McpServerReconciler } from './reconciler'
import { ContextCRD, McpServerCRD } from './types'
import { getErrorCode } from './utils'
@@ -76,6 +84,7 @@ vi.mock('node:dns/promises', () => ({
}))
vi.mock('./metrics', () => ({
+ mcpserverMissingSecret: { set: vi.fn() },
createsTotal: { inc: vi.fn() },
existenceReadsTotal: { inc: vi.fn() },
networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },
@@ -6274,4 +6283,394 @@ describe('NetworkPolicyReconciler', () => {
expect(store.has(orphanKey)).toBe(false)
})
})
+ describe('published runtime policy lifecycle (#604)', () => {
+ function lifecycle() {
+ config.netPolOrphanDeleteCapPercent = 100
+ const store = new Map<string, k8s.V1NetworkPolicy>()
+ const events: string[] = []
+ let revision = 0
+ const missing = () => Object.assign(new Error('not found'), { code: 404 })
+ const save = (namespace: string, body: k8s.V1NetworkPolicy) => {
+ const key = `${namespace}/${body.metadata!.name}`
+ const previous = store.get(key)
+ const result = structuredClone({
+ ...body,
+ metadata: {
+ ...body.metadata,
+ namespace,
+ uid: previous?.metadata?.uid ?? `${key}-${++revision}`,
+ resourceVersion: String(++revision),
+ },
+ })
+ store.set(key, result)
+ return result
+ }
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {
+ const key = `${namespace}/${body.metadata.name}`
+ if (store.has(key)) throw Object.assign(new Error('exists'), { code: 409 })
+ events.push(`create:${key}`)
+ return save(namespace, body)
+ })
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) =>
+ save(namespace, body)
+ )
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {
+ const found = store.get(`${namespace}/${name}`)
+ if (!found) throw missing()
+ return structuredClone(found)
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }) => ({
+ items: [...store.values()]
+ .filter(
+ policy =>
+ policy.metadata?.namespace === namespace &&
+ (!labelSelector ||
+ labelSelector.split(',').every((clause: string) => {
+ const [key, value] = clause.split('=')
+ return policy.metadata?.labels?.[key] === value
+ }))
+ )
+ .map(policy => structuredClone(policy)),
+ })
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, name, body }) => {
+ const key = `${namespace}/${name}`
+ const found = store.get(key)
+ if (!found) throw missing()
+ expect(body?.preconditions).toEqual({
+ uid: found.metadata!.uid,
+ resourceVersion: found.metadata!.resourceVersion,
+ })
+ events.push(`delete:${key}`)
+ store.delete(key)
+ return {}
+ }
+ )
+ const initial: McpServerCRD = {
+ name: 'lifecycle',
+ namespace: 'mcp-server',
+ uid: 'server-lifecycle',
+ generation: 1,
+ spec: {
+ contextRef: 'ctx',
+ image: 'fixture:v1',
+ envSecret: { name: 'fixture-env', keys: [{ secretKey: 'required', envVar: 'VALUE' }] },
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],
+ },
+ }
+ const cache = new Map([[initial.name, initial]])
+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async ({ name }) => {
+ const server = cache.get(name)
+ if (!server) throw missing()
+ return {
+ metadata: {
+ name,
+ namespace: server.namespace,
+ uid: server.uid,
+ generation: server.generation,
+ resourceVersion: '1',
+ },
+ spec: server.spec,
+ status: server.status,
+ }
+ })
+ mockCustomApi.getNamespacedCustomObjectStatus.mockImplementation(
+ mockCustomApi.getNamespacedCustomObject.getMockImplementation()!
+ )
+ const rec = makeReconciler(mockApi, cache, mockCustomApi)
+ // Use the stateful API directly: the general fixture caches LIST snapshots,
+ // which would mask a deleted resource during this lifecycle reproduction.
+ ;(rec as any).networkingApi = mockApi
+ const context: ContextCRD = {
+ name: 'ctx',
+ namespace: 'mcp-server',
+ spec: { contextId: 'ctx', mcpServers: [initial.name] },
+ }
+ const run = async () => {
+ const server = cache.get(initial.name)!
+ await rec.fullReconcile([context], [server], {
+ ensureDefaults: false,
+ resolveCurrentServer: name => cache.get(name),
+ })
+ await rec.reconcileExternalEgress(server, {
+ isCurrent: () => cache.get(server.name) === server,
+ })
+ }
+ return { rec, initial, cache, context, store, events, run }
+ }
+
+ it.each(['Disabled', 'SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied'])(
+ 'revokes once and restores all four families after %s',
+ async reason => {
+ const f = lifecycle()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ const policyKeys = [...f.store.keys()].sort()
+ const failed = structuredClone(f.initial)
+ if (reason === 'Disabled') failed.spec.enabled = false
+ else failed.status = { conditions: [{ type: 'SecretResolved', status: 'False', reason }] }
+ f.cache.set(failed.name, failed)
+ await f.run()
+ expect(f.store.size).toBe(0)
+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(4)
+ const settled = f.events.length
+ for (let cycle = 0; cycle < 3; cycle++) await f.run()
+ expect(f.events).toHaveLength(settled)
+ f.cache.set(f.initial.name, structuredClone(f.initial))
+ await f.run()
+ expect([...f.store.keys()].sort()).toEqual(policyKeys)
+ expect(f.events.filter(event => event.startsWith('create:'))).toHaveLength(8)
+ }
+ )
+
+ it.each(['SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied', 'ReadError'])(
+ 'keeps WRC policies stable for %s but honors disable',
+ async reason => {
+ const f = lifecycle()
+ const server = structuredClone(f.initial)
+ server.spec.managed = false
+ f.cache.set(server.name, server)
+ await f.run()
+ expect(f.store.size).toBe(4)
+ f.cache.set(server.name, {
+ ...server,
+ status: { conditions: [{ type: 'SecretResolved', status: 'False', reason }] },
+ })
+ for (let cycle = 0; cycle < 3; cycle++) await f.run()
+ expect(f.store.size).toBe(4)
+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(0)
+ f.cache.set(server.name, { ...server, spec: { ...server.spec, enabled: false } })
+ await f.run()
+ expect(f.store.size).toBe(0)
+ }
+ )
+
+ it('runs the real runtime owner between policy passes without recreating a revoked allow', async () => {
+ const f = lifecycle()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ const apps = createMockAppsApi()
+ const core = createMockCoreApi()
+ core.readNamespacedSecret.mockRejectedValue(
+ Object.assign(new Error('missing fixture'), { code: 404 })
+ )
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {
+ const patch = body.find(
+ (item: { path: string }) => item.path === '/status/conditions' || item.path === '/status'
+ )
+ if (patch)
+ f.cache.set(f.initial.name, {
+ ...f.cache.get(f.initial.name)!,
+ status: { conditions: patch.path === '/status' ? patch.value.conditions : patch.value },
+ })
+ return {}
+ })
+ const runtime = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ appsApi: asAppsApi(apps),
+ coreApi: asCoreApi(core),
+ customApi: asCustomApi(mockCustomApi as any),
+ })
+ await runtime.reconcile(f.initial)
+ expect(apps.deleteNamespacedDeployment).toHaveBeenCalledWith({
+ name: f.initial.name,
+ namespace: f.initial.namespace,
+ })
+ expect(f.cache.get(f.initial.name)?.status?.conditions).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'SecretResolved',
+ status: 'False',
+ reason: 'SecretNotFound',
+ }),
+ ])
+ )
+ expect(f.store.size).toBe(4) // Runtime owner has not deleted policies.
+ await f.run()
+ expect(f.store.size).toBe(0)
+ const stationary = [...f.events]
+ for (let cycle = 0; cycle < 3; cycle++) {
+ await runtime.reconcile(f.cache.get(f.initial.name)!)
+ await f.run()
+ }
+ expect(f.events).toEqual(stationary)
+ })
+
+ it('never deletes another owner policy selecting the same disabled server', async () => {
+ const f = lifecycle()
+ await f.run()
+ const ownedElsewhere: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'recipe-owned-lifecycle',
+ namespace: 'mcp-server',
+ uid: 'wrc-policy',
+ resourceVersion: '1',
+ labels: { [MANAGED_BY_LABEL]: 'workflow-recipes', 'clerum.io/recipe': 'fixture-recipe' },
+ },
+ spec: {
+ podSelector: { matchLabels: { [MCPSERVER_LABEL]: f.initial.name } },
+ policyTypes: ['Ingress'],
+ },
+ }
+ f.store.set('mcp-server/recipe-owned-lifecycle', ownedElsewhere)
+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })
+ await f.run()
+ expect([...f.store.values()]).toEqual([ownedElsewhere])
+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(4)
+ })
+
+ it('retries an external status publication failure without deleting healthy policies', async () => {
+ const f = lifecycle()
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(
+ Object.assign(new Error('temporary status failure'), { code: 503 })
+ )
+ await expect(f.run()).rejects.toThrow('temporary status failure')
+ expect(f.store.size).toBe(4)
+ await f.run()
+ expect(f.store.size).toBe(4)
+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(0)
+ })
+
+ it('retires an in-flight Context effect when only the runtime verdict changes', async () => {
+ const f = lifecycle()
+ const entered = deferred()
+ const release = deferred()
+ const create = mockApi.createNamespacedNetworkPolicy.getMockImplementation()!
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async args => {
+ entered.resolve()
+ await release.promise
+ return create(args)
+ })
+ const running = f.rec.reconcileContext(f.context, { isCurrent: () => true })
+ await entered.promise
+ f.cache.set(f.initial.name, {
+ ...f.initial,
+ status: {
+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],
+ },
+ })
+ release.resolve()
+ expect(await running).toBe(false)
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ await f.run()
+ expect(f.store.size).toBe(0)
+ })
+
+ it('retires revocation when recovery arrives before delete admission', async () => {
+ const f = lifecycle()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ f.cache.set(f.initial.name, {
+ ...f.initial,
+ status: {
+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],
+ },
+ })
+ const entered = deferred()
+ const release = deferred()
+ const list = mockApi.listNamespacedNetworkPolicy.getMockImplementation()!
+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async args => {
+ entered.resolve()
+ await release.promise
+ return list(args)
+ })
+ const pending = f.rec.reconcileContext(f.context, { isCurrent: () => true })
+ await entered.promise
+ f.cache.set(f.initial.name, structuredClone(f.initial))
+ release.resolve()
+ expect(await pending).toBe(false)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.store.size).toBe(4)
+ await f.run()
+ expect(f.store.size).toBe(4)
+ })
+
+ it('preserves HCC allows on a transient ReadError', async () => {
+ const f = lifecycle()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ f.cache.set(f.initial.name, {
+ ...f.initial,
+ status: { conditions: [{ type: 'SecretResolved', status: 'False', reason: 'ReadError' }] },
+ })
+ await f.run()
+ expect(f.store.size).toBe(4)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('preserves WRC allows when HCC rejects an absent optional mapping', async () => {
+ const f = lifecycle()
+ const reference = {
+ name: 'fixture-env',
+ keys: [{ secretKey: 'optional', envVar: 'OPTIONAL_VALUE', optional: true }],
+ }
+ const server = {
+ ...f.initial,
+ spec: { ...f.initial.spec, managed: false, envSecret: reference },
+ }
+ f.cache.set(server.name, server)
+ await f.run()
+ expect(f.store.size).toBe(4)
+ const apps = createMockAppsApi()
+ const core = createMockCoreApi()
+ core.readNamespacedSecret.mockResolvedValue({ data: {} })
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {
+ const patch = body.find(
+ (item: { path: string }) => item.path === '/status/conditions' || item.path === '/status'
+ )
+ if (patch)
+ f.cache.set(server.name, {
+ ...f.cache.get(server.name)!,
+ status: { conditions: patch.path === '/status' ? patch.value.conditions : patch.value },
+ })
+ return {}
+ })
+ const runtime = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ appsApi: asAppsApi(apps),
+ coreApi: asCoreApi(core),
+ customApi: asCustomApi(mockCustomApi as any),
+ })
+ await runtime.reconcile(server)
+ expect(f.cache.get(server.name)?.status?.conditions).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'SecretResolved',
+ status: 'False',
+ reason: 'SecretMissingKey',
+ }),
+ ])
+ )
+ expect(runtime.getStatus(server.name)).toMatchObject({ deployed: true, ready: false })
+ expect(apps.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(core.deleteNamespacedService).not.toHaveBeenCalled()
+ await f.run()
+ expect(f.store.size).toBe(4)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('restores allows when envSecret is removed with its failure condition unchanged', async () => {
+ const f = lifecycle()
+ await f.run()
+ const failed = {
+ ...f.initial,
+ status: {
+ conditions: [
+ { type: 'SecretResolved', status: 'False' as const, reason: 'SecretNotFound' },
+ ],
+ },
+ }
+ f.cache.set(failed.name, failed)
+ await f.run()
+ expect(f.store.size).toBe(0)
+ const recovered = structuredClone(failed)
+ delete recovered.spec.envSecret
+ f.cache.set(recovered.name, recovered)
+ await f.run()
+ expect(f.store.size).toBe(4)
+ })
+ })
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index a7ed8d507..d35782e62 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -37,7 +37,8 @@ import { accumulateHostExactHostEgress } from './externalEgressAccumulator'
import { hccLogger } from './logger'
import {
confirmAuthoritativeMcpServerAbsence,
- sameMcpServerDesiredRevision,
+ runtimeDesired,
+ sameMcpServerPolicyRevision,
} from './mcpServerSafety'
import {
netPolOrphanSweepCappedTotal,
@@ -1009,7 +1010,7 @@ export class NetworkPolicyReconciler {
const selected = selectedServers.get(serverName)
const current = this.serverCache.get(serverName)
if (!selected || !current) return selected === current
- return sameMcpServerDesiredRevision(selected, current)
+ return sameMcpServerPolicyRevision(selected, current)
})
if (!isCurrent()) return false
@@ -1027,6 +1028,14 @@ export class NetworkPolicyReconciler {
continue
}
+ if (!runtimeDesired(server)) {
+ hccLogger.debug('McpServer runtime not desired; context policy skipped', {
+ serverName,
+ contextId,
+ })
+ continue
+ }
+
const desired = this.buildContextAllowPolicies(context, server)
const ingressName = desired.ingress.metadata!.name!
await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)
@@ -1136,7 +1145,7 @@ export class NetworkPolicyReconciler {
const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()
for (const serverName of allowedServers) {
const server = this.serverCache.get(serverName)
- if (!server) continue
+ if (!server || !runtimeDesired(server)) continue
const desired = this.buildContextAllowPolicies(context, server)
desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)
desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)
@@ -1431,7 +1440,7 @@ export class NetworkPolicyReconciler {
}
const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))
- const desiredServerNames = new Set(servers.map(s => s.name))
+ const desiredServerNames = new Set(servers.filter(runtimeDesired).map(s => s.name))
const recordOrphanDelete = (lane: SafetyInventoryLane, deleted: boolean): void => {
if (!deleted) return
recordSafetyPassRevocation()
@@ -1725,7 +1734,7 @@ export class NetworkPolicyReconciler {
if (!serverCleanupAuthoritative()) return false
if (!options.resolveCurrentServer) return true
const latest = options.resolveCurrentServer(current.name)
- return latest !== undefined && sameMcpServerDesiredRevision(current, latest)
+ return latest !== undefined && sameMcpServerPolicyRevision(current, latest)
}
const completed = await this.reconcileExternalEgressSafety(
current,
@@ -2177,7 +2186,7 @@ export class NetworkPolicyReconciler {
): Promise<boolean> {
if (!isCurrent()) return false
const desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()
- for (const binding of server.spec.egressBindings ?? []) {
+ for (const binding of runtimeDesired(server) ? (server.spec.egressBindings ?? []) : []) {
const name = this.externalEgressPolicyName(server.name, binding)
if (!name) continue
const egressClass = binding.egressClass ?? 'exact-host'
@@ -2348,8 +2357,9 @@ export class NetworkPolicyReconciler {
}
if (!isCurrent()) return
+ const desired = runtimeDesired(server)
const bindings = server.spec.egressBindings
- if (!bindings || bindings.length === 0) {
+ if (!desired || !bindings || bindings.length === 0) {
try {
await this.cleanupExternalEgress(
server.name,
@@ -2373,9 +2383,11 @@ export class NetworkPolicyReconciler {
await this.writeExternalEgressStatus(
server,
[],
- 'True',
- 'NoEgressBindings',
- 'No external egress bindings declared',
+ desired ? 'True' : 'False',
+ desired ? 'NoEgressBindings' : 'RuntimeNotDesired',
+ desired
+ ? 'No external egress bindings declared'
+ : 'Runtime is not eligible for external egress',
isCurrent
)
return
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 13992b5ec..dbe51da3e 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -4,7 +4,6 @@ import {
asAppsApi,
asCoreApi,
asCustomApi,
- asNetworkingApi,
createMockAppsApi,
createMockCoreApi,
createMockCustomApi,
@@ -498,7 +497,6 @@ describe('PR-B B1 — validateSecret result shape', () => {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
- networkingApi: asNetworkingApi(networkingApi),
})
})
@@ -630,16 +628,24 @@ describe('PR-B B1 — validateSecret result shape', () => {
name: 'pg',
namespace: 'mcp-server',
})
- for (const namespace of ['mcp-server', 'mcp-host', 'rpc-proxy']) {
- expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: `np-${namespace}-pg`,
- namespace,
- })
- expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
- name: `np-${namespace}-other`,
- namespace,
+ expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.arrayContaining([
+ expect.objectContaining({
+ path: '/status/conditions',
+ value: expect.arrayContaining([
+ expect.objectContaining({
+ type: 'SecretResolved',
+ status: 'False',
+ reason: 'SecretNotFound',
+ }),
+ ]),
+ }),
+ ]),
})
- }
+ )
expect(reconciler.getStatus('pg')).toMatchObject({ deployed: false, ready: false })
})
@@ -657,7 +663,7 @@ describe('PR-B B1 — validateSecret result shape', () => {
})
await expect(reconciler.reconcile(server)).rejects.toThrow(
- 'Failed to delete runtime resources for McpServer "pg"'
+ 'Failed to delete runtime Kubernetes resources for McpServer "pg"'
)
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'pg',
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index 6be983991..e2400a3f5 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -25,6 +25,7 @@ import { hccLogger } from './logger'
import {
confirmAuthoritativeMcpServerAbsence,
sameMcpServerDesiredRevision,
+ shouldFailClosedForSecretFailure,
} from './mcpServerSafety'
import { createsTotal, mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
@@ -85,7 +86,6 @@ type McpServerReconcilerDeps = {
appsApi?: k8s.AppsV1Api
coreApi?: k8s.CoreV1Api
customApi?: k8s.CustomObjectsApi
- networkingApi?: k8s.NetworkingV1Api
/**
* TEST-ONLY escape hatch. When the inventory-authority fence is not wired
* (setInventoryAuthority / setResolveCurrentServer never called), mutation
@@ -259,7 +259,6 @@ export class McpServerReconciler {
private readonly appsApi: k8s.AppsV1Api
private readonly coreApi: k8s.CoreV1Api
private readonly customApi: k8s.CustomObjectsApi
- private readonly networkingApi?: k8s.NetworkingV1Api
/**
* Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.
@@ -312,9 +311,6 @@ export class McpServerReconciler {
this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
this.customApi = deps?.customApi ?? kc.makeApiClient(k8s.CustomObjectsApi)
- this.networkingApi =
- deps?.networkingApi ??
- (typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)
this.assumeInventoryAuthorityWhenUnconfigured =
deps?.assumeInventoryAuthorityWhenUnconfigured ?? false
}
@@ -1722,73 +1718,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async deleteNetworkPoliciesForServer(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
- if (!this.networkingApi) return
- const failures: unknown[] = []
- const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]
- for (const policyNamespace of namespaces) {
- try {
- const policies = await this.networkingApi.listNamespacedNetworkPolicy({
- namespace: policyNamespace,
- })
- for (const policy of policies.items ?? []) {
- if (policy.metadata?.labels?.[MCPSERVER_LABEL] !== name) continue
- const policyName = policy.metadata.name
- if (!policyName) continue
- try {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.networkingApi.deleteNamespacedNetworkPolicy({
- name: policyName,
- namespace: policyNamespace,
- })
- console.log(
- `[Reconciler] Deleted NetworkPolicy "${policyName}" in "${policyNamespace}"`
- )
- } catch (error: unknown) {
- if (getErrorCode(error) !== 404) {
- console.error(
- `[Reconciler] Failed to delete NetworkPolicy "${policyName}" in "${policyNamespace}":`,
- error
- )
- failures.push(error)
- }
- }
- }
- } catch (error: unknown) {
- if (getErrorCode(error) !== 404) {
- console.error(
- `[Reconciler] Failed to list NetworkPolicies in "${policyNamespace}" for "${name}":`,
- error
- )
- failures.push(error)
- }
- }
- }
-
- throwCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)
- }
-
private async deleteRuntimeResources(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
- const failures: unknown[] = []
- try {
- await this.deleteResources(name, namespace, deleteAllowed)
- } catch (error) {
- failures.push(error)
- }
- try {
- await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)
- } catch (error) {
- failures.push(error)
- }
- throwCleanupFailures(failures, `Failed to delete runtime resources for McpServer "${name}"`)
+ // Policy owners reconcile their own desired sets with ownership and
+ // resource-version preconditions; runtime cleanup must not race them.
+ await this.deleteResources(name, namespace, deleteAllowed)
}
// ─── CRD Status & Annotation Patching ──────────────────────────────
@@ -2183,7 +2120,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
- if (this.shouldFailClosedForSecretFailure(secretResult)) {
+ if (shouldFailClosedForSecretFailure(secretResult.reason)) {
await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
} else {
console.warn(
@@ -2374,16 +2311,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private shouldFailClosedForSecretFailure(
- result: Exclude<SecretValidationResult, { ok: true }>
- ): boolean {
- return (
- result.reason === 'SecretNotFound' ||
- result.reason === 'SecretMissingKey' ||
- result.reason === 'SecretAccessDenied'
- )
- }
-
private async reconcileWrcOwnedServer(
server: McpServerCRD,
isCurrent: () => boolean
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
new file mode 100644
index 000000000..035aa3f06
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+# Optional #604 lifecycle assertions inside the existing real watch-churn gate.
+# The gate owns API fault injection, branch provenance, lock and restoration.
+# This module owns only its labelled fixture and its policy event observer.
+
+NP604_CREATED=0
+NP604_WATCH_PID=''
+
+np604_kctl() {
+ kctl --request-timeout=30s "$@"
+}
+
+np604_setup() {
+ NP604_CONTEXT="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" 1)"
+ NP604_SERVER="$(truncate_rfc1123 "np604-${RUN_ID}-affected")"
+ NP604_CONTROL="$(truncate_rfc1123 "np604-${RUN_ID}-control")"
+ NP604_ENV="$(truncate_rfc1123 "np604-${RUN_ID}-env")"
+ NP604_EVIDENCE="${HCC_BRANCH_GATE_REPO_ROOT}/.local-notes/infra/runs/issue604-${RUN_ID}"
+ mkdir -p "$NP604_EVIDENCE"
+ NP604_CREATED=1
+ # Synthetic, non-production fixture data; never derive it from user state.
+ np604_kctl create secret generic "$NP604_ENV" -n "$MCP_NS" \
+ --from-literal=required=e2e-fixture --dry-run=client -o json |
+ jq --arg run "$RUN_ID" '.metadata.labels={"e2e.clerum.io/run":$run,"e2e.clerum.io/suite":"hcc-np604"}' |
+ np604_kctl apply -f - >/dev/null
+ local name
+ for name in "$NP604_SERVER" "$NP604_CONTROL"; do
+ np604_kctl apply -f - >/dev/null <<EOF
+apiVersion: clerum.io/v1alpha1
+kind: McpServer
+metadata:
+ name: ${name}
+ namespace: ${MCP_NS}
+ labels: {e2e.clerum.io/run: "${RUN_ID}", e2e.clerum.io/suite: hcc-np604}
+spec:
+ contextRef: ${NP604_CONTEXT}
+ image: clerum/mock-mcp-server:test
+ imagePullPolicy: Never
+ transport: {type: streamableHttp, port: 3000, url: "http://${name}.${MCP_NS}.svc.cluster.local:3000/mcp"}
+ egressBindings: [{cidr: 1.2.3.4/32, port: 443, protocol: TCP}]
+EOF
+ done
+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge \
+ -p "$(jq -cn --arg name "$NP604_ENV" '{spec:{envSecret:{name:$name,keys:[{secretKey:"required",envVar:"NP604_FIXTURE"}]}}}')" >/dev/null
+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \
+ -p "$(jq -cn --arg a "$NP604_SERVER" --arg c "$NP604_CONTROL" '{spec:{mcpServers:[$a,$c]}}')" >/dev/null
+}
+
+np604_snapshot() {
+ np604_kctl get networkpolicy -A -l "clerum.io/mcpserver=$1,clerum.io/managed-by=host-context-controller" -o json |
+ jq -Sc '[.items[] | {namespace:.metadata.namespace,name:.metadata.name,type:.metadata.labels["clerum.io/policy-type"],spec}] | sort_by(.namespace,.name)'
+}
+
+np604_ready() {
+ local name="$1"
+ np604_kctl get deployment "$name" -n "$MCP_NS" -o json 2>/dev/null |
+ jq -e '.status.observedGeneration >= .metadata.generation and (.status.readyReplicas // 0) > 0' >/dev/null &&
+ [ "$(np604_snapshot "$name" | jq length)" = 4 ]
+}
+
+np604_invoke() {
+ local name="$1" pod
+ # Real Context-labelled mcp-host created by the parent gate, not a probe
+ # with forged labels. Exercise the MCP protocol over the actual allow path.
+ pod="$(np604_kctl get pods -n "$HOST_NS" -l "clerum.io/context=${NP604_CONTEXT}" -o json |
+ jq -r '[.items[] | select(.metadata.deletionTimestamp == null) | select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))][0].metadata.name // empty')"
+ [ -n "$pod" ] || return 1
+ np604_kctl exec -i "$pod" -n "$HOST_NS" -- node --input-type=module - "$name" "$MCP_NS" <<'JS'
+const [name, namespace] = process.argv.slice(2)
+const url = `http://${name}.${namespace}.svc.cluster.local:3000/mcp`
+let session
+async function rpc(id, method, params) {
+ const headers = {'content-type':'application/json',accept:'application/json, text/event-stream'}
+ if (session) headers['mcp-session-id'] = session
+ const response = await fetch(url, {method:'POST', headers,
+ body:JSON.stringify({jsonrpc:'2.0',...(id === undefined ? {} : {id}),method,params}),
+ signal:AbortSignal.timeout(8000)})
+ if (!response.ok) throw new Error(`MCP request failed: ${response.status}`)
+ session = response.headers.get('mcp-session-id') ?? session
+ const body = await response.text()
+ if (id === undefined) return
+ const messages = response.headers.get('content-type')?.includes('text/event-stream')
+ ? body.split('\n').filter(line => line.startsWith('data:')).map(line => JSON.parse(line.slice(5)))
+ : [JSON.parse(body)]
+ const message = messages.find(item => item.id === id)
+ if (!message || message.error) throw new Error('MCP response missing or rejected')
+ return message.result
+}
+await rpc(1, 'initialize', {protocolVersion:'2024-11-05',capabilities:{},clientInfo:{name:'np604-real-flow',version:'1'}})
+await rpc(undefined, 'notifications/initialized', {})
+const result = await rpc(2, 'tools/call', {name:'add',arguments:{a:17,b:25}})
+if (result.isError || !result.content?.some(item => item.type === 'text' && item.text === '42')) {
+ throw new Error('MCP business result did not equal 42')
+}
+console.log('NP604_MCP_BUSINESS_RESULT=42')
+JS
+}
+
+np604_failed() {
+ local reason="$1" runtime
+ np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json |
+ jq -e --arg reason "$reason" 'any(.status.conditions[]?; .type=="SecretResolved" and .status=="False" and .reason==$reason)' >/dev/null || return 1
+ runtime="$(np604_kctl get deployment "$NP604_SERVER" -n "$MCP_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$runtime" ] && [ "$(np604_snapshot "$NP604_SERVER")" = '[]' ]
+}
+
+np604_observer_alive() {
+ [ -n "$NP604_WATCH_PID" ] && jobs -pr | awk -v pid="$NP604_WATCH_PID" '$1==pid {found=1} END {exit !found}'
+}
+
+np604_observer_witness() {
+ awk -v name="${MCP_NS}/ctx-${NP604_CONTEXT}-${NP604_CONTROL}" \
+ '$1=="MODIFIED" && $2==name {found=1} END {exit !found}' "$NP604_EVIDENCE/policy-events.txt"
+}
+
+np604_before_observation() {
+ wait_until 180 'NP604 affected runtime and four policy families' np604_ready "$NP604_SERVER" || die 'NP604 initial runtime did not converge'
+ wait_until 180 'NP604 control runtime and four policy families' np604_ready "$NP604_CONTROL" || die 'NP604 control did not converge'
+ wait_until 120 'NP604 initial business result from mcp-host' np604_invoke "$NP604_SERVER" || die 'NP604 initial invocation failed'
+ np604_invoke "$NP604_CONTROL" || die 'NP604 control invocation failed'
+ np604_snapshot "$NP604_SERVER" > "$NP604_EVIDENCE/desired-policies.json"
+ np604_kctl get networkpolicy deny-all-mcp-host -n "$HOST_NS" -o json |
+ jq -e '.spec.podSelector=={} and (.spec.policyTypes|index("Ingress"))!=null and (.spec.policyTypes|index("Egress"))!=null' >/dev/null || die 'NP604 default deny missing'
+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge -p '{"data":null}' >/dev/null
+ wait_until 120 'NP604 published failure and policy revocation' np604_failed SecretMissingKey || die 'NP604 did not revoke after failure'
+ np604_invoke "$NP604_CONTROL" || die 'NP604 failure affected healthy control'
+ local snapshot rv
+ snapshot="$(np604_kctl get networkpolicy -A -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o json)" || die 'NP604 observer inventory failed'
+ rv="$(jq -er '.metadata.resourceVersion' <<<"$snapshot")" || die 'NP604 observer missing version'
+ "$KUBECTL_BIN" --context="$E2E_KUBECONTEXT" --request-timeout=180s get networkpolicy -A \
+ -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" \
+ --watch-only --resource-version="$rv" --output-watch-events \
+ -o 'jsonpath={.type}{" "}{.object.metadata.namespace}{"/"}{.object.metadata.name}{"\n"}' \
+ > "$NP604_EVIDENCE/policy-events.txt" 2> "$NP604_EVIDENCE/observer-errors.txt" &
+ NP604_WATCH_PID=$!
+ # Metadata-only witness on our healthy fixture: proves the watch is live
+ # without creating a final policy or replacing HCC's desired policy spec.
+ np604_kctl annotate networkpolicy "ctx-${NP604_CONTEXT}-${NP604_CONTROL}" -n "$MCP_NS" \
+ "e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/null
+ wait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'
+ NP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
+}
+
+np604_after_observation() {
+ np604_observer_alive || die 'NP604 observer stopped before completing observation'
+ local current_cuts
+ current_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
+ [ "$((current_cuts - NP604_CUT_BASE))" -ge 3 ] || die 'NP604 needs three complete reconnect cycles in stable failed state'
+ awk -v target="$NP604_SERVER" '$1=="ADDED" && index($2,target) {bad=1} END {exit bad}' \
+ "$NP604_EVIDENCE/policy-events.txt" || die 'NP604 recreated a policy while runtime remained undesired'
+ kill "$NP604_WATCH_PID"
+ wait "$NP604_WATCH_PID" 2>/dev/null || true
+ NP604_WATCH_PID=''
+ np604_invoke "$NP604_CONTROL" || die 'NP604 control lost connectivity under churn'
+}
+
+np604_recover() {
+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge \
+ -p '{"stringData":{"required":"e2e-fixture"}}' >/dev/null
+ wait_until 180 'NP604 repaired runtime and policies' np604_ready "$NP604_SERVER" || die 'NP604 repair did not converge'
+ wait_until 120 'NP604 repaired business result' np604_invoke "$NP604_SERVER" || die 'NP604 repair did not restore invocation'
+ [ "$(np604_snapshot "$NP604_SERVER")" = "$(cat "$NP604_EVIDENCE/desired-policies.json")" ] || die 'NP604 recovered policy set differs'
+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge -p '{"data":null}' >/dev/null
+ wait_until 120 'NP604 second failure' np604_failed SecretMissingKey || die 'NP604 second failure did not converge'
+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge -p '{"spec":{"envSecret":null}}' >/dev/null
+ wait_until 180 'NP604 recovery after removing reference' np604_ready "$NP604_SERVER" || die 'NP604 removed reference did not recover'
+ np604_invoke "$NP604_SERVER" || die 'NP604 removed reference did not restore invocation'
+ [ "$(np604_snapshot "$NP604_SERVER")" = "$(cat "$NP604_EVIDENCE/desired-policies.json")" ] || die 'NP604 removed reference policy set differs'
+ printf 'NP604_LIFECYCLE=PASS\nevidence=%s\n' "$NP604_EVIDENCE"
+}
+
+np604_cleanup() {
+ if np604_observer_alive; then
+ kill "$NP604_WATCH_PID"
+ wait "$NP604_WATCH_PID" 2>/dev/null || true
+ fi
+ NP604_WATCH_PID=''
+ [ "$NP604_CREATED" = 1 ] || return 0
+ np604_kctl delete mcpserver,secret -n "$MCP_NS" \
+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null
+}
+
+np604_resources_absent() {
+ local remaining
+ remaining="$(np604_kctl get deployment,service,networkpolicy -A \
+ -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o name)" || return 1
+ [ -z "$remaining" ]
+}
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
index 414d48ab8..c568de800 100755
--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -179,6 +179,8 @@ EOF
done; } | kctl apply -f - >/dev/null
# McpServers (one stream) — inventory + NetworkPolicies, no heavy pods.
+ # Lifecycle suites supply their own real MCP fixtures and need no filler.
+ if (( n_mcp > 0 )); then
{ for ((i = 1; i <= n_mcp; i++)); do
ctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" $(((i % n_ctx) + 1)))"
cat <<EOF
@@ -194,6 +196,7 @@ spec:
transport: {type: streamableHttp, url: "http://127.0.0.1:3000/mcp", port: 3000}
EOF
done; } | kctl apply -f - >/dev/null
+ fi
# Hosts (one stream) — these materialize Deployments; keep the count modest.
{ for ((i = 1; i <= n_host; i++)); do
diff --git a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
new file mode 100755
index 000000000..cfeaa6345
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# #604 runtime proof. The underlying gate enforces branch/profile provenance,
+# holds its fault-injection lock and restores HCC before returning a verdict.
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+: "${MINIKUBE_PROFILE:?explicit branch-owned profile required}"
+: "${CONTROL_API_REAL_PG_CONTEXT:?explicit branch-owned context required}"
+[[ "$MINIKUBE_PROFILE" = "$CONTROL_API_REAL_PG_CONTEXT" ]] || {
+ echo 'FAIL: profile and context must match' >&2
+ exit 2
+}
+export KUBECONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
+export E2E_HCC_WATCH_FAULT_INJECTION=1
+export E2E_HCC_POLICY_LIFECYCLE=1
+export E2E_EXPECTED_PRE_GATE_GATE=minikube-t2
+# The two real MCP workloads in the lifecycle fixture provide the business
+# witness. Other synthetic McpServers would add unrelated crashing workloads.
+export FLEET_CONTEXTS=1 FLEET_MCPSERVERS=0 FLEET_HOSTS=1
+export CHURN_MIN_CUTS=3 CHURN_OBSERVE_SEC=45
+exec bash "$SCRIPT_DIR/e2e-hcc-watch-churn-readiness.sh"
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index c894191a1..61d231c67 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -39,6 +39,16 @@ source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-churn-fixture.sh"
+# #604 reuses the real TCP fault injection and provenance/restore contract.
+case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in
+ 0) ;;
+ 1)
+ # shellcheck source=scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+ source "${SCRIPT_DIR}/_lib/hcc-networkpolicy-lifecycle.sh"
+ ;;
+ *) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;
+esac
+
# ── Fail-closed guards (identical contract to the sibling HCC gates) ──
[ -n "$E2E_KUBECONTEXT" ] || {
echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2
@@ -123,6 +133,7 @@ PROXY_CREATED=0
PROBE_CREATED=0
FLEET_CREATED=0
HCC_PATCHED=0
+HCC_SCALED_DOWN=0
# Log-stream state consumed by hcc-watch-recovery-logs.sh (start/stop_hcc_recovery_log_stream).
# Declared here so `set -u` never trips when the stream helper first touches them.
HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn-stream.XXXXXX")"
@@ -275,6 +286,9 @@ cleanup() {
trap - EXIT
set +e
stop_hcc_recovery_log_stream
+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ np604_cleanup || cleanup_failed=1
+ fi
if [ "$HCC_PATCHED" = 1 ]; then
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1
wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
@@ -282,7 +296,11 @@ cleanup() {
[ "$FLEET_CREATED" = 1 ] && { delete_synthetic_fleet || cleanup_failed=1; }
if [ "$HCC_PATCHED" = 1 ]; then
restore_hcc_after_churn || restore_ok=0
- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1
+ fi
+ # Fleet creation can fail after scale-down but before the redirect patch.
+ # Restore replicas even then, without stripping untouched template env.
+ if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || restore_ok=0
fi
[ "$PROXY_CREATED" = 1 ] && kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1
@@ -292,6 +310,9 @@ cleanup() {
print_repair_instructions
cleanup_failed=1
}
+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] && [ "$NP604_CREATED" = 1 ] && [ "$restore_ok" = 1 ]; then
+ wait_until 120 "NP604 fixture resources removed by owners" np604_resources_absent || cleanup_failed=1
+ fi
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
print_results || status=1
[ "$cleanup_failed" = 0 ] || status=1
@@ -320,6 +341,7 @@ create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"
verify_hcc_proxy_network_policy
proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"
[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"
+HCC_SCALED_DOWN=1
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop before fleet creation"
@@ -327,6 +349,9 @@ wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop befor
log "Creating synthetic fleet: ${FLEET_CONTEXTS} Contexts / ${FLEET_MCPSERVERS} McpServers / ${FLEET_HOSTS} Hosts"
create_synthetic_fleet "$FLEET_CONTEXTS" "$FLEET_MCPSERVERS" "$FLEET_HOSTS"
ok "synthetic fleet created (${FLEET_MCPSERVERS} McpServers)"
+if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ np604_setup
+fi
# ── FASE D: start HCC under churn and poll /ready ──
log "Redirecting HCC through the churn proxy and starting it under load"
@@ -336,8 +361,8 @@ patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template
{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},
{name:"KUBERNETES_SERVICE_PORT",value:"443"},
{name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"
-kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null
HCC_PATCHED=1
+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
# Anchor the log stream to now so recovery-cycle assertions only see churn-era logs.
START_TIME="$(python3 -c 'import datetime; print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))')"
@@ -398,8 +423,14 @@ write_evidence_artifact() {
# watch drops /ready to 503 transiently while it re-syncs, so "no 503 after
# first 200" failed the CORRECT regime and measured the wrong thing.
if [ -n "$first_ready" ]; then
+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ np604_before_observation
+ fi
log "Observing /ready at 1Hz for ${CHURN_OBSERVE_SEC}s under continuous churn"
sample_ready_series "$CHURN_OBSERVE_SEC" churn 0
+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ np604_after_observation
+ fi
fi
# ── Pause the churn WITHOUT restarting the proxy or the HCC ──
@@ -495,5 +526,9 @@ if [ -n "$hcc_pod_at_stop" ] && [ "$hcc_pod_now" = "$hcc_pod_at_stop" ] &&
else
fail "HCC pod changed or restarted across the post-churn phase (${hcc_pod_at_stop:-none}/restarts=${hcc_restarts_at_stop:-?} -> ${hcc_pod_now:-none}/restarts=${hcc_restarts_now:-?}) — convergence could be a fresh boot, not in-situ recovery"
fi
+if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
+ [ -n "$first_ready" ] || die 'NP604 cannot run without a ready controller'
+ np604_recover
+fi
log "Evidence artifact: ${LOG_ARTIFACT}"
# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.
diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
new file mode 100755
index 000000000..593e593bf
--- /dev/null
+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+tmp="$(mktemp -d)"
+trap 'rm -rf -- "$tmp"' EXIT
+# shellcheck source=scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+source "$ROOT/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"
+export FLEET_SECRET=fixture-llm HOST_NS=mcp-host MCP_NS=mcp-server
+export FLEET_PREFIX=fixture RUN_ID=fixture HCC_IMAGE=fixture-image
+expected_mcp=0
+truncate_rfc1123() { printf '%s' "$1"; }
+die() { echo "$*" >&2; exit 1; }
+kctl() {
+ case "$1" in
+ create) printf '%s\n' 'apiVersion: v1' 'kind: Secret' ;;
+ label) cat ;;
+ apply)
+ local input
+ input="$(cat)"
+ [[ -n "$input" ]] || { echo 'FAIL: empty apply payload' >&2; return 1; }
+ printf '%s\n' "$input" >> "$tmp/applied"
+ ;;
+ get)
+ if (( expected_mcp > 0 )); then printf '%s\n' 'mcpserver/fixture'; fi
+ ;;
+ *) echo "FAIL: unexpected kubectl operation $1" >&2; return 1 ;;
+ esac
+}
+create_synthetic_fleet 1 0 1
+[[ "$FLEET_CREATED" = 1 ]]
+grep -q '^kind: Context' "$tmp/applied"
+grep -q '^kind: Host' "$tmp/applied"
+if grep -q '^kind: McpServer' "$tmp/applied"; then
+ echo 'FAIL: zero-sized MCP inventory emitted a filler resource' >&2
+ exit 1
+fi
+: > "$tmp/applied"
+expected_mcp=1
+create_synthetic_fleet 1 1 1
+grep -q '^kind: McpServer' "$tmp/applied"
+grep -q '^kind: Context' "$tmp/applied"
+grep -q '^kind: Host' "$tmp/applied"
+echo 'PASS: zero MCP inventory omits empty apply; nonzero inventory still creates resources'
+
+# Run the actual EXIT cleanup in a subshell with runtime calls replaced by
+# recording stubs. In particular, do not model the cleanup in the test.
+cleanup_source="$(sed -n '/^cleanup() {/,/^}/p' "$ROOT/scripts/e2e/e2e-hcc-watch-churn-readiness.sh")"
+[[ -n "$cleanup_source" ]]
+for patched in 0 1; do
+ (
+ HCC_PATCHED="$patched" HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1
+ HCC_DEPLOY=host-context-controller HCC_NS=control-plane
+ FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
+ E2E_HCC_POLICY_LIFECYCLE=0
+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe
+ HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"
+ kctl() { printf '%s\n' "$*" >> "$tmp/cleanup-$patched"; }
+ stop_hcc_recovery_log_stream() { :; }
+ wait_until() { :; }
+ restore_hcc_after_churn() { echo restore-template >> "$tmp/cleanup-$patched"; }
+ finalize_hcc_watch_gate_lock() { :; }
+ print_results() { :; }
+ eval "$cleanup_source"
+ cleanup
+ )
+ grep -q 'scale deployment host-context-controller -n control-plane --replicas=1' "$tmp/cleanup-$patched"
+done
+if grep -q restore-template "$tmp/cleanup-0"; then
+ echo 'FAIL: early failure rewrote a template the gate had not changed' >&2
+ exit 1
+fi
+grep -q restore-template "$tmp/cleanup-1"
+echo 'PASS: cleanup restores replicas before/after redirect and preserves an untouched template'