Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): converge policies with published runtime eligibility
Alfredo Lopez06355c987fdf
diff --git a/Makefile b/Makefileindex 4eea8240c..49fc7e38d 100644--- a/Makefile+++ b/Makefile@@ -671,6 +671,13 @@ minikube-t2: ## Full orchestrator: T0, Real PostgreSQL T1, then exact-head T2.PHONY: minikube-t2-np08-hcc-authorizationminikube-t2-np08-hcc-authorization: minikube-t2 ## Run canonical T2 including the required deployed NP-08 Host-to-HCC authorization journey+# Dedicated #604 certification: the bounded health lane runs the real+# policy lifecycle, business invocation and three watch reconnections.+.PHONY: minikube-t2-hcc-networkpolicy-lifecycle+minikube-t2-hcc-networkpolicy-lifecycle:+ @T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \+ T2_HEALTHCHECK_TIMEOUT_SECONDS=900 $(MAKE) minikube-t2+.PHONY: minikube-t2-runtimeminikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile@T2_RUN_T0=false T2_RUN_T1=false \diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 20238072a..331343bce 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.310",+ "version": "0.1.311","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.310",+ "version": "0.1.311","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex b409f572e..bc676b887 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.310",+ "version": "0.1.311","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 65eea1818..23c963224 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -3587,6 +3587,7 @@ describe('McpServerWatcher startup', () => {})await (watcher as any).startContextWatch('context-policy-rv')+ ;(watcher as any).contextCacheSynced = trueconst serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)await oldStarted.promise@@ -6118,7 +6119,11 @@ describe('McpServerWatcher external egress retries', () => {it('retries failed external egress reconciliation from an ADDED event', async () => {const watcher = new McpServerWatcher()+ // This test isolates the per-server retry; fleet convergence is covered+ // separately and would independently retry the same failed egress.+ vi.spyOn(watcher as any, 'runInitialNetworkPolicyConvergence').mockResolvedValue(undefined)markMcpServerInventoryAuthoritative(watcher)+ ;(watcher as any).contextCacheSynced = trueconst netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilerconst bindingReconciler = (watcher as any).bindingReconciler@@ -10934,6 +10939,97 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)})+ it('recovers policy verdict edges from re-LIST without invalidating on unrelated status (#604)', () => {+ const watcher = new McpServerWatcher()+ const state = watcher as any+ const server = {+ name: 'policy-server',+ namespace: 'mcp-server',+ uid: 'uid-policy',+ generation: 1,+ spec: {+ image: 'fixture:v1',+ contextRef: 'ctx',+ transport: { type: 'streamableHttp', port: 3000 },+ envSecret: { name: 'fixture-env', keys: [] },+ },+ status: {+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],+ },+ }+ state.installMcpServerSnapshot({ servers: [server], resourceVersion: '1' })+ const initialRevision = state.mcpServerDesiredRevision+ const recovered = {+ ...server,+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },+ }+ state.installMcpServerSnapshot({ servers: [recovered], resourceVersion: '2' })+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 1)+ state.installMcpServerSnapshot({ servers: [recovered], resourceVersion: '3' })+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 1)+ state.installMcpServerSnapshot({ servers: [server], resourceVersion: '4' })+ expect(state.mcpServerDesiredRevision).toBe(initialRevision + 2)+ })++ it('propagates only an effective WATCH verdict edge into policy work (#604)', async () => {+ const watcher = new McpServerWatcher()+ const state = watcher as any+ state.mcpServerCacheSynced = true+ state.contextCacheSynced = true+ const server = {+ name: 'policy-server',+ namespace: 'mcp-server',+ uid: 'uid-policy',+ generation: 1,+ spec: {+ image: 'fixture:v1',+ contextRef: 'ctx',+ transport: { type: 'streamableHttp', port: 3000 },+ envSecret: { name: 'fixture-env', keys: [] },+ },+ status: {+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],+ },+ }+ state.servers.set(server.name, server)+ const full = vi.spyOn(state, 'runInitialNetworkPolicyConvergence').mockResolvedValue(undefined)+ const retry = vi+ .spyOn(state.externalEgressCoordinator, 'scheduleRetry')+ .mockImplementation(() => {})+ const scoped = vi.spyOn(state, 'reconcileContextsForMcpServer').mockResolvedValue(undefined)+ const callback = state.getMcpServerWatchCallback()+ const event = (status: typeof server.status) => ({+ metadata: {+ name: server.name,+ namespace: server.namespace,+ uid: server.uid,+ generation: server.generation,+ },+ spec: server.spec,+ status,+ })+ await callback(+ 'MODIFIED',+ event({ conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] })+ )+ expect(full).toHaveBeenCalledTimes(1)+ expect(scoped).toHaveBeenCalledTimes(1)+ expect(retry).toHaveBeenCalledTimes(1)+ for (let repeat = 0; repeat < 3; repeat++) {+ await callback(+ 'MODIFIED',+ event({ conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] })+ )+ }+ expect(scoped).toHaveBeenCalledTimes(1)+ expect(retry).toHaveBeenCalledTimes(1)+ await callback('MODIFIED', event(server.status))+ expect(full).toHaveBeenCalledTimes(2)+ expect(scoped).toHaveBeenCalledTimes(2)+ expect(retry).toHaveBeenCalledTimes(2)+ await watcher.stop()+ })+it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {const watcher = new McpServerWatcher()const hostA = {diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 216143451..bef2ed366 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -41,6 +41,7 @@ import { K8sGfsApi } from './k8s/gfsK8sApi'import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'+import { hccLogger } from './logger'import type {AuthorityContext,AuthorityHost,@@ -52,7 +53,9 @@ import type {import {confirmAuthoritativeMcpServerAbsence,isMcpServerStatusOnlyUpdate,+ runtimeDesired,sameMcpServerDesiredRevision,+ sameMcpServerPolicyRevision,} from './mcpServerSafety'import {contextMetadataOnlyEventsTotal,@@ -845,7 +848,7 @@ export class McpServerWatcher implements McpServerProvider {listServers: () => [...this.servers.values()],getCurrentServer: name => this.servers.get(name),inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,- sameDesiredRevision: sameMcpServerDesiredRevision,+ sameDesiredRevision: sameMcpServerPolicyRevision,enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),replay: (type, server, retry) =>@@ -1406,7 +1409,7 @@ export class McpServerWatcher implements McpServerProvider {if (snapshot.servers.length !== this.servers.size) return truefor (const server of snapshot.servers) {const previous = this.servers.get(server.name)- if (previous === undefined || !sameMcpServerDesiredRevision(previous, server)) {+ if (previous === undefined || !sameMcpServerPolicyRevision(previous, server)) {return true}}@@ -3919,7 +3922,7 @@ export class McpServerWatcher implements McpServerProvider {let desiredStateChanged = falseif (type === 'ADDED' || type === 'MODIFIED') {desiredStateChanged =- previous === undefined || !sameMcpServerDesiredRevision(previous, server)+ previous === undefined || !sameMcpServerPolicyRevision(previous, server)this.servers.set(server.name, server)} else if (type === 'DELETED') {if (!previous?.uid || !server.uid || previous.uid === server.uid) {@@ -3931,11 +3934,25 @@ export class McpServerWatcher implements McpServerProvider {void this.runInitialNetworkPolicyConvergence()}- // HCC writes McpServer status during reconciliation. Those writes emit- // MODIFIED events but do not change the desired runtime or policy state.- // Suppress that self-induced work while still publishing the fresh status- // through discovery. Any UID/spec/annotation/label change remains live.+ // Most status writes do not change desired state. A runtime eligibility+ // edge does change policy intent: propagate it without re-entering the+ // runtime owner merely because it published status. Discovery still+ // receives every update; UID/spec/annotation/label changes remain live.if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {+ if (runtimeDesired(previous) !== runtimeDesired(server)) {+ // The revision bump above retires old policy work and requests a+ // certified safety pass; replay also retries per-server additions.+ this.externalEgressCoordinator.scheduleRetry('MODIFIED', server)+ try {+ await this.reconcileContextsForMcpServer(server, watchGeneration)+ } catch (err) {+ // The certified fleet pass and the server replay remain scheduled.+ hccLogger.error('Runtime verdict policy propagation failed', {+ serverName: server.name,+ err,+ })+ }+ }this.changeCallback?.()return}@@ -3950,6 +3967,42 @@ export class McpServerWatcher implements McpServerProvider {}}+ private async reconcileContextsForMcpServer(+ server: McpServerCRD,+ watchGeneration: number+ ): Promise<void> {+ const contextGeneration = this.contextWatchGeneration+ const isCurrent = (): boolean => {+ const current = this.servers.get(server.name)+ return (+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&+ current !== undefined &&+ sameMcpServerPolicyRevision(server, current) &&+ this.hasContextInventoryAuthority(contextGeneration)+ )+ }+ for (const selected of this.contexts.values()) {+ if (!isCurrent()) return+ if (!selected.spec.mcpServers?.includes(server.name)) continue+ const contextId = selected.spec.contextId+ await this.enqueueContextReconciliation(contextId, async () => {+ if (!isCurrent()) return+ const current = this.contexts.get(selected.name)+ if (+ current?.spec.contextId !== contextId ||+ !current.spec.mcpServers?.includes(server.name)+ )+ return+ const completed = await this.netPolReconciler.reconcileContext(current, {+ isCurrent: () => isCurrent() && this.contexts.get(current.name) === current,+ })+ if (completed === false && isCurrent()) {+ this.scheduleExternalEgressRetry('MODIFIED', server)+ }+ })+ }+ }+private async reconcileMcpServerWatchEvent(type: ExternalEgressWatchEventType,server: McpServerCRD,@@ -3973,11 +4026,6 @@ export class McpServerWatcher implements McpServerProvider {}const deleteAllowed = () =>this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)- const contextInventoryGeneration = this.contextWatchGeneration- const contextPolicyInventoryIsCurrent = (): boolean =>- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&- this.hasContextInventoryAuthority(contextInventoryGeneration)-// External egress is part of the workload's pre-start contract. Reconcile// it before HCC creates or updates any managed runtime Deployment so a// stdio MCP with egressBindings cannot start before ExternalEgressReady.@@ -4077,30 +4125,7 @@ export class McpServerWatcher implements McpServerProvider {if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) returnserver = this.servers.get(server.name)!}- // Re-reconcile cached Contexts after the server cache is populated.- for (const selectedContext of this.contexts.values()) {- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- if (!selectedContext.spec.mcpServers?.includes(server.name)) continue- const selectedContextId = selectedContext.spec.contextId- console.log(- `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`- )- await this.enqueueContextReconciliation(selectedContextId, async () => {- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- const currentContext = this.contexts.get(selectedContext.name)- if (- currentContext?.spec.contextId !== selectedContextId ||- !currentContext.spec.mcpServers?.includes(server.name)- ) {- return- }- await this.netPolReconciler.reconcileContext(currentContext, {- isCurrent: () =>- contextPolicyInventoryIsCurrent() &&- this.contexts.get(currentContext.name) === currentContext,- })- })- }+ await this.reconcileContextsForMcpServer(server, watchGeneration)} else if (type === 'DELETED') {const recipeName = server.labels?.['clerum.io/recipe'] ?? server.nameawait this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.tsindex 32a0329a0..08821c206 100644--- a/host-context-controller/src/mcpServerSafety.test.ts+++ b/host-context-controller/src/mcpServerSafety.test.ts@@ -2,7 +2,9 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest'import {confirmAuthoritativeMcpServerAbsence,isMcpServerStatusOnlyUpdate,+ runtimeDesired,sameMcpServerDesiredRevision,+ sameMcpServerPolicyRevision,} from './mcpServerSafety'import type { McpServerCRD } from './types'@@ -312,3 +314,57 @@ describe('confirmAuthoritativeMcpServerAbsence', () => {expect(resolveCurrent).toHaveBeenCalledTimes(2)})})++describe('runtimeDesired policy contract (#604)', () => {+ const reasons = ['SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied', 'ReadError']+ function withVerdict(reason: string, managed = true): McpServerCRD {+ const server = makeServer()+ server.spec = {+ ...server.spec,+ managed,+ envSecret: { name: 'fixture-env', keys: [{ secretKey: 'required', envVar: 'VALUE' }] },+ }+ server.status = { conditions: [{ type: 'SecretResolved', status: 'False', reason }] }+ return server+ }++ it.each(reasons)('preserves ownership semantics for %s', reason => {+ const managed = withVerdict(reason)+ const wrc = withVerdict(reason, false)+ expect(runtimeDesired(managed)).toBe(reason === 'ReadError')+ expect(runtimeDesired(wrc)).toBe(true)+ for (const server of [managed, wrc]) {+ server.spec.enabled = false+ expect(runtimeDesired(server)).toBe(false)+ }+ })++ it('recovers when the reference is removed despite a stale negative condition', () => {+ const server = withVerdict('SecretNotFound')+ expect(runtimeDesired(server)).toBe(false)+ delete server.spec.envSecret+ expect(runtimeDesired(server)).toBe(true)+ })++ it('retires policy effects, but not runtime effects, on a published verdict edge', () => {+ const previous = withVerdict('SecretNotFound')+ const current = withVerdict('SecretFound')+ expect(sameMcpServerDesiredRevision(previous, current)).toBe(true)+ expect(sameMcpServerPolicyRevision(previous, current)).toBe(false)+ expect(sameMcpServerPolicyRevision(current, previous)).toBe(false)+ expect(sameMcpServerPolicyRevision(previous, withVerdict('SecretMissingKey'))).toBe(true)+ expect(+ sameMcpServerPolicyRevision(+ withVerdict('SecretNotFound', false),+ withVerdict('SecretFound', false)+ )+ ).toBe(true)+ })++ it('does not require status for an otherwise desired server', () => {+ expect(runtimeDesired(makeServer())).toBe(true)+ const server = withVerdict('SecretNotFound')+ server.status = undefined+ expect(runtimeDesired(server)).toBe(true)+ })+})diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.tsindex 5b15bda16..bcdde5a58 100644--- a/host-context-controller/src/mcpServerSafety.ts+++ b/host-context-controller/src/mcpServerSafety.ts@@ -91,3 +91,33 @@ export async function confirmAuthoritativeMcpServerAbsence({return inventoryAuthoritative() && !resolveCurrent()}++/** The runtime owner is the only source of credential validation. */+export function shouldFailClosedForSecretFailure(reason: string | undefined): boolean {+ return (+ reason === 'SecretNotFound' || reason === 'SecretMissingKey' || reason === 'SecretAccessDenied'+ )+}++/**+ * HCC owns policies for both runtime owners, but its Secret verdict only+ * controls HCC-managed runtime. WRC permits optional keys HCC cannot judge.+ * A removed reference makes an old SecretResolved condition irrelevant.+ */+export function runtimeDesired(server: McpServerCRD): boolean {+ if (server.spec.enabled === false) return false+ if (server.spec.managed === false || !server.spec.envSecret) return true+ const condition = server.status?.conditions?.find(item => item.type === 'SecretResolved')+ return !shouldFailClosedForSecretFailure(condition?.reason)+}++/** Policy effects also depend on the effective published runtime verdict. */+export function sameMcpServerPolicyRevision(+ expected: McpServerCRD,+ current: McpServerCRD+): boolean {+ return (+ sameMcpServerDesiredRevision(expected, current) &&+ runtimeDesired(expected) === runtimeDesired(current)+ )+}diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 3dc6a0ea8..385abb2f2 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -4,6 +4,13 @@ import type { RecordWithTtl } from 'node:dns'import * as dns from 'node:dns/promises'import { readFileSync } from 'node:fs'import { join } from 'node:path'+import {+ asAppsApi,+ asCoreApi,+ asCustomApi,+ createMockAppsApi,+ createMockCoreApi,+} from '../test/__fixtures__/testMocks'import { asApiserverNetworkPolicy } from './__tests__/asApiserverNetworkPolicy'import { config } from './config'import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL, POLICY_TYPE_LABEL } from './constants'@@ -18,6 +25,7 @@ import {PUBLIC_EGRESS_EXCEPT_CIDRS,sameContextDesiredRevision,} from './networkPolicyReconciler'+import { McpServerReconciler } from './reconciler'import { ContextCRD, McpServerCRD } from './types'import { getErrorCode } from './utils'@@ -76,6 +84,7 @@ vi.mock('node:dns/promises', () => ({}))vi.mock('./metrics', () => ({+ mcpserverMissingSecret: { set: vi.fn() },createsTotal: { inc: vi.fn() },existenceReadsTotal: { inc: vi.fn() },networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },@@ -6274,4 +6283,394 @@ describe('NetworkPolicyReconciler', () => {expect(store.has(orphanKey)).toBe(false)})})+ describe('published runtime policy lifecycle (#604)', () => {+ function lifecycle() {+ config.netPolOrphanDeleteCapPercent = 100+ const store = new Map<string, k8s.V1NetworkPolicy>()+ const events: string[] = []+ let revision = 0+ const missing = () => Object.assign(new Error('not found'), { code: 404 })+ const save = (namespace: string, body: k8s.V1NetworkPolicy) => {+ const key = `${namespace}/${body.metadata!.name}`+ const previous = store.get(key)+ const result = structuredClone({+ ...body,+ metadata: {+ ...body.metadata,+ namespace,+ uid: previous?.metadata?.uid ?? `${key}-${++revision}`,+ resourceVersion: String(++revision),+ },+ })+ store.set(key, result)+ return result+ }+ mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {+ const key = `${namespace}/${body.metadata.name}`+ if (store.has(key)) throw Object.assign(new Error('exists'), { code: 409 })+ events.push(`create:${key}`)+ return save(namespace, body)+ })+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) =>+ save(namespace, body)+ )+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {+ const found = store.get(`${namespace}/${name}`)+ if (!found) throw missing()+ return structuredClone(found)+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }) => ({+ items: [...store.values()]+ .filter(+ policy =>+ policy.metadata?.namespace === namespace &&+ (!labelSelector ||+ labelSelector.split(',').every((clause: string) => {+ const [key, value] = clause.split('=')+ return policy.metadata?.labels?.[key] === value+ }))+ )+ .map(policy => structuredClone(policy)),+ })+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, name, body }) => {+ const key = `${namespace}/${name}`+ const found = store.get(key)+ if (!found) throw missing()+ expect(body?.preconditions).toEqual({+ uid: found.metadata!.uid,+ resourceVersion: found.metadata!.resourceVersion,+ })+ events.push(`delete:${key}`)+ store.delete(key)+ return {}+ }+ )+ const initial: McpServerCRD = {+ name: 'lifecycle',+ namespace: 'mcp-server',+ uid: 'server-lifecycle',+ generation: 1,+ spec: {+ contextRef: 'ctx',+ image: 'fixture:v1',+ envSecret: { name: 'fixture-env', keys: [{ secretKey: 'required', envVar: 'VALUE' }] },+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],+ },+ }+ const cache = new Map([[initial.name, initial]])+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async ({ name }) => {+ const server = cache.get(name)+ if (!server) throw missing()+ return {+ metadata: {+ name,+ namespace: server.namespace,+ uid: server.uid,+ generation: server.generation,+ resourceVersion: '1',+ },+ spec: server.spec,+ status: server.status,+ }+ })+ mockCustomApi.getNamespacedCustomObjectStatus.mockImplementation(+ mockCustomApi.getNamespacedCustomObject.getMockImplementation()!+ )+ const rec = makeReconciler(mockApi, cache, mockCustomApi)+ // Use the stateful API directly: the general fixture caches LIST snapshots,+ // which would mask a deleted resource during this lifecycle reproduction.+ ;(rec as any).networkingApi = mockApi+ const context: ContextCRD = {+ name: 'ctx',+ namespace: 'mcp-server',+ spec: { contextId: 'ctx', mcpServers: [initial.name] },+ }+ const run = async () => {+ const server = cache.get(initial.name)!+ await rec.fullReconcile([context], [server], {+ ensureDefaults: false,+ resolveCurrentServer: name => cache.get(name),+ })+ await rec.reconcileExternalEgress(server, {+ isCurrent: () => cache.get(server.name) === server,+ })+ }+ return { rec, initial, cache, context, store, events, run }+ }++ it.each(['Disabled', 'SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied'])(+ 'revokes once and restores all four families after %s',+ async reason => {+ const f = lifecycle()+ await f.run()+ expect(f.store.size).toBe(4)+ const policyKeys = [...f.store.keys()].sort()+ const failed = structuredClone(f.initial)+ if (reason === 'Disabled') failed.spec.enabled = false+ else failed.status = { conditions: [{ type: 'SecretResolved', status: 'False', reason }] }+ f.cache.set(failed.name, failed)+ await f.run()+ expect(f.store.size).toBe(0)+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(4)+ const settled = f.events.length+ for (let cycle = 0; cycle < 3; cycle++) await f.run()+ expect(f.events).toHaveLength(settled)+ f.cache.set(f.initial.name, structuredClone(f.initial))+ await f.run()+ expect([...f.store.keys()].sort()).toEqual(policyKeys)+ expect(f.events.filter(event => event.startsWith('create:'))).toHaveLength(8)+ }+ )++ it.each(['SecretNotFound', 'SecretMissingKey', 'SecretAccessDenied', 'ReadError'])(+ 'keeps WRC policies stable for %s but honors disable',+ async reason => {+ const f = lifecycle()+ const server = structuredClone(f.initial)+ server.spec.managed = false+ f.cache.set(server.name, server)+ await f.run()+ expect(f.store.size).toBe(4)+ f.cache.set(server.name, {+ ...server,+ status: { conditions: [{ type: 'SecretResolved', status: 'False', reason }] },+ })+ for (let cycle = 0; cycle < 3; cycle++) await f.run()+ expect(f.store.size).toBe(4)+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(0)+ f.cache.set(server.name, { ...server, spec: { ...server.spec, enabled: false } })+ await f.run()+ expect(f.store.size).toBe(0)+ }+ )++ it('runs the real runtime owner between policy passes without recreating a revoked allow', async () => {+ const f = lifecycle()+ await f.run()+ expect(f.store.size).toBe(4)+ const apps = createMockAppsApi()+ const core = createMockCoreApi()+ core.readNamespacedSecret.mockRejectedValue(+ Object.assign(new Error('missing fixture'), { code: 404 })+ )+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {+ const patch = body.find(+ (item: { path: string }) => item.path === '/status/conditions' || item.path === '/status'+ )+ if (patch)+ f.cache.set(f.initial.name, {+ ...f.cache.get(f.initial.name)!,+ status: { conditions: patch.path === '/status' ? patch.value.conditions : patch.value },+ })+ return {}+ })+ const runtime = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,+ appsApi: asAppsApi(apps),+ coreApi: asCoreApi(core),+ customApi: asCustomApi(mockCustomApi as any),+ })+ await runtime.reconcile(f.initial)+ expect(apps.deleteNamespacedDeployment).toHaveBeenCalledWith({+ name: f.initial.name,+ namespace: f.initial.namespace,+ })+ expect(f.cache.get(f.initial.name)?.status?.conditions).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: 'SecretResolved',+ status: 'False',+ reason: 'SecretNotFound',+ }),+ ])+ )+ expect(f.store.size).toBe(4) // Runtime owner has not deleted policies.+ await f.run()+ expect(f.store.size).toBe(0)+ const stationary = [...f.events]+ for (let cycle = 0; cycle < 3; cycle++) {+ await runtime.reconcile(f.cache.get(f.initial.name)!)+ await f.run()+ }+ expect(f.events).toEqual(stationary)+ })++ it('never deletes another owner policy selecting the same disabled server', async () => {+ const f = lifecycle()+ await f.run()+ const ownedElsewhere: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'recipe-owned-lifecycle',+ namespace: 'mcp-server',+ uid: 'wrc-policy',+ resourceVersion: '1',+ labels: { [MANAGED_BY_LABEL]: 'workflow-recipes', 'clerum.io/recipe': 'fixture-recipe' },+ },+ spec: {+ podSelector: { matchLabels: { [MCPSERVER_LABEL]: f.initial.name } },+ policyTypes: ['Ingress'],+ },+ }+ f.store.set('mcp-server/recipe-owned-lifecycle', ownedElsewhere)+ f.cache.set(f.initial.name, { ...f.initial, spec: { ...f.initial.spec, enabled: false } })+ await f.run()+ expect([...f.store.values()]).toEqual([ownedElsewhere])+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(4)+ })++ it('retries an external status publication failure without deleting healthy policies', async () => {+ const f = lifecycle()+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(+ Object.assign(new Error('temporary status failure'), { code: 503 })+ )+ await expect(f.run()).rejects.toThrow('temporary status failure')+ expect(f.store.size).toBe(4)+ await f.run()+ expect(f.store.size).toBe(4)+ expect(f.events.filter(event => event.startsWith('delete:'))).toHaveLength(0)+ })++ it('retires an in-flight Context effect when only the runtime verdict changes', async () => {+ const f = lifecycle()+ const entered = deferred()+ const release = deferred()+ const create = mockApi.createNamespacedNetworkPolicy.getMockImplementation()!+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async args => {+ entered.resolve()+ await release.promise+ return create(args)+ })+ const running = f.rec.reconcileContext(f.context, { isCurrent: () => true })+ await entered.promise+ f.cache.set(f.initial.name, {+ ...f.initial,+ status: {+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],+ },+ })+ release.resolve()+ expect(await running).toBe(false)+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ await f.run()+ expect(f.store.size).toBe(0)+ })++ it('retires revocation when recovery arrives before delete admission', async () => {+ const f = lifecycle()+ await f.run()+ expect(f.store.size).toBe(4)+ f.cache.set(f.initial.name, {+ ...f.initial,+ status: {+ conditions: [{ type: 'SecretResolved', status: 'False', reason: 'SecretNotFound' }],+ },+ })+ const entered = deferred()+ const release = deferred()+ const list = mockApi.listNamespacedNetworkPolicy.getMockImplementation()!+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async args => {+ entered.resolve()+ await release.promise+ return list(args)+ })+ const pending = f.rec.reconcileContext(f.context, { isCurrent: () => true })+ await entered.promise+ f.cache.set(f.initial.name, structuredClone(f.initial))+ release.resolve()+ expect(await pending).toBe(false)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.store.size).toBe(4)+ await f.run()+ expect(f.store.size).toBe(4)+ })++ it('preserves HCC allows on a transient ReadError', async () => {+ const f = lifecycle()+ await f.run()+ expect(f.store.size).toBe(4)+ f.cache.set(f.initial.name, {+ ...f.initial,+ status: { conditions: [{ type: 'SecretResolved', status: 'False', reason: 'ReadError' }] },+ })+ await f.run()+ expect(f.store.size).toBe(4)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('preserves WRC allows when HCC rejects an absent optional mapping', async () => {+ const f = lifecycle()+ const reference = {+ name: 'fixture-env',+ keys: [{ secretKey: 'optional', envVar: 'OPTIONAL_VALUE', optional: true }],+ }+ const server = {+ ...f.initial,+ spec: { ...f.initial.spec, managed: false, envSecret: reference },+ }+ f.cache.set(server.name, server)+ await f.run()+ expect(f.store.size).toBe(4)+ const apps = createMockAppsApi()+ const core = createMockCoreApi()+ core.readNamespacedSecret.mockResolvedValue({ data: {} })+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {+ const patch = body.find(+ (item: { path: string }) => item.path === '/status/conditions' || item.path === '/status'+ )+ if (patch)+ f.cache.set(server.name, {+ ...f.cache.get(server.name)!,+ status: { conditions: patch.path === '/status' ? patch.value.conditions : patch.value },+ })+ return {}+ })+ const runtime = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,+ appsApi: asAppsApi(apps),+ coreApi: asCoreApi(core),+ customApi: asCustomApi(mockCustomApi as any),+ })+ await runtime.reconcile(server)+ expect(f.cache.get(server.name)?.status?.conditions).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: 'SecretResolved',+ status: 'False',+ reason: 'SecretMissingKey',+ }),+ ])+ )+ expect(runtime.getStatus(server.name)).toMatchObject({ deployed: true, ready: false })+ expect(apps.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(core.deleteNamespacedService).not.toHaveBeenCalled()+ await f.run()+ expect(f.store.size).toBe(4)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('restores allows when envSecret is removed with its failure condition unchanged', async () => {+ const f = lifecycle()+ await f.run()+ const failed = {+ ...f.initial,+ status: {+ conditions: [+ { type: 'SecretResolved', status: 'False' as const, reason: 'SecretNotFound' },+ ],+ },+ }+ f.cache.set(failed.name, failed)+ await f.run()+ expect(f.store.size).toBe(0)+ const recovered = structuredClone(failed)+ delete recovered.spec.envSecret+ f.cache.set(recovered.name, recovered)+ await f.run()+ expect(f.store.size).toBe(4)+ })+ })})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex a7ed8d507..d35782e62 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -37,7 +37,8 @@ import { accumulateHostExactHostEgress } from './externalEgressAccumulator'import { hccLogger } from './logger'import {confirmAuthoritativeMcpServerAbsence,- sameMcpServerDesiredRevision,+ runtimeDesired,+ sameMcpServerPolicyRevision,} from './mcpServerSafety'import {netPolOrphanSweepCappedTotal,@@ -1009,7 +1010,7 @@ export class NetworkPolicyReconciler {const selected = selectedServers.get(serverName)const current = this.serverCache.get(serverName)if (!selected || !current) return selected === current- return sameMcpServerDesiredRevision(selected, current)+ return sameMcpServerPolicyRevision(selected, current)})if (!isCurrent()) return false@@ -1027,6 +1028,14 @@ export class NetworkPolicyReconciler {continue}+ if (!runtimeDesired(server)) {+ hccLogger.debug('McpServer runtime not desired; context policy skipped', {+ serverName,+ contextId,+ })+ continue+ }+const desired = this.buildContextAllowPolicies(context, server)const ingressName = desired.ingress.metadata!.name!await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)@@ -1136,7 +1145,7 @@ export class NetworkPolicyReconciler {const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()for (const serverName of allowedServers) {const server = this.serverCache.get(serverName)- if (!server) continue+ if (!server || !runtimeDesired(server)) continueconst desired = this.buildContextAllowPolicies(context, server)desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)@@ -1431,7 +1440,7 @@ export class NetworkPolicyReconciler {}const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))- const desiredServerNames = new Set(servers.map(s => s.name))+ const desiredServerNames = new Set(servers.filter(runtimeDesired).map(s => s.name))const recordOrphanDelete = (lane: SafetyInventoryLane, deleted: boolean): void => {if (!deleted) returnrecordSafetyPassRevocation()@@ -1725,7 +1734,7 @@ export class NetworkPolicyReconciler {if (!serverCleanupAuthoritative()) return falseif (!options.resolveCurrentServer) return trueconst latest = options.resolveCurrentServer(current.name)- return latest !== undefined && sameMcpServerDesiredRevision(current, latest)+ return latest !== undefined && sameMcpServerPolicyRevision(current, latest)}const completed = await this.reconcileExternalEgressSafety(current,@@ -2177,7 +2186,7 @@ export class NetworkPolicyReconciler {): Promise<boolean> {if (!isCurrent()) return falseconst desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()- for (const binding of server.spec.egressBindings ?? []) {+ for (const binding of runtimeDesired(server) ? (server.spec.egressBindings ?? []) : []) {const name = this.externalEgressPolicyName(server.name, binding)if (!name) continueconst egressClass = binding.egressClass ?? 'exact-host'@@ -2348,8 +2357,9 @@ export class NetworkPolicyReconciler {}if (!isCurrent()) return+ const desired = runtimeDesired(server)const bindings = server.spec.egressBindings- if (!bindings || bindings.length === 0) {+ if (!desired || !bindings || bindings.length === 0) {try {await this.cleanupExternalEgress(server.name,@@ -2373,9 +2383,11 @@ export class NetworkPolicyReconciler {await this.writeExternalEgressStatus(server,[],- 'True',- 'NoEgressBindings',- 'No external egress bindings declared',+ desired ? 'True' : 'False',+ desired ? 'NoEgressBindings' : 'RuntimeNotDesired',+ desired+ ? 'No external egress bindings declared'+ : 'Runtime is not eligible for external egress',isCurrent)returndiff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 13992b5ec..dbe51da3e 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -4,7 +4,6 @@ import {asAppsApi,asCoreApi,asCustomApi,- asNetworkingApi,createMockAppsApi,createMockCoreApi,createMockCustomApi,@@ -498,7 +497,6 @@ describe('PR-B B1 — validateSecret result shape', () => {appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),- networkingApi: asNetworkingApi(networkingApi),})})@@ -630,16 +628,24 @@ describe('PR-B B1 — validateSecret result shape', () => {name: 'pg',namespace: 'mcp-server',})- for (const namespace of ['mcp-server', 'mcp-host', 'rpc-proxy']) {- expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: `np-${namespace}-pg`,- namespace,- })- expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({- name: `np-${namespace}-other`,- namespace,+ expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.arrayContaining([+ expect.objectContaining({+ path: '/status/conditions',+ value: expect.arrayContaining([+ expect.objectContaining({+ type: 'SecretResolved',+ status: 'False',+ reason: 'SecretNotFound',+ }),+ ]),+ }),+ ]),})- }+ )expect(reconciler.getStatus('pg')).toMatchObject({ deployed: false, ready: false })})@@ -657,7 +663,7 @@ describe('PR-B B1 — validateSecret result shape', () => {})await expect(reconciler.reconcile(server)).rejects.toThrow(- 'Failed to delete runtime resources for McpServer "pg"'+ 'Failed to delete runtime Kubernetes resources for McpServer "pg"')expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({name: 'pg',diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex 6be983991..e2400a3f5 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -25,6 +25,7 @@ import { hccLogger } from './logger'import {confirmAuthoritativeMcpServerAbsence,sameMcpServerDesiredRevision,+ shouldFailClosedForSecretFailure,} from './mcpServerSafety'import { createsTotal, mcpserverMissingSecret } from './metrics'import { McpServerCRD, McpServerStatus } from './types'@@ -85,7 +86,6 @@ type McpServerReconcilerDeps = {appsApi?: k8s.AppsV1ApicoreApi?: k8s.CoreV1ApicustomApi?: k8s.CustomObjectsApi- networkingApi?: k8s.NetworkingV1Api/*** TEST-ONLY escape hatch. When the inventory-authority fence is not wired* (setInventoryAuthority / setResolveCurrentServer never called), mutation@@ -259,7 +259,6 @@ export class McpServerReconciler {private readonly appsApi: k8s.AppsV1Apiprivate readonly coreApi: k8s.CoreV1Apiprivate readonly customApi: k8s.CustomObjectsApi- private readonly networkingApi?: k8s.NetworkingV1Api/*** Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.@@ -312,9 +311,6 @@ export class McpServerReconciler {this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)this.customApi = deps?.customApi ?? kc.makeApiClient(k8s.CustomObjectsApi)- this.networkingApi =- deps?.networkingApi ??- (typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)this.assumeInventoryAuthorityWhenUnconfigured =deps?.assumeInventoryAuthorityWhenUnconfigured ?? false}@@ -1722,73 +1718,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async deleteNetworkPoliciesForServer(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {- if (!this.networkingApi) return- const failures: unknown[] = []- const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]- for (const policyNamespace of namespaces) {- try {- const policies = await this.networkingApi.listNamespacedNetworkPolicy({- namespace: policyNamespace,- })- for (const policy of policies.items ?? []) {- if (policy.metadata?.labels?.[MCPSERVER_LABEL] !== name) continue- const policyName = policy.metadata.name- if (!policyName) continue- try {- if (deleteAllowed && !(await deleteAllowed())) return- await this.networkingApi.deleteNamespacedNetworkPolicy({- name: policyName,- namespace: policyNamespace,- })- console.log(- `[Reconciler] Deleted NetworkPolicy "${policyName}" in "${policyNamespace}"`- )- } catch (error: unknown) {- if (getErrorCode(error) !== 404) {- console.error(- `[Reconciler] Failed to delete NetworkPolicy "${policyName}" in "${policyNamespace}":`,- error- )- failures.push(error)- }- }- }- } catch (error: unknown) {- if (getErrorCode(error) !== 404) {- console.error(- `[Reconciler] Failed to list NetworkPolicies in "${policyNamespace}" for "${name}":`,- error- )- failures.push(error)- }- }- }-- throwCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)- }-private async deleteRuntimeResources(name: string,namespace: string,deleteAllowed?: () => Promise<boolean>): Promise<void> {- const failures: unknown[] = []- try {- await this.deleteResources(name, namespace, deleteAllowed)- } catch (error) {- failures.push(error)- }- try {- await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)- } catch (error) {- failures.push(error)- }- throwCleanupFailures(failures, `Failed to delete runtime resources for McpServer "${name}"`)+ // Policy owners reconcile their own desired sets with ownership and+ // resource-version preconditions; runtime cleanup must not race them.+ await this.deleteResources(name, namespace, deleteAllowed)}// ─── CRD Status & Annotation Patching ──────────────────────────────@@ -2183,7 +2120,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.error(`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`)- if (this.shouldFailClosedForSecretFailure(secretResult)) {+ if (shouldFailClosedForSecretFailure(secretResult.reason)) {await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())} else {console.warn(@@ -2374,16 +2311,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private shouldFailClosedForSecretFailure(- result: Exclude<SecretValidationResult, { ok: true }>- ): boolean {- return (- result.reason === 'SecretNotFound' ||- result.reason === 'SecretMissingKey' ||- result.reason === 'SecretAccessDenied'- )- }-private async reconcileWrcOwnedServer(server: McpServerCRD,isCurrent: () => booleandiff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shnew file mode 100644index 000000000..035aa3f06--- /dev/null+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh@@ -0,0 +1,188 @@+#!/usr/bin/env bash+# Optional #604 lifecycle assertions inside the existing real watch-churn gate.+# The gate owns API fault injection, branch provenance, lock and restoration.+# This module owns only its labelled fixture and its policy event observer.++NP604_CREATED=0+NP604_WATCH_PID=''++np604_kctl() {+ kctl --request-timeout=30s "$@"+}++np604_setup() {+ NP604_CONTEXT="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" 1)"+ NP604_SERVER="$(truncate_rfc1123 "np604-${RUN_ID}-affected")"+ NP604_CONTROL="$(truncate_rfc1123 "np604-${RUN_ID}-control")"+ NP604_ENV="$(truncate_rfc1123 "np604-${RUN_ID}-env")"+ NP604_EVIDENCE="${HCC_BRANCH_GATE_REPO_ROOT}/.local-notes/infra/runs/issue604-${RUN_ID}"+ mkdir -p "$NP604_EVIDENCE"+ NP604_CREATED=1+ # Synthetic, non-production fixture data; never derive it from user state.+ np604_kctl create secret generic "$NP604_ENV" -n "$MCP_NS" \+ --from-literal=required=e2e-fixture --dry-run=client -o json |+ jq --arg run "$RUN_ID" '.metadata.labels={"e2e.clerum.io/run":$run,"e2e.clerum.io/suite":"hcc-np604"}' |+ np604_kctl apply -f - >/dev/null+ local name+ for name in "$NP604_SERVER" "$NP604_CONTROL"; do+ np604_kctl apply -f - >/dev/null <<EOF+apiVersion: clerum.io/v1alpha1+kind: McpServer+metadata:+ name: ${name}+ namespace: ${MCP_NS}+ labels: {e2e.clerum.io/run: "${RUN_ID}", e2e.clerum.io/suite: hcc-np604}+spec:+ contextRef: ${NP604_CONTEXT}+ image: clerum/mock-mcp-server:test+ imagePullPolicy: Never+ transport: {type: streamableHttp, port: 3000, url: "http://${name}.${MCP_NS}.svc.cluster.local:3000/mcp"}+ egressBindings: [{cidr: 1.2.3.4/32, port: 443, protocol: TCP}]+EOF+ done+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge \+ -p "$(jq -cn --arg name "$NP604_ENV" '{spec:{envSecret:{name:$name,keys:[{secretKey:"required",envVar:"NP604_FIXTURE"}]}}}')" >/dev/null+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \+ -p "$(jq -cn --arg a "$NP604_SERVER" --arg c "$NP604_CONTROL" '{spec:{mcpServers:[$a,$c]}}')" >/dev/null+}++np604_snapshot() {+ np604_kctl get networkpolicy -A -l "clerum.io/mcpserver=$1,clerum.io/managed-by=host-context-controller" -o json |+ jq -Sc '[.items[] | {namespace:.metadata.namespace,name:.metadata.name,type:.metadata.labels["clerum.io/policy-type"],spec}] | sort_by(.namespace,.name)'+}++np604_ready() {+ local name="$1"+ np604_kctl get deployment "$name" -n "$MCP_NS" -o json 2>/dev/null |+ jq -e '.status.observedGeneration >= .metadata.generation and (.status.readyReplicas // 0) > 0' >/dev/null &&+ [ "$(np604_snapshot "$name" | jq length)" = 4 ]+}++np604_invoke() {+ local name="$1" pod+ # Real Context-labelled mcp-host created by the parent gate, not a probe+ # with forged labels. Exercise the MCP protocol over the actual allow path.+ pod="$(np604_kctl get pods -n "$HOST_NS" -l "clerum.io/context=${NP604_CONTEXT}" -o json |+ jq -r '[.items[] | select(.metadata.deletionTimestamp == null) | select(any(.status.conditions[]?; .type=="Ready" and .status=="True"))][0].metadata.name // empty')"+ [ -n "$pod" ] || return 1+ np604_kctl exec -i "$pod" -n "$HOST_NS" -- node --input-type=module - "$name" "$MCP_NS" <<'JS'+const [name, namespace] = process.argv.slice(2)+const url = `http://${name}.${namespace}.svc.cluster.local:3000/mcp`+let session+async function rpc(id, method, params) {+ const headers = {'content-type':'application/json',accept:'application/json, text/event-stream'}+ if (session) headers['mcp-session-id'] = session+ const response = await fetch(url, {method:'POST', headers,+ body:JSON.stringify({jsonrpc:'2.0',...(id === undefined ? {} : {id}),method,params}),+ signal:AbortSignal.timeout(8000)})+ if (!response.ok) throw new Error(`MCP request failed: ${response.status}`)+ session = response.headers.get('mcp-session-id') ?? session+ const body = await response.text()+ if (id === undefined) return+ const messages = response.headers.get('content-type')?.includes('text/event-stream')+ ? body.split('\n').filter(line => line.startsWith('data:')).map(line => JSON.parse(line.slice(5)))+ : [JSON.parse(body)]+ const message = messages.find(item => item.id === id)+ if (!message || message.error) throw new Error('MCP response missing or rejected')+ return message.result+}+await rpc(1, 'initialize', {protocolVersion:'2024-11-05',capabilities:{},clientInfo:{name:'np604-real-flow',version:'1'}})+await rpc(undefined, 'notifications/initialized', {})+const result = await rpc(2, 'tools/call', {name:'add',arguments:{a:17,b:25}})+if (result.isError || !result.content?.some(item => item.type === 'text' && item.text === '42')) {+ throw new Error('MCP business result did not equal 42')+}+console.log('NP604_MCP_BUSINESS_RESULT=42')+JS+}++np604_failed() {+ local reason="$1" runtime+ np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json |+ jq -e --arg reason "$reason" 'any(.status.conditions[]?; .type=="SecretResolved" and .status=="False" and .reason==$reason)' >/dev/null || return 1+ runtime="$(np604_kctl get deployment "$NP604_SERVER" -n "$MCP_NS" --ignore-not-found -o name)" || return 1+ [ -z "$runtime" ] && [ "$(np604_snapshot "$NP604_SERVER")" = '[]' ]+}++np604_observer_alive() {+ [ -n "$NP604_WATCH_PID" ] && jobs -pr | awk -v pid="$NP604_WATCH_PID" '$1==pid {found=1} END {exit !found}'+}++np604_observer_witness() {+ awk -v name="${MCP_NS}/ctx-${NP604_CONTEXT}-${NP604_CONTROL}" \+ '$1=="MODIFIED" && $2==name {found=1} END {exit !found}' "$NP604_EVIDENCE/policy-events.txt"+}++np604_before_observation() {+ wait_until 180 'NP604 affected runtime and four policy families' np604_ready "$NP604_SERVER" || die 'NP604 initial runtime did not converge'+ wait_until 180 'NP604 control runtime and four policy families' np604_ready "$NP604_CONTROL" || die 'NP604 control did not converge'+ wait_until 120 'NP604 initial business result from mcp-host' np604_invoke "$NP604_SERVER" || die 'NP604 initial invocation failed'+ np604_invoke "$NP604_CONTROL" || die 'NP604 control invocation failed'+ np604_snapshot "$NP604_SERVER" > "$NP604_EVIDENCE/desired-policies.json"+ np604_kctl get networkpolicy deny-all-mcp-host -n "$HOST_NS" -o json |+ jq -e '.spec.podSelector=={} and (.spec.policyTypes|index("Ingress"))!=null and (.spec.policyTypes|index("Egress"))!=null' >/dev/null || die 'NP604 default deny missing'+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge -p '{"data":null}' >/dev/null+ wait_until 120 'NP604 published failure and policy revocation' np604_failed SecretMissingKey || die 'NP604 did not revoke after failure'+ np604_invoke "$NP604_CONTROL" || die 'NP604 failure affected healthy control'+ local snapshot rv+ snapshot="$(np604_kctl get networkpolicy -A -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o json)" || die 'NP604 observer inventory failed'+ rv="$(jq -er '.metadata.resourceVersion' <<<"$snapshot")" || die 'NP604 observer missing version'+ "$KUBECTL_BIN" --context="$E2E_KUBECONTEXT" --request-timeout=180s get networkpolicy -A \+ -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" \+ --watch-only --resource-version="$rv" --output-watch-events \+ -o 'jsonpath={.type}{" "}{.object.metadata.namespace}{"/"}{.object.metadata.name}{"\n"}' \+ > "$NP604_EVIDENCE/policy-events.txt" 2> "$NP604_EVIDENCE/observer-errors.txt" &+ NP604_WATCH_PID=$!+ # Metadata-only witness on our healthy fixture: proves the watch is live+ # without creating a final policy or replacing HCC's desired policy spec.+ np604_kctl annotate networkpolicy "ctx-${NP604_CONTEXT}-${NP604_CONTROL}" -n "$MCP_NS" \+ "e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/null+ wait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'+ NP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"+}++np604_after_observation() {+ np604_observer_alive || die 'NP604 observer stopped before completing observation'+ local current_cuts+ current_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"+ [ "$((current_cuts - NP604_CUT_BASE))" -ge 3 ] || die 'NP604 needs three complete reconnect cycles in stable failed state'+ awk -v target="$NP604_SERVER" '$1=="ADDED" && index($2,target) {bad=1} END {exit bad}' \+ "$NP604_EVIDENCE/policy-events.txt" || die 'NP604 recreated a policy while runtime remained undesired'+ kill "$NP604_WATCH_PID"+ wait "$NP604_WATCH_PID" 2>/dev/null || true+ NP604_WATCH_PID=''+ np604_invoke "$NP604_CONTROL" || die 'NP604 control lost connectivity under churn'+}++np604_recover() {+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge \+ -p '{"stringData":{"required":"e2e-fixture"}}' >/dev/null+ wait_until 180 'NP604 repaired runtime and policies' np604_ready "$NP604_SERVER" || die 'NP604 repair did not converge'+ wait_until 120 'NP604 repaired business result' np604_invoke "$NP604_SERVER" || die 'NP604 repair did not restore invocation'+ [ "$(np604_snapshot "$NP604_SERVER")" = "$(cat "$NP604_EVIDENCE/desired-policies.json")" ] || die 'NP604 recovered policy set differs'+ np604_kctl patch secret "$NP604_ENV" -n "$MCP_NS" --type=merge -p '{"data":null}' >/dev/null+ wait_until 120 'NP604 second failure' np604_failed SecretMissingKey || die 'NP604 second failure did not converge'+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge -p '{"spec":{"envSecret":null}}' >/dev/null+ wait_until 180 'NP604 recovery after removing reference' np604_ready "$NP604_SERVER" || die 'NP604 removed reference did not recover'+ np604_invoke "$NP604_SERVER" || die 'NP604 removed reference did not restore invocation'+ [ "$(np604_snapshot "$NP604_SERVER")" = "$(cat "$NP604_EVIDENCE/desired-policies.json")" ] || die 'NP604 removed reference policy set differs'+ printf 'NP604_LIFECYCLE=PASS\nevidence=%s\n' "$NP604_EVIDENCE"+}++np604_cleanup() {+ if np604_observer_alive; then+ kill "$NP604_WATCH_PID"+ wait "$NP604_WATCH_PID" 2>/dev/null || true+ fi+ NP604_WATCH_PID=''+ [ "$NP604_CREATED" = 1 ] || return 0+ np604_kctl delete mcpserver,secret -n "$MCP_NS" \+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null+}++np604_resources_absent() {+ local remaining+ remaining="$(np604_kctl get deployment,service,networkpolicy -A \+ -l "clerum.io/mcpserver in (${NP604_SERVER},${NP604_CONTROL})" -o name)" || return 1+ [ -z "$remaining" ]+}diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.shindex 414d48ab8..c568de800 100755--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh@@ -179,6 +179,8 @@ EOFdone; } | kctl apply -f - >/dev/null# McpServers (one stream) — inventory + NetworkPolicies, no heavy pods.+ # Lifecycle suites supply their own real MCP fixtures and need no filler.+ if (( n_mcp > 0 )); then{ for ((i = 1; i <= n_mcp; i++)); doctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" $(((i % n_ctx) + 1)))"cat <<EOF@@ -194,6 +196,7 @@ spec:transport: {type: streamableHttp, url: "http://127.0.0.1:3000/mcp", port: 3000}EOFdone; } | kctl apply -f - >/dev/null+ fi# Hosts (one stream) — these materialize Deployments; keep the count modest.{ for ((i = 1; i <= n_host; i++)); dodiff --git a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.shnew file mode 100755index 000000000..cfeaa6345--- /dev/null+++ b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh@@ -0,0 +1,20 @@+#!/usr/bin/env bash+set -euo pipefail+# #604 runtime proof. The underlying gate enforces branch/profile provenance,+# holds its fault-injection lock and restores HCC before returning a verdict.+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"+: "${MINIKUBE_PROFILE:?explicit branch-owned profile required}"+: "${CONTROL_API_REAL_PG_CONTEXT:?explicit branch-owned context required}"+[[ "$MINIKUBE_PROFILE" = "$CONTROL_API_REAL_PG_CONTEXT" ]] || {+ echo 'FAIL: profile and context must match' >&2+ exit 2+}+export KUBECONTEXT="$CONTROL_API_REAL_PG_CONTEXT"+export E2E_HCC_WATCH_FAULT_INJECTION=1+export E2E_HCC_POLICY_LIFECYCLE=1+export E2E_EXPECTED_PRE_GATE_GATE=minikube-t2+# The two real MCP workloads in the lifecycle fixture provide the business+# witness. Other synthetic McpServers would add unrelated crashing workloads.+export FLEET_CONTEXTS=1 FLEET_MCPSERVERS=0 FLEET_HOSTS=1+export CHURN_MIN_CUTS=3 CHURN_OBSERVE_SEC=45+exec bash "$SCRIPT_DIR/e2e-hcc-watch-churn-readiness.sh"diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.shindex c894191a1..61d231c67 100755--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh@@ -39,6 +39,16 @@ source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/hcc-watch-churn-fixture.sh"+# #604 reuses the real TCP fault injection and provenance/restore contract.+case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in+ 0) ;;+ 1)+ # shellcheck source=scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh+ source "${SCRIPT_DIR}/_lib/hcc-networkpolicy-lifecycle.sh"+ ;;+ *) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;+esac+# ── Fail-closed guards (identical contract to the sibling HCC gates) ──[ -n "$E2E_KUBECONTEXT" ] || {echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2@@ -123,6 +133,7 @@ PROXY_CREATED=0PROBE_CREATED=0FLEET_CREATED=0HCC_PATCHED=0+HCC_SCALED_DOWN=0# Log-stream state consumed by hcc-watch-recovery-logs.sh (start/stop_hcc_recovery_log_stream).# Declared here so `set -u` never trips when the stream helper first touches them.HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn-stream.XXXXXX")"@@ -275,6 +286,9 @@ cleanup() {trap - EXITset +estop_hcc_recovery_log_stream+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then+ np604_cleanup || cleanup_failed=1+ fiif [ "$HCC_PATCHED" = 1 ]; thenkctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0@@ -282,7 +296,11 @@ cleanup() {[ "$FLEET_CREATED" = 1 ] && { delete_synthetic_fleet || cleanup_failed=1; }if [ "$HCC_PATCHED" = 1 ]; thenrestore_hcc_after_churn || restore_ok=0- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1+ fi+ # Fleet creation can fail after scale-down but before the redirect patch.+ # Restore replicas even then, without stripping untouched template env.+ if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || restore_ok=0fi[ "$PROXY_CREATED" = 1 ] && kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1@@ -292,6 +310,9 @@ cleanup() {print_repair_instructionscleanup_failed=1}+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] && [ "$NP604_CREATED" = 1 ] && [ "$restore_ok" = 1 ]; then+ wait_until 120 "NP604 fixture resources removed by owners" np604_resources_absent || cleanup_failed=1+ fifinalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1print_results || status=1[ "$cleanup_failed" = 0 ] || status=1@@ -320,6 +341,7 @@ create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"verify_hcc_proxy_network_policyproxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"+HCC_SCALED_DOWN=1kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/nullwait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop before fleet creation"@@ -327,6 +349,9 @@ wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop beforlog "Creating synthetic fleet: ${FLEET_CONTEXTS} Contexts / ${FLEET_MCPSERVERS} McpServers / ${FLEET_HOSTS} Hosts"create_synthetic_fleet "$FLEET_CONTEXTS" "$FLEET_MCPSERVERS" "$FLEET_HOSTS"ok "synthetic fleet created (${FLEET_MCPSERVERS} McpServers)"+if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then+ np604_setup+fi# ── FASE D: start HCC under churn and poll /ready ──log "Redirecting HCC through the churn proxy and starting it under load"@@ -336,8 +361,8 @@ patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},{name:"KUBERNETES_SERVICE_PORT",value:"443"},{name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"-kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/nullHCC_PATCHED=1+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/nullkctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null# Anchor the log stream to now so recovery-cycle assertions only see churn-era logs.START_TIME="$(python3 -c 'import datetime; print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))')"@@ -398,8 +423,14 @@ write_evidence_artifact() {# watch drops /ready to 503 transiently while it re-syncs, so "no 503 after# first 200" failed the CORRECT regime and measured the wrong thing.if [ -n "$first_ready" ]; then+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then+ np604_before_observation+ filog "Observing /ready at 1Hz for ${CHURN_OBSERVE_SEC}s under continuous churn"sample_ready_series "$CHURN_OBSERVE_SEC" churn 0+ if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then+ np604_after_observation+ fifi# ── Pause the churn WITHOUT restarting the proxy or the HCC ──@@ -495,5 +526,9 @@ if [ -n "$hcc_pod_at_stop" ] && [ "$hcc_pod_now" = "$hcc_pod_at_stop" ] &&elsefail "HCC pod changed or restarted across the post-churn phase (${hcc_pod_at_stop:-none}/restarts=${hcc_restarts_at_stop:-?} -> ${hcc_pod_now:-none}/restarts=${hcc_restarts_now:-?}) — convergence could be a fresh boot, not in-situ recovery"fi+if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then+ [ -n "$first_ready" ] || die 'NP604 cannot run without a ready controller'+ np604_recover+filog "Evidence artifact: ${LOG_ARTIFACT}"# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.shnew file mode 100755index 000000000..593e593bf--- /dev/null+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh@@ -0,0 +1,73 @@+#!/usr/bin/env bash+set -euo pipefail+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+tmp="$(mktemp -d)"+trap 'rm -rf -- "$tmp"' EXIT+# shellcheck source=scripts/e2e/_lib/hcc-watch-churn-fixture.sh+source "$ROOT/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"+export FLEET_SECRET=fixture-llm HOST_NS=mcp-host MCP_NS=mcp-server+export FLEET_PREFIX=fixture RUN_ID=fixture HCC_IMAGE=fixture-image+expected_mcp=0+truncate_rfc1123() { printf '%s' "$1"; }+die() { echo "$*" >&2; exit 1; }+kctl() {+ case "$1" in+ create) printf '%s\n' 'apiVersion: v1' 'kind: Secret' ;;+ label) cat ;;+ apply)+ local input+ input="$(cat)"+ [[ -n "$input" ]] || { echo 'FAIL: empty apply payload' >&2; return 1; }+ printf '%s\n' "$input" >> "$tmp/applied"+ ;;+ get)+ if (( expected_mcp > 0 )); then printf '%s\n' 'mcpserver/fixture'; fi+ ;;+ *) echo "FAIL: unexpected kubectl operation $1" >&2; return 1 ;;+ esac+}+create_synthetic_fleet 1 0 1+[[ "$FLEET_CREATED" = 1 ]]+grep -q '^kind: Context' "$tmp/applied"+grep -q '^kind: Host' "$tmp/applied"+if grep -q '^kind: McpServer' "$tmp/applied"; then+ echo 'FAIL: zero-sized MCP inventory emitted a filler resource' >&2+ exit 1+fi+: > "$tmp/applied"+expected_mcp=1+create_synthetic_fleet 1 1 1+grep -q '^kind: McpServer' "$tmp/applied"+grep -q '^kind: Context' "$tmp/applied"+grep -q '^kind: Host' "$tmp/applied"+echo 'PASS: zero MCP inventory omits empty apply; nonzero inventory still creates resources'++# Run the actual EXIT cleanup in a subshell with runtime calls replaced by+# recording stubs. In particular, do not model the cleanup in the test.+cleanup_source="$(sed -n '/^cleanup() {/,/^}/p' "$ROOT/scripts/e2e/e2e-hcc-watch-churn-readiness.sh")"+[[ -n "$cleanup_source" ]]+for patched in 0 1; do+ (+ HCC_PATCHED="$patched" HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1+ HCC_DEPLOY=host-context-controller HCC_NS=control-plane+ FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0+ E2E_HCC_POLICY_LIFECYCLE=0+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe+ HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"+ kctl() { printf '%s\n' "$*" >> "$tmp/cleanup-$patched"; }+ stop_hcc_recovery_log_stream() { :; }+ wait_until() { :; }+ restore_hcc_after_churn() { echo restore-template >> "$tmp/cleanup-$patched"; }+ finalize_hcc_watch_gate_lock() { :; }+ print_results() { :; }+ eval "$cleanup_source"+ cleanup+ )+ grep -q 'scale deployment host-context-controller -n control-plane --replicas=1' "$tmp/cleanup-$patched"+done+if grep -q restore-template "$tmp/cleanup-0"; then+ echo 'FAIL: early failure rewrote a template the gate had not changed' >&2+ exit 1+fi+grep -q restore-template "$tmp/cleanup-1"+echo 'PASS: cleanup restores replicas before/after redirect and preserves an untouched template'