Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
perf(hcc): preserve pending work while skipping identical watch recovery
Alfredo Lopez9075e2b93c31
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex f00605ed9..f538d5876 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -449,6 +449,11 @@ jobs:- name: Test HCC watch-recovery gate coordinationrun: bash scripts/tests/test-hcc-watch-recovery-gate.sh+ - name: Test HCC watch recovery TLS proxy and configuration restoration+ run: |+ bash scripts/tests/test-hcc-watch-api-proxy.sh+ bash scripts/tests/test-hcc-watch-pr-a.sh+- name: Test HCC watch-churn readiness gate contractrun: bash scripts/tests/test-hcc-watch-churn-readiness-gate.shdiff --git a/Makefile b/Makefileindex cce68c7a4..9558b1b98 100644--- a/Makefile+++ b/Makefile@@ -680,6 +680,12 @@ minikube-t2-hcc-networkpolicy-lifecycle: ## Run canonical T2 with the HCC Networ@T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 $(MAKE) minikube-t2+.PHONY: minikube-t2-hcc-watch-recovery+minikube-t2-hcc-watch-recovery: ## Certify PR A recovery omission, runtime repair and API-gate recovery in owned Minikube+ @bash scripts/tests/test-hcc-watch-api-proxy.sh+ @bash scripts/tests/test-hcc-watch-pr-a.sh+ @E2E_HCC_PR_A=1 $(MAKE) minikube-t2-hcc-networkpolicy-lifecycle+.PHONY: minikube-t2-runtimeminikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile@T2_RUN_T0=false T2_RUN_T1=false \diff --git a/deploy/overlays/minikube/kustomization.yaml b/deploy/overlays/minikube/kustomization.yamlindex de4b04935..693af4dde 100644--- a/deploy/overlays/minikube/kustomization.yaml+++ b/deploy/overlays/minikube/kustomization.yaml@@ -81,6 +81,7 @@ images:newTag: testpatchesStrategicMerge:+ - patches/hcc-policy-resync.yaml- patches/resource-limits.yaml- patches/dynamic-images.yaml# Minikube is the owned T2/dev activation environment for GFS Upload v2.diff --git a/deploy/overlays/minikube/patches/hcc-policy-resync.yaml b/deploy/overlays/minikube/patches/hcc-policy-resync.yamlnew file mode 100644index 000000000..9dfde7192--- /dev/null+++ b/deploy/overlays/minikube/patches/hcc-policy-resync.yaml@@ -0,0 +1,15 @@+# Keep the existing periodic repair active in the owned development lane.+# The shared base remains unchanged; PR A does not change GKE cadence.+apiVersion: apps/v1+kind: Deployment+metadata:+ name: host-context-controller+ namespace: control-plane+spec:+ template:+ spec:+ containers:+ - name: host-context-controller+ env:+ - name: CONTEXT_MAPPER_NETPOL_RESYNC_SEC+ value: '3600'diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex dc045e364..7b01d0815 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.314",+ "version": "0.1.315","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.314",+ "version": "0.1.315","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 02c15dca1..ca8fc377d 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.314",+ "version": "0.1.315","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.tsindex d558af3eb..947a53c4f 100644--- a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts@@ -1,6 +1,13 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import type { V1DeleteOptions, V1NetworkPolicy, V1ObjectMeta } from '@kubernetes/client-node'-import { McpServerWatcher } from './k8sClient'+import * as http from 'node:http'+import { config } from './config'+import { McpServerWatcher, createMcpAuthorizationStore } from './k8sClient'+import { hccLogger } from './logger'+import type { McpApiAuthenticator } from './mcpApiAuthentication'+import { McpAuthorizationService } from './mcpAuthorization'+import { resolveProbeAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'+import { ContextMapperServer } from './server'import type { ContextCRD, McpServerCRD, McpServerCrdStatus } from './types'type ApiServer = Pick<McpServerCRD, 'spec' | 'status'> & { metadata: V1ObjectMeta }@@ -10,6 +17,8 @@ const fixture = vi.hoisted(() => ({policies: new Map<string, StoredPolicy>(),servers: new Map<string, ApiServer>(),revision: 0,+ contexts: new Map<string, { metadata: V1ObjectMeta; spec: ContextCRD['spec'] }>(),+ failPolicyType: undefined as string | undefined,watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),list: vi.fn(),}))@@ -17,6 +26,7 @@ const fixture = vi.hoisted(() => ({vi.mock('./config', () => ({config: {devMode: false,+ k8sApiCidrs: [],namespace: 'mcp-server',hostNamespace: 'mcp-host',rpcProxyNamespace: 'rpc-proxy',@@ -47,7 +57,15 @@ vi.mock('@kubernetes/client-node', async importOriginal => {}const custom = {listNamespacedCustomObject: fixture.list,- getNamespacedCustomObject: async ({ name }: { name: string }) => read(name),+ getNamespacedCustomObject: async ({ name, plural }: { name: string; plural: string }) => {+ if (plural === 'hosts')+ return {+ metadata: { name: 'host', namespace: 'mcp-host', uid: 'host-uid', resourceVersion: '1' },+ spec: { contextRef: 'ctx' },+ }+ if (plural === 'contexts') return structuredClone(fixture.contexts.get(name))+ return read(name)+ },getNamespacedCustomObjectStatus: async ({ name }: { name: string }) => read(name),patchNamespacedCustomObjectStatus: async ({name,@@ -83,6 +101,11 @@ vi.mock('@kubernetes/client-node', async importOriginal => {},}const save = (namespace: string, body: StoredPolicy) => {+ if (+ fixture.failPolicyType &&+ Object.values(body.metadata.labels ?? {}).includes(fixture.failPolicyType)+ )+ throw new Error('injected additive API failure')const key = `${namespace}/${body.metadata.name}`const old = fixture.policies.get(key)const policy = structuredClone({@@ -187,6 +210,12 @@ vi.mock('@kubernetes/client-node', async importOriginal => {// while keeping unrelated workload API writes out of this fixture.vi.mock('./reconciler', () => ({McpServerReconciler: class {+ hasPendingReconciliation() {+ return false+ }+ hasIncompleteReconciliation() {+ return false+ }setInventoryAuthority() {}setResolveCurrentServer() {}reconcile() {@@ -208,6 +237,12 @@ vi.mock('./reconciler', () => ({}))type Harness = {+ runInitialNetworkPolicyConvergence(options?: { cause?: string }): Promise<void>+ runInitialConvergence(lane: string): Promise<void>+ initialConvergenceRetryTimers: Map<string, unknown>+ recoverContextInventoryAndWatch(): Promise<boolean>+ enqueueMcpServerReconciliation(server: McpServerCRD, work: () => Promise<void>): Promise<void>+ servers: Map<string, McpServerCRD>contexts: Map<string, ContextCRD>contextCacheSynced: booleanmcpServerCacheSynced: boolean@@ -235,9 +270,13 @@ async function drain(state: Harness): Promise<void> {describe('WATCH and recovered LIST policy effects', () => {let watcher: McpServerWatcher+ let api: ContextMapperServer | undefinedbeforeEach(() => {vi.useFakeTimers()fixture.policies.clear()+ fixture.contexts.clear()+ fixture.failPolicyType = undefined+ config.netPolResyncIntervalSec = 0fixture.servers.clear()fixture.revision = 0fixture.list.mockImplementation(async ({ plural }: { plural: string }) => ({@@ -245,11 +284,16 @@ describe('WATCH and recovered LIST policy effects', () => {items:plural === 'mcpservers'? [...fixture.servers.values()].map(value => structuredClone(value))- : [],+ : plural === 'contexts'+ ? [...fixture.contexts.values()].map(value => structuredClone(value))+ : [],}))})afterEach(async () => {+ await api?.stop()+ api = undefinedawait watcher?.stop()+ vi.restoreAllMocks()vi.useRealTimers()})@@ -312,4 +356,528 @@ describe('WATCH and recovered LIST policy effects', () => {expect(fixture.list).toHaveBeenCalledWith(expect.objectContaining({ plural: 'mcpservers' }))})+ async function settled(): Promise<Harness> {+ config.netPolResyncIntervalSec = 3600+ fixture.contexts.set('ctx', {+ metadata: {+ name: 'ctx',+ namespace: 'mcp-server',+ uid: 'ctx-uid',+ generation: 1,+ resourceVersion: '1',+ },+ spec: { contextId: 'ctx', mcpServers: ['server'] },+ })+ const server: ApiServer = {+ metadata: {+ name: 'server',+ namespace: 'mcp-server',+ uid: 'server-uid',+ generation: 1,+ resourceVersion: '1',+ },+ spec: {+ image: 'fixture:v1',+ contextRef: 'ctx',+ transport: { type: 'streamableHttp', port: 3000 },+ envSecret: { name: 'fixture-env', keys: [] },+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],+ },+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },+ }+ fixture.servers.set('server', server)+ watcher = new McpServerWatcher()+ await watcher.start()+ const state = watcher as unknown as Harness+ await drain(state)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(fixture.policies.size).toBeGreaterThanOrEqual(4)+ return state+ }++ it('A-T3/T5: settled identical recoveries skip before scheduler admission with an active resync', async () => {+ const state = await settled()+ const admission = vi.spyOn(state, 'runInitialConvergence')+ const log = vi.spyOn(hccLogger, 'info')+ const before = structuredClone([...fixture.policies.entries()])+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toEqual([])+ expect([...fixture.policies.entries()]).toEqual(before)+ for (const kind of ['McpServer', 'Context'])+ expect(log).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({+ event: 'networkpolicy-recovery-decision',+ kind,+ decision: 'skip',+ reason: 'identical-complete',+ })+ )+ })++ it.each(['McpServer', 'Context', 'timer'] as const)(+ 'A-T4: %s repairs a real additive failure',+ async kind => {+ const state = await settled()+ const desired = [...fixture.policies.entries()].find(([, policy]) =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )!+ expect(desired).toBeDefined()+ const certificate = vi.spyOn(+ state as unknown as { recordNetworkPolicySafetyCertificate(value: unknown): boolean },+ 'recordNetworkPolicySafetyCertificate'+ )+ fixture.policies.delete(desired[0])+ fixture.failPolicyType = 'context-allow'+ await state.runInitialNetworkPolicyConvergence({ cause: 'context-change' })+ await drain(state)+ expect(certificate).toHaveBeenCalled()+ expect(+ certificate.mock.results.some(result => result.type === 'return' && result.value === true)+ ).toBe(true)+ expect(fixture.policies.has(desired[0])).toBe(false)+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ fixture.failPolicyType = undefined+ const now = Date.now()+ const request = vi.spyOn(state, 'runInitialNetworkPolicyConvergence')+ if (kind === 'timer') {+ await vi.advanceTimersByTimeAsync(5000)+ expect(request).toHaveBeenCalledWith(expect.objectContaining({ cause: 'retry' }))+ } else {+ expect(+ await (kind === 'McpServer'+ ? state.recoverMcpServerInventoryAndWatch()+ : state.recoverContextInventoryAndWatch())+ ).toBe(true)+ expect(Date.now()).toBe(now)+ }+ await drain(state)+ expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)+ }+ )++ it.each(['McpServer', 'Context'] as const)(+ 'A-T6: concurrent %s recoveries share one physical LIST and one decision',+ async kind => {+ const state = await settled()+ let release!: () => void+ const barrier = new Promise<void>(resolve => {+ release = resolve+ })+ const list = fixture.list.getMockImplementation()!+ const plural = kind === 'McpServer' ? 'mcpservers' : 'contexts'+ fixture.list.mockImplementation(async args => {+ if (args.plural === plural) await barrier+ return list(args)+ })+ fixture.list.mockClear()+ const log = vi.spyOn(hccLogger, 'info')+ const recover = () =>+ kind === 'McpServer'+ ? state.recoverMcpServerInventoryAndWatch()+ : state.recoverContextInventoryAndWatch()+ const first = recover(),+ second = recover()+ expect(first).toBe(second)+ release()+ expect(await first).toBe(true)+ await drain(state)+ expect(fixture.list.mock.calls.filter(([args]) => args.plural === plural)).toHaveLength(1)+ expect(+ log.mock.calls.filter(+ ([, fields]) =>+ (fields as { event?: string; kind?: string })?.event ===+ 'networkpolicy-recovery-decision' && (fields as { kind?: string }).kind === kind+ )+ ).toHaveLength(1)+ }+ )++ it('A-T3: inactive periodic recovery never permits an identical omission', async () => {+ const state = await settled()+ const internals = state as unknown as {+ netPolResyncTimer: ReturnType<typeof setInterval> | null+ }+ clearInterval(internals.netPolResyncTimer!)+ internals.netPolResyncTimer = null+ const admission = vi.spyOn(state, 'runInitialConvergence')+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toHaveLength(1)+ })++ it.each(['spec', 'uid', 'generation'] as const)(+ 'A-T3: changed server %s requests convergence and final policy matches current inventory',+ async change => {+ const state = await settled()+ const server = fixture.servers.get('server')!+ if (change === 'spec') server.spec.transport.port = 3011+ else if (change === 'uid') server.metadata.uid = 'replacement-server-uid'+ else server.metadata.generation = 2+ const admission = vi.spyOn(state, 'runInitialConvergence')+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toHaveLength(1)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ const contextPolicy = [...fixture.policies.values()].find(policy =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )!+ expect(+ contextPolicy.spec?.ingress+ ?.flatMap(rule => rule.ports ?? [])+ .some(port => port.port === (change === 'spec' ? 3011 : 3000))+ ).toBe(true)+ expect(state.servers.get('server')?.uid).toBe(server.metadata.uid)+ }+ )++ it('A-T8: a blocked server FIFO closes API authority while the probe stays fresh, then restores business policies', async () => {+ const state = await settled()+ const control = structuredClone(fixture.servers.get('server')!)+ control.metadata.name = 'control'+ control.metadata.uid = 'control-uid'+ fixture.servers.set('control', control)+ fixture.contexts.get('ctx')!.spec.mcpServers.push('control')+ await state.recoverMcpServerInventoryAndWatch()+ await state.recoverContextInventoryAndWatch()+ await drain(state)+ const controlPolicies = [...fixture.policies.entries()].filter(([, policy]) =>+ Object.values(policy.metadata.labels ?? {}).includes('control')+ )+ expect(controlPolicies.length).toBeGreaterThan(0)+ const apiGate = resolveProviderAuthoritativeFn(watcher)+ const probeGate = resolveProbeAuthoritativeFn(watcher)+ const authenticator = {+ authenticate: () => ({+ subject: 'host:test',+ hostName: 'host',+ hostUid: 'host-uid',+ namespace: 'mcp-host',+ jti: 'request-id',+ issuedAt: 1,+ expiresAt: Number.MAX_SAFE_INTEGER,+ audiences: ['host-context-controller'],+ }),+ } as unknown as McpApiAuthenticator+ const authorization = new McpAuthorizationService(createMcpAuthorizationStore(watcher))+ api = new ContextMapperServer(+ watcher,+ 0,+ undefined,+ undefined,+ apiGate,+ undefined,+ authenticator,+ authorization,+ () => watcher.getReadinessInventoryDetail(),+ probeGate+ )+ await api.start()+ api.setReady(true)+ const address = (api as unknown as { server: http.Server }).server.address() as { port: number }+ const request = (path: string): Promise<{ status: number; body: string }> =>+ new Promise((resolve, reject) => {+ http+ .get({ hostname: '127.0.0.1', port: address.port, path }, response => {+ let body = ''+ response.setEncoding('utf8')+ response.on('data', chunk => {+ body += chunk+ })+ response.on('end', () => resolve({ status: response.statusCode!, body }))+ })+ .on('error', reject)+ })+ expect((await request('/api/v2/hosts/self/mcpservers')).status).toBe(200)+ let release!: () => void, entered!: () => void+ const blocked = new Promise<void>(resolve => {+ release = resolve+ })+ const started = new Promise<void>(resolve => {+ entered = resolve+ })+ const queue = state.enqueueMcpServerReconciliation(state.servers.get('server')!, async () => {+ entered()+ await blocked+ })+ await started+ const original = structuredClone(fixture.servers.get('server')!)+ const changed = structuredClone(original)+ for (const condition of changed.status!.conditions!) {+ condition.status = 'False'+ condition.reason = condition.reason?.replace('Found', 'NotFound')+ }+ fixture.servers.set('server', changed)+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ expect(apiGate()).toBe(false)+ expect(probeGate()).toBe(true)+ expect((await request('/ready')).status).toBe(200)+ expect(await request('/api/v2/hosts/self/mcpservers')).toEqual({+ status: 503,+ body: JSON.stringify({ error: 'authorization_unavailable' }),+ })+ expect(+ [...fixture.policies.values()].some(policy =>+ Object.values(policy.metadata.labels ?? {}).includes('external-egress')+ )+ ).toBe(true)+ release()+ await queue+ await drain(state)+ expect(apiGate()).toBe(true)+ expect(probeGate()).toBe(true)+ expect(+ [...fixture.policies.values()].filter(+ policy =>+ Object.values(policy.metadata.labels ?? {}).includes('server') &&+ ['context-allow', 'rpc-proxy-egress', 'external-egress'].some(type =>+ Object.values(policy.metadata.labels ?? {}).includes(type)+ )+ )+ ).toEqual([])+ for (const [key, policy] of controlPolicies) expect(fixture.policies.get(key)).toEqual(policy)+ fixture.servers.set('server', original)+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ expect(apiGate()).toBe(true)+ const restored = await request('/api/v2/hosts/self/mcpservers')+ expect(restored.status).toBe(200)+ expect(JSON.parse(restored.body).servers).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ name: 'server' }),+ expect.objectContaining({ name: 'control' }),+ ])+ )+ expect(+ [...fixture.policies.values()].some(policy =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )+ ).toBe(true)+ })++ it.each(['before', 'after'] as const)(+ 'A-T5/T6: changes %s capture retain causes and converge the final revision',+ async order => {+ const state = await settled()+ const internal = state as unknown as {+ installMcpServerSnapshot(snapshot: { servers: McpServerCRD[] }): void+ netPolReconciler: { fullReconcile(...args: unknown[]): Promise<void> }+ }+ const log = vi.spyOn(hccLogger, 'info')+ const install = (port: number) => {+ fixture.servers.get('server')!.spec.transport.port = port+ const current = structuredClone(state.servers.get('server')!)+ current.spec.transport.port = port+ internal.installMcpServerSnapshot({ servers: [current] })+ }+ let entered!: () => void, release!: () => void+ const captured = new Promise<void>(resolve => {+ entered = resolve+ })+ const barrier = new Promise<void>(resolve => {+ release = resolve+ })+ const real = internal.netPolReconciler.fullReconcile.bind(internal.netPolReconciler)+ if (order === 'after')+ vi.spyOn(internal.netPolReconciler, 'fullReconcile').mockImplementationOnce(+ async (...args) => {+ entered()+ await barrier+ await real(...args)+ }+ )+ install(3001)+ const first = state.runInitialNetworkPolicyConvergence({ cause: 'mcp-change' })+ if (order === 'after') await captured+ install(3002)+ const second = state.runInitialNetworkPolicyConvergence({ cause: 'context-change' })+ release()+ await Promise.all([first, second])+ await drain(state)+ const starts = log.mock.calls+ .map(+ ([, fields]) =>+ fields as {+ event?: string+ causes: string[]+ serverRevision: number+ trailing: boolean+ }+ )+ .filter(fields => fields?.event === 'networkpolicy-pass-start')+ expect(starts.map(fields => fields.serverRevision)).toEqual(order === 'before' ? [3] : [2, 3])+ expect(starts.map(fields => fields.trailing)).toEqual(+ order === 'before' ? [false] : [false, true]+ )+ expect(starts.flatMap(fields => fields.causes).sort()).toEqual([+ 'context-change',+ 'mcp-change',+ ])+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ const policy = [...fixture.policies.values()].find(policy =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )!+ expect(+ policy.spec?.ingress?.flatMap(rule => rule.ports ?? []).some(port => port.port === 3002)+ ).toBe(true)+ }+ )++ it('A-T6: identical crossed recoveries do not add trailing work to an active settled pass', async () => {+ const state = await settled()+ const core = state as unknown as { runInitialNetworkPolicyConvergenceCore(): Promise<void> }+ const real = core.runInitialNetworkPolicyConvergenceCore.bind(core)+ let entered!: () => void, release!: () => void+ const completed = new Promise<void>(resolve => {+ entered = resolve+ })+ const barrier = new Promise<void>(resolve => {+ release = resolve+ })+ const passes = vi+ .spyOn(core, 'runInitialNetworkPolicyConvergenceCore')+ .mockImplementationOnce(async () => {+ await real()+ const runtime = state.initialConvergenceRuns.get('McpServer')+ if (runtime) await runtime.promise+ entered()+ await barrier+ })+ const active = state.runInitialNetworkPolicyConvergence({ cause: 'periodic-resync' })+ await completed+ const admission = vi.spyOn(state, 'runInitialConvergence')+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toEqual([])+ release()+ await active+ await drain(state)+ expect(passes).toHaveBeenCalledTimes(1)+ })++ it('A-T3: Context membership removal and restoration produce final allow policies', async () => {+ const state = await settled()+ const desired = [...fixture.policies.entries()].filter(([, policy]) =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )+ expect(desired.length).toBeGreaterThan(0)+ fixture.contexts.get('ctx')!.spec.mcpServers = []+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ await drain(state)+ for (const [key] of desired) expect(fixture.policies.has(key)).toBe(false)+ fixture.contexts.get('ctx')!.spec.mcpServers = ['server']+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ await drain(state)+ for (const [key, policy] of desired)+ expect(fixture.policies.get(key)?.spec).toEqual(policy.spec)+ })+ it('A-T5: all eight production entry points report their actual cause', async () => {+ const log = vi.spyOn(hccLogger, 'info')+ const state = await settled()+ const server = fixture.servers.get('server')!+ server.spec.transport.port = 3010+ await state.getMcpServerWatchCallback()('MODIFIED', structuredClone(server))+ await drain(state)+ const contextCallback = () =>+ fixture.watch.mock.calls+ .filter(([path]) => String(path).endsWith('/contexts'))+ .at(-1)![2] as (type: string, value: unknown) => Promise<void>+ const context = fixture.contexts.get('ctx')!+ context.metadata.generation = 2+ await contextCallback()('MODIFIED', structuredClone(context))+ await drain(state)+ server.metadata.generation = 2+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)+ await drain(state)+ context.metadata.generation = 3+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)+ await drain(state)+ const policy = [...fixture.policies.entries()].find(([, policy]) =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )!+ fixture.policies.delete(policy[0])+ fixture.failPolicyType = 'context-allow'+ await contextCallback()('MODIFIED', structuredClone(context))+ await drain(state)+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ fixture.failPolicyType = undefined+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ await vi.advanceTimersByTimeAsync(3_600_000)+ await drain(state)+ const requests = log.mock.calls+ .map(([, fields]) => fields as { event?: string; cause: string })+ .filter(fields => fields?.event === 'networkpolicy-request')+ expect([...new Set(requests.map(fields => fields.cause))].sort()).toEqual([+ 'context-change',+ 'context-reconcile-failure',+ 'context-recovery',+ 'mcp-change',+ 'mcp-recovery',+ 'periodic-resync',+ 'retry',+ 'startup',+ ])+ expect(fixture.policies.get(policy[0])?.spec).toEqual(policy[1].spec)+ })++ it('A-T6 regression: interrupted additive policy work cannot be marked complete after identical recovery', async () => {+ const state = await settled()+ const owner = (state as any).netPolReconciler+ const desired = [...fixture.policies.entries()].find(([, policy]) =>+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')+ )!+ fixture.policies.delete(desired[0])+ const defer = () => {+ let resolve!: () => void+ const promise = new Promise<void>(done => {+ resolve = done+ })+ return { promise, resolve }+ }+ const entered = defer(),+ run = defer(),+ innerDone = defer(),+ finish = defer(),+ watchEntered = defer(),+ watchFinish = defer()+ const real = owner.reconcileContext.bind(owner)+ vi.spyOn(owner, 'reconcileContext').mockImplementationOnce(async (...args) => {+ entered.resolve()+ await run.promise+ const result = await real(...args)+ innerDone.resolve()+ await finish.promise+ return result+ })+ const pass = state.runInitialNetworkPolicyConvergence({ cause: 'periodic-resync' })+ await entered.promise+ const runtime = state.initialConvergenceRuns.get('McpServer')+ if (runtime) await runtime.promise+ fixture.watch.mockImplementationOnce(async () => {+ watchEntered.resolve()+ await watchFinish.promise+ return { abort: vi.fn() }+ })+ const recovery = state.recoverContextInventoryAndWatch()+ await watchEntered.promise+ run.resolve()+ await innerDone.promise+ expect(fixture.policies.has(desired[0])).toBe(false)+ watchFinish.resolve()+ await recovery+ finish.resolve()+ await pass+ await drain(state)+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)+ })})diff --git a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.tsnew file mode 100644index 000000000..3664e368a--- /dev/null+++ b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts@@ -0,0 +1,565 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import type { V1Deployment, V1Service } from '@kubernetes/client-node'+import { McpServerWatcher } from './k8sClient'+import { hccLogger } from './logger'+import { registry } from './metrics'+import { McpServerReconciler } from './reconciler'++function deferred() {+ let resolve!: () => void+ const promise = new Promise<void>(done => {+ resolve = done+ })+ return { promise, resolve }+}++const fixture = vi.hoisted(() => ({+ services: new Map<string, V1Service>(),+ deployments: new Map<string, V1Deployment>(),+ writes: [] as string[],+ revision: 0,+ rolloutReady: true,+ failDeployment: false,+ serverExists: true,+ readInput: undefined as undefined | (() => Promise<unknown>),+ failPublication: undefined as undefined | ((body: any[]) => boolean),+ server: undefined as any,+ readService: undefined as undefined | (() => Promise<void>),+}))++vi.mock('./config', () => ({+ config: {+ devMode: false,+ namespace: 'mcp-server',+ hostNamespace: 'mcp-host',+ channelsNamespace: 'channels',+ llmHooksNamespace: 'llm-hooks',+ rpcProxyNamespace: 'rpc-proxy',+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],+ hostK8sRequestTimeoutMs: 30000,+ hostResyncIntervalSec: 0,+ netPolResyncIntervalSec: 3600,+ netPolDefaultsResyncIntervalSec: 0,+ externalEgressResyncIntervalSec: 0,+ allowedPluginImagePrefixes: ['ghcr.io/palmeradao/'],+ mcpServerImagePullPolicy: 'IfNotPresent',+ },+}))++vi.mock('@kubernetes/client-node', async importOriginal => {+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()+ const absent = () => Object.assign(new Error('fixture resource absent'), { code: 404 })+ const read = <T>(store: Map<string, T>, name: string): T => {+ const object = store.get(name)+ if (!object) throw absent()+ return structuredClone(object)+ }+ const save = (kind: string, store: Map<string, any>, namespace: string, body: any) => {+ if (kind === 'Deployment' && fixture.failDeployment)+ throw new Error('fixture deployment write unavailable')+ const object = structuredClone(body)+ object.metadata = {+ ...object.metadata,+ namespace,+ uid: `${kind}-${body.metadata.name}`,+ resourceVersion: String(++fixture.revision),+ generation: 1,+ }+ if (kind === 'Deployment')+ object.status = {+ observedGeneration: 1,+ replicas: 1,+ readyReplicas: fixture.rolloutReady ? 1 : 0,+ availableReplicas: fixture.rolloutReady ? 1 : 0,+ updatedReplicas: 1,+ }+ store.set(object.metadata.name, object)+ fixture.writes.push(`${kind}/${object.metadata.name}`)+ return structuredClone(object)+ }+ const core = {+ readNamespacedSecret: async () => {+ if (fixture.readInput) return fixture.readInput()+ throw absent()+ },+ readNamespacedConfigMap: async () => {+ throw absent()+ },+ deleteNamespacedService: async ({ name }: any) => {+ fixture.services.delete(name)+ },+ readNamespacedService: async ({ name }: any) => {+ await fixture.readService?.()+ return read(fixture.services, name)+ },+ createNamespacedService: async ({ namespace, body }: any) =>+ save('Service', fixture.services, namespace, body),+ replaceNamespacedService: async ({ namespace, body }: any) =>+ save('Service', fixture.services, namespace, body),+ }+ const apps = {+ deleteNamespacedDeployment: async ({ name }: any) => {+ fixture.deployments.delete(name)+ },+ listNamespacedDeployment: async () => ({+ items: [...fixture.deployments.values()].map(value => structuredClone(value)),+ }),+ readNamespacedDeployment: async ({ name }: any) => read(fixture.deployments, name),+ createNamespacedDeployment: async ({ namespace, body }: any) =>+ save('Deployment', fixture.deployments, namespace, body),+ replaceNamespacedDeployment: async ({ namespace, body }: any) =>+ save('Deployment', fixture.deployments, namespace, body),+ }+ const custom = {+ listNamespacedCustomObject: async ({ plural }: any) => ({+ metadata: { resourceVersion: String(++fixture.revision) },+ items:+ plural === 'mcpservers' && fixture.serverExists ? [structuredClone(fixture.server)] : [],+ }),+ getNamespacedCustomObject: async () => {+ if (!fixture.serverExists) throw absent()+ return structuredClone(fixture.server)+ },+ getNamespacedCustomObjectStatus: async () => structuredClone(fixture.server),+ patchNamespacedCustomObjectStatus: async ({ body }: any) => {+ if (fixture.failPublication?.(body))+ throw Object.assign(new Error('fixture status unavailable'), { code: 503 })+ for (const patch of body) {+ if (patch.op === 'test') {+ const field = patch.path.slice('/metadata/'.length)+ if (fixture.server.metadata[field] !== patch.value)+ throw Object.assign(new Error('fixture status conflict'), { code: 409 })+ } else if (patch.path === '/status') fixture.server.status = structuredClone(patch.value)+ else if (patch.path === '/status/conditions')+ fixture.server.status.conditions = structuredClone(patch.value)+ else throw new Error(`Unexpected status patch ${patch.path}`)+ }+ fixture.server.metadata.resourceVersion = String(++fixture.revision)+ return structuredClone(fixture.server)+ },+ }+ return {+ ...actual,+ KubeConfig: class {+ loadFromDefault() {}+ makeApiClient(type: unknown) {+ if (type === actual.CoreV1Api) return core+ if (type === actual.AppsV1Api) return apps+ if (type === actual.CustomObjectsApi) return custom+ return {}+ }+ },+ Watch: class {+ async watch() {+ return { abort() {} }+ }+ },+ }+})++async function until(predicate: () => boolean) {+ for (let turn = 0; turn < 200; turn++) {+ if (predicate()) return+ await Promise.resolve()+ }+ throw new Error('Expected runtime state did not settle')+}++async function drain(state: any) {+ for (let pass = 0; pass < 30; pass++) {+ await Promise.resolve()+ const pending = [...state.initialConvergenceRuns.values()].map((run: any) => run.promise)+ for (const entry of state.mcpServerReconciliationQueues.values()) pending.push(entry.queue.tail)+ if (!pending.length) return+ await Promise.all(pending)+ }+ throw new Error('Runtime queues did not drain')+}++function prepareWatcher(policyTail?: Promise<void>) {+ const watcher = new McpServerWatcher()+ const state = watcher as any+ expect(state.reconciler).toBeInstanceOf(McpServerReconciler)+ // Only unrelated controller effects are replaced. Runtime reconciliation,+ // serializers, authority fences and generated Service/Deployment bodies run.+ vi.spyOn(state.netPolReconciler, 'ensureDefaultPolicies').mockResolvedValue(undefined)+ vi.spyOn(state.netPolReconciler, 'hasCertifiedSafetyInventory').mockReturnValue(true)+ vi.spyOn(state.netPolReconciler, 'reconcileExternalEgress').mockResolvedValue(undefined)+ vi.spyOn(state.netPolReconciler, 'fullReconcile').mockImplementation(async (...args: any[]) => {+ args[2].onAuthoritativeRevocationComplete()+ await policyTail+ })+ for (const owner of [+ 'hostReconciler',+ 'sharedFileSystemReconciler',+ 'gfsReconciler',+ 'llmHookReconciler',+ ]) {+ vi.spyOn(state[owner], 'fullReconcile').mockResolvedValue(undefined)+ }+ vi.spyOn(state.hostReconciler, 'reconcileHosts').mockResolvedValue(undefined)+ return { watcher, state }+}++async function skippedTickCount() {+ const metric = await registry+ .getSingleMetric('clerum_hcc_netpol_resync_ticks_skipped_total')!+ .get()+ return metric.values.find(value => value.labels.reason === 'pass-in-flight')?.value ?? 0+}++describe('A-T7 real MCP runtime repair after identical recovery', () => {+ let watcher: McpServerWatcher | undefined+ beforeEach(() => {+ vi.useFakeTimers()+ fixture.services.clear()+ fixture.deployments.clear()+ fixture.writes.length = 0+ fixture.revision = 0+ fixture.rolloutReady = true+ fixture.failDeployment = false+ fixture.serverExists = true+ fixture.readInput = undefined+ fixture.failPublication = undefined+ fixture.readService = undefined+ fixture.server = {+ metadata: {+ name: 'runtime-witness',+ namespace: 'mcp-server',+ uid: 'runtime-witness-uid',+ generation: 1,+ resourceVersion: '1',+ },+ spec: {+ image: 'ghcr.io/palmeradao/runtime-test:v1',+ enabled: true,+ managed: true,+ transport: { type: 'http', port: 3000 },+ },+ }+ })+ afterEach(async () => {+ await watcher?.stop()+ vi.restoreAllMocks()+ vi.useRealTimers()+ })++ it('A-T7a a skipped tick does not repair drift; the next effective periodic tick repairs unchanged CRD intent', async () => {+ const policyTail = deferred()+ const harness = prepareWatcher(policyTail.promise)+ watcher = harness.watcher+ const { state } = harness+ const decisions = vi.spyOn(hccLogger, 'info')+ await watcher.start()+ await until(() => !state.runtimeRepairPending && fixture.deployments.size === 1)+ expect(fixture.writes).toEqual(['Service/runtime-witness', 'Deployment/runtime-witness'])+ const desired = structuredClone(fixture.server.spec)+ const desiredGeneration = fixture.server.metadata.generation+ fixture.services.delete('runtime-witness')+ const writesBefore = fixture.writes.length+ const skipsBefore = await skippedTickCount()+ await vi.advanceTimersByTimeAsync(3600_000)+ expect(await skippedTickCount()).toBe(skipsBefore + 1)+ expect(fixture.services.has('runtime-witness')).toBe(false)+ expect(fixture.writes).toHaveLength(writesBefore)+ policyTail.resolve()+ await drain(state)+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(decisions).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({+ event: 'networkpolicy-recovery-decision',+ decision: 'skip',+ reason: 'identical-complete',+ })+ )+ expect(fixture.services.has('runtime-witness')).toBe(false)+ expect(fixture.writes).toHaveLength(writesBefore)+ await vi.advanceTimersByTimeAsync(3600_000)+ await drain(state)+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)+ expect(fixture.writes.slice(writesBefore)).toContain('Service/runtime-witness')+ expect(fixture.server.spec).toEqual(desired)+ expect(fixture.server.metadata.generation).toBe(desiredGeneration)+ expect(state.runtimeRepairPending).toBe(false)+ })++ it('A-T7b a new watch generation retires a blocked effect and retains repair until a real resource write completes', async () => {+ const oldRead = deferred()+ const newRead = deferred()+ let reads = 0+ fixture.readService = async () => {+ reads+++ if (reads === 1) await oldRead.promise+ if (reads === 2) await newRead.promise+ }+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ const decisions = vi.spyOn(hccLogger, 'info')+ await watcher.start()+ await until(() => reads === 1 && !state.networkPolicyRepairPending)+ expect(state.runtimeRepairPending).toBe(true)+ const oldGeneration = state.mcpWatchGeneration+ const clockBefore = Date.now()+ await state.recoverMcpServerInventoryAndWatch()+ expect(state.mcpWatchGeneration).toBeGreaterThan(oldGeneration)+ oldRead.resolve()+ await until(() => reads === 2)+ expect(fixture.writes).toEqual([])+ expect(fixture.services.size).toBe(0)+ expect(state.runtimeRepairPending).toBe(true)+ newRead.resolve()+ await drain(state)+ expect(fixture.writes).toEqual(['Service/runtime-witness', 'Deployment/runtime-witness'])+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)+ expect(+ fixture.deployments.get('runtime-witness')?.spec?.template.spec?.containers[0].image+ ).toBe('ghcr.io/palmeradao/runtime-test:v1')+ expect(state.runtimeRepairPending).toBe(false)+ expect(decisions).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ decision: 'request', reason: 'work-pending' })+ )+ expect(Date.now()).toBe(clockBefore)+ })++ it('A-T7 regression: a swallowed runtime API failure must retain its pending repair', async () => {+ fixture.readService = async () => {+ throw new Error('transient upstream unavailable')+ }+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(fixture.services.size).toBe(0)+ expect(fixture.deployments.size).toBe(0)+ expect(state.reconciler.getStatus('runtime-witness')?.deployed).toBe(false)+ fixture.readService = undefined+ const decision = vi.spyOn(hccLogger, 'info')+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(decision).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ decision: 'request' })+ )+ expect(fixture.services.has('runtime-witness')).toBe(true)+ })++ it('A-T7 regression: identical recovery must preserve the active readiness poll', async () => {+ fixture.rolloutReady = false+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)+ expect(state.reconciler.hasPendingReconciliation()).toBe(true)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ expect(state.reconciler.readinessPolls.size).toBe(1)+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(false)+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ const deployment = fixture.deployments.get('runtime-witness')!+ deployment.status!.readyReplicas = 1+ deployment.status!.availableReplicas = 1+ await vi.advanceTimersByTimeAsync(5000)+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)+ expect(fixture.server.status.conditions).toEqual(+ expect.arrayContaining([expect.objectContaining({ type: 'DeploymentReady', status: 'True' })])+ )+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)+ })++ it.each(['disabled', 'wrc', 'missing-input', 'wrc-missing-input'] as const)(+ 'A-T7 control: intentional %s runtime absence is complete',+ async mode => {+ if (mode === 'disabled') fixture.server.spec.enabled = false+ else if (mode === 'wrc' || mode === 'wrc-missing-input') fixture.server.spec.managed = false+ if (mode.includes('missing-input'))+ fixture.server.spec.envSecret = { name: 'absent-fixture', keys: [] }+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(fixture.services.size).toBe(0)+ expect(fixture.deployments.size).toBe(0)+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ // Fold the status-derived eligibility result back through real LIST once.+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)+ const decisions = vi.spyOn(hccLogger, 'info')+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(decisions).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ decision: 'skip', reason: 'identical-complete' })+ )+ }+ )++ it('A-T7 control: watch or informer work retains its obligation outside a fleet pass', async () => {+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ fixture.services.delete('runtime-witness')+ const entered = deferred(),+ release = deferred()+ fixture.readService = async () => {+ entered.resolve()+ await release.promise+ }+ const work = state.reconciler.reconcile(state.servers.get('runtime-witness'))+ await entered.promise+ expect(state.runtimeRepairPending).toBe(false)+ expect(state.reconciler.hasPendingReconciliation()).toBe(true)+ const decisions = vi.spyOn(hccLogger, 'info')+ await state.recoverMcpServerInventoryAndWatch()+ release.resolve()+ await work+ await drain(state)+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)+ expect(decisions).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ decision: 'request', reason: 'work-pending' })+ )+ })++ it('A-T7 control: a readiness tick retired during LIST preserves repair after its poll entry disappears', async () => {+ fixture.rolloutReady = false+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(state.reconciler.readinessPolls.size).toBe(1)+ const entered = deferred(),+ release = deferred()+ vi.spyOn(state.watch, 'watch').mockImplementationOnce(async () => {+ entered.resolve()+ await release.promise+ return { abort() {} }+ })+ const recovery = state.recoverMcpServerInventoryAndWatch()+ await entered.promise+ await vi.advanceTimersByTimeAsync(5000)+ expect(state.reconciler.readinessPolls.size).toBe(0)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)+ release.resolve()+ await recovery+ await drain(state)+ const deployment = fixture.deployments.get('runtime-witness')!+ deployment.status!.readyReplicas = 1+ deployment.status!.availableReplicas = 1+ await vi.advanceTimersByTimeAsync(5000)+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)+ })+ it.each(['managed', 'wrc'] as const)(+ 'A-T7 regression: transient input read preserves %s repair',+ async mode => {+ fixture.server.spec.envSecret = { name: 'synthetic-input', keys: [] }+ if (mode === 'wrc') fixture.server.spec.managed = false+ fixture.readInput = async () => {+ throw Object.assign(new Error('transient fixture API failure'), { code: 503 })+ }+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(fixture.services.size).toBe(0)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)+ fixture.readInput = async () => ({+ metadata: { name: 'synthetic-input', resourceVersion: '1' },+ data: {},+ })+ const before = Date.now()+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(Date.now()).toBe(before)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ if (mode === 'managed')+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)+ else expect(fixture.services.size).toBe(0)+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)+ }+ )++ it('A-T7 regression: failed negative eligibility publication remains pending until repaired', async () => {+ fixture.server.spec.envSecret = { name: 'synthetic-input', keys: [] }+ fixture.readInput = async () => ({+ metadata: { name: 'synthetic-input', resourceVersion: '1' },+ data: {},+ })+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(fixture.services.size).toBe(1)+ expect(fixture.server.status.conditions).toEqual(+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'True' })])+ )+ fixture.readInput = undefined+ fixture.failPublication = body =>+ body.some((patch: any) => {+ const conditions =+ patch.path === '/status'+ ? patch.value?.conditions+ : patch.path === '/status/conditions'+ ? patch.value+ : undefined+ return conditions?.some(+ (condition: any) => condition.type === 'SecretResolved' && condition.status === 'False'+ )+ })+ await state.reconciler.reconcile(state.servers.get('runtime-witness'))+ expect(fixture.services.size).toBe(0)+ expect(fixture.deployments.size).toBe(0)+ expect(fixture.server.status.conditions).toEqual(+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'True' })])+ )+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)+ fixture.failPublication = undefined+ const before = Date.now()+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(Date.now()).toBe(before)+ expect(fixture.server.status.conditions).toEqual(+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'False' })])+ )+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)+ })++ it('A-T7 control: deleting a failed create cleans its Service-only remainder and discharges repair', async () => {+ fixture.failDeployment = true+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(fixture.services.size).toBe(1)+ expect(fixture.deployments.size).toBe(0)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)+ fixture.serverExists = false+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(fixture.services.size).toBe(0)+ expect(fixture.deployments.size).toBe(0)+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ })+})diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex a0c121cb7..772965505 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -13,6 +13,7 @@ import {listAllHosts,listAllSharedFileSystems,} from './k8sClient'+import { hccLogger } from './logger'import { registry } from './metrics'import {DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE,@@ -370,6 +371,12 @@ vi.mock('@kubernetes/client-node', () => {vi.mock('./reconciler', () => ({McpServerReconciler: class {+ hasPendingReconciliation() {+ return false+ }+ hasIncompleteReconciliation() {+ return false+ }fullReconcile = mocks.serverFullReconcilereconcile = vi.fn()reconcileDelete = vi.fn()@@ -1015,7 +1022,10 @@ describe('McpServerWatcher startup', () => {if (failureBoundary === 'LIST' && contextListAttempts === 1) {throw new Error('context discovery temporarily unavailable')}- return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }+ return {+ metadata: { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },+ items: [],+ }}if (plural === 'hosts') {return { metadata: { resourceVersion: 'host-rv' }, items: [] }@@ -1246,14 +1256,14 @@ describe('McpServerWatcher startup', () => {expect(mocks.watch).toHaveBeenNthCalledWith(1,expect.stringContaining('/mcpservers'),- { resourceVersion: 'mcp-inventory-rv' },+ { resourceVersion: 'mcp-inventory-rv', allowWatchBookmarks: true },expect.any(Function),expect.any(Function))expect(mocks.watch).toHaveBeenNthCalledWith(2,expect.stringContaining('/contexts'),- { resourceVersion: 'context-inventory-rv' },+ { resourceVersion: 'context-inventory-rv', allowWatchBookmarks: true },expect.any(Function),expect.any(Function))@@ -1413,10 +1423,16 @@ describe('McpServerWatcher startup', () => {}mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'mcpservers') {- return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }+ return {+ metadata: { resourceVersion: 'opaque/mcp:101', allowWatchBookmarks: true },+ items: [staleServer],+ }}if (plural === 'contexts') {- return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }+ return {+ metadata: { resourceVersion: 'opaque/context:202', allowWatchBookmarks: true },+ items: [staleContext],+ }}if (plural === 'communicationchannels') {return { metadata: { resourceVersion: 'cc-rv' }, items: [] }@@ -1425,11 +1441,14 @@ describe('McpServerWatcher startup', () => {})mocks.watch.mockImplementation(async (path, options, callback) => {if (path.endsWith('/mcpservers')) {- expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })+ expect(options).toEqual({ resourceVersion: 'opaque/mcp:101', allowWatchBookmarks: true })await callback('DELETED', staleServer)}if (path.endsWith('/contexts')) {- expect(options).toEqual({ resourceVersion: 'opaque/context:202' })+ expect(options).toEqual({+ resourceVersion: 'opaque/context:202',+ allowWatchBookmarks: true,+ })await callback('MODIFIED', currentContext)}return { abort: vi.fn() }@@ -1475,7 +1494,7 @@ describe('McpServerWatcher startup', () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'mcpservers') {return {- metadata: { resourceVersion: 'mcp-recovery-rv' },+ metadata: { resourceVersion: 'mcp-recovery-rv', allowWatchBookmarks: true },items: [{metadata: {@@ -1511,8 +1530,8 @@ describe('McpServerWatcher startup', () => {await vi.waitFor(() => expect(watchQueries).toHaveLength(2))expect(watchQueries).toEqual([- { resourceVersion: 'mcp-start-rv' },- { resourceVersion: 'mcp-recovery-rv' },+ { resourceVersion: 'mcp-start-rv', allowWatchBookmarks: true },+ { resourceVersion: 'mcp-recovery-rv', allowWatchBookmarks: true },])expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])expect(callbacks).toHaveLength(2)@@ -1685,7 +1704,7 @@ describe('McpServerWatcher startup', () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'contexts') {return {- metadata: { resourceVersion: 'context-recovery-rv' },+ metadata: { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },items: [{metadata: { name: 'recovered-context', namespace: 'mcp-server' },@@ -1708,8 +1727,8 @@ describe('McpServerWatcher startup', () => {await vi.waitFor(() => expect(watchQueries).toHaveLength(2))expect(watchQueries).toEqual([- { resourceVersion: 'context-start-rv' },- { resourceVersion: 'context-recovery-rv' },+ { resourceVersion: 'context-start-rv', allowWatchBookmarks: true },+ { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },])expect((watcher as any).contexts.get('recovered-context')).toEqual(expect.objectContaining({ name: 'recovered-context' })@@ -1848,7 +1867,9 @@ describe('McpServerWatcher startup', () => {)await server.start()server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({+ cause: 'startup',+ }) as Promise<void>;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(() => currentDeltaSafety.promise)@@ -1925,7 +1946,9 @@ describe('McpServerWatcher startup', () => {)await server.start()server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({+ cause: 'startup',+ }) as Promise<void>// `reconcileContext` reports that its authority fence broke, so the scoped// revocation never finished deleting this Context's stale allows.;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(@@ -2005,7 +2028,9 @@ describe('McpServerWatcher startup', () => {)await server.start()server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({+ cause: 'startup',+ }) as Promise<void>;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(() => uncertifiedDelta.promise)@@ -3422,7 +3447,7 @@ describe('McpServerWatcher startup', () => {})await (watcher as any).startContextWatch('recreated-context-rv')- const convergence = (watcher as any).runInitialNetworkPolicyConvergence()+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await orphanCleanupStarted.promiseconst recreation = contextWatchCallback!('ADDED', recreatedContext)await flushMicrotasks()@@ -4127,15 +4152,15 @@ describe('McpServerWatcher startup', () => {snapshots.push(contexts.map(context => context.name))})- const first = (watcher as any).runInitialNetworkPolicyConvergence()+ const first = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promise;(watcher as any).contexts.set('second-context', {name: 'second-context',namespace: 'mcp-server',spec: { contextId: 'second-context', mcpServers: [] },})- const second = (watcher as any).runInitialNetworkPolicyConvergence()- const third = (watcher as any).runInitialNetworkPolicyConvergence()+ const second = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })+ const third = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)releaseFirstPass.resolve(undefined)@@ -4277,7 +4302,7 @@ describe('McpServerWatcher startup', () => {await Promise.all([stalePass, trailing])expect(reconciler.reconcile).not.toHaveBeenCalled()- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(4);(watcher as any).servers.set('stale-server', {name: 'stale-server',namespace: 'mcp-server',@@ -4355,7 +4380,7 @@ describe('McpServerWatcher startup', () => {)})- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promiseconst revisionBefore = (watcher as any).contextDesiredRevision;(watcher as any).retireContextWatch()@@ -4418,6 +4443,7 @@ describe('McpServerWatcher startup', () => {{ lane: 'NetworkPolicy', kind: 'context' })mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {+ options.onAuthoritativeRevocationComplete?.()pass += 1if (pass === 1) {firstPassStarted.resolve(undefined)@@ -4439,13 +4465,13 @@ describe('McpServerWatcher startup', () => {)})- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promise;(watcher as any).contextCacheSynced = false;(watcher as any).mcpServerCacheSynced = false;(watcher as any).contextWatchGeneration = 12;(watcher as any).mcpWatchGeneration = 14- const trailing = (watcher as any).runInitialNetworkPolicyConvergence()+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })releaseFirstPass.resolve(undefined)await Promise.all([stalePass, trailing])@@ -4531,7 +4557,7 @@ describe('McpServerWatcher startup', () => {});(watcher as any).contextCacheSynced = true;(watcher as any).mcpServerCacheSynced = true- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(appliedContexts).toEqual(['current-context'])expect(appliedServers).toEqual(['current-server'])@@ -4593,7 +4619,7 @@ describe('McpServerWatcher startup', () => {])})- const pass = (watcher as any).runInitialNetworkPolicyConvergence()+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promiseconst contextRevision = (watcher as any).contextDesiredRevisionconst serverRevision = (watcher as any).mcpServerDesiredRevision@@ -4651,7 +4677,7 @@ describe('McpServerWatcher startup', () => {const watcher = new McpServerWatcher()seedNetworkPolicyPassInventory(watcher);(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})const successTimestampBefore = await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy'@@ -4670,7 +4696,7 @@ describe('McpServerWatcher startup', () => {options.onAuthoritativeRevocationComplete?.()})- const pass = (watcher as any).runInitialNetworkPolicyConvergence()+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promiseconst generationBefore = (watcher as any).contextWatchGeneration;(watcher as any).contextDesiredRevision += 1@@ -4717,7 +4743,7 @@ describe('McpServerWatcher startup', () => {const watcher = new McpServerWatcher()seedNetworkPolicyPassInventory(watcher);(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})const successTimestampBefore = await readInitialConvergenceMetric('clerum_hcc_initial_convergence_last_success_timestamp_seconds','NetworkPolicy'@@ -4736,7 +4762,7 @@ describe('McpServerWatcher startup', () => {options.onAuthoritativeRevocationComplete?.()})- const pass = (watcher as any).runInitialNetworkPolicyConvergence()+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await firstPassStarted.promiseconst generationBefore = (watcher as any).mcpWatchGeneration;(watcher as any).mcpServerDesiredRevision += 1@@ -4830,7 +4856,7 @@ describe('McpServerWatcher startup', () => {await Promise.all(pending)})- const pass = (watcher as any).runInitialNetworkPolicyConvergence()+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await effectsOffered.promiseexpect(passes).toBeGreaterThan(0)expect(offeredContextEffects).toBeGreaterThanOrEqual(1)@@ -4871,7 +4897,7 @@ describe('McpServerWatcher startup', () => {options.onAuthoritativeRevocationComplete()})- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })// Authority lost at the callback boundary (contextCacheSynced=false) → record// refuses, so the content-identity revocation counters stay at the "never@@ -4907,7 +4933,7 @@ describe('McpServerWatcher startup', () => {})- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')captured!.onExternalEgressRevoked!(server)@@ -4925,7 +4951,7 @@ describe('McpServerWatcher startup', () => {await (watcher as any).runInitialMcpServerConvergence()expect(mocks.serverFullReconcile).not.toHaveBeenCalled()- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())await watcher.stop()@@ -5207,7 +5233,10 @@ describe('McpServerWatcher startup', () => {{ lane: 'NetworkPolicy', result: 'deferred-unsynced' })- await (watcher as any).runInitialNetworkPolicyConvergence({ ensureDefaults: true })+ await (watcher as any).runInitialNetworkPolicyConvergence({+ ensureDefaults: true,+ cause: 'periodic-resync',+ })expect((watcher as any).netPolConvergenceEnsureDefaults).toBe(false)expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()@@ -5233,7 +5262,7 @@ describe('McpServerWatcher startup', () => {)).toBe(successTimestampBefore);(watcher as any).mcpServerCacheSynced = true- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(@@ -5272,7 +5301,7 @@ describe('McpServerWatcher startup', () => {{ lane: 'NetworkPolicy', result: 'certified' })- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)@@ -5303,7 +5332,7 @@ describe('McpServerWatcher startup', () => {mocks.netPolFullReconcile.mockClear();(watcher as any).mcpServerCacheSynced = false- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)await vi.advanceTimersByTimeAsync(5000)@@ -5373,7 +5402,7 @@ describe('McpServerWatcher startup', () => {server.setReady(true)try {- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()expect((await requestReadyOverHttp(server)).statusCode).toBe(503)const deferredScrape = await requestMetricsOverHttp(server)@@ -5490,7 +5519,7 @@ describe('McpServerWatcher startup', () => {{ lane: 'NetworkPolicy', result: 'aborted-bump' })- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {@@ -5510,8 +5539,8 @@ describe('McpServerWatcher startup', () => {})it('names the inventory-changed throw aborted-bump', async () => {- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const errorSpy = vi.spyOn(hccLogger, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})const watcher = new McpServerWatcher();(watcher as any).contextCacheSynced = true;(watcher as any).mcpServerCacheSynced = true@@ -5523,7 +5552,7 @@ describe('McpServerWatcher startup', () => {{ lane: 'NetworkPolicy', result: 'aborted-bump' })- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {@@ -5571,7 +5600,7 @@ describe('McpServerWatcher startup', () => {})- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(options?.contextInventoryAuthoritative()).toBe(true)expect(options?.serverInventoryAuthoritative()).toBe(true)@@ -5741,7 +5770,7 @@ describe('McpServerWatcher startup', () => {errorSpy.mockRestore()})- it('does not report a post-certification additive failure as a readiness safety failure', async () => {+ it('retains repair and invalidates authority after a post-revocation additive failure', async () => {vi.useFakeTimers()const additiveFailure = new AggregateError([new Error('Context policy API unavailable')],@@ -5754,10 +5783,11 @@ describe('McpServerWatcher startup', () => {options?: { onAuthoritativeRevocationComplete?: () => void }) => {options?.onAuthoritativeRevocationComplete?.()+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)throw additiveFailure})- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const errorSpy = vi.spyOn(hccLogger, 'error').mockImplementation(() => {})const watcher = new McpServerWatcher()stubAuthoritativeInventoryWatch(watcher, 'McpServer')stubAuthoritativeInventoryWatch(watcher, 'Context')@@ -5768,14 +5798,14 @@ describe('McpServerWatcher startup', () => {await watcher.start()await vi.advanceTimersByTimeAsync(0)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)expect(errorSpy).toHaveBeenCalledWith('[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',- additiveFailure+ { err: additiveFailure })expect(errorSpy).not.toHaveBeenCalledWith('[K8s] Initial NetworkPolicy background reconciliation failed:',- additiveFailure+ { err: additiveFailure })expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)@@ -10792,7 +10822,7 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-// unmistakable, and the fix must hold readiness through all of it.const CHURN_PASSES = 5for (let i = 0; i < CHURN_PASSES; i += 1) {- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await flushMicrotasks()}@@ -10866,7 +10896,7 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-server.setReady(true)const CHURN_PASSES = 5for (let i = 0; i < CHURN_PASSES; i += 1) {- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })await flushMicrotasks()}expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)@@ -11250,7 +11280,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptconst watcher = new McpServerWatcher();(watcher as any)[lane.peerSyncedField] = trueawait (watcher as any)[lane.restartMethod](lane.startSnapshot)- expect(watchQueries).toEqual([{ resourceVersion: lane.startRv }])+ expect(watchQueries).toEqual([{ resourceVersion: lane.startRv, allowWatchBookmarks: true }])doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))await flushMicrotasks(20)@@ -11259,8 +11289,8 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt// any timer. Under the old contract this stayed at 0 until +5000ms.expect(listCalls).toBe(1)expect(watchQueries).toEqual([- { resourceVersion: lane.startRv },- { resourceVersion: `${lane.recoveryRvPrefix}-1` },+ { resourceVersion: lane.startRv, allowWatchBookmarks: true },+ { resourceVersion: `${lane.recoveryRvPrefix}-1`, allowWatchBookmarks: true },])expect((watcher as any)[lane.syncedField]).toBe(true)expect((watcher as any)[lane.timerField]).toBeNull()@@ -11874,7 +11904,11 @@ describe('McpServerWatcher watch-recovery retry backoff (exponential, jittered,describe('McpServerWatcher NetworkPolicy periodic resync (#478)', () => {async function startWatcherForNetPolResync(): Promise<McpServerWatcher> {- mocks.netPolFullReconcile.mockClear()+ mocks.netPolFullReconcile+ .mockClear()+ .mockImplementation(async (_contexts, _servers, options) => {+ options?.onAuthoritativeRevocationComplete?.()+ })mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'communicationchannels') {@@ -11941,7 +11975,7 @@ describe('McpServerWatcher NetworkPolicy periodic resync (#478)', () => {expect.objectContaining({ ensureDefaults: true }))- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(3)expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(expect.any(Array),@@ -12291,7 +12325,7 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(2)expect((watcher as any).netPolConvergenceEnsureDefaults).toBe(false)- await (watcher as any).runInitialNetworkPolicyConvergence()+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(expect.any(Array),diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 5b3a4b286..47f15463c 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -234,6 +234,17 @@ type HostSnapshot = {type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'+type NetworkPolicyConvergenceCause =+ | 'mcp-recovery'+ | 'context-recovery'+ | 'periodic-resync'+ | 'startup'+ | 'retry'+ | 'mcp-change'+ | 'context-change'+ | 'context-reconcile-failure'+type CompletedInventoryRevision = { contextRevision: number; serverRevision: number }+type InitialConvergencePassResult =| 'certified'| 'aborted-authority'@@ -767,6 +778,22 @@ export class McpServerWatcher implements McpServerProvider {// the startup role the (now removed) generation equalities used to play.private networkPolicyRevocationContextRevision = -1private networkPolicyRevocationServerRevision = -1+ // Safety certification precedes additive policy work and runtime effects.+ // Only successful, current completions discharge these separate obligations.+ private completedNetworkPolicyRevision: CompletedInventoryRevision | null = null+ private completedRuntimeRevision: CompletedInventoryRevision | null = null+ private networkPolicyRepairPending = false+ private runtimeRepairPending = false+ private runtimeInventoryGeneration: number | null = null+ private readonly pendingNetworkPolicyCauses = new Set<NetworkPolicyConvergenceCause>()+ private networkPolicyPassSequence = 0+ private activeNetworkPolicyPass: {+ passId: number+ causes: NetworkPolicyConvergenceCause[]+ trailing: boolean+ contextRevision: number+ serverRevision: number+ } | null = nullprivate mcpServerCacheSynced = falseprivate contextCacheSynced = falseprivate mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null@@ -1490,6 +1517,7 @@ export class McpServerWatcher implements McpServerProvider {if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlightconst recovery = (async () => {+ const before = this.currentDesiredInventoryRevision()try {const snapshot = await listMcpServerSnapshot()if (this.stopped) return false@@ -1499,15 +1527,15 @@ export class McpServerWatcher implements McpServerProvider {this.mcpServerWatchRecoveryFailures = 0this.mcpServerWatchRecoveryRetryAfterMs = undefinedthis.changeCallback?.()- if (this.contextCacheSynced) {- void this.runInitialNetworkPolicyConvergence()+ if (this.shouldRequestNetworkPolicyRecovery('McpServer', before)) {+ void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-recovery' })}return true} catch (error) {this.mcpServerCacheSynced = falsethis.mcpServerWatchRecoveryFailures += 1this.mcpServerWatchRecoveryRetryAfterMs = getRetryAfterMs(error)- console.error('[K8s] McpServer cache recovery failed:', error)+ hccLogger.error('[K8s] McpServer cache recovery failed:', { err: error })return false}})()@@ -1526,6 +1554,7 @@ export class McpServerWatcher implements McpServerProvider {if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlightconst recovery = (async () => {+ const before = this.currentDesiredInventoryRevision()try {const snapshot = await listContextSnapshot()if (this.stopped) return false@@ -1535,7 +1564,9 @@ export class McpServerWatcher implements McpServerProvider {this.contextWatchRecoveryFailures = 0this.contextWatchRecoveryRetryAfterMs = undefinedthis.hostFleetScheduler.bumpInputRevision()- void this.runInitialNetworkPolicyConvergence()+ if (this.shouldRequestNetworkPolicyRecovery('Context', before)) {+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-recovery' })+ }void this.runInitialSharedFileSystemConvergence()if (this.hostCacheSynced) {const pendingCcLifecycleGeneration =@@ -1553,7 +1584,7 @@ export class McpServerWatcher implements McpServerProvider {this.contextCacheSynced = falsethis.contextWatchRecoveryFailures += 1this.contextWatchRecoveryRetryAfterMs = getRetryAfterMs(error)- console.error('[K8s] Context cache recovery failed:', error)+ hccLogger.error('[K8s] Context cache recovery failed:', { err: error })return false}})()@@ -2906,7 +2937,10 @@ export class McpServerWatcher implements McpServerProvider {netPolResyncTicksSkippedTotal.inc({ reason: 'pass-in-flight' })return}- void this.runInitialNetworkPolicyConvergence({ ensureDefaults: true })+ void this.runInitialNetworkPolicyConvergence({+ ensureDefaults: true,+ cause: 'periodic-resync',+ })}, netPolResyncSec * 1000)console.log(`[K8s] NetworkPolicy periodic resync enabled (every ${netPolResyncSec}s)`)} else {@@ -2946,7 +2980,7 @@ export class McpServerWatcher implements McpServerProvider {// its certification callback, before additive policy convergence completes.// This prevents the startup egress refresh from racing or undoing the safety// pass while keeping readiness independent from both additive fleets.- void this.runInitialNetworkPolicyConvergence()+ void this.runInitialNetworkPolicyConvergence({ cause: 'startup' })void initialSharedFileSystemInventory.then(inventoryComplete => {if (inventoryComplete && !this.stopped) {void this.runInitialSharedFileSystemConvergence()@@ -3006,6 +3040,60 @@ export class McpServerWatcher implements McpServerProvider {}}+ private currentDesiredInventoryRevision(): CompletedInventoryRevision {+ return {+ contextRevision: this.contextDesiredRevision,+ serverRevision: this.mcpServerDesiredRevision,+ }+ }++ private matchesCurrentInventory(revision: CompletedInventoryRevision | null): boolean {+ return (+ revision !== null &&+ revision.contextRevision === this.contextDesiredRevision &&+ revision.serverRevision === this.mcpServerDesiredRevision+ )+ }++ private shouldRequestNetworkPolicyRecovery(+ kind: 'McpServer' | 'Context',+ before: CompletedInventoryRevision+ ): boolean {+ let reason:+ | 'identical-complete'+ | 'relevant-change'+ | 'no-authority'+ | 'work-pending'+ | 'resync-disabled'+ const deferred = kind === 'McpServer' && !this.contextCacheSynced+ if (deferred) reason = 'no-authority'+ else if (!this.matchesCurrentInventory(before)) reason = 'relevant-change'+ else if (this.netPolResyncTimer === null) reason = 'resync-disabled'+ else if (this.currentNetworkPolicySafetyCertificate() === null) reason = 'no-authority'+ else if (+ !this.matchesCurrentInventory(this.completedNetworkPolicyRevision) ||+ !this.matchesCurrentInventory(this.completedRuntimeRevision) ||+ this.networkPolicyRepairPending ||+ this.runtimeRepairPending ||+ this.reconciler.hasPendingReconciliation() ||+ (this.runtimeInventoryGeneration !== null &&+ this.runtimeInventoryGeneration !== this.mcpWatchGeneration) ||+ this.initialConvergenceRetryTimers.has('NetworkPolicy') ||+ this.initialConvergenceRetryTimers.has('McpServer')+ ) {+ reason = 'work-pending'+ } else reason = 'identical-complete'+ const request = !deferred && reason !== 'identical-complete'+ hccLogger.info('[K8s] NetworkPolicy recovery decision', {+ event: 'networkpolicy-recovery-decision',+ kind,+ decision: deferred ? 'defer' : request ? 'request' : 'skip',+ reason,+ ...this.currentDesiredInventoryRevision(),+ })+ return request+ }+private runInitialMcpServerConvergence(): Promise<void> {return this.runInitialConvergence('McpServer')}@@ -3063,8 +3151,13 @@ export class McpServerWatcher implements McpServerProvider {private async runInitialMcpServerConvergenceCore(): Promise<void> {const safetyCertificate = this.currentNetworkPolicySafetyCertificate()+ // A retired run may settle normally without invoking work. Keep its repair+ // obligation until an authoritative pass actually completes every effect.+ this.runtimeRepairPending = trueif (safetyCertificate === null) return+ let effectsComplete = trueconst inventoryGeneration = this.mcpWatchGeneration+ this.runtimeInventoryGeneration = inventoryGenerationconst inventoryAuthoritative = () =>this.hasMcpServerInventoryAuthority(inventoryGeneration) &&this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)@@ -3075,7 +3168,7 @@ export class McpServerWatcher implements McpServerProvider {inventoryAuthoritative)if (this.stopped) return- console.log('[K8s] Running initial McpServer background reconciliation...')+ hccLogger.info('[K8s] Running initial McpServer background reconciliation...')await this.reconciler.fullReconcile(initialServers, {runEffect: async (serverName, work) => {const selected = this.servers.get(serverName)@@ -3083,29 +3176,58 @@ export class McpServerWatcher implements McpServerProvider {name: serverName,namespace: config.namespace,}- if (!(await initialExternalEgressGates.waitFor(laneOwner))) return- if (!inventoryAuthoritative()) return+ if (!(await initialExternalEgressGates.waitFor(laneOwner))) {+ effectsComplete = false+ return+ }+ if (!inventoryAuthoritative()) {+ effectsComplete = false+ return+ }await this.enqueueMcpServerReconciliation(laneOwner, async () => {- if (!inventoryAuthoritative()) return+ if (!inventoryAuthoritative()) {+ effectsComplete = false+ return+ }await work()})},})- if (!inventoryAuthoritative()) return+ if (+ !effectsComplete ||+ !inventoryAuthoritative() ||+ this.reconciler.hasIncompleteReconciliation()+ ) {+ this.scheduleInitialConvergenceRetry('McpServer')+ return+ }+ this.completedRuntimeRevision = safetyCertificate+ this.runtimeRepairPending = falseinitialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)this.clearInitialConvergenceRetry('McpServer')} catch (error) {- console.error('[K8s] Initial McpServer background reconciliation failed:', error)+ hccLogger.error('[K8s] Initial McpServer background reconciliation failed:', { err: error })this.scheduleInitialConvergenceRetry('McpServer')+ } finally {+ this.runtimeInventoryGeneration = null}}- private runInitialNetworkPolicyConvergence(options?: {+ private runInitialNetworkPolicyConvergence(options: {ensureDefaults?: boolean+ cause: NetworkPolicyConvergenceCause}): Promise<void> {if (options?.ensureDefaults) {this.netPolConvergenceEnsureDefaults = true}+ const cause = options.cause+ this.pendingNetworkPolicyCauses.add(cause)+ hccLogger.info('[K8s] NetworkPolicy convergence requested', {+ event: 'networkpolicy-request',+ cause,+ ...this.currentDesiredInventoryRevision(),+ admission: this.activeNetworkPolicyPass === null ? 'before-capture' : 'trailing',+ })return this.runInitialConvergence('NetworkPolicy')}@@ -3168,13 +3290,31 @@ export class McpServerWatcher implements McpServerProvider {this.initialConvergenceRuns.set(lane, run)run.promise = Promise.resolve().then(async () => {+ let trailing = falsedo {run.trailingRequested = falseif (lane === 'McpServer') {await this.runInitialMcpServerConvergenceCore()} else {- await this.runInitialNetworkPolicyConvergenceCore()+ const pass = {+ passId: ++this.networkPolicyPassSequence,+ causes: [...this.pendingNetworkPolicyCauses],+ trailing,+ ...this.currentDesiredInventoryRevision(),+ }+ this.pendingNetworkPolicyCauses.clear()+ this.activeNetworkPolicyPass = pass+ hccLogger.info('[K8s] NetworkPolicy pass started', {+ event: 'networkpolicy-pass-start',+ ...pass,+ })+ try {+ await this.runInitialNetworkPolicyConvergenceCore()+ } finally {+ this.activeNetworkPolicyPass = null+ }}+ trailing = true} while (!this.stopped && run.trailingRequested)// Retire the active slot synchronously before this promise settles.// A request arriving in the following settlement microtask must start@@ -3200,6 +3340,14 @@ export class McpServerWatcher implements McpServerProvider {// full convergence nor safe evidence of success. Recovery of either// missing LIST -> WATCH pair schedules a fresh current-cache pass.const startedAtMs = Date.now()+ const observePass = (result: InitialConvergencePassResult) => {+ observeInitialNetworkPolicyPass(startedAtMs, result)+ hccLogger.info('[K8s] NetworkPolicy pass finished', {+ event: 'networkpolicy-pass-result',+ ...this.activeNetworkPolicyPass,+ result,+ })+ }if (!this.contextCacheSynced || !this.mcpServerCacheSynced) {const unsynced = [!this.contextCacheSynced ? 'Context' : undefined,@@ -3207,17 +3355,19 @@ export class McpServerWatcher implements McpServerProvider {].filter((name): name is string => name !== undefined).join(' and ')- console.warn(+ hccLogger.warn(`[K8s] NetworkPolicy convergence request deferred: caches unsynced (${unsynced})`)initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'unsynced' })- observeInitialNetworkPolicyPass(startedAtMs, 'deferred-unsynced')+ observePass('deferred-unsynced')+ this.networkPolicyRepairPending = truethis.scheduleInitialConvergenceRetry('NetworkPolicy')// A timer tick may have requested defaults. This path never reached// fullReconcile, so drop the flag: event/retry entry stays false.this.netPolConvergenceEnsureDefaults = falsereturn}+ let effectsComplete = truelet authoritativeRevocationCompleted = falselet safetyCertificate: NetworkPolicySafetyCertificate | undefinedtry {@@ -3239,7 +3389,7 @@ export class McpServerWatcher implements McpServerProvider {const serverInventoryAuthoritative = () =>this.mcpServerCacheSynced &&this.mcpServerDesiredRevision === capturedSafetyCertificate.serverRevision- console.log('[K8s] Running initial NetworkPolicy background reconciliation...')+ hccLogger.info('[K8s] Running initial NetworkPolicy background reconciliation...')const ensureDefaults = this.netPolConvergenceEnsureDefaultsthis.netPolConvergenceEnsureDefaults = falseawait this.netPolReconciler.fullReconcile(initialContexts, initialServers, {@@ -3250,6 +3400,8 @@ export class McpServerWatcher implements McpServerProvider {runContextEffect: (contextId, work) =>this.enqueueContextReconciliation(contextId, async () => {if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {+ effectsComplete = false+ this.networkPolicyRepairPending = trueinitialConvergenceEffectsDroppedTotal.inc({lane: 'NetworkPolicy',kind: 'context',@@ -3264,6 +3416,8 @@ export class McpServerWatcher implements McpServerProvider {selected ?? { name: serverName, namespace: config.namespace },async () => {if (!serverInventoryAuthoritative()) {+ effectsComplete = false+ this.networkPolicyRepairPending = trueinitialConvergenceEffectsDroppedTotal.inc({lane: 'NetworkPolicy',kind: 'server',@@ -3297,38 +3451,47 @@ export class McpServerWatcher implements McpServerProvider {void this.runInitialMcpServerConvergence()},})- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {- console.warn(+ if (+ !effectsComplete ||+ !authoritativeRevocationCompleted ||+ !contextInventoryAuthoritative() ||+ !serverInventoryAuthoritative()+ ) {+ hccLogger.warn('[K8s] pass ended without certifying: inventory authority lost',this.networkPolicyInventoryMovement(capturedSafetyCertificate))initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })- observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')+ observePass('aborted-authority')+ this.networkPolicyRepairPending = truethis.scheduleInitialConvergenceRetry('NetworkPolicy')return}+ this.completedNetworkPolicyRevision = capturedSafetyCertificate+ this.networkPolicyRepairPending = falseinitialConvergenceLastSuccessTimestampSeconds.set({ lane: 'NetworkPolicy' },Date.now() / 1000)this.clearInitialConvergenceRetry('NetworkPolicy')- observeInitialNetworkPolicyPass(startedAtMs, 'certified')+ observePass('certified')} catch (error) {- console.error(+ hccLogger.error(authoritativeRevocationCompleted? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:': '[K8s] Initial NetworkPolicy background reconciliation failed:',- error+ { err: error })const abortedBump =error instanceof Error && error.message === DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGEif (abortedBump && safetyCertificate) {- console.warn(+ hccLogger.warn('[K8s] pass ended without certifying: desired inventory changed',this.networkPolicyInventoryMovement(safetyCertificate))}- observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')+ observePass(abortedBump ? 'aborted-bump' : 'failed')+ this.networkPolicyRepairPending = truethis.scheduleInitialConvergenceRetry('NetworkPolicy')}}@@ -3377,7 +3540,7 @@ export class McpServerWatcher implements McpServerProvider {void this.runInitialMcpServerConvergence()return}- void this.runInitialNetworkPolicyConvergence()+ void this.runInitialNetworkPolicyConvergence({ cause: 'retry' })}, delayMs)this.initialConvergenceRetryTimers.set(lane, timer)}@@ -3427,6 +3590,7 @@ export class McpServerWatcher implements McpServerProvider {type: string,apiObj: { metadata: { name: string; namespace?: string }; spec: SharedFileSystemSpec }) => {+ if (type === 'BOOKMARK') returnconst sfs: SharedFileSystemCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.hostNamespace,@@ -3511,6 +3675,7 @@ export class McpServerWatcher implements McpServerProvider {status?: GlobalFileSystemCRD['status']}) => {+ if (type === 'BOOKMARK') returnconst gfs: GlobalFileSystemCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || namespace,@@ -3575,6 +3740,7 @@ export class McpServerWatcher implements McpServerProvider {}) => Promise<void> {return async (type, apiObj) => {+ if (type === 'BOOKMARK') returnconst cc: CommunicationChannelCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.channelsNamespace,@@ -3902,6 +4068,7 @@ export class McpServerWatcher implements McpServerProvider {watchGeneration = this.mcpWatchGeneration): (type: string, apiObj: McpServerWatchObject) => Promise<void> {return async (type, apiObj) => {+ if (type === 'BOOKMARK') returnif (this.stopped || watchGeneration !== this.mcpWatchGeneration) returnconst server: McpServerCRD = {name: apiObj.metadata.name,@@ -3931,7 +4098,7 @@ export class McpServerWatcher implements McpServerProvider {}if (desiredStateChanged) {this.mcpServerDesiredRevision += 1- void this.runInitialNetworkPolicyConvergence()+ void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-change' })}// Most status writes do not change desired state. A runtime eligibility@@ -4172,7 +4339,12 @@ export class McpServerWatcher implements McpServerProvider {this.attemptMcpServerCacheRecovery()}- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)+ const request = await this.watch.watch(+ path,+ { resourceVersion, allowWatchBookmarks: true },+ watchCallback,+ doneCallback+ )if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {request.abort()return watchGeneration@@ -4340,6 +4512,7 @@ export class McpServerWatcher implements McpServerProvider {spec: ContextSpec}) => {+ if (type === 'BOOKMARK') returnif (this.stopped || watchGeneration !== this.contextWatchGeneration) returnconst context: ContextCRD = {name: apiObj.metadata.name,@@ -4399,7 +4572,7 @@ export class McpServerWatcher implements McpServerProvider {serverRevision: previousSafetyCertificate.serverRevision,}}- void this.runInitialNetworkPolicyConvergence()+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-change' })}// Metadata-only Context MODIFIED (label ping-pong, annotation churn)@@ -4512,7 +4685,7 @@ export class McpServerWatcher implements McpServerProvider {`[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,error)- void this.runInitialNetworkPolicyConvergence()+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-reconcile-failure' })return}@@ -4545,7 +4718,12 @@ export class McpServerWatcher implements McpServerProvider {this.attemptContextCacheRecovery()}- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)+ const request = await this.watch.watch(+ path,+ { resourceVersion, allowWatchBookmarks: true },+ watchCallback,+ doneCallback+ )if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {request.abort()return watchGenerationdiff --git a/host-context-controller/src/k8sClient.watchTransport.test.ts b/host-context-controller/src/k8sClient.watchTransport.test.tsnew file mode 100644index 000000000..f166326f8--- /dev/null+++ b/host-context-controller/src/k8sClient.watchTransport.test.ts@@ -0,0 +1,284 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import * as http from 'node:http'+import type { Socket } from 'node:net'+import { McpServerWatcher, getKubeConfig } from './k8sClient'++// Preserve the installed Watch and HTTP transport. Replace only ambient+// configuration loading with an anonymous, inert loopback configuration.+vi.mock('@kubernetes/client-node', async importOriginal => {+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()+ return {+ ...actual,+ KubeConfig: class extends actual.KubeConfig {+ loadFromDefault(): void {+ this.loadFromOptions({+ clusters: [{ name: 'loopback', server: 'http://127.0.0.1:1', skipTLSVerify: true }],+ users: [{ name: 'anonymous' }],+ contexts: [{ name: 'loopback', cluster: 'loopback', user: 'anonymous' }],+ currentContext: 'loopback',+ })+ }+ },+ }+})+vi.mock('./config', () => ({+ config: {+ devMode: false,+ namespace: 'mcp-server',+ hostNamespace: 'mcp-host',+ channelsNamespace: 'channels',+ llmHooksNamespace: 'llm-hooks',+ rpcProxyNamespace: 'rpc-proxy',+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],+ hostK8sRequestTimeoutMs: 30_000,+ hostResyncIntervalSec: 0,+ netPolResyncIntervalSec: 0,+ netPolDefaultsResyncIntervalSec: 0,+ externalEgressResyncIntervalSec: 0,+ governedTracingEnabled: false,+ },+}))++const lanes = [+ {+ plural: 'mcpservers',+ start: 'startMcpServerWatch',+ rv: '101',+ cache: 'servers',+ effect: 'enqueueMcpServerReconciliation',+ },+ {+ plural: 'contexts',+ start: 'startContextWatch',+ rv: '202',+ cache: 'contexts',+ effect: 'enqueueContextIdentityReconciliation',+ },+ {+ plural: 'sharedfilesystems',+ start: 'startSharedFileSystemWatch',+ cache: 'sharedFileSystems',+ effect: 'enqueueSharedFileSystemReconciliation',+ },+ {+ plural: 'globalfilesystems',+ start: 'startGlobalFileSystemWatch',+ cache: 'globalFileSystems',+ effect: 'enqueueGlobalFileSystemReconciliation',+ },+ {+ plural: 'communicationchannels',+ start: 'startCommunicationChannelWatch',+ rv: '505',+ cache: 'communicationChannels',+ effect: 'reconcileHostsReferencingCC',+ },+ { plural: 'llmhooks', start: 'startLlmHookWatch', cache: 'llmHooks', effect: 'hookReconcile' },+ {+ plural: 'hosts',+ start: 'startHostWatch',+ rv: '707',+ cache: 'hosts',+ effect: 'reconcileHostWatchEvent',+ },+] as const++type WatchCallback = (type: string, object: any) => Promise<void>+const watchers: McpServerWatcher[] = []++function createFixture() {+ const watcher = new McpServerWatcher()+ watchers.push(watcher)+ // These exact private methods are called by start(), which also performs+ // unrelated LIST/bootstrap work. Exercise their installed callbacks here.+ const state = watcher as any+ state.mcpServerCacheSynced = true+ state.contextCacheSynced = true+ state.hostCacheSynced = true+ state.ccCacheSynced = true+ const effects: Record<string, ReturnType<typeof vi.fn>> = {}+ for (const method of [+ 'enqueueMcpServerReconciliation',+ 'enqueueContextIdentityReconciliation',+ 'enqueueSharedFileSystemReconciliation',+ 'enqueueGlobalFileSystemReconciliation',+ 'runInitialNetworkPolicyConvergence',+ 'reconcileSharedFileSystemsReferencedByContext',+ 'reconcileHostWatchEvent',+ 'admitHostDependentEffects',+ 'requestHostFleetReconcile',+ ])+ effects[method] = vi.spyOn(state, method).mockResolvedValue(undefined)+ effects.reconcileHostsReferencingCC = vi+ .spyOn(state, 'reconcileHostsReferencingCC')+ .mockResolvedValue(true)+ for (const [name, owner, method] of [+ ['hookReconcile', 'llmHookReconciler', 'reconcile'],+ ['hookDelete', 'llmHookReconciler', 'reconcileDelete'],+ ['hookPolicies', 'llmHookReconciler', 'reconcileNetworkPoliciesForHooks'],+ ['hostEgress', 'llmHookReconciler', 'reconcileHostEgress'],+ ['channelRevision', 'hostReconciler', 'patchChannelReaderRevisionAnnotation'],+ ['sfsReconcile', 'sharedFileSystemReconciler', 'reconcile'],+ ['gfsReconcile', 'gfsReconciler', 'reconcile'],+ ['mcpReconcile', 'reconciler', 'reconcile'],+ ])+ effects[name] = vi.spyOn(state[owner], method).mockResolvedValue(undefined)+ effects.change = vi.fn()+ state.changeCallback = effects.change+ return { watcher, state, effects }+}++function snapshot(state: any) {+ return {+ caches: lanes.map(lane => [...state[lane.cache].entries()]),+ revisions: [+ state.mcpServerDesiredRevision,+ state.contextDesiredRevision,+ state.sharedFileSystemCacheRevision,+ state.globalFileSystemCacheRevision,+ state.hostDesiredRevision,+ state.hostWatchRevision,+ state.ccLifecycleGeneration,+ ],+ tasks: [+ state.mcpServerReconciliationQueues.size,+ state.contextReconciliationQueues.size,+ state.sharedFileSystemReconciliationQueues.size,+ state.globalFileSystemReconciliationQueues.size,+ state.latestHostWatchEventRevisions.size,+ ],+ }+}++function addedObject(plural: string) {+ return {+ metadata: {+ name: `${plural}-witness`,+ namespace: 'fixture',+ uid: `${plural}-uid`,+ generation: 1,+ resourceVersion: '900',+ },+ spec: {+ contextId: 'context-witness',+ contextRef: 'context-witness',+ hostRef: 'host-witness',+ sharedFileSystems: [],+ },+ }+}++afterEach(async () => {+ for (const watcher of watchers.splice(0)) await watcher.stop()+ vi.restoreAllMocks()+})++describe('A-T1 production watch callbacks', () => {+ it.each(lanes)(+ '$plural ignores metadata-only BOOKMARK and admits a named ADDED witness',+ async lane => {+ const { state, effects } = createFixture()+ let callback: WatchCallback | undefined+ vi.spyOn(state.watch, 'watch').mockImplementation(+ async (_path: unknown, _params: unknown, cb: unknown) => {+ callback = cb as WatchCallback+ return new AbortController()+ }+ )+ await state[lane.start]('rv' in lane ? lane.rv : undefined)+ expect(callback).toBeTypeOf('function')+ expect(state.stopped).toBe(false)+ const before = snapshot(state)+ await expect(+ callback!('BOOKMARK', { metadata: { resourceVersion: '899' } })+ ).resolves.toBeUndefined()+ expect(snapshot(state)).toEqual(before)+ for (const effect of Object.values(effects)) expect(effect).not.toHaveBeenCalled()++ // Same callback and fixture: stale generations and no-op callbacks fail+ // this positive control even if the BOOKMARK assertions pass.+ const object = addedObject(lane.plural)+ await callback!('ADDED', object)+ expect(state[lane.cache].get(object.metadata.name)).toMatchObject({+ name: object.metadata.name,+ })+ expect(effects[lane.effect]).toHaveBeenCalledTimes(1)+ const [firstArgument] = effects[lane.effect].mock.calls[0]+ if (lane.plural === 'contexts') expect(firstArgument).toEqual(['context-witness'])+ else if (lane.plural === 'communicationchannels') expect(firstArgument).toBe('host-witness')+ else if (lane.plural === 'hosts')+ expect(effects[lane.effect]).toHaveBeenCalledWith(+ 'ADDED',+ expect.objectContaining({ name: object.metadata.name }),+ expect.any(Number)+ )+ else if (lane.plural === 'sharedfilesystems' || lane.plural === 'globalfilesystems')+ expect(firstArgument).toBe(object.metadata.name)+ else expect(firstArgument).toMatchObject({ name: object.metadata.name })+ }+ )+})++describe('A-T2a installed Kubernetes Watch HTTP transport', () => {+ it('sends seven distinct watch requests, retaining RVs and requesting bookmarks only for MCP/Context', async () => {+ const requests: URL[] = []+ const sockets = new Set<Socket>()+ const server = http.createServer((request, response) => {+ requests.push(new URL(request.url!, 'http://127.0.0.1'))+ response.writeHead(200, { 'content-type': 'application/json' })+ response.flushHeaders()+ })+ server.on('connection', socket => {+ sockets.add(socket)+ socket.once('close', () => sockets.delete(socket))+ })+ await new Promise<void>((resolve, reject) => {+ server.once('error', reject)+ server.listen(0, '127.0.0.1', resolve)+ })+ const address = server.address()+ if (!address || typeof address === 'string') throw new Error('Loopback server did not bind')+ const kubeConfig = getKubeConfig()!+ kubeConfig.loadFromOptions({+ clusters: kubeConfig.getClusters().map(cluster => ({+ ...cluster,+ server: `http://127.0.0.1:${address.port}`,+ })),+ users: kubeConfig.getUsers(),+ contexts: kubeConfig.getContexts(),+ currentContext: kubeConfig.getCurrentContext(),+ })+ const { watcher, state } = createFixture()+ try {+ for (const lane of lanes) await state[lane.start]('rv' in lane ? lane.rv : undefined)+ expect(requests).toHaveLength(7)+ expect(new Set(requests.map(url => url.pathname)).size).toBe(7)+ for (const lane of lanes) {+ const matches = requests.filter(url => url.pathname.endsWith(`/${lane.plural}`))+ expect(matches, lane.plural).toHaveLength(1)+ const url = matches[0]+ expect(url.searchParams.get('watch'), lane.plural).toBe('true')+ expect(url.searchParams.get('allowWatchBookmarks'), lane.plural).toBe(+ lane.plural === 'mcpservers' || lane.plural === 'contexts' ? 'true' : null+ )+ expect(url.searchParams.get('resourceVersion'), lane.plural).toBe(+ 'rv' in lane ? lane.rv : null+ )+ }+ } finally {+ await watcher.stop()+ for (const socket of sockets) socket.destroy()+ await new Promise<void>((resolve, reject) => {+ const deadline = setTimeout(+ () => reject(new Error('Loopback server close exceeded 2 seconds')),+ 2000+ )+ server.close(error => {+ clearTimeout(deadline)+ if (error) reject(error)+ else resolve()+ })+ })+ }+ }, 10_000)+})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex d029f9d70..b87f4b42b 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -5167,7 +5167,9 @@ describe('NetworkPolicyReconciler', () => {],})- await fullPass+ await expect(fullPass).rejects.toThrow(+ 'One or more additive Context NetworkPolicy reconciliations failed'+ )expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 69ae979c9..01c7342de 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -1807,7 +1807,10 @@ export class NetworkPolicyReconciler {const latest = options.resolveCurrentContext(current.name)return latest !== undefined && sameContextDesiredRevision(current, latest)}- await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })+ const completed = await this.reconcileContext(current, {+ isCurrent: contextEffectIsCurrent,+ })+ if (!completed) throw new Error(DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE)})} catch (error) {contextAdditiveFailures.push(error)diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.tsindex 1153a0709..ff3d86767 100644--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts@@ -615,7 +615,7 @@ describe('lost delete fence: per-request 503 while /ready stays 200', () => {})mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await w.runInitialNetworkPolicyConvergence()+ await w.runInitialNetworkPolicyConvergence({ cause: 'startup' })expect(lists).toBeGreaterThan(0)expect(w.contextWatchGeneration).toBe(generationBefore.context + 2)diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex e2400a3f5..e5a52b7a9 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -307,6 +307,39 @@ export class McpServerReconciler {*/private readonly inFlight: Map<string, Promise<void>> = new Map()+ // A fulfilled reconcile promise can mean an authority guard retired work or+ // a resource API error was reported through status. Keep that distinction+ // independently of discovery readiness and the per-name promise queue.+ private readonly pendingReconciliations = new Map<string, { namespace: string }>()++ hasIncompleteReconciliation(): boolean {+ return this.pendingReconciliations.size > 0+ }++ hasPendingReconciliation(): boolean {+ return (+ this.hasIncompleteReconciliation() ||+ this.inFlight.size > 0 ||+ [...this.readinessPolls.values()].some(+ poll => (poll.timer !== undefined || poll.ticking) && this.readinessPollStillDesired(poll)+ )+ )+ }++ private readinessPollStillDesired(poll: ReadinessPollState): boolean {+ const current = this.resolveCurrentServer?.(poll.server.name)+ return (+ this.resolveCurrentServer === undefined ||+ (current !== undefined && sameMcpServerDesiredRevision(poll.server, current))+ )+ }++ private retainRetiredReadinessPoll(poll: ReadinessPollState): void {+ if (this.readinessPollStillDesired(poll)) {+ this.pendingReconciliations.set(poll.server.name, { namespace: poll.server.namespace })+ }+ }+constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)@@ -587,12 +620,14 @@ export class McpServerReconciler {// status — on behalf of a stale desired state. End the window quietly;// the current reconcile arms its own window after its Deployment write.if (!state.isCurrent()) {+ this.retainRetiredReadinessPoll(state)state.ticking = falseif (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)return}const rollout = await this.readDeploymentRollout(name, namespace)if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {+ this.retainRetiredReadinessPoll(state)state.ticking = falseif (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)return@@ -1722,10 +1757,22 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvname: string,namespace: string,deleteAllowed?: () => Promise<boolean>- ): Promise<void> {- // Policy owners reconcile their own desired sets with ownership and- // resource-version preconditions; runtime cleanup must not race them.- await this.deleteResources(name, namespace, deleteAllowed)+ ): Promise<boolean> {+ // A guarded no-op is incomplete even if authority returns before the+ // outer call settles. Ownership-based skips remain legitimate outcomes.+ let complete = true+ await this.deleteResources(+ name,+ namespace,+ deleteAllowed+ ? async () => {+ const allowed = await deleteAllowed()+ if (!allowed) complete = false+ return allowed+ }+ : undefined+ )+ return complete}// ─── CRD Status & Annotation Patching ──────────────────────────────@@ -1850,7 +1897,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvcondition: Omit<McpServerCondition, 'lastTransitionTime'>,isCurrent: () => boolean = () => true): Promise<boolean> {- if (!isCurrent()) return false+ if (!isCurrent()) return this.recordIncompletePublication(server)const now = new Date().toISOString()// Update the missing-secret gauge for SecretResolved conditions.@@ -1875,7 +1922,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// if anything changed since our read; we then re-read and retry, so no// writer's condition is lost.for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {- if (!isCurrent()) return false+ if (!isCurrent()) return this.recordIncompletePublication(server)// Re-read on every optimistic-conflict retry. Merging against the// latest resourceVersion preserves conditions written by peer// controllers between our read and patch.@@ -1901,15 +1948,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`)- return false+ return this.recordIncompletePublication(server)}console.warn(`[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,error)- return false+ return this.recordIncompletePublication(server)}- if (!isCurrent()) return false+ if (!isCurrent()) return this.recordIncompletePublication(server)const prior = existingConditions.find(c => c.type === condition.type)const observedGeneration = condition.observedGeneration ?? server.generation@@ -1980,7 +2027,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// The generated client prefers JSON Patch for this endpoint. Send an// actual patch document so status writes keep working across client// upgrades instead of relying on merge-patch object bodies.- if (!isCurrent()) return false+ if (!isCurrent()) return this.recordIncompletePublication(server)await this.customApi.patchNamespacedCustomObjectStatus({group: 'clerum.io',version: 'v1alpha1',@@ -1996,7 +2043,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`)- return false+ return this.recordIncompletePublication(server)}// 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer// won the race. Re-read and retry rather than lose our condition. Any@@ -2012,9 +2059,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv`[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,error)- return false+ return this.recordIncompletePublication(server)}}+ return this.recordIncompletePublication(server)+ }++ private recordIncompletePublication(server: McpServerCRD): false {+ // Keep the changed-boolean API: an unchanged condition is successful but+ // returns false. Only a failed/retired write replaces the obligation, so+ // an enclosing reconcile cannot clear that failure on normal return.+ this.pendingReconciliations.set(server.name, { namespace: server.namespace })return false}@@ -2039,12 +2094,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn current !== undefined && sameMcpServerDesiredRevision(server, current)}const key = server.name+ const obligation = { namespace: server.namespace }+ this.pendingReconciliations.set(key, obligation)const prev = this.inFlight.get(key) ?? Promise.resolve()const next = prev.catch(() => undefined).then(async () => {if (!isCurrent()) return- await this.performReconcile(server, isCurrent)+ const completed = await this.performReconcile(server, isCurrent)+ if (completed && this.pendingReconciliations.get(key) === obligation) {+ this.pendingReconciliations.delete(key)+ }})this.inFlight.set(key, next)try {@@ -2056,12 +2116,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {- if (!isCurrent()) return+ private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<boolean> {+ if (!isCurrent()) return falsethis.beginStatusTracking(server)// G7: Enforce managed field immutability — once set, cannot change- if (!isCurrent()) return+ if (!isCurrent()) return falseconst currentManaged = server.spec.managed ?? trueconst previousManaged = this.managedSnapshot.get(server.name)if (previousManaged !== undefined && previousManaged !== currentManaged) {@@ -2069,15 +2129,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +`This is not allowed — delete and recreate the McpServer to change ownership.`)- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, {deployed: false,ready: false,message: `managed field is immutable (was ${previousManaged}, attempted ${currentManaged})`,})- return+ return isCurrent()}- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.managedSnapshot.set(server.name, currentManaged)// If disabled, respect ownership before any cleanup. managed:false means@@ -2087,13 +2147,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.log(`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`)- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())+ if (+ !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>+ isCurrent()+ ))+ )+ return false} else {console.log(`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`)}- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })await this.writeStatusCondition(server,@@ -2105,29 +2170,33 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- return+ return isCurrent()}if (!currentManaged) {- await this.reconcileWrcOwnedServer(server, isCurrent)- return+ return this.reconcileWrcOwnedServer(server, isCurrent)}// Validate secret before creating/updatingconst secretResult = await this.validateSecret(server)- if (!isCurrent()) return+ if (!isCurrent()) return falseif (!secretResult.ok) {console.error(`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`)if (shouldFailClosedForSecretFailure(secretResult.reason)) {- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())+ if (+ !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>+ isCurrent()+ ))+ )+ return false} else {console.warn(`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`)}- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, {deployed: false,ready: false,@@ -2153,7 +2222,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- return+ return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()}// Ensure resources exist and are up to date@@ -2186,27 +2255,27 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, { deployed: false, ready: false, message })- return+ return isCurrent()}console.warn(`[Reconciler] ${message} for "${server.name}" — audit mode, allowing (set CONTEXT_MAPPER_ENFORCE_IMAGE_ALLOWLIST=true to block)`)}}- if (!isCurrent()) return+ if (!isCurrent()) return falseawait this.ensureService(server, isCurrent)- if (!isCurrent()) return+ if (!isCurrent()) return falseawait this.ensureDeployment(server, secretResult.revision, isCurrent)} catch (err) {- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, {deployed: false,ready: false,message: `Resource sync failed: ${err instanceof Error ? err.message : String(err)}`,})- return+ return false}// G2: Complete pre-deploy handshake — set network-ready annotation@@ -2223,19 +2292,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvtry {await this.setNetworkReadyAnnotation(server, isCurrent)} catch (err) {- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, {deployed: false,ready: false,message: `Pre-deploy handshake failed: ${err instanceof Error ? err.message : String(err)}`,})- return+ return false}}// Check readiness after reconciliationconst rollout = await this.readDeploymentRollout(server.name, server.namespace)- if (!isCurrent()) return+ if (!isCurrent()) return falseconst ready = rollout.readythis.setStatus(server.name, {deployed: true,@@ -2309,20 +2378,21 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvif (!ready && isCurrent()) {this.pollReadiness(server, 5000, 24, isCurrent)}+ return isCurrent()}private async reconcileWrcOwnedServer(server: McpServerCRD,isCurrent: () => boolean- ): Promise<void> {- if (!isCurrent()) return+ ): Promise<boolean> {+ if (!isCurrent()) return falseconsole.log(`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +'skipping HCC runtime creation/deletion and updating discovery status.')const secretResult = await this.validateSecret(server)- if (!isCurrent()) return+ if (!isCurrent()) return falseif (!secretResult.ok) {this.setStatus(server.name, {deployed: true,@@ -2351,10 +2421,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- return+ return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()}- if (!isCurrent()) return+ if (!isCurrent()) return falsethis.setStatus(server.name, {deployed: true,ready: true,@@ -2382,6 +2452,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)+ return isCurrent()}/**@@ -2390,10 +2461,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvasync reconcileDelete(name: string, namespace: string): Promise<void> {console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)const capturedAuthority = this.inventoryAuthority()- await this.deleteRuntimeResources(name, namespace, () =>+ const obligation = { namespace }+ this.pendingReconciliations.set(name, obligation)+ const complete = await this.deleteRuntimeResources(name, namespace, () =>this.orphanDeleteAllowed(name, namespace, capturedAuthority))- this.clearStatus(name)+ if (complete && this.pendingReconciliations.get(name) === obligation) {+ this.pendingReconciliations.delete(name)+ this.clearStatus(name)+ }}private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {@@ -2487,10 +2563,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)// Delete orphaned deployments (managed by us but no longer desired)- for (const deployment of existingDeployments) {- const name = deployment.metadata?.name || ''+ const cleanupTargets = new Map(+ existingDeployments.map(deployment => [+ deployment.metadata?.name || '',+ deployment.metadata?.namespace || config.namespace,+ ])+ )+ const presentNames = new Set(desiredServers.map(server => server.name))+ for (const [name, pending] of this.pendingReconciliations) {+ if (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)+ }+ for (const [name, namespace] of cleanupTargets) {if (!desiredNames.has(name)) {- const namespace = deployment.metadata?.namespace || config.namespace+ if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })await runEffect(name, async () => {if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {console.warn(@@ -2499,9 +2584,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn}console.log(`[Reconciler] Removing orphaned resources for "${name}"`)- await this.deleteRuntimeResources(name, namespace, () =>- this.orphanDeleteAllowed(name, namespace, capturedAuthority)- )+ await this.reconcileDelete(name, namespace)})}}diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shindex 6cc161d10..c794dbd6f 100755--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh@@ -144,11 +144,20 @@ np604_before_observation() {np604_kctl annotate networkpolicy "ctx-${NP604_CONTEXT}-${NP604_CONTROL}" -n "$MCP_NS" \"e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/nullwait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ HCC_PR_A_MCP_SKIP_BASE="$(hcc_pr_a_decision_count McpServer skip)"+ HCC_PR_A_CONTEXT_SKIP_BASE="$(hcc_pr_a_decision_count Context skip)"+ fiNP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"}np604_after_observation() {np604_observer_alive || die 'NP604 observer stopped before completing observation'+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ [ "$(( $(hcc_pr_a_decision_count McpServer skip) - HCC_PR_A_MCP_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical MCP omissions'+ [ "$(( $(hcc_pr_a_decision_count Context skip) - HCC_PR_A_CONTEXT_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical Context omissions'+ printf 'PR_A_IDENTICAL_RECOVERIES=PASS (at least 3 per watch)\n'+ filocal current_cutscurrent_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"[ "$((current_cuts - NP604_CUT_BASE))" -ge 3 ] || die 'NP604 needs three complete reconnect cycles in stable failed state'diff --git a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs b/scripts/e2e/_lib/hcc-watch-api-proxy.mjsnew file mode 100644index 000000000..162e37bd1--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs@@ -0,0 +1,192 @@+// Development-only transport fixture. No fabricated upstream success responses.+import fs from 'node:fs'+import https from 'node:https'+import { pathToFileURL } from 'node:url'++export function validateCommand(command, allowedPaths) {+ if (!command || typeof command.id !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.id))+ throw new Error('invalid_control_id')+ if (command.action === 'arm') {+ if (+ !allowedPaths.includes(command.path) ||+ command.method !== 'GET' ||+ !Number.isInteger(command.durationMs) ||+ command.durationMs < 1000 ||+ command.durationMs > 25000+ ) {+ throw new Error('invalid_pause_scope')+ }+ } else if (command.action === 'cut') {+ if (!['McpServer', 'Context', 'both'].includes(command.kind))+ throw new Error('invalid_cut_scope')+ } else if (command.action === 'release') {+ if (typeof command.pauseId !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.pauseId))+ throw new Error('invalid_pause_id')+ } else throw new Error('invalid_control_action')+ return command+}++export function createProxy({+ key,+ cert,+ upstreamCa,+ upstreamHost = 'kubernetes.default.svc',+ upstreamPort = 443,+ allowedPaths,+ controlDir,+ periodMs,+ minAgeMs,+}) {+ const streams = new Set()+ let pause = null+ let commandId = null+ const writeRecord = (name, fields) => {+ fs.writeFileSync(`${controlDir}/${name}.next`, JSON.stringify(fields), { mode: 0o600 })+ fs.renameSync(`${controlDir}/${name}.next`, `${controlDir}/${name}.json`)+ }+ const acknowledge = fields => writeRecord('ack', fields)+ const recordPause = (id, state) => writeRecord('pause', { id, state, at: Date.now() })+ const finishPause = reason => {+ if (!pause) return+ const old = pause+ pause = null+ clearTimeout(old.timer)+ recordPause(old.id, reason)+ old.resume?.()+ acknowledge({ id: old.id, state: reason })+ }+ const server = https.createServer({ key, cert }, (request, response) => {+ const url = new URL(request.url, 'https://fixture.invalid')+ if (!request.url.startsWith('/') || request.url.startsWith('//')) {+ response.destroy()+ return+ }+ const watch = url.searchParams.get('watch') === 'true'+ const kind = url.pathname.endsWith('/mcpservers')+ ? 'McpServer'+ : url.pathname.endsWith('/contexts')+ ? 'Context'+ : 'other'+ let upstream+ const stream = {+ request,+ response,+ kind,+ watch,+ born: Date.now(),+ close: () => {+ upstream?.destroy()+ response.destroy()+ request.destroy()+ },+ }+ streams.add(stream)+ response.once('close', () => {+ streams.delete(stream)+ upstream?.destroy()+ })+ const forward = () => {+ if (response.destroyed) return+ // Authentication is forwarded only in process memory to the verified API.+ upstream = https.request(+ {+ hostname: upstreamHost,+ port: upstreamPort,+ method: request.method,+ path: request.url,+ ca: upstreamCa,+ rejectUnauthorized: true,+ headers: { ...request.headers, host: upstreamHost },+ agent: false,+ },+ incoming => {+ response.writeHead(incoming.statusCode, incoming.headers)+ incoming.pipe(response)+ }+ )+ upstream.on('error', () => stream.close())+ request.pipe(upstream)+ }+ if (+ pause &&+ !pause.intercepted &&+ request.method === 'GET' &&+ !watch &&+ url.pathname === pause.path+ ) {+ pause.intercepted = true+ pause.resume = forward+ recordPause(pause.id, 'intercepted')+ acknowledge({ id: pause.id, state: 'intercepted', method: 'GET', path: pause.path })+ } else forward()+ })+ const poll = setInterval(() => {+ try {+ if (!fs.existsSync(`${controlDir}/command.json`)) return+ const command = validateCommand(+ JSON.parse(fs.readFileSync(`${controlDir}/command.json`, 'utf8')),+ allowedPaths+ )+ if (command.id === commandId) return+ commandId = command.id+ if (command.action === 'arm') {+ if (pause) throw new Error('pause_already_active')+ pause = {+ ...command,+ intercepted: false,+ timer: setTimeout(() => finishPause('expired'), command.durationMs),+ }+ recordPause(command.id, 'armed')+ acknowledge({ id: command.id, state: 'armed' })+ } else if (command.action === 'release') {+ if (!pause || pause.id !== command.pauseId || !pause.intercepted)+ throw new Error('pause_not_held')+ finishPause('released')+ acknowledge({ id: command.id, state: 'released' })+ } else {+ let count = 0+ for (const stream of streams) {+ if (+ stream.watch &&+ (command.kind === 'both'+ ? ['McpServer', 'Context'].includes(stream.kind)+ : stream.kind === command.kind)+ ) {+ stream.close()+ count+++ }+ }+ acknowledge({ id: command.id, state: 'cut', count })+ }+ } catch {+ acknowledge({ id: commandId, state: 'rejected' })+ }+ }, 100)+ const churn = setInterval(() => {+ if (fs.existsSync(`${controlDir}/paused`)) return+ for (const stream of streams)+ if (stream.watch && Date.now() - stream.born >= minAgeMs) stream.close()+ }, periodMs)+ const close = () => {+ clearInterval(poll)+ clearInterval(churn)+ finishPause('shutdown')+ for (const stream of streams) stream.close()+ server.close()+ }+ return { server, close }+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const proxy = createProxy({+ key: fs.readFileSync('/fixture-tls/tls.key'),+ cert: fs.readFileSync('/fixture-tls/tls.crt'),+ upstreamCa: fs.readFileSync('/upstream-ca/ca.crt'),+ allowedPaths: JSON.parse(process.env.PAUSE_PATHS),+ controlDir: '/churn-ctl',+ periodMs: Number(process.env.CHURN_PERIOD_MS),+ minAgeMs: Number(process.env.CHURN_MIN_AGE_MS),+ })+ proxy.server.listen(8443, '0.0.0.0')+ process.on('SIGTERM', proxy.close)+}diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.shindex 7d2f21ad8..91b1bc588 100755--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh@@ -245,6 +245,10 @@ delete_synthetic_fleet() {# Non-destructive restore: undo the env overrides and hostAliases redirect.restore_hcc_after_churn() {+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ hcc_pr_a_restore+ return $?+ filocal failed=0kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || failed=1diff --git a/scripts/e2e/_lib/hcc-watch-config.mjs b/scripts/e2e/_lib/hcc-watch-config.mjsnew file mode 100644index 000000000..468a11d1b--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-config.mjs@@ -0,0 +1,96 @@+// Persist only the public fields this development fixture changes.+import fs from 'node:fs'+import { pathToFileURL } from 'node:url'++export const changedEnvironment = [+ 'CONTEXT_MAPPER_K8S_API_CIDRS',+ 'CONTEXT_MAPPER_NETPOL_RESYNC_SEC',+ 'KUBECONFIG',+]+const own = (object, key) => Object.prototype.hasOwnProperty.call(object, key)+export function snapshot(deployment) {+ const pod = deployment.spec.template.spec+ const container = pod.containers.find(item => item.name === 'host-context-controller')+ if (!deployment.metadata.uid || !container) throw new Error('deployment_identity_missing')+ if (+ (pod.volumes ?? []).some(item => item.name === 'hcc-pr-a-config') ||+ (container.volumeMounts ?? []).some(item => item.name === 'hcc-pr-a-config')+ )+ throw new Error('fixture_mount_collision')+ return {+ uid: deployment.metadata.uid,+ envPresent: own(container, 'env'),+ env: (container.env ?? []).filter(item => changedEnvironment.includes(item.name)),+ volumesPresent: own(pod, 'volumes'),+ volumes: pod.volumes ?? [],+ mountsPresent: own(container, 'volumeMounts'),+ mounts: container.volumeMounts ?? [],+ }+}+export function restorePatch(original, deployment) {+ if (deployment.metadata.uid !== original.uid) throw new Error('deployment_uid_changed')+ const pod = deployment.spec.template.spec+ const index = pod.containers.findIndex(item => item.name === 'host-context-controller')+ if (index < 0) throw new Error('deployment_container_missing')+ const container = pod.containers[index]+ for (const [current, previous] of [+ [pod.volumes ?? [], original.volumes],+ [container.volumeMounts ?? [], original.mounts],+ ]) {+ if (+ JSON.stringify(current.filter(item => item.name !== 'hcc-pr-a-config')) !==+ JSON.stringify(previous)+ ) {+ throw new Error('unrelated_configuration_changed')+ }+ }+ if (+ typeof deployment.metadata.resourceVersion !== 'string' ||+ !deployment.metadata.resourceVersion+ ) {+ throw new Error('deployment_resource_version_missing')+ }+ const patches = [+ { op: 'test', path: '/metadata/uid', value: original.uid },+ { op: 'test', path: '/metadata/resourceVersion', value: deployment.metadata.resourceVersion },+ ]+ const set = (path, present, existed, value) => {+ if (present) patches.push({ op: 'add', path, value })+ else if (existed) patches.push({ op: 'remove', path })+ }+ const env = [+ ...(container.env ?? []).filter(item => !changedEnvironment.includes(item.name)),+ ...original.env,+ ]+ set(+ `/spec/template/spec/containers/${index}/env`,+ original.envPresent || env.length > 0,+ own(container, 'env'),+ env+ )+ set(+ `/spec/template/spec/containers/${index}/volumeMounts`,+ original.mountsPresent,+ own(container, 'volumeMounts'),+ original.mounts+ )+ set('/spec/template/spec/volumes', original.volumesPresent, own(pod, 'volumes'), original.volumes)+ return patches+}+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [action, originalPath] = process.argv.slice(2)+ const deployment = JSON.parse(fs.readFileSync(0, 'utf8'))+ if (action === 'snapshot') process.stdout.write(JSON.stringify(snapshot(deployment)))+ else if (action === 'restore')+ process.stdout.write(+ JSON.stringify(restorePatch(JSON.parse(fs.readFileSync(originalPath, 'utf8')), deployment))+ )+ else if (action === 'verify') {+ const original = JSON.parse(fs.readFileSync(originalPath, 'utf8'))+ const current = snapshot(deployment)+ // Environment ordering has no semantic meaning; each exact entry does.+ for (const value of [original, current]) value.env.sort((a, b) => a.name.localeCompare(b.name))+ if (JSON.stringify(current) !== JSON.stringify(original))+ throw new Error('configuration_not_restored')+ } else throw new Error('unknown_configuration_action')+}diff --git a/scripts/e2e/_lib/hcc-watch-gate-probe.mjs b/scripts/e2e/_lib/hcc-watch-gate-probe.mjsnew file mode 100644index 000000000..d9cdf20f6--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-gate-probe.mjs@@ -0,0 +1,31 @@+// Appended to np08-runtime-access.mjs and executed only inside the owned Host.+const expectedStatus = Number(process.env.HCC_PR_A_EXPECT_STATUS)+const observed = await requestWithRuntimeAccess({+ mountedAccessValue: process.env.MCP_HOST_RUNTIME_ACCESS_TOKEN,+ timeoutMs: 5000,+ request: async value => {+ const response = await fetch(+ `${contextMapperBaseUrl(process.env)}/api/v2/hosts/self/mcpservers`,+ {+ headers: { authorization: `Bearer ${value}` },+ redirect: 'error',+ signal: AbortSignal.timeout(5000),+ }+ )+ const body = await response.json()+ if (+ response.status === 200 &&+ (!Array.isArray(body.servers) ||+ !body.servers.some(server => server.name === process.env.HCC_PR_A_CONTROL) ||+ body.servers.some(server => server.name === process.env.HCC_PR_A_AFFECTED) !==+ (process.env.HCC_PR_A_AFFECTED_VISIBLE !== 'false'))+ ) {+ throw new Error('fixture_inventory_missing')+ }+ if (response.status === 503 && body.error !== 'authorization_unavailable')+ throw new Error('unexpected_503_class')+ return { status: response.status }+ },+})+if (observed.result.status !== expectedStatus) throw new Error('protected_gate_status_mismatch')+console.log(`PR_A_PROTECTED_API_STATUS=${expectedStatus} AT=${Date.now()}`)diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.shindex a94799e7a..33383fdf2 100755--- a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh@@ -75,6 +75,11 @@ hcc_lifecycle_fixture_absent() {remaining="$(kctl get pod -n "$HCC_NS" -l "app=${PROXY_NAME}" -o name)" || return 1[ -z "$remaining" ] || return 1fi+ if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then+ remaining="$(kctl get secret,configmap -n "$HCC_NS" \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1+ [ -z "$remaining" ] || return 1+ fiif [ "$PROBE_CREATED" = 1 ]; thenremaining="$(kctl get pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1[ -z "$remaining" ] || return 1@@ -103,10 +108,15 @@ cleanup_hcc_lifecycle() ([ -z "${HCC_LOG_STREAM_PID:-}" ] || kill "$HCC_LOG_STREAM_PID" 2>/dev/null[ -z "${NP604_WATCH_PID:-}" ] || kill "$NP604_WATCH_PID" 2>/dev/nullNP604_WATCH_PID=''- if [ "$HCC_PATCHED" = 1 ]; then+ if [ "${E2E_HCC_PR_A:-0}" = 1 ] &&+ { [ "$HCC_PATCHED" = 1 ] || [ "$HCC_SCALED_DOWN" = 1 ]; } && ! hcc_pr_a_identity_current; then+ restore_ok=0+ fi+ if [ "$HCC_PATCHED" = 1 ] && { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; thenrestore_hcc_after_churn || restore_ok=0fi- if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then+ if { [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; } &&+ { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; thenkctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \--replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \@@ -133,6 +143,11 @@ cleanup_hcc_lifecycle() (kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \-n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1fi+ if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then+ kctl delete secret,configmap -n "$HCC_NS" \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1+ fiwait_until 100 'lifecycle fixture resources removed' hcc_lifecycle_fixture_absent || cleanup_failed=1fi@@ -140,6 +155,9 @@ cleanup_hcc_lifecycle() (finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1print_results || cleanup_failed=1if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi+ if [ "$cleanup_failed" = 0 ] && [ -n "${HCC_PR_A_CONFIG_SNAPSHOT:-}" ]; then+ rm -f "$HCC_PR_A_CONFIG_SNAPSHOT"+ firm -f "$HCC_LOG_BUFFER" "$READY_SERIES"return "$status")diff --git a/scripts/e2e/_lib/hcc-watch-pr-a.sh b/scripts/e2e/_lib/hcc-watch-pr-a.shnew file mode 100644index 000000000..bcb0736a1--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-pr-a.sh@@ -0,0 +1,253 @@+#!/usr/bin/env bash+# PR A extends the existing branch-owned lifecycle fault controller.+HCC_PR_A_RESYNC_SEC=60+HCC_PR_A_TLS_CREATED=0+HCC_PR_A_CONFIG_SNAPSHOT=''+HCC_PR_A_LIB="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"++hcc_pr_a_proxy_pod() {+ kctl get pods -n "$HCC_NS" -l "app=${PROXY_NAME}" -o json |+ jq -er '[.items[]|select(.metadata.deletionTimestamp==null)|select(any(.status.conditions[]?;.type=="Ready" and .status=="True"))]|if length==1 then .[0].metadata.name else error("ambiguous proxy") end'+}++hcc_pr_a_command() {+ local action=$1 value=${2:-} pod command+ pod="$(hcc_pr_a_proxy_pod)" || return 1+ HCC_PR_A_COMMAND_ID="${RUN_ID}-$(date +%s)-${RANDOM}"+ command="$(jq -cn --arg id "$HCC_PR_A_COMMAND_ID" --arg action "$action" --arg value "$value" \+ '{id:$id,action:$action} + (if $action=="arm" then {path:$value,method:"GET",durationMs:25000} elif $action=="cut" then {kind:$value} elif $action=="release" then {pauseId:$value} else {} end)')"+ printf '%s' "$command" | kctl exec -i "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \+ 'const fs=require("fs");const value=fs.readFileSync(0);fs.writeFileSync("/churn-ctl/next.json",value,{mode:384});fs.renameSync("/churn-ctl/next.json","/churn-ctl/command.json")'+}++hcc_pr_a_ack() {+ local state=$1 pod+ pod="$(hcc_pr_a_proxy_pod)" || return 1+ kctl exec "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \+ 'const fs=require("fs");try{const a=JSON.parse(fs.readFileSync("/churn-ctl/ack.json"));if(a.id!==process.argv[1]||a.state!==process.argv[2]||(a.state==="cut"&&a.count<1))process.exit(1)}catch{process.exit(1)}' \+ "$HCC_PR_A_COMMAND_ID" "$state" >/dev/null 2>&1+}++hcc_pr_a_cut() {+ hcc_pr_a_command cut "$1" || return 1+ wait_until 5 'acknowledged live watch cut' hcc_pr_a_ack cut+}++hcc_pr_a_pause_held() {+ local pause_id=$1 pod+ pod="$(hcc_pr_a_proxy_pod)" || return 1+ kctl exec "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \+ 'const fs=require("fs");try{const p=JSON.parse(fs.readFileSync("/churn-ctl/pause.json"));if(p.id!==process.argv[1]||p.state!=="intercepted")process.exit(1)}catch{process.exit(1)}' \+ "$pause_id" >/dev/null 2>&1+}++hcc_pr_a_logs() {+ jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result")' "$HCC_LOG_BUFFER"+}++hcc_pr_a_decision_after() {+ local baseline=$1 kind=$2 decision=$3+ require_hcc_recovery_log_stream || return 1+ [ "$(hcc_pr_a_decision_count "$kind" "$decision")" -gt "$baseline" ]+}++hcc_pr_a_decision_count() {+ hcc_pr_a_logs | jq -s --arg kind "$1" --arg decision "$2" \+ '[.[]|select(.event=="networkpolicy-recovery-decision" and .kind==$kind and .decision==$decision)]|length'+}++hcc_pr_a_periodic_count() {+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified" and (.causes|index("periodic-resync"))!=null)]|length'+}+hcc_pr_a_periodic_after() { [ "$(hcc_pr_a_periodic_count)" -gt "$1" ]; }++hcc_pr_a_periodic_started_count() {+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-start" and (.causes|index("periodic-resync"))!=null)]|length'+}++hcc_pr_a_runtime_completed() {+ local captured pod+ captured="$(hcc_pr_a_logs | jq -sr '[.[]|select(.event=="networkpolicy-pass-start" and (.causes|index("periodic-resync"))!=null)][-1].ts')"+ [ "$captured" != null ] || return 1+ pod="$(running_hcc_pod)" || return 1+ kctl exec -i "pod/$pod" -n "$HCC_NS" -c host-context-controller -- node - \+ "$captured" < "$HCC_PR_A_LIB/hcc-watch-runtime-metric.cjs" >/dev/null 2>&1+}++hcc_pr_a_certified_count() {+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified")]|length'+}+hcc_pr_a_certified_after() { [ "$(hcc_pr_a_certified_count)" -gt "$1" ]; }++hcc_pr_a_run() {+ local periodic baseline context_spec certified affected_policy control_policy context_servers pause_id+ require_hcc_recovery_log_stream || die 'PR A requires live structured logs'+ hcc_pr_a_gate 200 || die 'PR A protected inventory unavailable before fault'+ control_policy="$(np604_snapshot "$NP604_CONTROL")"+ periodic="$(hcc_pr_a_periodic_count)"+ wait_until 75 'PR A real periodic completion' hcc_pr_a_periodic_after "$periodic" || die 'PR A periodic path inactive'+ wait_until 15 'PR A periodic runtime effects complete' hcc_pr_a_runtime_completed || die 'PR A runtime did not complete before drift'+ periodic="$(hcc_pr_a_periodic_count)"+ context_spec="$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')"+ hcc_pr_a_remove_service+ baseline="$(hcc_pr_a_decision_count McpServer skip)"+ hcc_pr_a_cut McpServer || die 'PR A did not cut live MCP watch'+ wait_until 12 'PR A identical recovery omitted' hcc_pr_a_decision_after "$baseline" McpServer skip || die 'PR A identical recovery not omitted'+ hcc_pr_a_service_absent || die 'PR A drift repaired before periodic witness'+ [ "$context_spec" = "$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')" ] || die 'PR A drift changed CRD'+ wait_until 75 'PR A drift repaired by periodic pass' hcc_pr_a_periodic_after "$periodic" || die 'PR A no periodic repair pass'+ wait_until 15 'PR A actual Service recreation' hcc_pr_a_service_restored || die 'PR A periodic pass omitted runtime effect'+ wait_until 10 'PR A repaired runtime complete' hcc_pr_a_runtime_completed || die 'PR A repaired runtime not complete'+ np604_invoke "$NP604_SERVER" || die 'PR A drift recovery business result failed'+ printf 'PR_A_DRIFT_REPAIR=PASS\n'++ periodic="$(hcc_pr_a_periodic_started_count)"+ certified="$(hcc_pr_a_certified_count)"+ hcc_pr_a_remove_service+ # ensureService precedes ensureDeployment in the real reconciler. Holding+ # the Service existence GET prevents the missing effect from being repaired+ # before the watch generation retires its mutationAllowed predicate.+ hcc_pr_a_command arm "/api/v1/namespaces/${MCP_NS}/services/${NP604_SERVER}"+ pause_id=$HCC_PR_A_COMMAND_ID+ wait_until 5 'runtime pause armed' hcc_pr_a_ack armed || die 'PR A runtime pause not armed'+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \+ -p '{"spec":{"description":"PR A runtime interruption"}}' >/dev/null+ wait_until 10 'runtime GET intercepted' hcc_pr_a_ack intercepted || die 'PR A runtime GET not intercepted'+ wait_until 5 'NP finishes while runtime held' hcc_pr_a_certified_after "$certified" || die 'PR A NP did not finish before runtime retirement'+ hcc_pr_a_pause_held "$pause_id" || die 'PR A runtime pause expired before retirement'+ hcc_pr_a_service_absent || die 'PR A missing interrupted effect witness'+ baseline="$(hcc_pr_a_decision_count McpServer request)"+ hcc_pr_a_cut McpServer || die 'PR A runtime watch retirement missing'+ wait_until 5 'retired runtime recovery admitted' hcc_pr_a_decision_after "$baseline" McpServer request || die 'PR A dropped runtime recovery'+ hcc_pr_a_pause_held "$pause_id" || die 'PR A runtime pause expired before recovery completed'+ hcc_pr_a_command release "$pause_id"+ wait_until 5 'runtime pause released' hcc_pr_a_ack released || die 'PR A release not acknowledged'+ wait_until 20 'retired runtime effect repaired' hcc_pr_a_service_restored || die 'PR A retired runtime not repaired'+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || die 'PR A interrupted repair confounded by periodic tick'+ np604_invoke "$NP604_SERVER" || die 'PR A interrupted runtime business result failed'+ printf 'PR_A_INTERRUPTED_RUNTIME=PASS\n'++ affected_policy="$(np604_snapshot "$NP604_SERVER" | jq -er '[.[]|select(.type=="external-egress")]|if length==1 then .[0].name else error("egress identity ambiguous") end')"+ hcc_pr_a_command arm "/apis/networking.k8s.io/v1/namespaces/${MCP_NS}/networkpolicies/${affected_policy}"+ pause_id=$HCC_PR_A_COMMAND_ID+ wait_until 5 'egress pause armed' hcc_pr_a_ack armed || die 'PR A egress pause not armed'+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge \+ -p '{"spec":{"description":"PR A queue interception"}}' >/dev/null+ wait_until 10 'egress GET intercepted' hcc_pr_a_ack intercepted || die 'PR A egress operation not intercepted'+ context_servers="$(np604_kctl get context "$NP604_CONTEXT" -n "$MCP_NS" -o json | jq -c '.spec.mcpServers')"+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \+ -p "$(jq -cn --argjson names "$context_servers" --arg server "$NP604_SERVER" '{spec:{mcpServers:($names|map(select(.!=$server)))}}')" >/dev/null+ wait_until 5 'protected API closes while queue held' hcc_pr_a_gate 503 || die 'PR A protected gate never closed'+ printf 'PR_A_READY_DURING_API_503=%s\n' "$(ready_status "$(running_hcc_pod)")"+ hcc_pr_a_pause_held "$pause_id" || die 'PR A queue pause expired before the API witness'+ hcc_pr_a_command release "$pause_id"+ wait_until 5 'egress pause released' hcc_pr_a_ack released || die 'PR A egress release not acknowledged'+ wait_until 25 'protected API recertifies without revoked server' hcc_pr_a_gate 200 false || die 'PR A protected gate did not reopen with revoked inventory'+ np604_snapshot "$NP604_SERVER" | jq -e 'all(.[]; .type=="external-egress")' >/dev/null || die 'PR A stale Context allows remained'+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \+ -p "$(jq -cn --argjson names "$context_servers" '{spec:{mcpServers:$names}}')" >/dev/null+ wait_until 20 'protected API restores desired binding' hcc_pr_a_gate 200 || die 'PR A desired binding not restored'+ np604_invoke "$NP604_SERVER" || die 'PR A affected business result failed after API recovery'+ np604_invoke "$NP604_CONTROL" || die 'PR A control business result failed after API recovery'+ [ "$(np604_snapshot "$NP604_CONTROL")" = "$control_policy" ] || die 'PR A control policies changed'+ printf 'PR_A_PROTECTED_API_RECOVERY=PASS\n'+ hcc_pr_a_logs > "$NP604_EVIDENCE/pr-a-observations.jsonl"+}++hcc_pr_a_service_absent() {+ local resource+ resource="$(np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" --ignore-not-found -o name)" || return 1+ [ -z "$resource" ]+}++hcc_pr_a_service_restored() {+ np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" -o json |+ jq -e --arg uid "$HCC_PR_A_SERVICE_UID" --arg name "$NP604_SERVER" \+ '.metadata.uid!=$uid and .metadata.labels["clerum.io/mcpserver"]==$name and any(.spec.ports[]; .port==3000)' >/dev/null+}++hcc_pr_a_remove_service() {+ HCC_PR_A_SERVICE_UID="$(np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" -o json |+ jq -er --arg name "$NP604_SERVER" 'select(.metadata.labels["clerum.io/mcpserver"]==$name)|.metadata.uid')" || die 'PR A Service ownership missing'+ np604_kctl delete service "$NP604_SERVER" -n "$MCP_NS" --wait=true --timeout=10s >/dev/null+ hcc_pr_a_service_absent || die 'PR A Service never became absent'+}++hcc_pr_a_gate() {+ local expected=$1 visible=${2:-true} pod+ pod="$(np604_kctl get pods -n "$HOST_NS" -l "clerum.io/context=${NP604_CONTEXT}" -o json |+ jq -er '[.items[]|select(.metadata.deletionTimestamp==null)|select(any(.status.conditions[]?;.type=="Ready" and .status=="True"))][0].metadata.name')" || return 1+ { cat "$HCC_PR_A_LIB/np08-runtime-access.mjs"; cat "$HCC_PR_A_LIB/hcc-watch-gate-probe.mjs"; } |+ np604_kctl exec -i "$pod" -n "$HOST_NS" -- env "HCC_PR_A_EXPECT_STATUS=$expected" \+ "HCC_PR_A_AFFECTED=$NP604_SERVER" "HCC_PR_A_CONTROL=$NP604_CONTROL" \+ "HCC_PR_A_AFFECTED_VISIBLE=$visible" node --input-type=module -+}++hcc_pr_a_preflight() {+ local command_name+ for command_name in node openssl; do+ command -v "$command_name" >/dev/null || die "PR A requires installed ${command_name}"+ done+ [ "$(node -p 'process.versions.node.split(".")[0]')" = 24 ] || die 'PR A requires Node 24'+ openssl req -help 2>&1 | grep -q -- '-addext' || die 'PR A requires openssl req -addext'+ HCC_PR_A_CONFIG_SNAPSHOT="$(mktemp "${TMPDIR:-/tmp}/hcc-pr-a-public-config.XXXXXX")"+ chmod 600 "$HCC_PR_A_CONFIG_SNAPSHOT"+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" snapshot > "$HCC_PR_A_CONFIG_SNAPSHOT"+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || die 'PR A public configuration capture failed'+ printf 'PR_A_TEST_RESYNC_SECONDS=%s (development test only)\n' "$HCC_PR_A_RESYNC_SEC"+}++hcc_pr_a_identity_current() {+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || return 1+ local current+ current="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.uid}')" || return 1+ [ "$current" = "$(jq -r '.uid' "$HCC_PR_A_CONFIG_SNAPSHOT")" ]+}++hcc_pr_a_enable_proxy() {+ local patch paths server+ server="$(truncate_rfc1123 "np604-${RUN_ID}-affected")"+ paths="$(jq -cn --arg ns "$MCP_NS" --arg server "$server" '[+ ("/apis/apps/v1/namespaces/"+$ns+"/deployments/"+$server),+ ("/api/v1/namespaces/"+$ns+"/services/"+$server),+ ("/apis/networking.k8s.io/v1/namespaces/"+$ns+"/networkpolicies/ext-egress-"+$server+"-1-2-3-4-32-443")]')"+ # Consumed by the lifecycle cleanup owner.+ # shellcheck disable=SC2034+ HCC_PR_A_TLS_CREATED=1+ # Generated private material stays in this anonymous pipe, never an artifact.+ node "$HCC_PR_A_LIB/hcc-watch-tls-manifest.mjs" "$PROXY_NAME" "$HCC_NS" "$RUN_ID" \+ "$HCC_PR_A_LIB/hcc-watch-api-proxy.mjs" | kctl create -f - >/dev/null 2>&1 || die 'PR A TLS fixture creation failed'+ patch="$(jq -cn --arg name "$PROXY_NAME" --arg paths "$paths" '{spec:{template:{spec:{+ securityContext:{fsGroup:1000}, containers:[{name:"proxy",command:["node","/fixture/proxy.mjs"],args:[],+ env:[{name:"PAUSE_PATHS",value:$paths}],volumeMounts:[+ {name:"fixture-code",mountPath:"/fixture",readOnly:true},+ {name:"fixture-tls",mountPath:"/fixture-tls",readOnly:true},+ {name:"upstream-ca",mountPath:"/upstream-ca",readOnly:true}]}],+ volumes:[{name:"fixture-code",configMap:{name:$name}},+ {name:"fixture-tls","secret":{secretName:$name,defaultMode:288}},+ {name:"upstream-ca",configMap:{name:"kube-root-ca.crt"}}]}}}}')"+ kctl patch deployment "$PROXY_NAME" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null+ kctl rollout status deployment "$PROXY_NAME" -n "$HCC_NS" --timeout=60s >/dev/null || die 'PR A TLS proxy not Ready'+}++hcc_pr_a_redirect_patch() {+ jq -cn --arg uid "$(jq -r '.uid' "$HCC_PR_A_CONFIG_SNAPSHOT")" \+ --arg name "$PROXY_NAME" --arg cidr "$K8S_API_CIDR" --arg seconds "$HCC_PR_A_RESYNC_SEC" '{metadata:{uid:$uid},spec:{template:{spec:{+ containers:[{name:"host-context-controller",env:[+ {name:"KUBECONFIG",valueFrom:null,value:"/hcc-pr-a/config.json"},+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",valueFrom:null,value:$cidr},+ {name:"CONTEXT_MAPPER_NETPOL_RESYNC_SEC",valueFrom:null,value:$seconds}],+ volumeMounts:[{name:"hcc-pr-a-config",mountPath:"/hcc-pr-a",readOnly:true}]}],+ volumes:[{name:"hcc-pr-a-config",configMap:{name:$name}}]}}}}'+}++hcc_pr_a_restore() {+ local patch+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || return 1+ patch="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" restore "$HCC_PR_A_CONFIG_SNAPSHOT")" || return 1+ kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=json -p "$patch" >/dev/null || return 1+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" verify "$HCC_PR_A_CONFIG_SNAPSHOT"+}diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.shindex ecb97cbdc..7d813c4f3 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh@@ -167,6 +167,7 @@ EOFverify_hcc_proxy_network_policy() {local proxy_dns="${PROXY_NAME}.${HCC_NS}.svc"local proxy_ip positive_probe negative_probe probe_status+ local positive_servername='kubernetes.default.svc' proxy_public_ca=''proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')" ||die "could not resolve the proxy Service ClusterIP"@@ -175,12 +176,20 @@ verify_hcc_proxy_network_policy() {positive_probe="$(cat <<'NODE'const fs=require('fs'),https=require('https');const root='/var/run/secrets/kubernetes.io/serviceaccount/';-const request=https.request({host:process.argv[1],port:443,path:'/version',servername:'kubernetes.default.svc',ca:fs.readFileSync(root+'ca.crt'),headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});+const request=https.request({host:process.argv[1],port:443,path:'/version',servername:process.argv[2]||'kubernetes.default.svc',ca:process.argv[3]?Buffer.from(process.argv[3],'base64'):fs.readFileSync(root+'ca.crt'),rejectUnauthorized:true,headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});request.setTimeout(5000,()=>request.destroy(new Error('timeout')));request.on('error',()=>process.exit(3));request.end();NODE)"+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ # This is the public certificate HCC's temporary kubeconfig will trust.+ # Authentication still comes from the original mounted service account.+ proxy_public_ca="$(kctl get configmap "$PROXY_NAME" -n "$HCC_NS" -o json |+ jq -er '.data["config.json"]|fromjson|.clusters[0].cluster["certificate-authority-data"]')" ||+ die 'PR A proxy public trust configuration missing'+ positive_servername=$proxy_dns+ fikctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \- node -e "$positive_probe" "$proxy_dns" >/dev/null ||+ node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null ||die "HCC cannot reach the Kubernetes API through the isolated proxy"PROBE_CREATED=1diff --git a/scripts/e2e/_lib/hcc-watch-runtime-metric.cjs b/scripts/e2e/_lib/hcc-watch-runtime-metric.cjsnew file mode 100644index 000000000..cd61ddb7d--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-runtime-metric.cjs@@ -0,0 +1,29 @@+// Executed on stdin inside the HCC image's documented working directory.+// Reuse the deployed config parser; never infer a different listener port.+const { config } = require('./dist/config')+const http = require('node:http')+const threshold = Date.parse(process.argv[2]) / 1000+if (!Number.isFinite(threshold)) process.exit(1)+http+ .get(+ {+ hostname: '127.0.0.1',+ port: config.port,+ path: '/metrics',+ signal: AbortSignal.timeout(5000),+ },+ response => {+ let body = ''+ response.on('data', chunk => {+ body += chunk+ })+ response.on('error', () => process.exit(1))+ response.on('end', () => {+ const sample = body.match(+ /^clerum_hcc_initial_convergence_last_success_timestamp_seconds\{lane="McpServer"\} ([0-9.e+]+)$/m+ )+ if (response.statusCode !== 200 || !sample || Number(sample[1]) < threshold) process.exit(1)+ })+ }+ )+ .on('error', () => process.exit(1))diff --git a/scripts/e2e/_lib/hcc-watch-tls-manifest.mjs b/scripts/e2e/_lib/hcc-watch-tls-manifest.mjsnew file mode 100644index 000000000..8af6cda67--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-tls-manifest.mjs@@ -0,0 +1,94 @@+// Public configuration and an ephemeral test key go directly to kubectl stdin.+// Never redirect this program's output to an artifact or log.+import { spawnSync } from 'node:child_process'+import { generateKeyPairSync } from 'node:crypto'+import { readFileSync } from 'node:fs'++const [name, namespace, run, proxySource] = process.argv.slice(2)+for (const value of [name, namespace, run])+ if (!/^[a-z0-9][a-z0-9.-]{0,62}$/.test(value ?? '')) throw new Error('invalid_fixture_identity')+const { privateKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+ publicKeyEncoding: { type: 'spki', format: 'pem' },+})+const signed = spawnSync(+ 'openssl',+ [+ 'req',+ '-new',+ '-x509',+ '-key',+ '/dev/stdin',+ '-days',+ '1',+ '-subj',+ `/CN=${name}.${namespace}.svc`,+ '-addext',+ `subjectAltName=DNS:${name}.${namespace}.svc`,+ '-addext',+ 'basicConstraints=critical,CA:TRUE',+ '-addext',+ 'keyUsage=critical,digitalSignature,keyEncipherment,keyCertSign',+ ],+ { input: privateKey, encoding: 'utf8', timeout: 15000, maxBuffer: 1048576 }+)+if (signed.status !== 0 || !signed.stdout.includes('BEGIN CERTIFICATE'))+ throw new Error('fixture_certificate_generation_failed')+const cert = signed.stdout+const labels = { 'e2e.clerum.io/suite': 'hcc-watch-churn', 'e2e.clerum.io/run': run }+const metadata = { name, namespace, labels }+const config = {+ apiVersion: 'v1',+ kind: 'Config',+ clusters: [+ {+ name: 'fixture',+ cluster: {+ server: `https://${name}.${namespace}.svc:443`,+ 'certificate-authority-data': Buffer.from(cert).toString('base64'),+ },+ },+ ],+ users: [+ {+ name: 'inClusterUser',+ user: {+ 'auth-provider': {+ name: 'tokenFile',+ config: {+ tokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',+ },+ },+ },+ },+ ],+ contexts: [+ { name: 'fixture', context: { cluster: 'fixture', user: 'inClusterUser', namespace } },+ ],+ 'current-context': 'fixture',+}+process.stdout.write(+ JSON.stringify({+ apiVersion: 'v1',+ kind: 'List',+ items: [+ {+ apiVersion: 'v1',+ kind: 'Secret',+ metadata,+ type: 'kubernetes.io/tls',+ stringData: { 'tls.key': privateKey, 'tls.crt': cert },+ },+ {+ apiVersion: 'v1',+ kind: 'ConfigMap',+ metadata,+ data: {+ 'config.json': JSON.stringify(config),+ 'proxy.mjs': readFileSync(proxySource, 'utf8'),+ },+ },+ ],+ })+)diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.shindex 0dbc5f6fa..285ec6e0f 100755--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh@@ -50,6 +50,24 @@ case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in*) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;esac+case "${E2E_HCC_PR_A:-0}" in+ 0) ;;+ 1)+ [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] || { echo 'PR A requires lifecycle mode' >&2; exit 2; }+ source "${SCRIPT_DIR}/_lib/hcc-watch-pr-a.sh"+ # Work600 + cleanup270 leaves30 seconds within the normal900 runner budget.+ HCC_PR_A_WORK_DEADLINE=600+ kctl() {+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))+ [ "$remaining" -gt 0 ] || return 124+ [ "$remaining" -le 90 ] || remaining=90+ node "${SCRIPT_DIR}/_lib/hcc-cleanup-command.mjs" "$remaining" "$KUBECTL_BIN" \+ --context "$E2E_KUBECONTEXT" --request-timeout=30s "$@"+ }+ ;;+ *) echo 'E2E_HCC_PR_A must be 0 or 1' >&2; exit 2 ;;+esac+# ── Fail-closed guards (identical contract to the sibling HCC gates) ──[ -n "$E2E_KUBECONTEXT" ] || {echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2@@ -163,6 +181,11 @@ wait_until() {local timeout=$1 description=$2shift 2local deadline now+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))+ [ "$remaining" -gt 0 ] || return 124+ [ "$timeout" -le "$remaining" ] || timeout=$remaining+ fideadline=$(($(date +%s) + timeout))while :; do"$@" && return 0@@ -211,6 +234,11 @@ hcc_restart_count() {sample_ready_series() {local duration=$1 phase=$2 stop_on_ready=$3local s_deadline t0 now pod status+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))+ [ "$remaining" -gt 0 ] || return 124+ [ "$duration" -le "$remaining" ] || duration=$remaining+ fis_deadline=$(($(date +%s) + duration))while t0="$(date +%s)"; [ "$t0" -lt "$s_deadline" ]; dostatus=503@@ -265,6 +293,11 @@ first_200_epoch() {}print_repair_instructions() {+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ printf 'PR A cleanup failed; lock and dependent fixtures retained.\nContext: %s\nHCC: %s/%s\nExact public restoration snapshot: %s\nInspect identity and restore from this snapshot; do not remove environment entries blindly.\n' \+ "$E2E_KUBECONTEXT" "$HCC_NS" "$HCC_DEPLOY" "${HCC_PR_A_CONFIG_SNAPSHOT:-missing}" >&2+ return+ ficat >&2 <<EOFHCC watch-churn gate cleanup could not restore a verified clean state.Context: ${E2E_KUBECONTEXT}@@ -351,9 +384,10 @@ trap 'exit 131' QUIT# ── FASE A: guards + snapshot ──require_branch_owned_hcc_gate "$HCC_NS"acquire_hcc_watch_gate_lock+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then hcc_pr_a_preflight; fiORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"-[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||+[ "${E2E_HCC_PR_A:-0}" = 1 ] || [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||die "HCC already has hostAliases; refusing a non-restorable injection"HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"@@ -365,6 +399,7 @@ K8S_API_CIDR="${api_ip}/32"# ── FASE B: redirect through the self-flapping proxy (HCC still healthy) ──log "Creating self-flapping API proxy (period=${CHURN_PERIOD_MS}ms, min-age=${CHURN_MIN_AGE_MS}ms)"create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then hcc_pr_a_enable_proxy; fiverify_hcc_proxy_network_policyproxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"@@ -388,6 +423,7 @@ patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},{name:"KUBERNETES_SERVICE_PORT",value:"443"},{name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then patch="$(hcc_pr_a_redirect_patch)"; fiHCC_PATCHED=1kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/nullkctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null@@ -493,7 +529,7 @@ fi# consumed READINESS_BUDGET_SEC of churn.require_hcc_recovery_log_stream ||warn "HCC log stream ended before the verdict; counting the frozen buffer (fail-closed: an undercount can only turn this gate RED)"-stop_hcc_recovery_log_stream+[ "${E2E_HCC_PR_A:-0}" = 1 ] || stop_hcc_recovery_log_streamrecount_churn_evidenceread -r churn_total churn_200 churn_503 churn_maxstreak churn_transitions <<<"$(series_metrics churn)"read -r hold_total _ hold_503 _ _ <<<"$(series_metrics post-hold)"@@ -551,6 +587,10 @@ fiif [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then[ -n "$first_ready" ] || die 'NP604 cannot run without a ready controller'np604_recover+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then+ hcc_pr_a_run+ stop_hcc_recovery_log_stream+ fifilog "Evidence artifact: ${LOG_ARTIFACT}"# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.diff --git a/scripts/tests/hcc-watch-api-proxy.test.mjs b/scripts/tests/hcc-watch-api-proxy.test.mjsnew file mode 100644index 000000000..2919efd12--- /dev/null+++ b/scripts/tests/hcc-watch-api-proxy.test.mjs@@ -0,0 +1,661 @@+import assert from 'node:assert/strict'+import { spawnSync } from 'node:child_process'+import { X509Certificate, generateKeyPairSync } from 'node:crypto'+import { once } from 'node:events'+import { mkdtempSync, readFileSync, rmSync, watch, writeFileSync } from 'node:fs'+import https from 'node:https'+import { tmpdir } from 'node:os'+import { dirname, join, resolve } from 'node:path'+import { before, test } from 'node:test'+import { fileURLToPath } from 'node:url'+import { createProxy, validateCommand } from '../e2e/_lib/hcc-watch-api-proxy.mjs'+import { changedEnvironment, restorePatch, snapshot } from '../e2e/_lib/hcc-watch-config.mjs'++const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..')+const allowedPath = '/api/v1/namespaces/mcp-server/services/runtime-witness'+const mcpPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/mcpservers?watch=true'+const contextPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/contexts?watch=true'+const otherPath = '/api/v1/namespaces/mcp-server/pods?watch=true'+let front+let upstreamTls+let publicConfig++function certificate() {+ const { privateKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ })+ const result = spawnSync(+ 'openssl',+ [+ 'req',+ '-new',+ '-x509',+ '-key',+ '/dev/stdin',+ '-days',+ '1',+ '-subj',+ '/CN=localhost',+ '-addext',+ 'subjectAltName=DNS:localhost,IP:127.0.0.1',+ '-addext',+ 'basicConstraints=critical,CA:TRUE',+ ],+ {+ input: privateKey,+ encoding: 'utf8',+ timeout: 15000,+ maxBuffer: 1048576,+ }+ )+ assert.equal(result.status, 0, 'in-memory certificate generation succeeds')+ return { key: privateKey, cert: result.stdout }+}++before(() => {+ const result = spawnSync(+ process.execPath,+ [+ join(root, 'scripts/e2e/_lib/hcc-watch-tls-manifest.mjs'),+ 'proxy-fixture',+ 'test-fixture',+ 'run-fixture',+ join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'),+ ],+ {+ encoding: 'utf8',+ timeout: 20000,+ maxBuffer: 1048576,+ }+ )+ // Captured generator output stays in memory and is never printed in diagnostics.+ assert.equal(result.status, 0, 'fixture manifest generator succeeds')+ let manifest+ try {+ manifest = JSON.parse(result.stdout)+ } catch {+ throw new Error('fixture generator did not emit JSON')+ }+ assert.equal(manifest.kind, 'List')+ assert.equal(manifest.apiVersion, 'v1')+ assert.equal(manifest.items.length, 2)+ const tlsResource = manifest.items.find(item => item.kind === 'Secret')+ const configMap = manifest.items.find(item => item.kind === 'ConfigMap')+ assert.equal(tlsResource?.type, 'kubernetes.io/tls')+ assert.equal(tlsResource?.metadata.namespace, 'test-fixture')+ assert.equal(tlsResource?.metadata.labels['e2e.clerum.io/run'], 'run-fixture')+ assert.equal(typeof tlsResource?.stringData['tls.key'], 'string')+ front = { key: tlsResource.stringData['tls.key'], cert: tlsResource.stringData['tls.crt'] }+ const cert = new X509Certificate(front.cert)+ assert.equal(cert.checkHost('proxy-fixture.test-fixture.svc'), 'proxy-fixture.test-fixture.svc')+ assert.equal(cert.ca, true)+ publicConfig = JSON.parse(configMap.data['config.json'])+ assert.equal(+ configMap.data['proxy.mjs'] ===+ readFileSync(join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'), 'utf8'),+ true+ )+ assert.equal(+ JSON.stringify(configMap).includes(JSON.stringify(front.key).slice(1, -1)),+ false,+ 'public config excludes signing material'+ )+ upstreamTls = certificate()+})++function bounded(promise, message, timeoutMs = 4000) {+ let timer+ return Promise.race([+ promise,+ new Promise((_, reject) => {+ timer = setTimeout(() => reject(new Error(message)), timeoutMs)+ }),+ ]).finally(() => clearTimeout(timer))+}+function ack(controlDir, id, state, filename = 'ack.json') {+ return new Promise((resolveAck, reject) => {+ let observer+ const done = (error, value) => {+ clearTimeout(timer)+ observer?.close()+ if (error) reject(error)+ else resolveAck(value)+ }+ const read = () => {+ let value+ try {+ value = JSON.parse(readFileSync(join(controlDir, filename), 'utf8'))+ } catch {+ return+ }+ if (value.id === id && value.state === state) done(null, value)+ }+ const timer = setTimeout(+ () => done(new Error(`control acknowledgement missing: ${state}`)),+ 4000+ )+ observer = watch(controlDir, read)+ read()+ })+}+async function fixture(run, { wrongCa = false } = {}) {+ const controlDir = mkdtempSync(join(tmpdir(), 'hcc-proxy-test-'))+ const requests = []+ const responses = new Map()+ const upstream = https.createServer(upstreamTls, (request, response) => {+ requests.push({+ path: request.url,+ method: request.method,+ authorization: request.headers.authorization,+ })+ if (request.url === '/opaque') {+ response.destroy()+ return+ }+ if (request.url.includes('watch=true')) {+ responses.set(request.url, response)+ response.writeHead(200, { 'content-type': 'application/json' })+ response.write('watch-start\n')+ } else {+ response.writeHead(request.url === '/status' ? 418 : 200, { 'content-type': 'text/plain' })+ response.end('upstream-response')+ }+ })+ upstream.on('tlsClientError', () => {})+ await new Promise(resolveListen => upstream.listen(0, '127.0.0.1', resolveListen))+ const proxy = createProxy({+ ...front,+ upstreamCa: wrongCa ? front.cert : upstreamTls.cert,+ upstreamHost: '127.0.0.1',+ upstreamPort: upstream.address().port,+ allowedPaths: [allowedPath],+ controlDir,+ periodMs: 60000,+ minAgeMs: 60000,+ })+ await new Promise(resolveListen => proxy.server.listen(0, '127.0.0.1', resolveListen))+ const clients = new Set()+ const request = (path, method = 'GET') => {+ let responseResolve+ let responseReject+ const opened = new Promise((resolveResponse, reject) => {+ responseResolve = resolveResponse+ responseReject = reject+ })+ const req = https.request({+ hostname: '127.0.0.1',+ port: proxy.server.address().port,+ servername: 'proxy-fixture.test-fixture.svc',+ ca: front.cert,+ rejectUnauthorized: true,+ method,+ path,+ headers: { authorization: 'Bearer test' },+ agent: false,+ })+ clients.add(req)+ let body = ''+ const completed = new Promise((resolveResponse, reject) => {+ req.on('response', response => {+ responseResolve(response)+ response.on('data', chunk => {+ body += chunk+ })+ response.on('end', () => resolveResponse({ status: response.statusCode, body }))+ response.on('error', reject)+ })+ req.on('error', error => {+ responseReject(error)+ reject(error)+ })+ })+ opened.catch(() => {})+ completed.catch(() => {})+ req.end()+ return {+ req,+ opened,+ completed,+ get body() {+ return body+ },+ }+ }+ const command = async (value, state) => {+ const waiting = ack(controlDir, value.id, state)+ writeFileSync(join(controlDir, 'command.json'), JSON.stringify(value), { mode: 0o600 })+ return waiting+ }+ let closed = false+ const close = async () => {+ if (closed) return+ closed = true+ const stopped = once(proxy.server, 'close')+ proxy.close()+ await bounded(stopped, 'proxy server did not close')+ }+ try {+ await run({ request, command, controlDir, requests, responses, close })+ } finally {+ try {+ await close()+ } finally {+ for (const client of clients) client.destroy()+ upstream.closeAllConnections()+ await bounded(+ new Promise(resolveClose => upstream.close(resolveClose)),+ 'upstream did not close'+ )+ rmSync(controlDir, { recursive: true, force: true })+ }+ }+}+async function watchStarted(stream) {+ const response = await bounded(stream.opened, 'watch response not opened')+ if (!stream.body.includes('watch-start'))+ await bounded(once(response, 'data'), 'watch did not flow')+ assert.equal(stream.body.includes('watch-start'), true)+ return response+}++test('generator emits public CA trust and service-account tokenFile provider', () => {+ const cluster = publicConfig.clusters[0].cluster+ assert.equal(cluster.server, 'https://proxy-fixture.test-fixture.svc:443')+ assert.equal(+ Buffer.from(cluster['certificate-authority-data'], 'base64').equals(Buffer.from(front.cert)),+ true+ )+ assert.equal('insecure-skip-tls-verify' in cluster, false)+ const user = publicConfig.users[0].user+ assert.equal(user['auth-provider'].name, 'tokenFile')+ assert.equal(+ user['auth-provider'].config.tokenFile,+ '/var/run/secrets/kubernetes.io/serviceaccount/token'+ )+ assert.deepEqual(Object.keys(user), ['auth-provider'])+ assert.equal(publicConfig['current-context'], 'fixture')+})+test('commands reject invalid id, method, path, duration and cut scope', () => {+ const valid = { action: 'arm', id: 'valid-1', method: 'GET', path: allowedPath, durationMs: 1000 }+ assert.equal(validateCommand(valid, [allowedPath]), valid)+ for (const change of [+ { id: '../escape' },+ { id: true },+ { id: 123 },+ { method: 'POST' },+ { path: '/api/v1/pods' },+ { durationMs: 999 },+ { durationMs: 25001 },+ { durationMs: 1000.5 },+ { action: 'unknown' },+ ]) {+ assert.throws(() => validateCommand({ ...valid, ...change }, [allowedPath]))+ }+ assert.throws(() => validateCommand({ action: 'cut', id: 'cut-1', kind: 'Pod' }, [allowedPath]))+})+test(+ 'verified HTTPS forwards status and synthetic authorization; upstream errors are opaque',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, requests }) => {+ const result = await bounded(request('/status').completed, 'short response absent')+ assert.equal(result.status, 418)+ assert.equal(result.body, 'upstream-response')+ assert.equal(requests[0].authorization === 'Bearer test', true)+ await assert.rejects(bounded(request('/opaque').completed, 'error response absent'), {+ code: 'ECONNRESET',+ })+ })+ }+)+test('wrong upstream CA cannot produce a successful response', { timeout: 10000 }, async () => {+ await fixture(+ async ({ request, requests }) => {+ await assert.rejects(+ bounded(request('/status').completed, 'untrusted response did not settle'),+ { code: 'ECONNRESET' }+ )+ assert.equal(requests.length, 0)+ },+ { wrongCa: true }+ )+})+test(+ 'exact GET pause acknowledges interception and release while watches keep flowing',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, command, controlDir, requests, responses }) => {+ const stream = request(mcpPath)+ const response = await watchStarted(stream)+ await command(+ { action: 'arm', id: 'arm-release', path: allowedPath, method: 'GET', durationMs: 5000 },+ 'armed'+ )+ assert.equal(+ (await bounded(request('/status').completed, 'unrelated short request paused')).status,+ 418+ )+ assert.equal(+ (await bounded(request(allowedPath, 'POST').completed, 'POST was incorrectly paused'))+ .status,+ 200+ )+ const intercepted = ack(controlDir, 'arm-release', 'intercepted')+ const held = request(allowedPath)+ await intercepted+ assert.equal(+ requests.some(value => value.path === allowedPath && value.method === 'GET'),+ false+ )+ const flowing = once(response, 'data')+ responses.get(mcpPath).write('watch-during-pause\n')+ await bounded(flowing, 'watch stalled behind short request')+ assert.equal(stream.body.includes('watch-during-pause'), true)+ await command({ action: 'release', id: 'release-1', pauseId: 'arm-release' }, 'released')+ assert.equal((await bounded(held.completed, 'held GET did not resume')).status, 200)+ assert.equal(+ requests.filter(value => value.path === allowedPath && value.method === 'GET').length,+ 1+ )+ })+ }+)+test(+ 'minimum pause expires and resumes the actual intercepted request',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, command, controlDir, requests }) => {+ await command(+ { action: 'arm', id: 'arm-expiry', path: allowedPath, method: 'GET', durationMs: 1000 },+ 'armed'+ )+ const intercepted = ack(controlDir, 'arm-expiry', 'intercepted')+ const held = request(allowedPath)+ await intercepted+ assert.equal(+ requests.some(value => value.path === allowedPath && value.method === 'GET'),+ false+ )+ await ack(controlDir, 'arm-expiry', 'expired', 'pause.json')+ await command({ action: 'release', id: 'release-expired', pauseId: 'arm-expiry' }, 'rejected')+ assert.equal(+ JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state,+ 'expired'+ )+ assert.equal((await bounded(held.completed, 'expired GET did not resume')).status, 200)+ })+ }+)+test(+ 'cut targets only the selected watch kind and preserves short requests and other streams',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, command, controlDir, responses }) => {+ const mcp = request(mcpPath)+ const context = request(contextPath)+ const other = request(otherPath)+ await watchStarted(mcp)+ const contextResponse = await watchStarted(context)+ const otherResponse = await watchStarted(other)+ await command(+ { action: 'arm', id: 'arm-cut', path: allowedPath, method: 'GET', durationMs: 5000 },+ 'armed'+ )+ const intercepted = ack(controlDir, 'arm-cut', 'intercepted')+ const held = request(allowedPath)+ await intercepted+ const pauseBeforeCut = await ack(controlDir, 'arm-cut', 'intercepted', 'pause.json')+ const cut = await command({ action: 'cut', id: 'cut-mcp', kind: 'McpServer' }, 'cut')+ assert.deepEqual(+ JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')),+ pauseBeforeCut+ )+ assert.equal(cut.count, 1)+ await assert.rejects(bounded(mcp.completed, 'MCP watch did not close'))+ for (const [path, response, stream] of [+ [contextPath, contextResponse, context],+ [otherPath, otherResponse, other],+ ]) {+ const flowing = once(response, 'data')+ responses.get(path).write('survived-cut\n')+ await bounded(flowing, 'unselected stream was cut')+ assert.equal(stream.body.includes('survived-cut'), true)+ }+ await command({ action: 'release', id: 'release-cut', pauseId: 'arm-cut' }, 'released')+ assert.equal((await bounded(held.completed, 'short request was cut')).status, 200)+ const both = await command({ action: 'cut', id: 'cut-both', kind: 'both' }, 'cut')+ assert.equal(both.count, 1)+ await assert.rejects(bounded(context.completed, 'Context watch survived both cut'))+ const flowing = once(otherResponse, 'data')+ responses.get(otherPath).write('survived-both\n')+ await bounded(flowing, 'both cut unrelated watch')+ assert.equal(other.body.includes('survived-both'), true)+ })+ }+)+test(+ 'close tears down active watches and an armed intercepted request',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, command, controlDir, close }) => {+ const stream = request(mcpPath)+ await watchStarted(stream)+ await command(+ { action: 'arm', id: 'arm-close', path: allowedPath, method: 'GET', durationMs: 25000 },+ 'armed'+ )+ const intercepted = ack(controlDir, 'arm-close', 'intercepted')+ const held = request(allowedPath)+ await intercepted+ await close()+ await assert.rejects(bounded(stream.completed, 'watch survived shutdown'))+ await assert.rejects(bounded(held.completed, 'held request survived shutdown'))+ })+ }+)++// Apply the emitted JSON Patch to a cloned in-memory Deployment.+function applyPatch(deployment, patches) {+ const result = structuredClone(deployment)+ for (const patch of patches) {+ const keys = patch.path.slice(1).split('/')+ const parent = keys.slice(0, -1).reduce((value, key) => value[key], result)+ const key = keys.at(-1)+ if (patch.op === 'test') assert.deepEqual(parent[key], patch.value)+ else if (patch.op === 'add') parent[key] = structuredClone(patch.value)+ else if (patch.op === 'remove') delete parent[key]+ else throw new Error('unexpected restore operation')+ }+ return result+}+function deploymentFixture() {+ return {+ metadata: { uid: 'deployment-fixture-uid', resourceVersion: '1' },+ spec: {+ template: {+ spec: {+ containers: [+ { name: 'sidecar', env: [{ name: 'SIDECAR', value: 'untouched' }] },+ { name: 'host-context-controller' },+ ],+ },+ },+ },+ }+}+function injectFixture(deployment) {+ const result = structuredClone(deployment)+ const pod = result.spec.template.spec+ const container = pod.containers.find(value => value.name === 'host-context-controller')+ container.env = [+ ...(container.env ?? []).filter(value => !changedEnvironment.includes(value.name)),+ ...changedEnvironment.map(name => ({ name, value: 'fixture-value' })),+ ]+ pod.volumes = [+ ...(pod.volumes ?? []),+ { name: 'hcc-pr-a-config', configMap: { name: 'fixture-config' } },+ ]+ container.volumeMounts = [+ ...(container.volumeMounts ?? []),+ { name: 'hcc-pr-a-config', mountPath: '/fixture-config' },+ ]+ return result+}+test('config restore removes fixture-added fields that were originally absent', () => {+ const original = deploymentFixture()+ const saved = snapshot(original)+ assert.equal(saved.envPresent, false)+ assert.equal(saved.volumesPresent, false)+ assert.equal(saved.mountsPresent, false)+ assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_HOST'), false)+ assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_PORT'), false)+ assert.equal('aliases' in saved, false)+ assert.equal('aliasesPresent' in saved, false)+ const restored = applyPatch(injectFixture(original), restorePatch(saved, injectFixture(original)))+ assert.deepEqual(restored, original)+})+test('config snapshot is public-only and restore preserves unrelated environment edits', () => {+ const original = deploymentFixture()+ const pod = original.spec.template.spec+ const container = pod.containers[1]+ container.env = [+ { name: 'KUBECONFIG', value: '/previous/config' },+ { name: 'UNRELATED', value: 'not-in-public-snapshot' },+ ]+ container.volumeMounts = [{ name: 'existing', mountPath: '/existing' }]+ pod.volumes = [{ name: 'existing', emptyDir: {} }]+ pod.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]+ const saved = snapshot(original)+ assert.deepEqual(saved.env, [{ name: 'KUBECONFIG', value: '/previous/config' }])+ assert.equal(JSON.stringify(saved).includes('not-in-public-snapshot'), false)+ const changed = injectFixture(original)+ changed.spec.template.spec.containers[1].env.find(value => value.name === 'UNRELATED').value =+ 'concurrent-public-edit'+ const restored = applyPatch(changed, restorePatch(saved, changed))+ const restoredContainer = restored.spec.template.spec.containers[1]+ assert.equal(+ restoredContainer.env.find(value => value.name === 'UNRELATED').value,+ 'concurrent-public-edit'+ )+ assert.equal(+ restoredContainer.env.find(value => value.name === 'KUBECONFIG').value,+ '/previous/config'+ )+ assert.deepEqual(snapshot(restored), saved)+ assert.deepEqual(+ restored.spec.template.spec.containers[0],+ original.spec.template.spec.containers[0]+ )+})+test('config snapshot and restore reject collisions, identity replacement and unrelated mount changes', () => {+ const original = deploymentFixture()+ const saved = snapshot(original)+ assert.throws(() => snapshot(injectFixture(original)), /fixture_mount_collision/)+ const replaced = injectFixture(original)+ replaced.metadata.uid = 'different-deployment'+ assert.throws(() => restorePatch(saved, replaced), /deployment_uid_changed/)+ const removed = injectFixture(original)+ removed.spec.template.spec.containers = []+ assert.throws(() => restorePatch(saved, removed), /deployment_container_missing/)+ for (const field of ['volumes', 'volumeMounts']) {+ const changed = injectFixture(original)+ const target =+ field === 'volumes' ? changed.spec.template.spec : changed.spec.template.spec.containers[1]+ target[field].push({ name: 'unrelated-new-mount' })+ assert.throws(() => restorePatch(saved, changed), /unrelated_configuration_changed/)+ }+})++test('release requires a valid original pause identity', () => {+ for (const pauseId of [undefined, '', true, 123, '../wrong']) {+ assert.throws(() =>+ validateCommand({ action: 'release', id: 'release-invalid', pauseId }, [allowedPath])+ )+ }+ assert.equal(+ validateCommand({ action: 'release', id: 'release-valid', pauseId: 'arm-valid' }, [allowedPath])+ .pauseId,+ 'arm-valid'+ )+})++test(+ 'absent or mismatched pause cannot be released or resume an intercepted GET',+ { timeout: 10000 },+ async () => {+ await fixture(async ({ request, command, controlDir, requests }) => {+ await command({ action: 'release', id: 'release-absent', pauseId: 'never-armed' }, 'rejected')+ await command(+ { action: 'arm', id: 'arm-identity', path: allowedPath, method: 'GET', durationMs: 5000 },+ 'armed'+ )+ await command(+ { action: 'release', id: 'release-not-intercepted', pauseId: 'arm-identity' },+ 'rejected'+ )+ assert.equal(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state, 'armed')+ const intercepted = ack(controlDir, 'arm-identity', 'intercepted', 'pause.json')+ const held = request(allowedPath)+ const ledger = await intercepted+ await command(+ { action: 'release', id: 'release-wrong', pauseId: 'different-arm' },+ 'rejected'+ )+ assert.deepEqual(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')), ledger)+ assert.equal(+ requests.some(value => value.path === allowedPath),+ false+ )+ await command(+ { action: 'release', id: 'release-correct', pauseId: 'arm-identity' },+ 'released'+ )+ const released = await ack(controlDir, 'arm-identity', 'released', 'pause.json')+ assert.equal(Number.isFinite(released.at), true)+ assert.equal(+ (await bounded(held.completed, 'correctly released GET did not resume')).status,+ 200+ )+ })+ }+)++test('config restore compare-and-swap rejects a newer Deployment revision', () => {+ const original = deploymentFixture()+ const changed = injectFixture(original)+ changed.metadata.resourceVersion = '2'+ const missingRevision = structuredClone(changed)+ delete missingRevision.metadata.resourceVersion+ assert.throws(+ () => restorePatch(snapshot(original), missingRevision),+ /deployment_resource_version_missing/+ )+ const patches = restorePatch(snapshot(original), changed)+ assert.equal(+ patches.some(+ patch =>+ patch.op === 'test' && patch.path === '/metadata/resourceVersion' && patch.value === '2'+ ),+ true+ )+ const concurrent = structuredClone(changed)+ concurrent.metadata.resourceVersion = '3'+ assert.throws(() => applyPatch(concurrent, patches))+ assert.equal(applyPatch(changed, patches).metadata.resourceVersion, '2')+})++test('config restoration preserves new host aliases outside fixture ownership', () => {+ const original = deploymentFixture()+ original.spec.template.spec.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]+ const changed = injectFixture(original)+ const aliases = [{ ip: '127.0.0.4', hostnames: ['concurrent.test'] }]+ changed.spec.template.spec.hostAliases = aliases+ const patches = restorePatch(snapshot(original), changed)+ assert.equal(+ patches.some(patch => patch.path.includes('hostAliases')),+ false+ )+ assert.deepEqual(applyPatch(changed, patches).spec.template.spec.hostAliases, aliases)+})diff --git a/scripts/tests/test-hcc-watch-api-proxy.sh b/scripts/tests/test-hcc-watch-api-proxy.shnew file mode 100755index 000000000..4fea92878--- /dev/null+++ b/scripts/tests/test-hcc-watch-api-proxy.sh@@ -0,0 +1,8 @@+#!/usr/bin/env bash+set -euo pipefail+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"+command -v openssl >/dev/null || { echo 'FAIL: openssl is required' >&2; exit 1; }+node -e 'if (Number(process.versions.node.split(".")[0]) !== 24) { console.error("FAIL: Node 24 is required"); process.exit(1) }'+# Test certificates and the generator output stay in Node memory; never tee or+# redirect the generator itself to any artifact. The reporter prints test results.+exec node --test --test-timeout=15000 "$SCRIPT_DIR/hcc-watch-api-proxy.test.mjs"diff --git a/scripts/tests/test-hcc-watch-pr-a.sh b/scripts/tests/test-hcc-watch-pr-a.shnew file mode 100644index 000000000..6b704eb1f--- /dev/null+++ b/scripts/tests/test-hcc-watch-pr-a.sh@@ -0,0 +1,109 @@+#!/usr/bin/env bash+set -euo pipefail+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+tmp="$(mktemp -d)"+trap 'rm -rf -- "$tmp"' EXIT+source "$ROOT/scripts/e2e/_lib/hcc-watch-pr-a.sh"+HCC_LOG_BUFFER="$tmp/logs"+require_hcc_recovery_log_stream() { return 0; }+printf '%s\n' '{"event":"networkpolicy-recovery-decision","kind":"McpServer","decision":"skip"}' \+ '{"event":"networkpolicy-recovery-decision","kind":"Context","decision":"defer"}' > "$HCC_LOG_BUFFER"+[[ "$(hcc_pr_a_decision_count McpServer skip)" = 1 ]]+hcc_pr_a_decision_after 0 McpServer skip+if hcc_pr_a_decision_after 1 McpServer skip; then echo 'FAIL: old recovery reused'; exit 1; fi+if hcc_pr_a_decision_after 0 Context skip; then echo 'FAIL: deferral counted as optimization'; exit 1; fi+: > "$HCC_LOG_BUFFER"+if hcc_pr_a_decision_after 0 McpServer skip; then echo 'FAIL: empty observations passed'; exit 1; fi+printf 'PASS: recovery observation needs a new explicit omission of the exact kind\n'++NP604_SERVER=fixture-affected MCP_NS=mcp-server HCC_PR_A_SERVICE_UID=old+np604_kctl() { cat "$tmp/service"; }+for variation in new old wrong-label wrong-port; do+ uid=new name=fixture-affected port=3000+ case "$variation" in old) uid=old;; wrong-label) name=other;; wrong-port) port=3001;; esac+ jq -cn --arg uid "$uid" --arg name "$name" --argjson port "$port" \+ '{metadata:{uid:$uid,labels:{"clerum.io/mcpserver":$name}},spec:{ports:[{port:$port}]}}' > "$tmp/service"+ if hcc_pr_a_service_restored; then+ [[ "$variation" = new ]] || { echo "FAIL: accepted $variation Service"; exit 1; }+ else+ [[ "$variation" != new ]] || { echo 'FAIL: valid repaired Service rejected'; exit 1; }+ fi+done+printf 'service/fixture-affected\n' > "$tmp/service"+if hcc_pr_a_service_absent; then echo 'FAIL: delete request replaced absence proof'; exit 1; fi+: > "$tmp/service"+hcc_pr_a_service_absent+printf 'PASS: actual absence and new correctly scoped Service are required\n'++# Execute the scenario until its first arm command with harmless boundary+# doubles. This proves the selected read is before Service repair, rather+# than searching a comment or passing through an unexecuted branch.+(+ NP604_CONTROL=fixture-control+ hcc_pr_a_gate() { :; }+ np604_snapshot() { printf '[]'; }+ hcc_pr_a_periodic_count() { printf 1; }+ hcc_pr_a_periodic_started_count() { printf 1; }+ hcc_pr_a_certified_count() { printf 1; }+ hcc_pr_a_remove_service() { :; }+ hcc_pr_a_cut() { :; }+ hcc_pr_a_service_absent() { :; }+ np604_kctl() { printf '{"metadata":{"uid":"fixture","generation":1},"spec":{}}'; }+ np604_invoke() { :; }+ wait_until() { :; }+ hcc_pr_a_command() { printf '%s\n' "$2" > "$tmp/first-arm"; exit 0; }+ die() { echo "FAIL: unexpected scenario rejection $*" >&2; exit 1; }+ hcc_pr_a_run > "$tmp/scenario-output"+)+[[ "$(cat "$tmp/first-arm")" = '/api/v1/namespaces/mcp-server/services/fixture-affected' ]]+printf 'PASS: interrupted runtime holds the Service GET before its missing effect\n'++# Execute the proxy patch builder without generating material or contacting+# Kubernetes; its allowlist must include the fixture read chosen above.+(+ RUN_ID=fixture PROXY_NAME=fixture-proxy HCC_NS=control-plane+ truncate_rfc1123() { printf '%s' "$1"; }+ node() { printf '{}'; }+ kctl() {+ case "$1" in+ create) cat >/dev/null;;+ patch) printf '%s' "${@: -1}" > "$tmp/proxy-patch";;+ rollout) :;;+ *) exit 1;;+ esac+ }+ hcc_pr_a_enable_proxy+)+jq -e '.spec.template.spec.containers[0].env[0].value|fromjson|index("/api/v1/namespaces/mcp-server/services/np604-fixture-affected")!=null' "$tmp/proxy-patch" >/dev/null+jq -e '.spec.template.spec.containers[0].env[0].value|fromjson|index("/apis/networking.k8s.io/v1/namespaces/mcp-server/networkpolicies/ext-egress-np604-fixture-affected-1-2-3-4-32-443")!=null' "$tmp/proxy-patch" >/dev/null+printf 'PASS: proxy allowlist covers only the declared fixture operations\n'++# Run the shared positive probe builder in both modes up to its actual exec+# boundary. No credentials are read; only the exact public trust arguments+# supplied to the in-pod HTTPS client are observed here.+for mode in 0 1; do+ (+ source "$ROOT/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh"+ E2E_HCC_PR_A=$mode PROXY_NAME=fixture-proxy HCC_NS=control-plane HCC_DEPLOY=host-context-controller+ kctl() {+ case "$1/$2" in+ get/service) printf '10.0.0.1';;+ get/configmap)+ jq -cn '{data:{"config.json":"{\"clusters\":[{\"cluster\":{\"certificate-authority-data\":\"cHVibGljLXRlc3QtY2E=\"}}]}"}}';;+ exec/*) printf '%s\n' "$@" > "$tmp/trust-$mode"; exit 0;;+ *) exit 1;;+ esac+ }+ verify_hcc_proxy_network_policy+ )+ grep -q 'rejectUnauthorized:true' "$tmp/trust-$mode"+ [[ "$(tail -3 "$tmp/trust-$mode" | head -1)" = 'fixture-proxy.control-plane.svc' ]]+ if [[ "$mode" = 1 ]]; then+ [[ "$(tail -2 "$tmp/trust-$mode" | head -1)" = 'fixture-proxy.control-plane.svc' ]]+ [[ "$(tail -1 "$tmp/trust-$mode")" = 'cHVibGljLXRlc3QtY2E=' ]]+ else+ [[ "$(tail -2 "$tmp/trust-$mode" | head -1)" = 'kubernetes.default.svc' ]]+ [[ -z "$(tail -1 "$tmp/trust-$mode")" ]]+ fi+done+printf 'PASS: PR A verifies its public fixture CA/name; PR B preserves Kubernetes trust\n'