Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

perf(hcc): preserve pending work while skipping identical watch recovery

Alfredo Lopez9075e2b93c31
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index f00605ed9..f538d5876 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -449,6 +449,11 @@ jobs:
- name: Test HCC watch-recovery gate coordination
run: bash scripts/tests/test-hcc-watch-recovery-gate.sh
+ - name: Test HCC watch recovery TLS proxy and configuration restoration
+ run: |
+ bash scripts/tests/test-hcc-watch-api-proxy.sh
+ bash scripts/tests/test-hcc-watch-pr-a.sh
+
- name: Test HCC watch-churn readiness gate contract
run: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh
diff --git a/Makefile b/Makefile
index cce68c7a4..9558b1b98 100644
--- a/Makefile
+++ b/Makefile
@@ -680,6 +680,12 @@ minikube-t2-hcc-networkpolicy-lifecycle: ## Run canonical T2 with the HCC Networ
@T2_HEALTHCHECK_COMMAND='bash scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh' \
T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 $(MAKE) minikube-t2
+.PHONY: minikube-t2-hcc-watch-recovery
+minikube-t2-hcc-watch-recovery: ## Certify PR A recovery omission, runtime repair and API-gate recovery in owned Minikube
+ @bash scripts/tests/test-hcc-watch-api-proxy.sh
+ @bash scripts/tests/test-hcc-watch-pr-a.sh
+ @E2E_HCC_PR_A=1 $(MAKE) minikube-t2-hcc-networkpolicy-lifecycle
+
.PHONY: minikube-t2-runtime
minikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile
@T2_RUN_T0=false T2_RUN_T1=false \
diff --git a/deploy/overlays/minikube/kustomization.yaml b/deploy/overlays/minikube/kustomization.yaml
index de4b04935..693af4dde 100644
--- a/deploy/overlays/minikube/kustomization.yaml
+++ b/deploy/overlays/minikube/kustomization.yaml
@@ -81,6 +81,7 @@ images:
newTag: test
patchesStrategicMerge:
+ - patches/hcc-policy-resync.yaml
- patches/resource-limits.yaml
- patches/dynamic-images.yaml
# Minikube is the owned T2/dev activation environment for GFS Upload v2.
diff --git a/deploy/overlays/minikube/patches/hcc-policy-resync.yaml b/deploy/overlays/minikube/patches/hcc-policy-resync.yaml
new file mode 100644
index 000000000..9dfde7192
--- /dev/null
+++ b/deploy/overlays/minikube/patches/hcc-policy-resync.yaml
@@ -0,0 +1,15 @@
+# Keep the existing periodic repair active in the owned development lane.
+# The shared base remains unchanged; PR A does not change GKE cadence.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: host-context-controller
+ namespace: control-plane
+spec:
+ template:
+ spec:
+ containers:
+ - name: host-context-controller
+ env:
+ - name: CONTEXT_MAPPER_NETPOL_RESYNC_SEC
+ value: '3600'
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index dc045e364..7b01d0815 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.314",
+ "version": "0.1.315",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.314",
+ "version": "0.1.315",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 02c15dca1..ca8fc377d 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.314",
+ "version": "0.1.315",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
index d558af3eb..947a53c4f 100644
--- a/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
+++ b/host-context-controller/src/k8sClient.policyEligibility.integration.test.ts
@@ -1,6 +1,13 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { V1DeleteOptions, V1NetworkPolicy, V1ObjectMeta } from '@kubernetes/client-node'
-import { McpServerWatcher } from './k8sClient'
+import * as http from 'node:http'
+import { config } from './config'
+import { McpServerWatcher, createMcpAuthorizationStore } from './k8sClient'
+import { hccLogger } from './logger'
+import type { McpApiAuthenticator } from './mcpApiAuthentication'
+import { McpAuthorizationService } from './mcpAuthorization'
+import { resolveProbeAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
+import { ContextMapperServer } from './server'
import type { ContextCRD, McpServerCRD, McpServerCrdStatus } from './types'
type ApiServer = Pick<McpServerCRD, 'spec' | 'status'> & { metadata: V1ObjectMeta }
@@ -10,6 +17,8 @@ const fixture = vi.hoisted(() => ({
policies: new Map<string, StoredPolicy>(),
servers: new Map<string, ApiServer>(),
revision: 0,
+ contexts: new Map<string, { metadata: V1ObjectMeta; spec: ContextCRD['spec'] }>(),
+ failPolicyType: undefined as string | undefined,
watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),
list: vi.fn(),
}))
@@ -17,6 +26,7 @@ const fixture = vi.hoisted(() => ({
vi.mock('./config', () => ({
config: {
devMode: false,
+ k8sApiCidrs: [],
namespace: 'mcp-server',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
@@ -47,7 +57,15 @@ vi.mock('@kubernetes/client-node', async importOriginal => {
}
const custom = {
listNamespacedCustomObject: fixture.list,
- getNamespacedCustomObject: async ({ name }: { name: string }) => read(name),
+ getNamespacedCustomObject: async ({ name, plural }: { name: string; plural: string }) => {
+ if (plural === 'hosts')
+ return {
+ metadata: { name: 'host', namespace: 'mcp-host', uid: 'host-uid', resourceVersion: '1' },
+ spec: { contextRef: 'ctx' },
+ }
+ if (plural === 'contexts') return structuredClone(fixture.contexts.get(name))
+ return read(name)
+ },
getNamespacedCustomObjectStatus: async ({ name }: { name: string }) => read(name),
patchNamespacedCustomObjectStatus: async ({
name,
@@ -83,6 +101,11 @@ vi.mock('@kubernetes/client-node', async importOriginal => {
},
}
const save = (namespace: string, body: StoredPolicy) => {
+ if (
+ fixture.failPolicyType &&
+ Object.values(body.metadata.labels ?? {}).includes(fixture.failPolicyType)
+ )
+ throw new Error('injected additive API failure')
const key = `${namespace}/${body.metadata.name}`
const old = fixture.policies.get(key)
const policy = structuredClone({
@@ -187,6 +210,12 @@ vi.mock('@kubernetes/client-node', async importOriginal => {
// while keeping unrelated workload API writes out of this fixture.
vi.mock('./reconciler', () => ({
McpServerReconciler: class {
+ hasPendingReconciliation() {
+ return false
+ }
+ hasIncompleteReconciliation() {
+ return false
+ }
setInventoryAuthority() {}
setResolveCurrentServer() {}
reconcile() {
@@ -208,6 +237,12 @@ vi.mock('./reconciler', () => ({
}))
type Harness = {
+ runInitialNetworkPolicyConvergence(options?: { cause?: string }): Promise<void>
+ runInitialConvergence(lane: string): Promise<void>
+ initialConvergenceRetryTimers: Map<string, unknown>
+ recoverContextInventoryAndWatch(): Promise<boolean>
+ enqueueMcpServerReconciliation(server: McpServerCRD, work: () => Promise<void>): Promise<void>
+ servers: Map<string, McpServerCRD>
contexts: Map<string, ContextCRD>
contextCacheSynced: boolean
mcpServerCacheSynced: boolean
@@ -235,9 +270,13 @@ async function drain(state: Harness): Promise<void> {
describe('WATCH and recovered LIST policy effects', () => {
let watcher: McpServerWatcher
+ let api: ContextMapperServer | undefined
beforeEach(() => {
vi.useFakeTimers()
fixture.policies.clear()
+ fixture.contexts.clear()
+ fixture.failPolicyType = undefined
+ config.netPolResyncIntervalSec = 0
fixture.servers.clear()
fixture.revision = 0
fixture.list.mockImplementation(async ({ plural }: { plural: string }) => ({
@@ -245,11 +284,16 @@ describe('WATCH and recovered LIST policy effects', () => {
items:
plural === 'mcpservers'
? [...fixture.servers.values()].map(value => structuredClone(value))
- : [],
+ : plural === 'contexts'
+ ? [...fixture.contexts.values()].map(value => structuredClone(value))
+ : [],
}))
})
afterEach(async () => {
+ await api?.stop()
+ api = undefined
await watcher?.stop()
+ vi.restoreAllMocks()
vi.useRealTimers()
})
@@ -312,4 +356,528 @@ describe('WATCH and recovered LIST policy effects', () => {
expect(fixture.list).toHaveBeenCalledWith(expect.objectContaining({ plural: 'mcpservers' }))
}
)
+ async function settled(): Promise<Harness> {
+ config.netPolResyncIntervalSec = 3600
+ fixture.contexts.set('ctx', {
+ metadata: {
+ name: 'ctx',
+ namespace: 'mcp-server',
+ uid: 'ctx-uid',
+ generation: 1,
+ resourceVersion: '1',
+ },
+ spec: { contextId: 'ctx', mcpServers: ['server'] },
+ })
+ const server: ApiServer = {
+ metadata: {
+ name: 'server',
+ namespace: 'mcp-server',
+ uid: 'server-uid',
+ generation: 1,
+ resourceVersion: '1',
+ },
+ spec: {
+ image: 'fixture:v1',
+ contextRef: 'ctx',
+ transport: { type: 'streamableHttp', port: 3000 },
+ envSecret: { name: 'fixture-env', keys: [] },
+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443 }],
+ },
+ status: { conditions: [{ type: 'SecretResolved', status: 'True', reason: 'SecretFound' }] },
+ }
+ fixture.servers.set('server', server)
+ watcher = new McpServerWatcher()
+ await watcher.start()
+ const state = watcher as unknown as Harness
+ await drain(state)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(fixture.policies.size).toBeGreaterThanOrEqual(4)
+ return state
+ }
+
+ it('A-T3/T5: settled identical recoveries skip before scheduler admission with an active resync', async () => {
+ const state = await settled()
+ const admission = vi.spyOn(state, 'runInitialConvergence')
+ const log = vi.spyOn(hccLogger, 'info')
+ const before = structuredClone([...fixture.policies.entries()])
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toEqual([])
+ expect([...fixture.policies.entries()]).toEqual(before)
+ for (const kind of ['McpServer', 'Context'])
+ expect(log).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({
+ event: 'networkpolicy-recovery-decision',
+ kind,
+ decision: 'skip',
+ reason: 'identical-complete',
+ })
+ )
+ })
+
+ it.each(['McpServer', 'Context', 'timer'] as const)(
+ 'A-T4: %s repairs a real additive failure',
+ async kind => {
+ const state = await settled()
+ const desired = [...fixture.policies.entries()].find(([, policy]) =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )!
+ expect(desired).toBeDefined()
+ const certificate = vi.spyOn(
+ state as unknown as { recordNetworkPolicySafetyCertificate(value: unknown): boolean },
+ 'recordNetworkPolicySafetyCertificate'
+ )
+ fixture.policies.delete(desired[0])
+ fixture.failPolicyType = 'context-allow'
+ await state.runInitialNetworkPolicyConvergence({ cause: 'context-change' })
+ await drain(state)
+ expect(certificate).toHaveBeenCalled()
+ expect(
+ certificate.mock.results.some(result => result.type === 'return' && result.value === true)
+ ).toBe(true)
+ expect(fixture.policies.has(desired[0])).toBe(false)
+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)
+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ fixture.failPolicyType = undefined
+ const now = Date.now()
+ const request = vi.spyOn(state, 'runInitialNetworkPolicyConvergence')
+ if (kind === 'timer') {
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(request).toHaveBeenCalledWith(expect.objectContaining({ cause: 'retry' }))
+ } else {
+ expect(
+ await (kind === 'McpServer'
+ ? state.recoverMcpServerInventoryAndWatch()
+ : state.recoverContextInventoryAndWatch())
+ ).toBe(true)
+ expect(Date.now()).toBe(now)
+ }
+ await drain(state)
+ expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)
+ }
+ )
+
+ it.each(['McpServer', 'Context'] as const)(
+ 'A-T6: concurrent %s recoveries share one physical LIST and one decision',
+ async kind => {
+ const state = await settled()
+ let release!: () => void
+ const barrier = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const list = fixture.list.getMockImplementation()!
+ const plural = kind === 'McpServer' ? 'mcpservers' : 'contexts'
+ fixture.list.mockImplementation(async args => {
+ if (args.plural === plural) await barrier
+ return list(args)
+ })
+ fixture.list.mockClear()
+ const log = vi.spyOn(hccLogger, 'info')
+ const recover = () =>
+ kind === 'McpServer'
+ ? state.recoverMcpServerInventoryAndWatch()
+ : state.recoverContextInventoryAndWatch()
+ const first = recover(),
+ second = recover()
+ expect(first).toBe(second)
+ release()
+ expect(await first).toBe(true)
+ await drain(state)
+ expect(fixture.list.mock.calls.filter(([args]) => args.plural === plural)).toHaveLength(1)
+ expect(
+ log.mock.calls.filter(
+ ([, fields]) =>
+ (fields as { event?: string; kind?: string })?.event ===
+ 'networkpolicy-recovery-decision' && (fields as { kind?: string }).kind === kind
+ )
+ ).toHaveLength(1)
+ }
+ )
+
+ it('A-T3: inactive periodic recovery never permits an identical omission', async () => {
+ const state = await settled()
+ const internals = state as unknown as {
+ netPolResyncTimer: ReturnType<typeof setInterval> | null
+ }
+ clearInterval(internals.netPolResyncTimer!)
+ internals.netPolResyncTimer = null
+ const admission = vi.spyOn(state, 'runInitialConvergence')
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toHaveLength(1)
+ })
+
+ it.each(['spec', 'uid', 'generation'] as const)(
+ 'A-T3: changed server %s requests convergence and final policy matches current inventory',
+ async change => {
+ const state = await settled()
+ const server = fixture.servers.get('server')!
+ if (change === 'spec') server.spec.transport.port = 3011
+ else if (change === 'uid') server.metadata.uid = 'replacement-server-uid'
+ else server.metadata.generation = 2
+ const admission = vi.spyOn(state, 'runInitialConvergence')
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toHaveLength(1)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ const contextPolicy = [...fixture.policies.values()].find(policy =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )!
+ expect(
+ contextPolicy.spec?.ingress
+ ?.flatMap(rule => rule.ports ?? [])
+ .some(port => port.port === (change === 'spec' ? 3011 : 3000))
+ ).toBe(true)
+ expect(state.servers.get('server')?.uid).toBe(server.metadata.uid)
+ }
+ )
+
+ it('A-T8: a blocked server FIFO closes API authority while the probe stays fresh, then restores business policies', async () => {
+ const state = await settled()
+ const control = structuredClone(fixture.servers.get('server')!)
+ control.metadata.name = 'control'
+ control.metadata.uid = 'control-uid'
+ fixture.servers.set('control', control)
+ fixture.contexts.get('ctx')!.spec.mcpServers.push('control')
+ await state.recoverMcpServerInventoryAndWatch()
+ await state.recoverContextInventoryAndWatch()
+ await drain(state)
+ const controlPolicies = [...fixture.policies.entries()].filter(([, policy]) =>
+ Object.values(policy.metadata.labels ?? {}).includes('control')
+ )
+ expect(controlPolicies.length).toBeGreaterThan(0)
+ const apiGate = resolveProviderAuthoritativeFn(watcher)
+ const probeGate = resolveProbeAuthoritativeFn(watcher)
+ const authenticator = {
+ authenticate: () => ({
+ subject: 'host:test',
+ hostName: 'host',
+ hostUid: 'host-uid',
+ namespace: 'mcp-host',
+ jti: 'request-id',
+ issuedAt: 1,
+ expiresAt: Number.MAX_SAFE_INTEGER,
+ audiences: ['host-context-controller'],
+ }),
+ } as unknown as McpApiAuthenticator
+ const authorization = new McpAuthorizationService(createMcpAuthorizationStore(watcher))
+ api = new ContextMapperServer(
+ watcher,
+ 0,
+ undefined,
+ undefined,
+ apiGate,
+ undefined,
+ authenticator,
+ authorization,
+ () => watcher.getReadinessInventoryDetail(),
+ probeGate
+ )
+ await api.start()
+ api.setReady(true)
+ const address = (api as unknown as { server: http.Server }).server.address() as { port: number }
+ const request = (path: string): Promise<{ status: number; body: string }> =>
+ new Promise((resolve, reject) => {
+ http
+ .get({ hostname: '127.0.0.1', port: address.port, path }, response => {
+ let body = ''
+ response.setEncoding('utf8')
+ response.on('data', chunk => {
+ body += chunk
+ })
+ response.on('end', () => resolve({ status: response.statusCode!, body }))
+ })
+ .on('error', reject)
+ })
+ expect((await request('/api/v2/hosts/self/mcpservers')).status).toBe(200)
+ let release!: () => void, entered!: () => void
+ const blocked = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const started = new Promise<void>(resolve => {
+ entered = resolve
+ })
+ const queue = state.enqueueMcpServerReconciliation(state.servers.get('server')!, async () => {
+ entered()
+ await blocked
+ })
+ await started
+ const original = structuredClone(fixture.servers.get('server')!)
+ const changed = structuredClone(original)
+ for (const condition of changed.status!.conditions!) {
+ condition.status = 'False'
+ condition.reason = condition.reason?.replace('Found', 'NotFound')
+ }
+ fixture.servers.set('server', changed)
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ expect(apiGate()).toBe(false)
+ expect(probeGate()).toBe(true)
+ expect((await request('/ready')).status).toBe(200)
+ expect(await request('/api/v2/hosts/self/mcpservers')).toEqual({
+ status: 503,
+ body: JSON.stringify({ error: 'authorization_unavailable' }),
+ })
+ expect(
+ [...fixture.policies.values()].some(policy =>
+ Object.values(policy.metadata.labels ?? {}).includes('external-egress')
+ )
+ ).toBe(true)
+ release()
+ await queue
+ await drain(state)
+ expect(apiGate()).toBe(true)
+ expect(probeGate()).toBe(true)
+ expect(
+ [...fixture.policies.values()].filter(
+ policy =>
+ Object.values(policy.metadata.labels ?? {}).includes('server') &&
+ ['context-allow', 'rpc-proxy-egress', 'external-egress'].some(type =>
+ Object.values(policy.metadata.labels ?? {}).includes(type)
+ )
+ )
+ ).toEqual([])
+ for (const [key, policy] of controlPolicies) expect(fixture.policies.get(key)).toEqual(policy)
+ fixture.servers.set('server', original)
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ expect(apiGate()).toBe(true)
+ const restored = await request('/api/v2/hosts/self/mcpservers')
+ expect(restored.status).toBe(200)
+ expect(JSON.parse(restored.body).servers).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ name: 'server' }),
+ expect.objectContaining({ name: 'control' }),
+ ])
+ )
+ expect(
+ [...fixture.policies.values()].some(policy =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )
+ ).toBe(true)
+ })
+
+ it.each(['before', 'after'] as const)(
+ 'A-T5/T6: changes %s capture retain causes and converge the final revision',
+ async order => {
+ const state = await settled()
+ const internal = state as unknown as {
+ installMcpServerSnapshot(snapshot: { servers: McpServerCRD[] }): void
+ netPolReconciler: { fullReconcile(...args: unknown[]): Promise<void> }
+ }
+ const log = vi.spyOn(hccLogger, 'info')
+ const install = (port: number) => {
+ fixture.servers.get('server')!.spec.transport.port = port
+ const current = structuredClone(state.servers.get('server')!)
+ current.spec.transport.port = port
+ internal.installMcpServerSnapshot({ servers: [current] })
+ }
+ let entered!: () => void, release!: () => void
+ const captured = new Promise<void>(resolve => {
+ entered = resolve
+ })
+ const barrier = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const real = internal.netPolReconciler.fullReconcile.bind(internal.netPolReconciler)
+ if (order === 'after')
+ vi.spyOn(internal.netPolReconciler, 'fullReconcile').mockImplementationOnce(
+ async (...args) => {
+ entered()
+ await barrier
+ await real(...args)
+ }
+ )
+ install(3001)
+ const first = state.runInitialNetworkPolicyConvergence({ cause: 'mcp-change' })
+ if (order === 'after') await captured
+ install(3002)
+ const second = state.runInitialNetworkPolicyConvergence({ cause: 'context-change' })
+ release()
+ await Promise.all([first, second])
+ await drain(state)
+ const starts = log.mock.calls
+ .map(
+ ([, fields]) =>
+ fields as {
+ event?: string
+ causes: string[]
+ serverRevision: number
+ trailing: boolean
+ }
+ )
+ .filter(fields => fields?.event === 'networkpolicy-pass-start')
+ expect(starts.map(fields => fields.serverRevision)).toEqual(order === 'before' ? [3] : [2, 3])
+ expect(starts.map(fields => fields.trailing)).toEqual(
+ order === 'before' ? [false] : [false, true]
+ )
+ expect(starts.flatMap(fields => fields.causes).sort()).toEqual([
+ 'context-change',
+ 'mcp-change',
+ ])
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ const policy = [...fixture.policies.values()].find(policy =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )!
+ expect(
+ policy.spec?.ingress?.flatMap(rule => rule.ports ?? []).some(port => port.port === 3002)
+ ).toBe(true)
+ }
+ )
+
+ it('A-T6: identical crossed recoveries do not add trailing work to an active settled pass', async () => {
+ const state = await settled()
+ const core = state as unknown as { runInitialNetworkPolicyConvergenceCore(): Promise<void> }
+ const real = core.runInitialNetworkPolicyConvergenceCore.bind(core)
+ let entered!: () => void, release!: () => void
+ const completed = new Promise<void>(resolve => {
+ entered = resolve
+ })
+ const barrier = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const passes = vi
+ .spyOn(core, 'runInitialNetworkPolicyConvergenceCore')
+ .mockImplementationOnce(async () => {
+ await real()
+ const runtime = state.initialConvergenceRuns.get('McpServer')
+ if (runtime) await runtime.promise
+ entered()
+ await barrier
+ })
+ const active = state.runInitialNetworkPolicyConvergence({ cause: 'periodic-resync' })
+ await completed
+ const admission = vi.spyOn(state, 'runInitialConvergence')
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ expect(admission.mock.calls.filter(([lane]) => lane === 'NetworkPolicy')).toEqual([])
+ release()
+ await active
+ await drain(state)
+ expect(passes).toHaveBeenCalledTimes(1)
+ })
+
+ it('A-T3: Context membership removal and restoration produce final allow policies', async () => {
+ const state = await settled()
+ const desired = [...fixture.policies.entries()].filter(([, policy]) =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )
+ expect(desired.length).toBeGreaterThan(0)
+ fixture.contexts.get('ctx')!.spec.mcpServers = []
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ await drain(state)
+ for (const [key] of desired) expect(fixture.policies.has(key)).toBe(false)
+ fixture.contexts.get('ctx')!.spec.mcpServers = ['server']
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ await drain(state)
+ for (const [key, policy] of desired)
+ expect(fixture.policies.get(key)?.spec).toEqual(policy.spec)
+ })
+ it('A-T5: all eight production entry points report their actual cause', async () => {
+ const log = vi.spyOn(hccLogger, 'info')
+ const state = await settled()
+ const server = fixture.servers.get('server')!
+ server.spec.transport.port = 3010
+ await state.getMcpServerWatchCallback()('MODIFIED', structuredClone(server))
+ await drain(state)
+ const contextCallback = () =>
+ fixture.watch.mock.calls
+ .filter(([path]) => String(path).endsWith('/contexts'))
+ .at(-1)![2] as (type: string, value: unknown) => Promise<void>
+ const context = fixture.contexts.get('ctx')!
+ context.metadata.generation = 2
+ await contextCallback()('MODIFIED', structuredClone(context))
+ await drain(state)
+ server.metadata.generation = 2
+ expect(await state.recoverMcpServerInventoryAndWatch()).toBe(true)
+ await drain(state)
+ context.metadata.generation = 3
+ expect(await state.recoverContextInventoryAndWatch()).toBe(true)
+ await drain(state)
+ const policy = [...fixture.policies.entries()].find(([, policy]) =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )!
+ fixture.policies.delete(policy[0])
+ fixture.failPolicyType = 'context-allow'
+ await contextCallback()('MODIFIED', structuredClone(context))
+ await drain(state)
+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ fixture.failPolicyType = undefined
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ await vi.advanceTimersByTimeAsync(3_600_000)
+ await drain(state)
+ const requests = log.mock.calls
+ .map(([, fields]) => fields as { event?: string; cause: string })
+ .filter(fields => fields?.event === 'networkpolicy-request')
+ expect([...new Set(requests.map(fields => fields.cause))].sort()).toEqual([
+ 'context-change',
+ 'context-reconcile-failure',
+ 'context-recovery',
+ 'mcp-change',
+ 'mcp-recovery',
+ 'periodic-resync',
+ 'retry',
+ 'startup',
+ ])
+ expect(fixture.policies.get(policy[0])?.spec).toEqual(policy[1].spec)
+ })
+
+ it('A-T6 regression: interrupted additive policy work cannot be marked complete after identical recovery', async () => {
+ const state = await settled()
+ const owner = (state as any).netPolReconciler
+ const desired = [...fixture.policies.entries()].find(([, policy]) =>
+ Object.values(policy.metadata.labels ?? {}).includes('context-allow')
+ )!
+ fixture.policies.delete(desired[0])
+ const defer = () => {
+ let resolve!: () => void
+ const promise = new Promise<void>(done => {
+ resolve = done
+ })
+ return { promise, resolve }
+ }
+ const entered = defer(),
+ run = defer(),
+ innerDone = defer(),
+ finish = defer(),
+ watchEntered = defer(),
+ watchFinish = defer()
+ const real = owner.reconcileContext.bind(owner)
+ vi.spyOn(owner, 'reconcileContext').mockImplementationOnce(async (...args) => {
+ entered.resolve()
+ await run.promise
+ const result = await real(...args)
+ innerDone.resolve()
+ await finish.promise
+ return result
+ })
+ const pass = state.runInitialNetworkPolicyConvergence({ cause: 'periodic-resync' })
+ await entered.promise
+ const runtime = state.initialConvergenceRuns.get('McpServer')
+ if (runtime) await runtime.promise
+ fixture.watch.mockImplementationOnce(async () => {
+ watchEntered.resolve()
+ await watchFinish.promise
+ return { abort: vi.fn() }
+ })
+ const recovery = state.recoverContextInventoryAndWatch()
+ await watchEntered.promise
+ run.resolve()
+ await innerDone.promise
+ expect(fixture.policies.has(desired[0])).toBe(false)
+ watchFinish.resolve()
+ await recovery
+ finish.resolve()
+ await pass
+ await drain(state)
+ expect(state.initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ expect(fixture.policies.get(desired[0])?.spec).toEqual(desired[1].spec)
+ })
})
diff --git a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts
new file mode 100644
index 000000000..3664e368a
--- /dev/null
+++ b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts
@@ -0,0 +1,565 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { V1Deployment, V1Service } from '@kubernetes/client-node'
+import { McpServerWatcher } from './k8sClient'
+import { hccLogger } from './logger'
+import { registry } from './metrics'
+import { McpServerReconciler } from './reconciler'
+
+function deferred() {
+ let resolve!: () => void
+ const promise = new Promise<void>(done => {
+ resolve = done
+ })
+ return { promise, resolve }
+}
+
+const fixture = vi.hoisted(() => ({
+ services: new Map<string, V1Service>(),
+ deployments: new Map<string, V1Deployment>(),
+ writes: [] as string[],
+ revision: 0,
+ rolloutReady: true,
+ failDeployment: false,
+ serverExists: true,
+ readInput: undefined as undefined | (() => Promise<unknown>),
+ failPublication: undefined as undefined | ((body: any[]) => boolean),
+ server: undefined as any,
+ readService: undefined as undefined | (() => Promise<void>),
+}))
+
+vi.mock('./config', () => ({
+ config: {
+ devMode: false,
+ namespace: 'mcp-server',
+ hostNamespace: 'mcp-host',
+ channelsNamespace: 'channels',
+ llmHooksNamespace: 'llm-hooks',
+ rpcProxyNamespace: 'rpc-proxy',
+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],
+ hostK8sRequestTimeoutMs: 30000,
+ hostResyncIntervalSec: 0,
+ netPolResyncIntervalSec: 3600,
+ netPolDefaultsResyncIntervalSec: 0,
+ externalEgressResyncIntervalSec: 0,
+ allowedPluginImagePrefixes: ['ghcr.io/palmeradao/'],
+ mcpServerImagePullPolicy: 'IfNotPresent',
+ },
+}))
+
+vi.mock('@kubernetes/client-node', async importOriginal => {
+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()
+ const absent = () => Object.assign(new Error('fixture resource absent'), { code: 404 })
+ const read = <T>(store: Map<string, T>, name: string): T => {
+ const object = store.get(name)
+ if (!object) throw absent()
+ return structuredClone(object)
+ }
+ const save = (kind: string, store: Map<string, any>, namespace: string, body: any) => {
+ if (kind === 'Deployment' && fixture.failDeployment)
+ throw new Error('fixture deployment write unavailable')
+ const object = structuredClone(body)
+ object.metadata = {
+ ...object.metadata,
+ namespace,
+ uid: `${kind}-${body.metadata.name}`,
+ resourceVersion: String(++fixture.revision),
+ generation: 1,
+ }
+ if (kind === 'Deployment')
+ object.status = {
+ observedGeneration: 1,
+ replicas: 1,
+ readyReplicas: fixture.rolloutReady ? 1 : 0,
+ availableReplicas: fixture.rolloutReady ? 1 : 0,
+ updatedReplicas: 1,
+ }
+ store.set(object.metadata.name, object)
+ fixture.writes.push(`${kind}/${object.metadata.name}`)
+ return structuredClone(object)
+ }
+ const core = {
+ readNamespacedSecret: async () => {
+ if (fixture.readInput) return fixture.readInput()
+ throw absent()
+ },
+ readNamespacedConfigMap: async () => {
+ throw absent()
+ },
+ deleteNamespacedService: async ({ name }: any) => {
+ fixture.services.delete(name)
+ },
+ readNamespacedService: async ({ name }: any) => {
+ await fixture.readService?.()
+ return read(fixture.services, name)
+ },
+ createNamespacedService: async ({ namespace, body }: any) =>
+ save('Service', fixture.services, namespace, body),
+ replaceNamespacedService: async ({ namespace, body }: any) =>
+ save('Service', fixture.services, namespace, body),
+ }
+ const apps = {
+ deleteNamespacedDeployment: async ({ name }: any) => {
+ fixture.deployments.delete(name)
+ },
+ listNamespacedDeployment: async () => ({
+ items: [...fixture.deployments.values()].map(value => structuredClone(value)),
+ }),
+ readNamespacedDeployment: async ({ name }: any) => read(fixture.deployments, name),
+ createNamespacedDeployment: async ({ namespace, body }: any) =>
+ save('Deployment', fixture.deployments, namespace, body),
+ replaceNamespacedDeployment: async ({ namespace, body }: any) =>
+ save('Deployment', fixture.deployments, namespace, body),
+ }
+ const custom = {
+ listNamespacedCustomObject: async ({ plural }: any) => ({
+ metadata: { resourceVersion: String(++fixture.revision) },
+ items:
+ plural === 'mcpservers' && fixture.serverExists ? [structuredClone(fixture.server)] : [],
+ }),
+ getNamespacedCustomObject: async () => {
+ if (!fixture.serverExists) throw absent()
+ return structuredClone(fixture.server)
+ },
+ getNamespacedCustomObjectStatus: async () => structuredClone(fixture.server),
+ patchNamespacedCustomObjectStatus: async ({ body }: any) => {
+ if (fixture.failPublication?.(body))
+ throw Object.assign(new Error('fixture status unavailable'), { code: 503 })
+ for (const patch of body) {
+ if (patch.op === 'test') {
+ const field = patch.path.slice('/metadata/'.length)
+ if (fixture.server.metadata[field] !== patch.value)
+ throw Object.assign(new Error('fixture status conflict'), { code: 409 })
+ } else if (patch.path === '/status') fixture.server.status = structuredClone(patch.value)
+ else if (patch.path === '/status/conditions')
+ fixture.server.status.conditions = structuredClone(patch.value)
+ else throw new Error(`Unexpected status patch ${patch.path}`)
+ }
+ fixture.server.metadata.resourceVersion = String(++fixture.revision)
+ return structuredClone(fixture.server)
+ },
+ }
+ return {
+ ...actual,
+ KubeConfig: class {
+ loadFromDefault() {}
+ makeApiClient(type: unknown) {
+ if (type === actual.CoreV1Api) return core
+ if (type === actual.AppsV1Api) return apps
+ if (type === actual.CustomObjectsApi) return custom
+ return {}
+ }
+ },
+ Watch: class {
+ async watch() {
+ return { abort() {} }
+ }
+ },
+ }
+})
+
+async function until(predicate: () => boolean) {
+ for (let turn = 0; turn < 200; turn++) {
+ if (predicate()) return
+ await Promise.resolve()
+ }
+ throw new Error('Expected runtime state did not settle')
+}
+
+async function drain(state: any) {
+ for (let pass = 0; pass < 30; pass++) {
+ await Promise.resolve()
+ const pending = [...state.initialConvergenceRuns.values()].map((run: any) => run.promise)
+ for (const entry of state.mcpServerReconciliationQueues.values()) pending.push(entry.queue.tail)
+ if (!pending.length) return
+ await Promise.all(pending)
+ }
+ throw new Error('Runtime queues did not drain')
+}
+
+function prepareWatcher(policyTail?: Promise<void>) {
+ const watcher = new McpServerWatcher()
+ const state = watcher as any
+ expect(state.reconciler).toBeInstanceOf(McpServerReconciler)
+ // Only unrelated controller effects are replaced. Runtime reconciliation,
+ // serializers, authority fences and generated Service/Deployment bodies run.
+ vi.spyOn(state.netPolReconciler, 'ensureDefaultPolicies').mockResolvedValue(undefined)
+ vi.spyOn(state.netPolReconciler, 'hasCertifiedSafetyInventory').mockReturnValue(true)
+ vi.spyOn(state.netPolReconciler, 'reconcileExternalEgress').mockResolvedValue(undefined)
+ vi.spyOn(state.netPolReconciler, 'fullReconcile').mockImplementation(async (...args: any[]) => {
+ args[2].onAuthoritativeRevocationComplete()
+ await policyTail
+ })
+ for (const owner of [
+ 'hostReconciler',
+ 'sharedFileSystemReconciler',
+ 'gfsReconciler',
+ 'llmHookReconciler',
+ ]) {
+ vi.spyOn(state[owner], 'fullReconcile').mockResolvedValue(undefined)
+ }
+ vi.spyOn(state.hostReconciler, 'reconcileHosts').mockResolvedValue(undefined)
+ return { watcher, state }
+}
+
+async function skippedTickCount() {
+ const metric = await registry
+ .getSingleMetric('clerum_hcc_netpol_resync_ticks_skipped_total')!
+ .get()
+ return metric.values.find(value => value.labels.reason === 'pass-in-flight')?.value ?? 0
+}
+
+describe('A-T7 real MCP runtime repair after identical recovery', () => {
+ let watcher: McpServerWatcher | undefined
+ beforeEach(() => {
+ vi.useFakeTimers()
+ fixture.services.clear()
+ fixture.deployments.clear()
+ fixture.writes.length = 0
+ fixture.revision = 0
+ fixture.rolloutReady = true
+ fixture.failDeployment = false
+ fixture.serverExists = true
+ fixture.readInput = undefined
+ fixture.failPublication = undefined
+ fixture.readService = undefined
+ fixture.server = {
+ metadata: {
+ name: 'runtime-witness',
+ namespace: 'mcp-server',
+ uid: 'runtime-witness-uid',
+ generation: 1,
+ resourceVersion: '1',
+ },
+ spec: {
+ image: 'ghcr.io/palmeradao/runtime-test:v1',
+ enabled: true,
+ managed: true,
+ transport: { type: 'http', port: 3000 },
+ },
+ }
+ })
+ afterEach(async () => {
+ await watcher?.stop()
+ vi.restoreAllMocks()
+ vi.useRealTimers()
+ })
+
+ it('A-T7a a skipped tick does not repair drift; the next effective periodic tick repairs unchanged CRD intent', async () => {
+ const policyTail = deferred()
+ const harness = prepareWatcher(policyTail.promise)
+ watcher = harness.watcher
+ const { state } = harness
+ const decisions = vi.spyOn(hccLogger, 'info')
+ await watcher.start()
+ await until(() => !state.runtimeRepairPending && fixture.deployments.size === 1)
+ expect(fixture.writes).toEqual(['Service/runtime-witness', 'Deployment/runtime-witness'])
+ const desired = structuredClone(fixture.server.spec)
+ const desiredGeneration = fixture.server.metadata.generation
+ fixture.services.delete('runtime-witness')
+ const writesBefore = fixture.writes.length
+ const skipsBefore = await skippedTickCount()
+ await vi.advanceTimersByTimeAsync(3600_000)
+ expect(await skippedTickCount()).toBe(skipsBefore + 1)
+ expect(fixture.services.has('runtime-witness')).toBe(false)
+ expect(fixture.writes).toHaveLength(writesBefore)
+ policyTail.resolve()
+ await drain(state)
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(decisions).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({
+ event: 'networkpolicy-recovery-decision',
+ decision: 'skip',
+ reason: 'identical-complete',
+ })
+ )
+ expect(fixture.services.has('runtime-witness')).toBe(false)
+ expect(fixture.writes).toHaveLength(writesBefore)
+ await vi.advanceTimersByTimeAsync(3600_000)
+ await drain(state)
+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)
+ expect(fixture.writes.slice(writesBefore)).toContain('Service/runtime-witness')
+ expect(fixture.server.spec).toEqual(desired)
+ expect(fixture.server.metadata.generation).toBe(desiredGeneration)
+ expect(state.runtimeRepairPending).toBe(false)
+ })
+
+ it('A-T7b a new watch generation retires a blocked effect and retains repair until a real resource write completes', async () => {
+ const oldRead = deferred()
+ const newRead = deferred()
+ let reads = 0
+ fixture.readService = async () => {
+ reads++
+ if (reads === 1) await oldRead.promise
+ if (reads === 2) await newRead.promise
+ }
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ const decisions = vi.spyOn(hccLogger, 'info')
+ await watcher.start()
+ await until(() => reads === 1 && !state.networkPolicyRepairPending)
+ expect(state.runtimeRepairPending).toBe(true)
+ const oldGeneration = state.mcpWatchGeneration
+ const clockBefore = Date.now()
+ await state.recoverMcpServerInventoryAndWatch()
+ expect(state.mcpWatchGeneration).toBeGreaterThan(oldGeneration)
+ oldRead.resolve()
+ await until(() => reads === 2)
+ expect(fixture.writes).toEqual([])
+ expect(fixture.services.size).toBe(0)
+ expect(state.runtimeRepairPending).toBe(true)
+ newRead.resolve()
+ await drain(state)
+ expect(fixture.writes).toEqual(['Service/runtime-witness', 'Deployment/runtime-witness'])
+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)
+ expect(
+ fixture.deployments.get('runtime-witness')?.spec?.template.spec?.containers[0].image
+ ).toBe('ghcr.io/palmeradao/runtime-test:v1')
+ expect(state.runtimeRepairPending).toBe(false)
+ expect(decisions).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ decision: 'request', reason: 'work-pending' })
+ )
+ expect(Date.now()).toBe(clockBefore)
+ })
+
+ it('A-T7 regression: a swallowed runtime API failure must retain its pending repair', async () => {
+ fixture.readService = async () => {
+ throw new Error('transient upstream unavailable')
+ }
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(fixture.services.size).toBe(0)
+ expect(fixture.deployments.size).toBe(0)
+ expect(state.reconciler.getStatus('runtime-witness')?.deployed).toBe(false)
+ fixture.readService = undefined
+ const decision = vi.spyOn(hccLogger, 'info')
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(decision).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ decision: 'request' })
+ )
+ expect(fixture.services.has('runtime-witness')).toBe(true)
+ })
+
+ it('A-T7 regression: identical recovery must preserve the active readiness poll', async () => {
+ fixture.rolloutReady = false
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(true)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ expect(state.reconciler.readinessPolls.size).toBe(1)
+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(false)
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ const deployment = fixture.deployments.get('runtime-witness')!
+ deployment.status!.readyReplicas = 1
+ deployment.status!.availableReplicas = 1
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)
+ expect(fixture.server.status.conditions).toEqual(
+ expect.arrayContaining([expect.objectContaining({ type: 'DeploymentReady', status: 'True' })])
+ )
+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)
+ })
+
+ it.each(['disabled', 'wrc', 'missing-input', 'wrc-missing-input'] as const)(
+ 'A-T7 control: intentional %s runtime absence is complete',
+ async mode => {
+ if (mode === 'disabled') fixture.server.spec.enabled = false
+ else if (mode === 'wrc' || mode === 'wrc-missing-input') fixture.server.spec.managed = false
+ if (mode.includes('missing-input'))
+ fixture.server.spec.envSecret = { name: 'absent-fixture', keys: [] }
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(fixture.services.size).toBe(0)
+ expect(fixture.deployments.size).toBe(0)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ // Fold the status-derived eligibility result back through real LIST once.
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)
+ const decisions = vi.spyOn(hccLogger, 'info')
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(decisions).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ decision: 'skip', reason: 'identical-complete' })
+ )
+ }
+ )
+
+ it('A-T7 control: watch or informer work retains its obligation outside a fleet pass', async () => {
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ fixture.services.delete('runtime-witness')
+ const entered = deferred(),
+ release = deferred()
+ fixture.readService = async () => {
+ entered.resolve()
+ await release.promise
+ }
+ const work = state.reconciler.reconcile(state.servers.get('runtime-witness'))
+ await entered.promise
+ expect(state.runtimeRepairPending).toBe(false)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(true)
+ const decisions = vi.spyOn(hccLogger, 'info')
+ await state.recoverMcpServerInventoryAndWatch()
+ release.resolve()
+ await work
+ await drain(state)
+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)
+ expect(decisions).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ decision: 'request', reason: 'work-pending' })
+ )
+ })
+
+ it('A-T7 control: a readiness tick retired during LIST preserves repair after its poll entry disappears', async () => {
+ fixture.rolloutReady = false
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(state.reconciler.readinessPolls.size).toBe(1)
+ const entered = deferred(),
+ release = deferred()
+ vi.spyOn(state.watch, 'watch').mockImplementationOnce(async () => {
+ entered.resolve()
+ await release.promise
+ return { abort() {} }
+ })
+ const recovery = state.recoverMcpServerInventoryAndWatch()
+ await entered.promise
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(state.reconciler.readinessPolls.size).toBe(0)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)
+ release.resolve()
+ await recovery
+ await drain(state)
+ const deployment = fixture.deployments.get('runtime-witness')!
+ deployment.status!.readyReplicas = 1
+ deployment.status!.availableReplicas = 1
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)
+ })
+ it.each(['managed', 'wrc'] as const)(
+ 'A-T7 regression: transient input read preserves %s repair',
+ async mode => {
+ fixture.server.spec.envSecret = { name: 'synthetic-input', keys: [] }
+ if (mode === 'wrc') fixture.server.spec.managed = false
+ fixture.readInput = async () => {
+ throw Object.assign(new Error('transient fixture API failure'), { code: 503 })
+ }
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(fixture.services.size).toBe(0)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)
+ fixture.readInput = async () => ({
+ metadata: { name: 'synthetic-input', resourceVersion: '1' },
+ data: {},
+ })
+ const before = Date.now()
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(Date.now()).toBe(before)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ if (mode === 'managed')
+ expect(fixture.services.get('runtime-witness')?.spec?.ports?.[0].port).toBe(3000)
+ else expect(fixture.services.size).toBe(0)
+ expect(state.reconciler.getStatus('runtime-witness').ready).toBe(true)
+ }
+ )
+
+ it('A-T7 regression: failed negative eligibility publication remains pending until repaired', async () => {
+ fixture.server.spec.envSecret = { name: 'synthetic-input', keys: [] }
+ fixture.readInput = async () => ({
+ metadata: { name: 'synthetic-input', resourceVersion: '1' },
+ data: {},
+ })
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(fixture.services.size).toBe(1)
+ expect(fixture.server.status.conditions).toEqual(
+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'True' })])
+ )
+ fixture.readInput = undefined
+ fixture.failPublication = body =>
+ body.some((patch: any) => {
+ const conditions =
+ patch.path === '/status'
+ ? patch.value?.conditions
+ : patch.path === '/status/conditions'
+ ? patch.value
+ : undefined
+ return conditions?.some(
+ (condition: any) => condition.type === 'SecretResolved' && condition.status === 'False'
+ )
+ })
+ await state.reconciler.reconcile(state.servers.get('runtime-witness'))
+ expect(fixture.services.size).toBe(0)
+ expect(fixture.deployments.size).toBe(0)
+ expect(fixture.server.status.conditions).toEqual(
+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'True' })])
+ )
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)
+ fixture.failPublication = undefined
+ const before = Date.now()
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(Date.now()).toBe(before)
+ expect(fixture.server.status.conditions).toEqual(
+ expect.arrayContaining([expect.objectContaining({ type: 'SecretResolved', status: 'False' })])
+ )
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)
+ })
+
+ it('A-T7 control: deleting a failed create cleans its Service-only remainder and discharges repair', async () => {
+ fixture.failDeployment = true
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(fixture.services.size).toBe(1)
+ expect(fixture.deployments.size).toBe(0)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(true)
+ fixture.serverExists = false
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(fixture.services.size).toBe(0)
+ expect(fixture.deployments.size).toBe(0)
+ expect(state.reconciler.hasPendingReconciliation()).toBe(false)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ })
+})
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index a0c121cb7..772965505 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -13,6 +13,7 @@ import {
listAllHosts,
listAllSharedFileSystems,
} from './k8sClient'
+import { hccLogger } from './logger'
import { registry } from './metrics'
import {
DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE,
@@ -370,6 +371,12 @@ vi.mock('@kubernetes/client-node', () => {
vi.mock('./reconciler', () => ({
McpServerReconciler: class {
+ hasPendingReconciliation() {
+ return false
+ }
+ hasIncompleteReconciliation() {
+ return false
+ }
fullReconcile = mocks.serverFullReconcile
reconcile = vi.fn()
reconcileDelete = vi.fn()
@@ -1015,7 +1022,10 @@ describe('McpServerWatcher startup', () => {
if (failureBoundary === 'LIST' && contextListAttempts === 1) {
throw new Error('context discovery temporarily unavailable')
}
- return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }
+ return {
+ metadata: { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },
+ items: [],
+ }
}
if (plural === 'hosts') {
return { metadata: { resourceVersion: 'host-rv' }, items: [] }
@@ -1246,14 +1256,14 @@ describe('McpServerWatcher startup', () => {
expect(mocks.watch).toHaveBeenNthCalledWith(
1,
expect.stringContaining('/mcpservers'),
- { resourceVersion: 'mcp-inventory-rv' },
+ { resourceVersion: 'mcp-inventory-rv', allowWatchBookmarks: true },
expect.any(Function),
expect.any(Function)
)
expect(mocks.watch).toHaveBeenNthCalledWith(
2,
expect.stringContaining('/contexts'),
- { resourceVersion: 'context-inventory-rv' },
+ { resourceVersion: 'context-inventory-rv', allowWatchBookmarks: true },
expect.any(Function),
expect.any(Function)
)
@@ -1413,10 +1423,16 @@ describe('McpServerWatcher startup', () => {
}
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'mcpservers') {
- return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }
+ return {
+ metadata: { resourceVersion: 'opaque/mcp:101', allowWatchBookmarks: true },
+ items: [staleServer],
+ }
}
if (plural === 'contexts') {
- return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }
+ return {
+ metadata: { resourceVersion: 'opaque/context:202', allowWatchBookmarks: true },
+ items: [staleContext],
+ }
}
if (plural === 'communicationchannels') {
return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
@@ -1425,11 +1441,14 @@ describe('McpServerWatcher startup', () => {
})
mocks.watch.mockImplementation(async (path, options, callback) => {
if (path.endsWith('/mcpservers')) {
- expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })
+ expect(options).toEqual({ resourceVersion: 'opaque/mcp:101', allowWatchBookmarks: true })
await callback('DELETED', staleServer)
}
if (path.endsWith('/contexts')) {
- expect(options).toEqual({ resourceVersion: 'opaque/context:202' })
+ expect(options).toEqual({
+ resourceVersion: 'opaque/context:202',
+ allowWatchBookmarks: true,
+ })
await callback('MODIFIED', currentContext)
}
return { abort: vi.fn() }
@@ -1475,7 +1494,7 @@ describe('McpServerWatcher startup', () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'mcpservers') {
return {
- metadata: { resourceVersion: 'mcp-recovery-rv' },
+ metadata: { resourceVersion: 'mcp-recovery-rv', allowWatchBookmarks: true },
items: [
{
metadata: {
@@ -1511,8 +1530,8 @@ describe('McpServerWatcher startup', () => {
await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
expect(watchQueries).toEqual([
- { resourceVersion: 'mcp-start-rv' },
- { resourceVersion: 'mcp-recovery-rv' },
+ { resourceVersion: 'mcp-start-rv', allowWatchBookmarks: true },
+ { resourceVersion: 'mcp-recovery-rv', allowWatchBookmarks: true },
])
expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])
expect(callbacks).toHaveLength(2)
@@ -1685,7 +1704,7 @@ describe('McpServerWatcher startup', () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'contexts') {
return {
- metadata: { resourceVersion: 'context-recovery-rv' },
+ metadata: { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },
items: [
{
metadata: { name: 'recovered-context', namespace: 'mcp-server' },
@@ -1708,8 +1727,8 @@ describe('McpServerWatcher startup', () => {
await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
expect(watchQueries).toEqual([
- { resourceVersion: 'context-start-rv' },
- { resourceVersion: 'context-recovery-rv' },
+ { resourceVersion: 'context-start-rv', allowWatchBookmarks: true },
+ { resourceVersion: 'context-recovery-rv', allowWatchBookmarks: true },
])
expect((watcher as any).contexts.get('recovered-context')).toEqual(
expect.objectContaining({ name: 'recovered-context' })
@@ -1848,7 +1867,9 @@ describe('McpServerWatcher startup', () => {
)
await server.start()
server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({
+ cause: 'startup',
+ }) as Promise<void>
;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
() => currentDeltaSafety.promise
)
@@ -1925,7 +1946,9 @@ describe('McpServerWatcher startup', () => {
)
await server.start()
server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({
+ cause: 'startup',
+ }) as Promise<void>
// `reconcileContext` reports that its authority fence broke, so the scoped
// revocation never finished deleting this Context's stale allows.
;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
@@ -2005,7 +2028,9 @@ describe('McpServerWatcher startup', () => {
)
await server.start()
server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence({
+ cause: 'startup',
+ }) as Promise<void>
;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
() => uncertifiedDelta.promise
)
@@ -3422,7 +3447,7 @@ describe('McpServerWatcher startup', () => {
})
await (watcher as any).startContextWatch('recreated-context-rv')
- const convergence = (watcher as any).runInitialNetworkPolicyConvergence()
+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await orphanCleanupStarted.promise
const recreation = contextWatchCallback!('ADDED', recreatedContext)
await flushMicrotasks()
@@ -4127,15 +4152,15 @@ describe('McpServerWatcher startup', () => {
snapshots.push(contexts.map(context => context.name))
})
- const first = (watcher as any).runInitialNetworkPolicyConvergence()
+ const first = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
;(watcher as any).contexts.set('second-context', {
name: 'second-context',
namespace: 'mcp-server',
spec: { contextId: 'second-context', mcpServers: [] },
})
- const second = (watcher as any).runInitialNetworkPolicyConvergence()
- const third = (watcher as any).runInitialNetworkPolicyConvergence()
+ const second = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
+ const third = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
releaseFirstPass.resolve(undefined)
@@ -4277,7 +4302,7 @@ describe('McpServerWatcher startup', () => {
await Promise.all([stalePass, trailing])
expect(reconciler.reconcile).not.toHaveBeenCalled()
- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(4)
;(watcher as any).servers.set('stale-server', {
name: 'stale-server',
namespace: 'mcp-server',
@@ -4355,7 +4380,7 @@ describe('McpServerWatcher startup', () => {
)
})
- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
const revisionBefore = (watcher as any).contextDesiredRevision
;(watcher as any).retireContextWatch()
@@ -4418,6 +4443,7 @@ describe('McpServerWatcher startup', () => {
{ lane: 'NetworkPolicy', kind: 'context' }
)
mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
+ options.onAuthoritativeRevocationComplete?.()
pass += 1
if (pass === 1) {
firstPassStarted.resolve(undefined)
@@ -4439,13 +4465,13 @@ describe('McpServerWatcher startup', () => {
)
})
- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
;(watcher as any).contextCacheSynced = false
;(watcher as any).mcpServerCacheSynced = false
;(watcher as any).contextWatchGeneration = 12
;(watcher as any).mcpWatchGeneration = 14
- const trailing = (watcher as any).runInitialNetworkPolicyConvergence()
+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
releaseFirstPass.resolve(undefined)
await Promise.all([stalePass, trailing])
@@ -4531,7 +4557,7 @@ describe('McpServerWatcher startup', () => {
})
;(watcher as any).contextCacheSynced = true
;(watcher as any).mcpServerCacheSynced = true
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(appliedContexts).toEqual(['current-context'])
expect(appliedServers).toEqual(['current-server'])
@@ -4593,7 +4619,7 @@ describe('McpServerWatcher startup', () => {
])
})
- const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
const contextRevision = (watcher as any).contextDesiredRevision
const serverRevision = (watcher as any).mcpServerDesiredRevision
@@ -4651,7 +4677,7 @@ describe('McpServerWatcher startup', () => {
const watcher = new McpServerWatcher()
seedNetworkPolicyPassInventory(watcher)
;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})
const successTimestampBefore = await readInitialConvergenceMetric(
'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
'NetworkPolicy'
@@ -4670,7 +4696,7 @@ describe('McpServerWatcher startup', () => {
options.onAuthoritativeRevocationComplete?.()
})
- const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
const generationBefore = (watcher as any).contextWatchGeneration
;(watcher as any).contextDesiredRevision += 1
@@ -4717,7 +4743,7 @@ describe('McpServerWatcher startup', () => {
const watcher = new McpServerWatcher()
seedNetworkPolicyPassInventory(watcher)
;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})
const successTimestampBefore = await readInitialConvergenceMetric(
'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
'NetworkPolicy'
@@ -4736,7 +4762,7 @@ describe('McpServerWatcher startup', () => {
options.onAuthoritativeRevocationComplete?.()
})
- const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await firstPassStarted.promise
const generationBefore = (watcher as any).mcpWatchGeneration
;(watcher as any).mcpServerDesiredRevision += 1
@@ -4830,7 +4856,7 @@ describe('McpServerWatcher startup', () => {
await Promise.all(pending)
})
- const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await effectsOffered.promise
expect(passes).toBeGreaterThan(0)
expect(offeredContextEffects).toBeGreaterThanOrEqual(1)
@@ -4871,7 +4897,7 @@ describe('McpServerWatcher startup', () => {
options.onAuthoritativeRevocationComplete()
})
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
// Authority lost at the callback boundary (contextCacheSynced=false) → record
// refuses, so the content-identity revocation counters stay at the "never
@@ -4907,7 +4933,7 @@ describe('McpServerWatcher startup', () => {
}
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')
captured!.onExternalEgressRevoked!(server)
@@ -4925,7 +4951,7 @@ describe('McpServerWatcher startup', () => {
await (watcher as any).runInitialMcpServerConvergence()
expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
await watcher.stop()
@@ -5207,7 +5233,10 @@ describe('McpServerWatcher startup', () => {
{ lane: 'NetworkPolicy', result: 'deferred-unsynced' }
)
- await (watcher as any).runInitialNetworkPolicyConvergence({ ensureDefaults: true })
+ await (watcher as any).runInitialNetworkPolicyConvergence({
+ ensureDefaults: true,
+ cause: 'periodic-resync',
+ })
expect((watcher as any).netPolConvergenceEnsureDefaults).toBe(false)
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
@@ -5233,7 +5262,7 @@ describe('McpServerWatcher startup', () => {
)
).toBe(successTimestampBefore)
;(watcher as any).mcpServerCacheSynced = true
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
@@ -5272,7 +5301,7 @@ describe('McpServerWatcher startup', () => {
{ lane: 'NetworkPolicy', result: 'certified' }
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
@@ -5303,7 +5332,7 @@ describe('McpServerWatcher startup', () => {
mocks.netPolFullReconcile.mockClear()
;(watcher as any).mcpServerCacheSynced = false
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
await vi.advanceTimersByTimeAsync(5000)
@@ -5373,7 +5402,7 @@ describe('McpServerWatcher startup', () => {
server.setReady(true)
try {
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
const deferredScrape = await requestMetricsOverHttp(server)
@@ -5490,7 +5519,7 @@ describe('McpServerWatcher startup', () => {
{ lane: 'NetworkPolicy', result: 'aborted-bump' }
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(
await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
@@ -5510,8 +5539,8 @@ describe('McpServerWatcher startup', () => {
})
it('names the inventory-changed throw aborted-bump', async () => {
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const errorSpy = vi.spyOn(hccLogger, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(hccLogger, 'warn').mockImplementation(() => {})
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
;(watcher as any).mcpServerCacheSynced = true
@@ -5523,7 +5552,7 @@ describe('McpServerWatcher startup', () => {
{ lane: 'NetworkPolicy', result: 'aborted-bump' }
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(
await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
@@ -5571,7 +5600,7 @@ describe('McpServerWatcher startup', () => {
}
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(options?.contextInventoryAuthoritative()).toBe(true)
expect(options?.serverInventoryAuthoritative()).toBe(true)
@@ -5741,7 +5770,7 @@ describe('McpServerWatcher startup', () => {
errorSpy.mockRestore()
})
- it('does not report a post-certification additive failure as a readiness safety failure', async () => {
+ it('retains repair and invalidates authority after a post-revocation additive failure', async () => {
vi.useFakeTimers()
const additiveFailure = new AggregateError(
[new Error('Context policy API unavailable')],
@@ -5754,10 +5783,11 @@ describe('McpServerWatcher startup', () => {
options?: { onAuthoritativeRevocationComplete?: () => void }
) => {
options?.onAuthoritativeRevocationComplete?.()
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
throw additiveFailure
}
)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const errorSpy = vi.spyOn(hccLogger, 'error').mockImplementation(() => {})
const watcher = new McpServerWatcher()
stubAuthoritativeInventoryWatch(watcher, 'McpServer')
stubAuthoritativeInventoryWatch(watcher, 'Context')
@@ -5768,14 +5798,14 @@ describe('McpServerWatcher startup', () => {
await watcher.start()
await vi.advanceTimersByTimeAsync(0)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
expect(errorSpy).toHaveBeenCalledWith(
'[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',
- additiveFailure
+ { err: additiveFailure }
)
expect(errorSpy).not.toHaveBeenCalledWith(
'[K8s] Initial NetworkPolicy background reconciliation failed:',
- additiveFailure
+ { err: additiveFailure }
)
expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
@@ -10792,7 +10822,7 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
// unmistakable, and the fix must hold readiness through all of it.
const CHURN_PASSES = 5
for (let i = 0; i < CHURN_PASSES; i += 1) {
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await flushMicrotasks()
}
@@ -10866,7 +10896,7 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
server.setReady(true)
const CHURN_PASSES = 5
for (let i = 0; i < CHURN_PASSES; i += 1) {
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
await flushMicrotasks()
}
expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)
@@ -11250,7 +11280,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt
const watcher = new McpServerWatcher()
;(watcher as any)[lane.peerSyncedField] = true
await (watcher as any)[lane.restartMethod](lane.startSnapshot)
- expect(watchQueries).toEqual([{ resourceVersion: lane.startRv }])
+ expect(watchQueries).toEqual([{ resourceVersion: lane.startRv, allowWatchBookmarks: true }])
doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
await flushMicrotasks(20)
@@ -11259,8 +11289,8 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt
// any timer. Under the old contract this stayed at 0 until +5000ms.
expect(listCalls).toBe(1)
expect(watchQueries).toEqual([
- { resourceVersion: lane.startRv },
- { resourceVersion: `${lane.recoveryRvPrefix}-1` },
+ { resourceVersion: lane.startRv, allowWatchBookmarks: true },
+ { resourceVersion: `${lane.recoveryRvPrefix}-1`, allowWatchBookmarks: true },
])
expect((watcher as any)[lane.syncedField]).toBe(true)
expect((watcher as any)[lane.timerField]).toBeNull()
@@ -11874,7 +11904,11 @@ describe('McpServerWatcher watch-recovery retry backoff (exponential, jittered,
describe('McpServerWatcher NetworkPolicy periodic resync (#478)', () => {
async function startWatcherForNetPolResync(): Promise<McpServerWatcher> {
- mocks.netPolFullReconcile.mockClear()
+ mocks.netPolFullReconcile
+ .mockClear()
+ .mockImplementation(async (_contexts, _servers, options) => {
+ options?.onAuthoritativeRevocationComplete?.()
+ })
mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'communicationchannels') {
@@ -11941,7 +11975,7 @@ describe('McpServerWatcher NetworkPolicy periodic resync (#478)', () => {
expect.objectContaining({ ensureDefaults: true })
)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(3)
expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
expect.any(Array),
@@ -12291,7 +12325,7 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {
expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(2)
expect((watcher as any).netPolConvergenceEnsureDefaults).toBe(false)
- await (watcher as any).runInitialNetworkPolicyConvergence()
+ await (watcher as any).runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
expect.any(Array),
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 5b3a4b286..47f15463c 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -234,6 +234,17 @@ type HostSnapshot = {
type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'
type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'
+type NetworkPolicyConvergenceCause =
+ | 'mcp-recovery'
+ | 'context-recovery'
+ | 'periodic-resync'
+ | 'startup'
+ | 'retry'
+ | 'mcp-change'
+ | 'context-change'
+ | 'context-reconcile-failure'
+type CompletedInventoryRevision = { contextRevision: number; serverRevision: number }
+
type InitialConvergencePassResult =
| 'certified'
| 'aborted-authority'
@@ -767,6 +778,22 @@ export class McpServerWatcher implements McpServerProvider {
// the startup role the (now removed) generation equalities used to play.
private networkPolicyRevocationContextRevision = -1
private networkPolicyRevocationServerRevision = -1
+ // Safety certification precedes additive policy work and runtime effects.
+ // Only successful, current completions discharge these separate obligations.
+ private completedNetworkPolicyRevision: CompletedInventoryRevision | null = null
+ private completedRuntimeRevision: CompletedInventoryRevision | null = null
+ private networkPolicyRepairPending = false
+ private runtimeRepairPending = false
+ private runtimeInventoryGeneration: number | null = null
+ private readonly pendingNetworkPolicyCauses = new Set<NetworkPolicyConvergenceCause>()
+ private networkPolicyPassSequence = 0
+ private activeNetworkPolicyPass: {
+ passId: number
+ causes: NetworkPolicyConvergenceCause[]
+ trailing: boolean
+ contextRevision: number
+ serverRevision: number
+ } | null = null
private mcpServerCacheSynced = false
private contextCacheSynced = false
private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
@@ -1490,6 +1517,7 @@ export class McpServerWatcher implements McpServerProvider {
if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight
const recovery = (async () => {
+ const before = this.currentDesiredInventoryRevision()
try {
const snapshot = await listMcpServerSnapshot()
if (this.stopped) return false
@@ -1499,15 +1527,15 @@ export class McpServerWatcher implements McpServerProvider {
this.mcpServerWatchRecoveryFailures = 0
this.mcpServerWatchRecoveryRetryAfterMs = undefined
this.changeCallback?.()
- if (this.contextCacheSynced) {
- void this.runInitialNetworkPolicyConvergence()
+ if (this.shouldRequestNetworkPolicyRecovery('McpServer', before)) {
+ void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-recovery' })
}
return true
} catch (error) {
this.mcpServerCacheSynced = false
this.mcpServerWatchRecoveryFailures += 1
this.mcpServerWatchRecoveryRetryAfterMs = getRetryAfterMs(error)
- console.error('[K8s] McpServer cache recovery failed:', error)
+ hccLogger.error('[K8s] McpServer cache recovery failed:', { err: error })
return false
}
})()
@@ -1526,6 +1554,7 @@ export class McpServerWatcher implements McpServerProvider {
if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight
const recovery = (async () => {
+ const before = this.currentDesiredInventoryRevision()
try {
const snapshot = await listContextSnapshot()
if (this.stopped) return false
@@ -1535,7 +1564,9 @@ export class McpServerWatcher implements McpServerProvider {
this.contextWatchRecoveryFailures = 0
this.contextWatchRecoveryRetryAfterMs = undefined
this.hostFleetScheduler.bumpInputRevision()
- void this.runInitialNetworkPolicyConvergence()
+ if (this.shouldRequestNetworkPolicyRecovery('Context', before)) {
+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-recovery' })
+ }
void this.runInitialSharedFileSystemConvergence()
if (this.hostCacheSynced) {
const pendingCcLifecycleGeneration =
@@ -1553,7 +1584,7 @@ export class McpServerWatcher implements McpServerProvider {
this.contextCacheSynced = false
this.contextWatchRecoveryFailures += 1
this.contextWatchRecoveryRetryAfterMs = getRetryAfterMs(error)
- console.error('[K8s] Context cache recovery failed:', error)
+ hccLogger.error('[K8s] Context cache recovery failed:', { err: error })
return false
}
})()
@@ -2906,7 +2937,10 @@ export class McpServerWatcher implements McpServerProvider {
netPolResyncTicksSkippedTotal.inc({ reason: 'pass-in-flight' })
return
}
- void this.runInitialNetworkPolicyConvergence({ ensureDefaults: true })
+ void this.runInitialNetworkPolicyConvergence({
+ ensureDefaults: true,
+ cause: 'periodic-resync',
+ })
}, netPolResyncSec * 1000)
console.log(`[K8s] NetworkPolicy periodic resync enabled (every ${netPolResyncSec}s)`)
} else {
@@ -2946,7 +2980,7 @@ export class McpServerWatcher implements McpServerProvider {
// its certification callback, before additive policy convergence completes.
// This prevents the startup egress refresh from racing or undoing the safety
// pass while keeping readiness independent from both additive fleets.
- void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialNetworkPolicyConvergence({ cause: 'startup' })
void initialSharedFileSystemInventory.then(inventoryComplete => {
if (inventoryComplete && !this.stopped) {
void this.runInitialSharedFileSystemConvergence()
@@ -3006,6 +3040,60 @@ export class McpServerWatcher implements McpServerProvider {
}
}
+ private currentDesiredInventoryRevision(): CompletedInventoryRevision {
+ return {
+ contextRevision: this.contextDesiredRevision,
+ serverRevision: this.mcpServerDesiredRevision,
+ }
+ }
+
+ private matchesCurrentInventory(revision: CompletedInventoryRevision | null): boolean {
+ return (
+ revision !== null &&
+ revision.contextRevision === this.contextDesiredRevision &&
+ revision.serverRevision === this.mcpServerDesiredRevision
+ )
+ }
+
+ private shouldRequestNetworkPolicyRecovery(
+ kind: 'McpServer' | 'Context',
+ before: CompletedInventoryRevision
+ ): boolean {
+ let reason:
+ | 'identical-complete'
+ | 'relevant-change'
+ | 'no-authority'
+ | 'work-pending'
+ | 'resync-disabled'
+ const deferred = kind === 'McpServer' && !this.contextCacheSynced
+ if (deferred) reason = 'no-authority'
+ else if (!this.matchesCurrentInventory(before)) reason = 'relevant-change'
+ else if (this.netPolResyncTimer === null) reason = 'resync-disabled'
+ else if (this.currentNetworkPolicySafetyCertificate() === null) reason = 'no-authority'
+ else if (
+ !this.matchesCurrentInventory(this.completedNetworkPolicyRevision) ||
+ !this.matchesCurrentInventory(this.completedRuntimeRevision) ||
+ this.networkPolicyRepairPending ||
+ this.runtimeRepairPending ||
+ this.reconciler.hasPendingReconciliation() ||
+ (this.runtimeInventoryGeneration !== null &&
+ this.runtimeInventoryGeneration !== this.mcpWatchGeneration) ||
+ this.initialConvergenceRetryTimers.has('NetworkPolicy') ||
+ this.initialConvergenceRetryTimers.has('McpServer')
+ ) {
+ reason = 'work-pending'
+ } else reason = 'identical-complete'
+ const request = !deferred && reason !== 'identical-complete'
+ hccLogger.info('[K8s] NetworkPolicy recovery decision', {
+ event: 'networkpolicy-recovery-decision',
+ kind,
+ decision: deferred ? 'defer' : request ? 'request' : 'skip',
+ reason,
+ ...this.currentDesiredInventoryRevision(),
+ })
+ return request
+ }
+
private runInitialMcpServerConvergence(): Promise<void> {
return this.runInitialConvergence('McpServer')
}
@@ -3063,8 +3151,13 @@ export class McpServerWatcher implements McpServerProvider {
private async runInitialMcpServerConvergenceCore(): Promise<void> {
const safetyCertificate = this.currentNetworkPolicySafetyCertificate()
+ // A retired run may settle normally without invoking work. Keep its repair
+ // obligation until an authoritative pass actually completes every effect.
+ this.runtimeRepairPending = true
if (safetyCertificate === null) return
+ let effectsComplete = true
const inventoryGeneration = this.mcpWatchGeneration
+ this.runtimeInventoryGeneration = inventoryGeneration
const inventoryAuthoritative = () =>
this.hasMcpServerInventoryAuthority(inventoryGeneration) &&
this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)
@@ -3075,7 +3168,7 @@ export class McpServerWatcher implements McpServerProvider {
inventoryAuthoritative
)
if (this.stopped) return
- console.log('[K8s] Running initial McpServer background reconciliation...')
+ hccLogger.info('[K8s] Running initial McpServer background reconciliation...')
await this.reconciler.fullReconcile(initialServers, {
runEffect: async (serverName, work) => {
const selected = this.servers.get(serverName)
@@ -3083,29 +3176,58 @@ export class McpServerWatcher implements McpServerProvider {
name: serverName,
namespace: config.namespace,
}
- if (!(await initialExternalEgressGates.waitFor(laneOwner))) return
- if (!inventoryAuthoritative()) return
+ if (!(await initialExternalEgressGates.waitFor(laneOwner))) {
+ effectsComplete = false
+ return
+ }
+ if (!inventoryAuthoritative()) {
+ effectsComplete = false
+ return
+ }
await this.enqueueMcpServerReconciliation(laneOwner, async () => {
- if (!inventoryAuthoritative()) return
+ if (!inventoryAuthoritative()) {
+ effectsComplete = false
+ return
+ }
await work()
})
},
})
- if (!inventoryAuthoritative()) return
+ if (
+ !effectsComplete ||
+ !inventoryAuthoritative() ||
+ this.reconciler.hasIncompleteReconciliation()
+ ) {
+ this.scheduleInitialConvergenceRetry('McpServer')
+ return
+ }
+ this.completedRuntimeRevision = safetyCertificate
+ this.runtimeRepairPending = false
initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)
this.clearInitialConvergenceRetry('McpServer')
} catch (error) {
- console.error('[K8s] Initial McpServer background reconciliation failed:', error)
+ hccLogger.error('[K8s] Initial McpServer background reconciliation failed:', { err: error })
this.scheduleInitialConvergenceRetry('McpServer')
+ } finally {
+ this.runtimeInventoryGeneration = null
}
}
- private runInitialNetworkPolicyConvergence(options?: {
+ private runInitialNetworkPolicyConvergence(options: {
ensureDefaults?: boolean
+ cause: NetworkPolicyConvergenceCause
}): Promise<void> {
if (options?.ensureDefaults) {
this.netPolConvergenceEnsureDefaults = true
}
+ const cause = options.cause
+ this.pendingNetworkPolicyCauses.add(cause)
+ hccLogger.info('[K8s] NetworkPolicy convergence requested', {
+ event: 'networkpolicy-request',
+ cause,
+ ...this.currentDesiredInventoryRevision(),
+ admission: this.activeNetworkPolicyPass === null ? 'before-capture' : 'trailing',
+ })
return this.runInitialConvergence('NetworkPolicy')
}
@@ -3168,13 +3290,31 @@ export class McpServerWatcher implements McpServerProvider {
this.initialConvergenceRuns.set(lane, run)
run.promise = Promise.resolve()
.then(async () => {
+ let trailing = false
do {
run.trailingRequested = false
if (lane === 'McpServer') {
await this.runInitialMcpServerConvergenceCore()
} else {
- await this.runInitialNetworkPolicyConvergenceCore()
+ const pass = {
+ passId: ++this.networkPolicyPassSequence,
+ causes: [...this.pendingNetworkPolicyCauses],
+ trailing,
+ ...this.currentDesiredInventoryRevision(),
+ }
+ this.pendingNetworkPolicyCauses.clear()
+ this.activeNetworkPolicyPass = pass
+ hccLogger.info('[K8s] NetworkPolicy pass started', {
+ event: 'networkpolicy-pass-start',
+ ...pass,
+ })
+ try {
+ await this.runInitialNetworkPolicyConvergenceCore()
+ } finally {
+ this.activeNetworkPolicyPass = null
+ }
}
+ trailing = true
} while (!this.stopped && run.trailingRequested)
// Retire the active slot synchronously before this promise settles.
// A request arriving in the following settlement microtask must start
@@ -3200,6 +3340,14 @@ export class McpServerWatcher implements McpServerProvider {
// full convergence nor safe evidence of success. Recovery of either
// missing LIST -> WATCH pair schedules a fresh current-cache pass.
const startedAtMs = Date.now()
+ const observePass = (result: InitialConvergencePassResult) => {
+ observeInitialNetworkPolicyPass(startedAtMs, result)
+ hccLogger.info('[K8s] NetworkPolicy pass finished', {
+ event: 'networkpolicy-pass-result',
+ ...this.activeNetworkPolicyPass,
+ result,
+ })
+ }
if (!this.contextCacheSynced || !this.mcpServerCacheSynced) {
const unsynced = [
!this.contextCacheSynced ? 'Context' : undefined,
@@ -3207,17 +3355,19 @@ export class McpServerWatcher implements McpServerProvider {
]
.filter((name): name is string => name !== undefined)
.join(' and ')
- console.warn(
+ hccLogger.warn(
`[K8s] NetworkPolicy convergence request deferred: caches unsynced (${unsynced})`
)
initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'unsynced' })
- observeInitialNetworkPolicyPass(startedAtMs, 'deferred-unsynced')
+ observePass('deferred-unsynced')
+ this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
// A timer tick may have requested defaults. This path never reached
// fullReconcile, so drop the flag: event/retry entry stays false.
this.netPolConvergenceEnsureDefaults = false
return
}
+ let effectsComplete = true
let authoritativeRevocationCompleted = false
let safetyCertificate: NetworkPolicySafetyCertificate | undefined
try {
@@ -3239,7 +3389,7 @@ export class McpServerWatcher implements McpServerProvider {
const serverInventoryAuthoritative = () =>
this.mcpServerCacheSynced &&
this.mcpServerDesiredRevision === capturedSafetyCertificate.serverRevision
- console.log('[K8s] Running initial NetworkPolicy background reconciliation...')
+ hccLogger.info('[K8s] Running initial NetworkPolicy background reconciliation...')
const ensureDefaults = this.netPolConvergenceEnsureDefaults
this.netPolConvergenceEnsureDefaults = false
await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
@@ -3250,6 +3400,8 @@ export class McpServerWatcher implements McpServerProvider {
runContextEffect: (contextId, work) =>
this.enqueueContextReconciliation(contextId, async () => {
if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
+ effectsComplete = false
+ this.networkPolicyRepairPending = true
initialConvergenceEffectsDroppedTotal.inc({
lane: 'NetworkPolicy',
kind: 'context',
@@ -3264,6 +3416,8 @@ export class McpServerWatcher implements McpServerProvider {
selected ?? { name: serverName, namespace: config.namespace },
async () => {
if (!serverInventoryAuthoritative()) {
+ effectsComplete = false
+ this.networkPolicyRepairPending = true
initialConvergenceEffectsDroppedTotal.inc({
lane: 'NetworkPolicy',
kind: 'server',
@@ -3297,38 +3451,47 @@ export class McpServerWatcher implements McpServerProvider {
void this.runInitialMcpServerConvergence()
},
})
- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
- console.warn(
+ if (
+ !effectsComplete ||
+ !authoritativeRevocationCompleted ||
+ !contextInventoryAuthoritative() ||
+ !serverInventoryAuthoritative()
+ ) {
+ hccLogger.warn(
'[K8s] pass ended without certifying: inventory authority lost',
this.networkPolicyInventoryMovement(capturedSafetyCertificate)
)
initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })
- observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')
+ observePass('aborted-authority')
+ this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
return
}
+ this.completedNetworkPolicyRevision = capturedSafetyCertificate
+ this.networkPolicyRepairPending = false
initialConvergenceLastSuccessTimestampSeconds.set(
{ lane: 'NetworkPolicy' },
Date.now() / 1000
)
this.clearInitialConvergenceRetry('NetworkPolicy')
- observeInitialNetworkPolicyPass(startedAtMs, 'certified')
+ observePass('certified')
} catch (error) {
- console.error(
+ hccLogger.error(
authoritativeRevocationCompleted
? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'
: '[K8s] Initial NetworkPolicy background reconciliation failed:',
- error
+ { err: error }
)
const abortedBump =
error instanceof Error && error.message === DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE
if (abortedBump && safetyCertificate) {
- console.warn(
+ hccLogger.warn(
'[K8s] pass ended without certifying: desired inventory changed',
this.networkPolicyInventoryMovement(safetyCertificate)
)
}
- observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')
+ observePass(abortedBump ? 'aborted-bump' : 'failed')
+ this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
}
}
@@ -3377,7 +3540,7 @@ export class McpServerWatcher implements McpServerProvider {
void this.runInitialMcpServerConvergence()
return
}
- void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialNetworkPolicyConvergence({ cause: 'retry' })
}, delayMs)
this.initialConvergenceRetryTimers.set(lane, timer)
}
@@ -3427,6 +3590,7 @@ export class McpServerWatcher implements McpServerProvider {
type: string,
apiObj: { metadata: { name: string; namespace?: string }; spec: SharedFileSystemSpec }
) => {
+ if (type === 'BOOKMARK') return
const sfs: SharedFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.hostNamespace,
@@ -3511,6 +3675,7 @@ export class McpServerWatcher implements McpServerProvider {
status?: GlobalFileSystemCRD['status']
}
) => {
+ if (type === 'BOOKMARK') return
const gfs: GlobalFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || namespace,
@@ -3575,6 +3740,7 @@ export class McpServerWatcher implements McpServerProvider {
}
) => Promise<void> {
return async (type, apiObj) => {
+ if (type === 'BOOKMARK') return
const cc: CommunicationChannelCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.channelsNamespace,
@@ -3902,6 +4068,7 @@ export class McpServerWatcher implements McpServerProvider {
watchGeneration = this.mcpWatchGeneration
): (type: string, apiObj: McpServerWatchObject) => Promise<void> {
return async (type, apiObj) => {
+ if (type === 'BOOKMARK') return
if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
const server: McpServerCRD = {
name: apiObj.metadata.name,
@@ -3931,7 +4098,7 @@ export class McpServerWatcher implements McpServerProvider {
}
if (desiredStateChanged) {
this.mcpServerDesiredRevision += 1
- void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-change' })
}
// Most status writes do not change desired state. A runtime eligibility
@@ -4172,7 +4339,12 @@ export class McpServerWatcher implements McpServerProvider {
this.attemptMcpServerCacheRecovery()
}
- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
+ const request = await this.watch.watch(
+ path,
+ { resourceVersion, allowWatchBookmarks: true },
+ watchCallback,
+ doneCallback
+ )
if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {
request.abort()
return watchGeneration
@@ -4340,6 +4512,7 @@ export class McpServerWatcher implements McpServerProvider {
spec: ContextSpec
}
) => {
+ if (type === 'BOOKMARK') return
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
const context: ContextCRD = {
name: apiObj.metadata.name,
@@ -4399,7 +4572,7 @@ export class McpServerWatcher implements McpServerProvider {
serverRevision: previousSafetyCertificate.serverRevision,
}
}
- void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-change' })
}
// Metadata-only Context MODIFIED (label ping-pong, annotation churn)
@@ -4512,7 +4685,7 @@ export class McpServerWatcher implements McpServerProvider {
`[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
error
)
- void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialNetworkPolicyConvergence({ cause: 'context-reconcile-failure' })
return
}
@@ -4545,7 +4718,12 @@ export class McpServerWatcher implements McpServerProvider {
this.attemptContextCacheRecovery()
}
- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
+ const request = await this.watch.watch(
+ path,
+ { resourceVersion, allowWatchBookmarks: true },
+ watchCallback,
+ doneCallback
+ )
if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {
request.abort()
return watchGeneration
diff --git a/host-context-controller/src/k8sClient.watchTransport.test.ts b/host-context-controller/src/k8sClient.watchTransport.test.ts
new file mode 100644
index 000000000..f166326f8
--- /dev/null
+++ b/host-context-controller/src/k8sClient.watchTransport.test.ts
@@ -0,0 +1,284 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import * as http from 'node:http'
+import type { Socket } from 'node:net'
+import { McpServerWatcher, getKubeConfig } from './k8sClient'
+
+// Preserve the installed Watch and HTTP transport. Replace only ambient
+// configuration loading with an anonymous, inert loopback configuration.
+vi.mock('@kubernetes/client-node', async importOriginal => {
+ const actual = await importOriginal<typeof import('@kubernetes/client-node')>()
+ return {
+ ...actual,
+ KubeConfig: class extends actual.KubeConfig {
+ loadFromDefault(): void {
+ this.loadFromOptions({
+ clusters: [{ name: 'loopback', server: 'http://127.0.0.1:1', skipTLSVerify: true }],
+ users: [{ name: 'anonymous' }],
+ contexts: [{ name: 'loopback', cluster: 'loopback', user: 'anonymous' }],
+ currentContext: 'loopback',
+ })
+ }
+ },
+ }
+})
+vi.mock('./config', () => ({
+ config: {
+ devMode: false,
+ namespace: 'mcp-server',
+ hostNamespace: 'mcp-host',
+ channelsNamespace: 'channels',
+ llmHooksNamespace: 'llm-hooks',
+ rpcProxyNamespace: 'rpc-proxy',
+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'rpc-proxy'],
+ hostK8sRequestTimeoutMs: 30_000,
+ hostResyncIntervalSec: 0,
+ netPolResyncIntervalSec: 0,
+ netPolDefaultsResyncIntervalSec: 0,
+ externalEgressResyncIntervalSec: 0,
+ governedTracingEnabled: false,
+ },
+}))
+
+const lanes = [
+ {
+ plural: 'mcpservers',
+ start: 'startMcpServerWatch',
+ rv: '101',
+ cache: 'servers',
+ effect: 'enqueueMcpServerReconciliation',
+ },
+ {
+ plural: 'contexts',
+ start: 'startContextWatch',
+ rv: '202',
+ cache: 'contexts',
+ effect: 'enqueueContextIdentityReconciliation',
+ },
+ {
+ plural: 'sharedfilesystems',
+ start: 'startSharedFileSystemWatch',
+ cache: 'sharedFileSystems',
+ effect: 'enqueueSharedFileSystemReconciliation',
+ },
+ {
+ plural: 'globalfilesystems',
+ start: 'startGlobalFileSystemWatch',
+ cache: 'globalFileSystems',
+ effect: 'enqueueGlobalFileSystemReconciliation',
+ },
+ {
+ plural: 'communicationchannels',
+ start: 'startCommunicationChannelWatch',
+ rv: '505',
+ cache: 'communicationChannels',
+ effect: 'reconcileHostsReferencingCC',
+ },
+ { plural: 'llmhooks', start: 'startLlmHookWatch', cache: 'llmHooks', effect: 'hookReconcile' },
+ {
+ plural: 'hosts',
+ start: 'startHostWatch',
+ rv: '707',
+ cache: 'hosts',
+ effect: 'reconcileHostWatchEvent',
+ },
+] as const
+
+type WatchCallback = (type: string, object: any) => Promise<void>
+const watchers: McpServerWatcher[] = []
+
+function createFixture() {
+ const watcher = new McpServerWatcher()
+ watchers.push(watcher)
+ // These exact private methods are called by start(), which also performs
+ // unrelated LIST/bootstrap work. Exercise their installed callbacks here.
+ const state = watcher as any
+ state.mcpServerCacheSynced = true
+ state.contextCacheSynced = true
+ state.hostCacheSynced = true
+ state.ccCacheSynced = true
+ const effects: Record<string, ReturnType<typeof vi.fn>> = {}
+ for (const method of [
+ 'enqueueMcpServerReconciliation',
+ 'enqueueContextIdentityReconciliation',
+ 'enqueueSharedFileSystemReconciliation',
+ 'enqueueGlobalFileSystemReconciliation',
+ 'runInitialNetworkPolicyConvergence',
+ 'reconcileSharedFileSystemsReferencedByContext',
+ 'reconcileHostWatchEvent',
+ 'admitHostDependentEffects',
+ 'requestHostFleetReconcile',
+ ])
+ effects[method] = vi.spyOn(state, method).mockResolvedValue(undefined)
+ effects.reconcileHostsReferencingCC = vi
+ .spyOn(state, 'reconcileHostsReferencingCC')
+ .mockResolvedValue(true)
+ for (const [name, owner, method] of [
+ ['hookReconcile', 'llmHookReconciler', 'reconcile'],
+ ['hookDelete', 'llmHookReconciler', 'reconcileDelete'],
+ ['hookPolicies', 'llmHookReconciler', 'reconcileNetworkPoliciesForHooks'],
+ ['hostEgress', 'llmHookReconciler', 'reconcileHostEgress'],
+ ['channelRevision', 'hostReconciler', 'patchChannelReaderRevisionAnnotation'],
+ ['sfsReconcile', 'sharedFileSystemReconciler', 'reconcile'],
+ ['gfsReconcile', 'gfsReconciler', 'reconcile'],
+ ['mcpReconcile', 'reconciler', 'reconcile'],
+ ])
+ effects[name] = vi.spyOn(state[owner], method).mockResolvedValue(undefined)
+ effects.change = vi.fn()
+ state.changeCallback = effects.change
+ return { watcher, state, effects }
+}
+
+function snapshot(state: any) {
+ return {
+ caches: lanes.map(lane => [...state[lane.cache].entries()]),
+ revisions: [
+ state.mcpServerDesiredRevision,
+ state.contextDesiredRevision,
+ state.sharedFileSystemCacheRevision,
+ state.globalFileSystemCacheRevision,
+ state.hostDesiredRevision,
+ state.hostWatchRevision,
+ state.ccLifecycleGeneration,
+ ],
+ tasks: [
+ state.mcpServerReconciliationQueues.size,
+ state.contextReconciliationQueues.size,
+ state.sharedFileSystemReconciliationQueues.size,
+ state.globalFileSystemReconciliationQueues.size,
+ state.latestHostWatchEventRevisions.size,
+ ],
+ }
+}
+
+function addedObject(plural: string) {
+ return {
+ metadata: {
+ name: `${plural}-witness`,
+ namespace: 'fixture',
+ uid: `${plural}-uid`,
+ generation: 1,
+ resourceVersion: '900',
+ },
+ spec: {
+ contextId: 'context-witness',
+ contextRef: 'context-witness',
+ hostRef: 'host-witness',
+ sharedFileSystems: [],
+ },
+ }
+}
+
+afterEach(async () => {
+ for (const watcher of watchers.splice(0)) await watcher.stop()
+ vi.restoreAllMocks()
+})
+
+describe('A-T1 production watch callbacks', () => {
+ it.each(lanes)(
+ '$plural ignores metadata-only BOOKMARK and admits a named ADDED witness',
+ async lane => {
+ const { state, effects } = createFixture()
+ let callback: WatchCallback | undefined
+ vi.spyOn(state.watch, 'watch').mockImplementation(
+ async (_path: unknown, _params: unknown, cb: unknown) => {
+ callback = cb as WatchCallback
+ return new AbortController()
+ }
+ )
+ await state[lane.start]('rv' in lane ? lane.rv : undefined)
+ expect(callback).toBeTypeOf('function')
+ expect(state.stopped).toBe(false)
+ const before = snapshot(state)
+ await expect(
+ callback!('BOOKMARK', { metadata: { resourceVersion: '899' } })
+ ).resolves.toBeUndefined()
+ expect(snapshot(state)).toEqual(before)
+ for (const effect of Object.values(effects)) expect(effect).not.toHaveBeenCalled()
+
+ // Same callback and fixture: stale generations and no-op callbacks fail
+ // this positive control even if the BOOKMARK assertions pass.
+ const object = addedObject(lane.plural)
+ await callback!('ADDED', object)
+ expect(state[lane.cache].get(object.metadata.name)).toMatchObject({
+ name: object.metadata.name,
+ })
+ expect(effects[lane.effect]).toHaveBeenCalledTimes(1)
+ const [firstArgument] = effects[lane.effect].mock.calls[0]
+ if (lane.plural === 'contexts') expect(firstArgument).toEqual(['context-witness'])
+ else if (lane.plural === 'communicationchannels') expect(firstArgument).toBe('host-witness')
+ else if (lane.plural === 'hosts')
+ expect(effects[lane.effect]).toHaveBeenCalledWith(
+ 'ADDED',
+ expect.objectContaining({ name: object.metadata.name }),
+ expect.any(Number)
+ )
+ else if (lane.plural === 'sharedfilesystems' || lane.plural === 'globalfilesystems')
+ expect(firstArgument).toBe(object.metadata.name)
+ else expect(firstArgument).toMatchObject({ name: object.metadata.name })
+ }
+ )
+})
+
+describe('A-T2a installed Kubernetes Watch HTTP transport', () => {
+ it('sends seven distinct watch requests, retaining RVs and requesting bookmarks only for MCP/Context', async () => {
+ const requests: URL[] = []
+ const sockets = new Set<Socket>()
+ const server = http.createServer((request, response) => {
+ requests.push(new URL(request.url!, 'http://127.0.0.1'))
+ response.writeHead(200, { 'content-type': 'application/json' })
+ response.flushHeaders()
+ })
+ server.on('connection', socket => {
+ sockets.add(socket)
+ socket.once('close', () => sockets.delete(socket))
+ })
+ await new Promise<void>((resolve, reject) => {
+ server.once('error', reject)
+ server.listen(0, '127.0.0.1', resolve)
+ })
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('Loopback server did not bind')
+ const kubeConfig = getKubeConfig()!
+ kubeConfig.loadFromOptions({
+ clusters: kubeConfig.getClusters().map(cluster => ({
+ ...cluster,
+ server: `http://127.0.0.1:${address.port}`,
+ })),
+ users: kubeConfig.getUsers(),
+ contexts: kubeConfig.getContexts(),
+ currentContext: kubeConfig.getCurrentContext(),
+ })
+ const { watcher, state } = createFixture()
+ try {
+ for (const lane of lanes) await state[lane.start]('rv' in lane ? lane.rv : undefined)
+ expect(requests).toHaveLength(7)
+ expect(new Set(requests.map(url => url.pathname)).size).toBe(7)
+ for (const lane of lanes) {
+ const matches = requests.filter(url => url.pathname.endsWith(`/${lane.plural}`))
+ expect(matches, lane.plural).toHaveLength(1)
+ const url = matches[0]
+ expect(url.searchParams.get('watch'), lane.plural).toBe('true')
+ expect(url.searchParams.get('allowWatchBookmarks'), lane.plural).toBe(
+ lane.plural === 'mcpservers' || lane.plural === 'contexts' ? 'true' : null
+ )
+ expect(url.searchParams.get('resourceVersion'), lane.plural).toBe(
+ 'rv' in lane ? lane.rv : null
+ )
+ }
+ } finally {
+ await watcher.stop()
+ for (const socket of sockets) socket.destroy()
+ await new Promise<void>((resolve, reject) => {
+ const deadline = setTimeout(
+ () => reject(new Error('Loopback server close exceeded 2 seconds')),
+ 2000
+ )
+ server.close(error => {
+ clearTimeout(deadline)
+ if (error) reject(error)
+ else resolve()
+ })
+ })
+ }
+ }, 10_000)
+})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index d029f9d70..b87f4b42b 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -5167,7 +5167,9 @@ describe('NetworkPolicyReconciler', () => {
],
})
- await fullPass
+ await expect(fullPass).rejects.toThrow(
+ 'One or more additive Context NetworkPolicy reconciliations failed'
+ )
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index 69ae979c9..01c7342de 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -1807,7 +1807,10 @@ export class NetworkPolicyReconciler {
const latest = options.resolveCurrentContext(current.name)
return latest !== undefined && sameContextDesiredRevision(current, latest)
}
- await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })
+ const completed = await this.reconcileContext(current, {
+ isCurrent: contextEffectIsCurrent,
+ })
+ if (!completed) throw new Error(DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE)
})
} catch (error) {
contextAdditiveFailures.push(error)
diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.ts
index 1153a0709..ff3d86767 100644
--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts
+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts
@@ -615,7 +615,7 @@ describe('lost delete fence: per-request 503 while /ready stays 200', () => {
})
mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await w.runInitialNetworkPolicyConvergence()
+ await w.runInitialNetworkPolicyConvergence({ cause: 'startup' })
expect(lists).toBeGreaterThan(0)
expect(w.contextWatchGeneration).toBe(generationBefore.context + 2)
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index e2400a3f5..e5a52b7a9 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -307,6 +307,39 @@ export class McpServerReconciler {
*/
private readonly inFlight: Map<string, Promise<void>> = new Map()
+ // A fulfilled reconcile promise can mean an authority guard retired work or
+ // a resource API error was reported through status. Keep that distinction
+ // independently of discovery readiness and the per-name promise queue.
+ private readonly pendingReconciliations = new Map<string, { namespace: string }>()
+
+ hasIncompleteReconciliation(): boolean {
+ return this.pendingReconciliations.size > 0
+ }
+
+ hasPendingReconciliation(): boolean {
+ return (
+ this.hasIncompleteReconciliation() ||
+ this.inFlight.size > 0 ||
+ [...this.readinessPolls.values()].some(
+ poll => (poll.timer !== undefined || poll.ticking) && this.readinessPollStillDesired(poll)
+ )
+ )
+ }
+
+ private readinessPollStillDesired(poll: ReadinessPollState): boolean {
+ const current = this.resolveCurrentServer?.(poll.server.name)
+ return (
+ this.resolveCurrentServer === undefined ||
+ (current !== undefined && sameMcpServerDesiredRevision(poll.server, current))
+ )
+ }
+
+ private retainRetiredReadinessPoll(poll: ReadinessPollState): void {
+ if (this.readinessPollStillDesired(poll)) {
+ this.pendingReconciliations.set(poll.server.name, { namespace: poll.server.namespace })
+ }
+ }
+
constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {
this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
@@ -587,12 +620,14 @@ export class McpServerReconciler {
// status — on behalf of a stale desired state. End the window quietly;
// the current reconcile arms its own window after its Deployment write.
if (!state.isCurrent()) {
+ this.retainRetiredReadinessPoll(state)
state.ticking = false
if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
return
}
const rollout = await this.readDeploymentRollout(name, namespace)
if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {
+ this.retainRetiredReadinessPoll(state)
state.ticking = false
if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
return
@@ -1722,10 +1757,22 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
- // Policy owners reconcile their own desired sets with ownership and
- // resource-version preconditions; runtime cleanup must not race them.
- await this.deleteResources(name, namespace, deleteAllowed)
+ ): Promise<boolean> {
+ // A guarded no-op is incomplete even if authority returns before the
+ // outer call settles. Ownership-based skips remain legitimate outcomes.
+ let complete = true
+ await this.deleteResources(
+ name,
+ namespace,
+ deleteAllowed
+ ? async () => {
+ const allowed = await deleteAllowed()
+ if (!allowed) complete = false
+ return allowed
+ }
+ : undefined
+ )
+ return complete
}
// ─── CRD Status & Annotation Patching ──────────────────────────────
@@ -1850,7 +1897,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
condition: Omit<McpServerCondition, 'lastTransitionTime'>,
isCurrent: () => boolean = () => true
): Promise<boolean> {
- if (!isCurrent()) return false
+ if (!isCurrent()) return this.recordIncompletePublication(server)
const now = new Date().toISOString()
// Update the missing-secret gauge for SecretResolved conditions.
@@ -1875,7 +1922,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// if anything changed since our read; we then re-read and retry, so no
// writer's condition is lost.
for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
- if (!isCurrent()) return false
+ if (!isCurrent()) return this.recordIncompletePublication(server)
// Re-read on every optimistic-conflict retry. Merging against the
// latest resourceVersion preserves conditions written by peer
// controllers between our read and patch.
@@ -1901,15 +1948,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`
)
- return false
+ return this.recordIncompletePublication(server)
}
console.warn(
`[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,
error
)
- return false
+ return this.recordIncompletePublication(server)
}
- if (!isCurrent()) return false
+ if (!isCurrent()) return this.recordIncompletePublication(server)
const prior = existingConditions.find(c => c.type === condition.type)
const observedGeneration = condition.observedGeneration ?? server.generation
@@ -1980,7 +2027,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// The generated client prefers JSON Patch for this endpoint. Send an
// actual patch document so status writes keep working across client
// upgrades instead of relying on merge-patch object bodies.
- if (!isCurrent()) return false
+ if (!isCurrent()) return this.recordIncompletePublication(server)
await this.customApi.patchNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
@@ -1996,7 +2043,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
)
- return false
+ return this.recordIncompletePublication(server)
}
// 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
// won the race. Re-read and retry rather than lose our condition. Any
@@ -2012,9 +2059,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
`[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
error
)
- return false
+ return this.recordIncompletePublication(server)
}
}
+ return this.recordIncompletePublication(server)
+ }
+
+ private recordIncompletePublication(server: McpServerCRD): false {
+ // Keep the changed-boolean API: an unchanged condition is successful but
+ // returns false. Only a failed/retired write replaces the obligation, so
+ // an enclosing reconcile cannot clear that failure on normal return.
+ this.pendingReconciliations.set(server.name, { namespace: server.namespace })
return false
}
@@ -2039,12 +2094,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return current !== undefined && sameMcpServerDesiredRevision(server, current)
}
const key = server.name
+ const obligation = { namespace: server.namespace }
+ this.pendingReconciliations.set(key, obligation)
const prev = this.inFlight.get(key) ?? Promise.resolve()
const next = prev
.catch(() => undefined)
.then(async () => {
if (!isCurrent()) return
- await this.performReconcile(server, isCurrent)
+ const completed = await this.performReconcile(server, isCurrent)
+ if (completed && this.pendingReconciliations.get(key) === obligation) {
+ this.pendingReconciliations.delete(key)
+ }
})
this.inFlight.set(key, next)
try {
@@ -2056,12 +2116,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {
- if (!isCurrent()) return
+ private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<boolean> {
+ if (!isCurrent()) return false
this.beginStatusTracking(server)
// G7: Enforce managed field immutability — once set, cannot change
- if (!isCurrent()) return
+ if (!isCurrent()) return false
const currentManaged = server.spec.managed ?? true
const previousManaged = this.managedSnapshot.get(server.name)
if (previousManaged !== undefined && previousManaged !== currentManaged) {
@@ -2069,15 +2129,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +
`This is not allowed — delete and recreate the McpServer to change ownership.`
)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `managed field is immutable (was ${previousManaged}, attempted ${currentManaged})`,
})
- return
+ return isCurrent()
}
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.managedSnapshot.set(server.name, currentManaged)
// If disabled, respect ownership before any cleanup. managed:false means
@@ -2087,13 +2147,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.log(
`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
)
- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
+ if (
+ !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>
+ isCurrent()
+ ))
+ )
+ return false
} else {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
)
}
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })
await this.writeStatusCondition(
server,
@@ -2105,29 +2170,33 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- return
+ return isCurrent()
}
if (!currentManaged) {
- await this.reconcileWrcOwnedServer(server, isCurrent)
- return
+ return this.reconcileWrcOwnedServer(server, isCurrent)
}
// Validate secret before creating/updating
const secretResult = await this.validateSecret(server)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
if (!secretResult.ok) {
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
if (shouldFailClosedForSecretFailure(secretResult.reason)) {
- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
+ if (
+ !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>
+ isCurrent()
+ ))
+ )
+ return false
} else {
console.warn(
`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
)
}
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -2153,7 +2222,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- return
+ return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()
}
// Ensure resources exist and are up to date
@@ -2186,27 +2255,27 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, { deployed: false, ready: false, message })
- return
+ return isCurrent()
}
console.warn(
`[Reconciler] ${message} for "${server.name}" — audit mode, allowing (set CONTEXT_MAPPER_ENFORCE_IMAGE_ALLOWLIST=true to block)`
)
}
}
- if (!isCurrent()) return
+ if (!isCurrent()) return false
await this.ensureService(server, isCurrent)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
await this.ensureDeployment(server, secretResult.revision, isCurrent)
} catch (err) {
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `Resource sync failed: ${err instanceof Error ? err.message : String(err)}`,
})
- return
+ return false
}
// G2: Complete pre-deploy handshake — set network-ready annotation
@@ -2223,19 +2292,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
try {
await this.setNetworkReadyAnnotation(server, isCurrent)
} catch (err) {
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `Pre-deploy handshake failed: ${err instanceof Error ? err.message : String(err)}`,
})
- return
+ return false
}
}
// Check readiness after reconciliation
const rollout = await this.readDeploymentRollout(server.name, server.namespace)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
const ready = rollout.ready
this.setStatus(server.name, {
deployed: true,
@@ -2309,20 +2378,21 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
if (!ready && isCurrent()) {
this.pollReadiness(server, 5000, 24, isCurrent)
}
+ return isCurrent()
}
private async reconcileWrcOwnedServer(
server: McpServerCRD,
isCurrent: () => boolean
- ): Promise<void> {
- if (!isCurrent()) return
+ ): Promise<boolean> {
+ if (!isCurrent()) return false
console.log(
`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +
'skipping HCC runtime creation/deletion and updating discovery status.'
)
const secretResult = await this.validateSecret(server)
- if (!isCurrent()) return
+ if (!isCurrent()) return false
if (!secretResult.ok) {
this.setStatus(server.name, {
deployed: true,
@@ -2351,10 +2421,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- return
+ return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()
}
- if (!isCurrent()) return
+ if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: true,
ready: true,
@@ -2382,6 +2452,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
+ return isCurrent()
}
/**
@@ -2390,10 +2461,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
async reconcileDelete(name: string, namespace: string): Promise<void> {
console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)
const capturedAuthority = this.inventoryAuthority()
- await this.deleteRuntimeResources(name, namespace, () =>
+ const obligation = { namespace }
+ this.pendingReconciliations.set(name, obligation)
+ const complete = await this.deleteRuntimeResources(name, namespace, () =>
this.orphanDeleteAllowed(name, namespace, capturedAuthority)
)
- this.clearStatus(name)
+ if (complete && this.pendingReconciliations.get(name) === obligation) {
+ this.pendingReconciliations.delete(name)
+ this.clearStatus(name)
+ }
}
private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {
@@ -2487,10 +2563,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
// Delete orphaned deployments (managed by us but no longer desired)
- for (const deployment of existingDeployments) {
- const name = deployment.metadata?.name || ''
+ const cleanupTargets = new Map(
+ existingDeployments.map(deployment => [
+ deployment.metadata?.name || '',
+ deployment.metadata?.namespace || config.namespace,
+ ])
+ )
+ const presentNames = new Set(desiredServers.map(server => server.name))
+ for (const [name, pending] of this.pendingReconciliations) {
+ if (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)
+ }
+ for (const [name, namespace] of cleanupTargets) {
if (!desiredNames.has(name)) {
- const namespace = deployment.metadata?.namespace || config.namespace
+ if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })
await runEffect(name, async () => {
if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {
console.warn(
@@ -2499,9 +2584,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return
}
console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
- await this.deleteRuntimeResources(name, namespace, () =>
- this.orphanDeleteAllowed(name, namespace, capturedAuthority)
- )
+ await this.reconcileDelete(name, namespace)
})
}
}
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
index 6cc161d10..c794dbd6f 100755
--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -144,11 +144,20 @@ np604_before_observation() {
np604_kctl annotate networkpolicy "ctx-${NP604_CONTEXT}-${NP604_CONTROL}" -n "$MCP_NS" \
"e2e.clerum.io/observer=${RUN_ID}" --overwrite >/dev/null
wait_until 15 'NP604 policy observer positive witness' np604_observer_witness || die 'NP604 observer is not live'
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ HCC_PR_A_MCP_SKIP_BASE="$(hcc_pr_a_decision_count McpServer skip)"
+ HCC_PR_A_CONTEXT_SKIP_BASE="$(hcc_pr_a_decision_count Context skip)"
+ fi
NP604_CUT_BASE="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
}
np604_after_observation() {
np604_observer_alive || die 'NP604 observer stopped before completing observation'
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ [ "$(( $(hcc_pr_a_decision_count McpServer skip) - HCC_PR_A_MCP_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical MCP omissions'
+ [ "$(( $(hcc_pr_a_decision_count Context skip) - HCC_PR_A_CONTEXT_SKIP_BASE ))" -ge 3 ] || die 'PR A requires 3 completed identical Context omissions'
+ printf 'PR_A_IDENTICAL_RECOVERIES=PASS (at least 3 per watch)\n'
+ fi
local current_cuts
current_cuts="$(count_buffer '\[K8s\] Context watch ended; recovering authoritative inventory')"
[ "$((current_cuts - NP604_CUT_BASE))" -ge 3 ] || die 'NP604 needs three complete reconnect cycles in stable failed state'
diff --git a/scripts/e2e/_lib/hcc-watch-api-proxy.mjs b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs
new file mode 100644
index 000000000..162e37bd1
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-api-proxy.mjs
@@ -0,0 +1,192 @@
+// Development-only transport fixture. No fabricated upstream success responses.
+import fs from 'node:fs'
+import https from 'node:https'
+import { pathToFileURL } from 'node:url'
+
+export function validateCommand(command, allowedPaths) {
+ if (!command || typeof command.id !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.id))
+ throw new Error('invalid_control_id')
+ if (command.action === 'arm') {
+ if (
+ !allowedPaths.includes(command.path) ||
+ command.method !== 'GET' ||
+ !Number.isInteger(command.durationMs) ||
+ command.durationMs < 1000 ||
+ command.durationMs > 25000
+ ) {
+ throw new Error('invalid_pause_scope')
+ }
+ } else if (command.action === 'cut') {
+ if (!['McpServer', 'Context', 'both'].includes(command.kind))
+ throw new Error('invalid_cut_scope')
+ } else if (command.action === 'release') {
+ if (typeof command.pauseId !== 'string' || !/^[a-zA-Z0-9-]{1,80}$/.test(command.pauseId))
+ throw new Error('invalid_pause_id')
+ } else throw new Error('invalid_control_action')
+ return command
+}
+
+export function createProxy({
+ key,
+ cert,
+ upstreamCa,
+ upstreamHost = 'kubernetes.default.svc',
+ upstreamPort = 443,
+ allowedPaths,
+ controlDir,
+ periodMs,
+ minAgeMs,
+}) {
+ const streams = new Set()
+ let pause = null
+ let commandId = null
+ const writeRecord = (name, fields) => {
+ fs.writeFileSync(`${controlDir}/${name}.next`, JSON.stringify(fields), { mode: 0o600 })
+ fs.renameSync(`${controlDir}/${name}.next`, `${controlDir}/${name}.json`)
+ }
+ const acknowledge = fields => writeRecord('ack', fields)
+ const recordPause = (id, state) => writeRecord('pause', { id, state, at: Date.now() })
+ const finishPause = reason => {
+ if (!pause) return
+ const old = pause
+ pause = null
+ clearTimeout(old.timer)
+ recordPause(old.id, reason)
+ old.resume?.()
+ acknowledge({ id: old.id, state: reason })
+ }
+ const server = https.createServer({ key, cert }, (request, response) => {
+ const url = new URL(request.url, 'https://fixture.invalid')
+ if (!request.url.startsWith('/') || request.url.startsWith('//')) {
+ response.destroy()
+ return
+ }
+ const watch = url.searchParams.get('watch') === 'true'
+ const kind = url.pathname.endsWith('/mcpservers')
+ ? 'McpServer'
+ : url.pathname.endsWith('/contexts')
+ ? 'Context'
+ : 'other'
+ let upstream
+ const stream = {
+ request,
+ response,
+ kind,
+ watch,
+ born: Date.now(),
+ close: () => {
+ upstream?.destroy()
+ response.destroy()
+ request.destroy()
+ },
+ }
+ streams.add(stream)
+ response.once('close', () => {
+ streams.delete(stream)
+ upstream?.destroy()
+ })
+ const forward = () => {
+ if (response.destroyed) return
+ // Authentication is forwarded only in process memory to the verified API.
+ upstream = https.request(
+ {
+ hostname: upstreamHost,
+ port: upstreamPort,
+ method: request.method,
+ path: request.url,
+ ca: upstreamCa,
+ rejectUnauthorized: true,
+ headers: { ...request.headers, host: upstreamHost },
+ agent: false,
+ },
+ incoming => {
+ response.writeHead(incoming.statusCode, incoming.headers)
+ incoming.pipe(response)
+ }
+ )
+ upstream.on('error', () => stream.close())
+ request.pipe(upstream)
+ }
+ if (
+ pause &&
+ !pause.intercepted &&
+ request.method === 'GET' &&
+ !watch &&
+ url.pathname === pause.path
+ ) {
+ pause.intercepted = true
+ pause.resume = forward
+ recordPause(pause.id, 'intercepted')
+ acknowledge({ id: pause.id, state: 'intercepted', method: 'GET', path: pause.path })
+ } else forward()
+ })
+ const poll = setInterval(() => {
+ try {
+ if (!fs.existsSync(`${controlDir}/command.json`)) return
+ const command = validateCommand(
+ JSON.parse(fs.readFileSync(`${controlDir}/command.json`, 'utf8')),
+ allowedPaths
+ )
+ if (command.id === commandId) return
+ commandId = command.id
+ if (command.action === 'arm') {
+ if (pause) throw new Error('pause_already_active')
+ pause = {
+ ...command,
+ intercepted: false,
+ timer: setTimeout(() => finishPause('expired'), command.durationMs),
+ }
+ recordPause(command.id, 'armed')
+ acknowledge({ id: command.id, state: 'armed' })
+ } else if (command.action === 'release') {
+ if (!pause || pause.id !== command.pauseId || !pause.intercepted)
+ throw new Error('pause_not_held')
+ finishPause('released')
+ acknowledge({ id: command.id, state: 'released' })
+ } else {
+ let count = 0
+ for (const stream of streams) {
+ if (
+ stream.watch &&
+ (command.kind === 'both'
+ ? ['McpServer', 'Context'].includes(stream.kind)
+ : stream.kind === command.kind)
+ ) {
+ stream.close()
+ count++
+ }
+ }
+ acknowledge({ id: command.id, state: 'cut', count })
+ }
+ } catch {
+ acknowledge({ id: commandId, state: 'rejected' })
+ }
+ }, 100)
+ const churn = setInterval(() => {
+ if (fs.existsSync(`${controlDir}/paused`)) return
+ for (const stream of streams)
+ if (stream.watch && Date.now() - stream.born >= minAgeMs) stream.close()
+ }, periodMs)
+ const close = () => {
+ clearInterval(poll)
+ clearInterval(churn)
+ finishPause('shutdown')
+ for (const stream of streams) stream.close()
+ server.close()
+ }
+ return { server, close }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const proxy = createProxy({
+ key: fs.readFileSync('/fixture-tls/tls.key'),
+ cert: fs.readFileSync('/fixture-tls/tls.crt'),
+ upstreamCa: fs.readFileSync('/upstream-ca/ca.crt'),
+ allowedPaths: JSON.parse(process.env.PAUSE_PATHS),
+ controlDir: '/churn-ctl',
+ periodMs: Number(process.env.CHURN_PERIOD_MS),
+ minAgeMs: Number(process.env.CHURN_MIN_AGE_MS),
+ })
+ proxy.server.listen(8443, '0.0.0.0')
+ process.on('SIGTERM', proxy.close)
+}
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
index 7d2f21ad8..91b1bc588 100755
--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -245,6 +245,10 @@ delete_synthetic_fleet() {
# Non-destructive restore: undo the env overrides and hostAliases redirect.
restore_hcc_after_churn() {
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ hcc_pr_a_restore
+ return $?
+ fi
local failed=0
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || failed=1
diff --git a/scripts/e2e/_lib/hcc-watch-config.mjs b/scripts/e2e/_lib/hcc-watch-config.mjs
new file mode 100644
index 000000000..468a11d1b
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-config.mjs
@@ -0,0 +1,96 @@
+// Persist only the public fields this development fixture changes.
+import fs from 'node:fs'
+import { pathToFileURL } from 'node:url'
+
+export const changedEnvironment = [
+ 'CONTEXT_MAPPER_K8S_API_CIDRS',
+ 'CONTEXT_MAPPER_NETPOL_RESYNC_SEC',
+ 'KUBECONFIG',
+]
+const own = (object, key) => Object.prototype.hasOwnProperty.call(object, key)
+export function snapshot(deployment) {
+ const pod = deployment.spec.template.spec
+ const container = pod.containers.find(item => item.name === 'host-context-controller')
+ if (!deployment.metadata.uid || !container) throw new Error('deployment_identity_missing')
+ if (
+ (pod.volumes ?? []).some(item => item.name === 'hcc-pr-a-config') ||
+ (container.volumeMounts ?? []).some(item => item.name === 'hcc-pr-a-config')
+ )
+ throw new Error('fixture_mount_collision')
+ return {
+ uid: deployment.metadata.uid,
+ envPresent: own(container, 'env'),
+ env: (container.env ?? []).filter(item => changedEnvironment.includes(item.name)),
+ volumesPresent: own(pod, 'volumes'),
+ volumes: pod.volumes ?? [],
+ mountsPresent: own(container, 'volumeMounts'),
+ mounts: container.volumeMounts ?? [],
+ }
+}
+export function restorePatch(original, deployment) {
+ if (deployment.metadata.uid !== original.uid) throw new Error('deployment_uid_changed')
+ const pod = deployment.spec.template.spec
+ const index = pod.containers.findIndex(item => item.name === 'host-context-controller')
+ if (index < 0) throw new Error('deployment_container_missing')
+ const container = pod.containers[index]
+ for (const [current, previous] of [
+ [pod.volumes ?? [], original.volumes],
+ [container.volumeMounts ?? [], original.mounts],
+ ]) {
+ if (
+ JSON.stringify(current.filter(item => item.name !== 'hcc-pr-a-config')) !==
+ JSON.stringify(previous)
+ ) {
+ throw new Error('unrelated_configuration_changed')
+ }
+ }
+ if (
+ typeof deployment.metadata.resourceVersion !== 'string' ||
+ !deployment.metadata.resourceVersion
+ ) {
+ throw new Error('deployment_resource_version_missing')
+ }
+ const patches = [
+ { op: 'test', path: '/metadata/uid', value: original.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: deployment.metadata.resourceVersion },
+ ]
+ const set = (path, present, existed, value) => {
+ if (present) patches.push({ op: 'add', path, value })
+ else if (existed) patches.push({ op: 'remove', path })
+ }
+ const env = [
+ ...(container.env ?? []).filter(item => !changedEnvironment.includes(item.name)),
+ ...original.env,
+ ]
+ set(
+ `/spec/template/spec/containers/${index}/env`,
+ original.envPresent || env.length > 0,
+ own(container, 'env'),
+ env
+ )
+ set(
+ `/spec/template/spec/containers/${index}/volumeMounts`,
+ original.mountsPresent,
+ own(container, 'volumeMounts'),
+ original.mounts
+ )
+ set('/spec/template/spec/volumes', original.volumesPresent, own(pod, 'volumes'), original.volumes)
+ return patches
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [action, originalPath] = process.argv.slice(2)
+ const deployment = JSON.parse(fs.readFileSync(0, 'utf8'))
+ if (action === 'snapshot') process.stdout.write(JSON.stringify(snapshot(deployment)))
+ else if (action === 'restore')
+ process.stdout.write(
+ JSON.stringify(restorePatch(JSON.parse(fs.readFileSync(originalPath, 'utf8')), deployment))
+ )
+ else if (action === 'verify') {
+ const original = JSON.parse(fs.readFileSync(originalPath, 'utf8'))
+ const current = snapshot(deployment)
+ // Environment ordering has no semantic meaning; each exact entry does.
+ for (const value of [original, current]) value.env.sort((a, b) => a.name.localeCompare(b.name))
+ if (JSON.stringify(current) !== JSON.stringify(original))
+ throw new Error('configuration_not_restored')
+ } else throw new Error('unknown_configuration_action')
+}
diff --git a/scripts/e2e/_lib/hcc-watch-gate-probe.mjs b/scripts/e2e/_lib/hcc-watch-gate-probe.mjs
new file mode 100644
index 000000000..d9cdf20f6
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-gate-probe.mjs
@@ -0,0 +1,31 @@
+// Appended to np08-runtime-access.mjs and executed only inside the owned Host.
+const expectedStatus = Number(process.env.HCC_PR_A_EXPECT_STATUS)
+const observed = await requestWithRuntimeAccess({
+ mountedAccessValue: process.env.MCP_HOST_RUNTIME_ACCESS_TOKEN,
+ timeoutMs: 5000,
+ request: async value => {
+ const response = await fetch(
+ `${contextMapperBaseUrl(process.env)}/api/v2/hosts/self/mcpservers`,
+ {
+ headers: { authorization: `Bearer ${value}` },
+ redirect: 'error',
+ signal: AbortSignal.timeout(5000),
+ }
+ )
+ const body = await response.json()
+ if (
+ response.status === 200 &&
+ (!Array.isArray(body.servers) ||
+ !body.servers.some(server => server.name === process.env.HCC_PR_A_CONTROL) ||
+ body.servers.some(server => server.name === process.env.HCC_PR_A_AFFECTED) !==
+ (process.env.HCC_PR_A_AFFECTED_VISIBLE !== 'false'))
+ ) {
+ throw new Error('fixture_inventory_missing')
+ }
+ if (response.status === 503 && body.error !== 'authorization_unavailable')
+ throw new Error('unexpected_503_class')
+ return { status: response.status }
+ },
+})
+if (observed.result.status !== expectedStatus) throw new Error('protected_gate_status_mismatch')
+console.log(`PR_A_PROTECTED_API_STATUS=${expectedStatus} AT=${Date.now()}`)
diff --git a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
index a94799e7a..33383fdf2 100755
--- a/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
+++ b/scripts/e2e/_lib/hcc-watch-lifecycle-cleanup.sh
@@ -75,6 +75,11 @@ hcc_lifecycle_fixture_absent() {
remaining="$(kctl get pod -n "$HCC_NS" -l "app=${PROXY_NAME}" -o name)" || return 1
[ -z "$remaining" ] || return 1
fi
+ if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then
+ remaining="$(kctl get secret,configmap -n "$HCC_NS" \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" -o name)" || return 1
+ [ -z "$remaining" ] || return 1
+ fi
if [ "$PROBE_CREATED" = 1 ]; then
remaining="$(kctl get pod "$PROBE_NAME" -n "$HCC_NS" --ignore-not-found -o name)" || return 1
[ -z "$remaining" ] || return 1
@@ -103,10 +108,15 @@ cleanup_hcc_lifecycle() (
[ -z "${HCC_LOG_STREAM_PID:-}" ] || kill "$HCC_LOG_STREAM_PID" 2>/dev/null
[ -z "${NP604_WATCH_PID:-}" ] || kill "$NP604_WATCH_PID" 2>/dev/null
NP604_WATCH_PID=''
- if [ "$HCC_PATCHED" = 1 ]; then
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ] &&
+ { [ "$HCC_PATCHED" = 1 ] || [ "$HCC_SCALED_DOWN" = 1 ]; } && ! hcc_pr_a_identity_current; then
+ restore_ok=0
+ fi
+ if [ "$HCC_PATCHED" = 1 ] && { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; then
restore_hcc_after_churn || restore_ok=0
fi
- if [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; then
+ if { [ "$HCC_SCALED_DOWN" = 1 ] || [ "$HCC_PATCHED" = 1 ]; } &&
+ { [ "${E2E_HCC_PR_A:-0}" != 1 ] || [ "$restore_ok" = 1 ]; }; then
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
--replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \
@@ -133,6 +143,11 @@ cleanup_hcc_lifecycle() (
kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" "$PROBE_EGRESS_NP" \
-n "$HCC_NS" --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
fi
+ if [ "${HCC_PR_A_TLS_CREATED:-0}" = 1 ]; then
+ kctl delete secret,configmap -n "$HCC_NS" \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_failed=1
+ fi
wait_until 100 'lifecycle fixture resources removed' hcc_lifecycle_fixture_absent || cleanup_failed=1
fi
@@ -140,6 +155,9 @@ cleanup_hcc_lifecycle() (
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
print_results || cleanup_failed=1
if [ "$status" -eq 0 ] && [ "$cleanup_failed" != 0 ]; then status=1; fi
+ if [ "$cleanup_failed" = 0 ] && [ -n "${HCC_PR_A_CONFIG_SNAPSHOT:-}" ]; then
+ rm -f "$HCC_PR_A_CONFIG_SNAPSHOT"
+ fi
rm -f "$HCC_LOG_BUFFER" "$READY_SERIES"
return "$status"
)
diff --git a/scripts/e2e/_lib/hcc-watch-pr-a.sh b/scripts/e2e/_lib/hcc-watch-pr-a.sh
new file mode 100644
index 000000000..bcb0736a1
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-pr-a.sh
@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+# PR A extends the existing branch-owned lifecycle fault controller.
+HCC_PR_A_RESYNC_SEC=60
+HCC_PR_A_TLS_CREATED=0
+HCC_PR_A_CONFIG_SNAPSHOT=''
+HCC_PR_A_LIB="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+hcc_pr_a_proxy_pod() {
+ kctl get pods -n "$HCC_NS" -l "app=${PROXY_NAME}" -o json |
+ jq -er '[.items[]|select(.metadata.deletionTimestamp==null)|select(any(.status.conditions[]?;.type=="Ready" and .status=="True"))]|if length==1 then .[0].metadata.name else error("ambiguous proxy") end'
+}
+
+hcc_pr_a_command() {
+ local action=$1 value=${2:-} pod command
+ pod="$(hcc_pr_a_proxy_pod)" || return 1
+ HCC_PR_A_COMMAND_ID="${RUN_ID}-$(date +%s)-${RANDOM}"
+ command="$(jq -cn --arg id "$HCC_PR_A_COMMAND_ID" --arg action "$action" --arg value "$value" \
+ '{id:$id,action:$action} + (if $action=="arm" then {path:$value,method:"GET",durationMs:25000} elif $action=="cut" then {kind:$value} elif $action=="release" then {pauseId:$value} else {} end)')"
+ printf '%s' "$command" | kctl exec -i "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \
+ 'const fs=require("fs");const value=fs.readFileSync(0);fs.writeFileSync("/churn-ctl/next.json",value,{mode:384});fs.renameSync("/churn-ctl/next.json","/churn-ctl/command.json")'
+}
+
+hcc_pr_a_ack() {
+ local state=$1 pod
+ pod="$(hcc_pr_a_proxy_pod)" || return 1
+ kctl exec "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \
+ 'const fs=require("fs");try{const a=JSON.parse(fs.readFileSync("/churn-ctl/ack.json"));if(a.id!==process.argv[1]||a.state!==process.argv[2]||(a.state==="cut"&&a.count<1))process.exit(1)}catch{process.exit(1)}' \
+ "$HCC_PR_A_COMMAND_ID" "$state" >/dev/null 2>&1
+}
+
+hcc_pr_a_cut() {
+ hcc_pr_a_command cut "$1" || return 1
+ wait_until 5 'acknowledged live watch cut' hcc_pr_a_ack cut
+}
+
+hcc_pr_a_pause_held() {
+ local pause_id=$1 pod
+ pod="$(hcc_pr_a_proxy_pod)" || return 1
+ kctl exec "pod/$pod" -n "$HCC_NS" -c proxy -- node -e \
+ 'const fs=require("fs");try{const p=JSON.parse(fs.readFileSync("/churn-ctl/pause.json"));if(p.id!==process.argv[1]||p.state!=="intercepted")process.exit(1)}catch{process.exit(1)}' \
+ "$pause_id" >/dev/null 2>&1
+}
+
+hcc_pr_a_logs() {
+ jq -Rrc 'fromjson? | select(.event=="networkpolicy-recovery-decision" or .event=="networkpolicy-request" or .event=="networkpolicy-pass-start" or .event=="networkpolicy-pass-result")' "$HCC_LOG_BUFFER"
+}
+
+hcc_pr_a_decision_after() {
+ local baseline=$1 kind=$2 decision=$3
+ require_hcc_recovery_log_stream || return 1
+ [ "$(hcc_pr_a_decision_count "$kind" "$decision")" -gt "$baseline" ]
+}
+
+hcc_pr_a_decision_count() {
+ hcc_pr_a_logs | jq -s --arg kind "$1" --arg decision "$2" \
+ '[.[]|select(.event=="networkpolicy-recovery-decision" and .kind==$kind and .decision==$decision)]|length'
+}
+
+hcc_pr_a_periodic_count() {
+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified" and (.causes|index("periodic-resync"))!=null)]|length'
+}
+hcc_pr_a_periodic_after() { [ "$(hcc_pr_a_periodic_count)" -gt "$1" ]; }
+
+hcc_pr_a_periodic_started_count() {
+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-start" and (.causes|index("periodic-resync"))!=null)]|length'
+}
+
+hcc_pr_a_runtime_completed() {
+ local captured pod
+ captured="$(hcc_pr_a_logs | jq -sr '[.[]|select(.event=="networkpolicy-pass-start" and (.causes|index("periodic-resync"))!=null)][-1].ts')"
+ [ "$captured" != null ] || return 1
+ pod="$(running_hcc_pod)" || return 1
+ kctl exec -i "pod/$pod" -n "$HCC_NS" -c host-context-controller -- node - \
+ "$captured" < "$HCC_PR_A_LIB/hcc-watch-runtime-metric.cjs" >/dev/null 2>&1
+}
+
+hcc_pr_a_certified_count() {
+ hcc_pr_a_logs | jq -s '[.[]|select(.event=="networkpolicy-pass-result" and .result=="certified")]|length'
+}
+hcc_pr_a_certified_after() { [ "$(hcc_pr_a_certified_count)" -gt "$1" ]; }
+
+hcc_pr_a_run() {
+ local periodic baseline context_spec certified affected_policy control_policy context_servers pause_id
+ require_hcc_recovery_log_stream || die 'PR A requires live structured logs'
+ hcc_pr_a_gate 200 || die 'PR A protected inventory unavailable before fault'
+ control_policy="$(np604_snapshot "$NP604_CONTROL")"
+ periodic="$(hcc_pr_a_periodic_count)"
+ wait_until 75 'PR A real periodic completion' hcc_pr_a_periodic_after "$periodic" || die 'PR A periodic path inactive'
+ wait_until 15 'PR A periodic runtime effects complete' hcc_pr_a_runtime_completed || die 'PR A runtime did not complete before drift'
+ periodic="$(hcc_pr_a_periodic_count)"
+ context_spec="$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')"
+ hcc_pr_a_remove_service
+ baseline="$(hcc_pr_a_decision_count McpServer skip)"
+ hcc_pr_a_cut McpServer || die 'PR A did not cut live MCP watch'
+ wait_until 12 'PR A identical recovery omitted' hcc_pr_a_decision_after "$baseline" McpServer skip || die 'PR A identical recovery not omitted'
+ hcc_pr_a_service_absent || die 'PR A drift repaired before periodic witness'
+ [ "$context_spec" = "$(np604_kctl get mcpserver "$NP604_SERVER" -n "$MCP_NS" -o json | jq -Sc '{uid:.metadata.uid,generation:.metadata.generation,spec}')" ] || die 'PR A drift changed CRD'
+ wait_until 75 'PR A drift repaired by periodic pass' hcc_pr_a_periodic_after "$periodic" || die 'PR A no periodic repair pass'
+ wait_until 15 'PR A actual Service recreation' hcc_pr_a_service_restored || die 'PR A periodic pass omitted runtime effect'
+ wait_until 10 'PR A repaired runtime complete' hcc_pr_a_runtime_completed || die 'PR A repaired runtime not complete'
+ np604_invoke "$NP604_SERVER" || die 'PR A drift recovery business result failed'
+ printf 'PR_A_DRIFT_REPAIR=PASS\n'
+
+ periodic="$(hcc_pr_a_periodic_started_count)"
+ certified="$(hcc_pr_a_certified_count)"
+ hcc_pr_a_remove_service
+ # ensureService precedes ensureDeployment in the real reconciler. Holding
+ # the Service existence GET prevents the missing effect from being repaired
+ # before the watch generation retires its mutationAllowed predicate.
+ hcc_pr_a_command arm "/api/v1/namespaces/${MCP_NS}/services/${NP604_SERVER}"
+ pause_id=$HCC_PR_A_COMMAND_ID
+ wait_until 5 'runtime pause armed' hcc_pr_a_ack armed || die 'PR A runtime pause not armed'
+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \
+ -p '{"spec":{"description":"PR A runtime interruption"}}' >/dev/null
+ wait_until 10 'runtime GET intercepted' hcc_pr_a_ack intercepted || die 'PR A runtime GET not intercepted'
+ wait_until 5 'NP finishes while runtime held' hcc_pr_a_certified_after "$certified" || die 'PR A NP did not finish before runtime retirement'
+ hcc_pr_a_pause_held "$pause_id" || die 'PR A runtime pause expired before retirement'
+ hcc_pr_a_service_absent || die 'PR A missing interrupted effect witness'
+ baseline="$(hcc_pr_a_decision_count McpServer request)"
+ hcc_pr_a_cut McpServer || die 'PR A runtime watch retirement missing'
+ wait_until 5 'retired runtime recovery admitted' hcc_pr_a_decision_after "$baseline" McpServer request || die 'PR A dropped runtime recovery'
+ hcc_pr_a_pause_held "$pause_id" || die 'PR A runtime pause expired before recovery completed'
+ hcc_pr_a_command release "$pause_id"
+ wait_until 5 'runtime pause released' hcc_pr_a_ack released || die 'PR A release not acknowledged'
+ wait_until 20 'retired runtime effect repaired' hcc_pr_a_service_restored || die 'PR A retired runtime not repaired'
+ [ "$(hcc_pr_a_periodic_started_count)" = "$periodic" ] || die 'PR A interrupted repair confounded by periodic tick'
+ np604_invoke "$NP604_SERVER" || die 'PR A interrupted runtime business result failed'
+ printf 'PR_A_INTERRUPTED_RUNTIME=PASS\n'
+
+ affected_policy="$(np604_snapshot "$NP604_SERVER" | jq -er '[.[]|select(.type=="external-egress")]|if length==1 then .[0].name else error("egress identity ambiguous") end')"
+ hcc_pr_a_command arm "/apis/networking.k8s.io/v1/namespaces/${MCP_NS}/networkpolicies/${affected_policy}"
+ pause_id=$HCC_PR_A_COMMAND_ID
+ wait_until 5 'egress pause armed' hcc_pr_a_ack armed || die 'PR A egress pause not armed'
+ np604_kctl patch mcpserver "$NP604_SERVER" -n "$MCP_NS" --type=merge \
+ -p '{"spec":{"description":"PR A queue interception"}}' >/dev/null
+ wait_until 10 'egress GET intercepted' hcc_pr_a_ack intercepted || die 'PR A egress operation not intercepted'
+ context_servers="$(np604_kctl get context "$NP604_CONTEXT" -n "$MCP_NS" -o json | jq -c '.spec.mcpServers')"
+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \
+ -p "$(jq -cn --argjson names "$context_servers" --arg server "$NP604_SERVER" '{spec:{mcpServers:($names|map(select(.!=$server)))}}')" >/dev/null
+ wait_until 5 'protected API closes while queue held' hcc_pr_a_gate 503 || die 'PR A protected gate never closed'
+ printf 'PR_A_READY_DURING_API_503=%s\n' "$(ready_status "$(running_hcc_pod)")"
+ hcc_pr_a_pause_held "$pause_id" || die 'PR A queue pause expired before the API witness'
+ hcc_pr_a_command release "$pause_id"
+ wait_until 5 'egress pause released' hcc_pr_a_ack released || die 'PR A egress release not acknowledged'
+ wait_until 25 'protected API recertifies without revoked server' hcc_pr_a_gate 200 false || die 'PR A protected gate did not reopen with revoked inventory'
+ np604_snapshot "$NP604_SERVER" | jq -e 'all(.[]; .type=="external-egress")' >/dev/null || die 'PR A stale Context allows remained'
+ np604_kctl patch context "$NP604_CONTEXT" -n "$MCP_NS" --type=merge \
+ -p "$(jq -cn --argjson names "$context_servers" '{spec:{mcpServers:$names}}')" >/dev/null
+ wait_until 20 'protected API restores desired binding' hcc_pr_a_gate 200 || die 'PR A desired binding not restored'
+ np604_invoke "$NP604_SERVER" || die 'PR A affected business result failed after API recovery'
+ np604_invoke "$NP604_CONTROL" || die 'PR A control business result failed after API recovery'
+ [ "$(np604_snapshot "$NP604_CONTROL")" = "$control_policy" ] || die 'PR A control policies changed'
+ printf 'PR_A_PROTECTED_API_RECOVERY=PASS\n'
+ hcc_pr_a_logs > "$NP604_EVIDENCE/pr-a-observations.jsonl"
+}
+
+hcc_pr_a_service_absent() {
+ local resource
+ resource="$(np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" --ignore-not-found -o name)" || return 1
+ [ -z "$resource" ]
+}
+
+hcc_pr_a_service_restored() {
+ np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" -o json |
+ jq -e --arg uid "$HCC_PR_A_SERVICE_UID" --arg name "$NP604_SERVER" \
+ '.metadata.uid!=$uid and .metadata.labels["clerum.io/mcpserver"]==$name and any(.spec.ports[]; .port==3000)' >/dev/null
+}
+
+hcc_pr_a_remove_service() {
+ HCC_PR_A_SERVICE_UID="$(np604_kctl get service "$NP604_SERVER" -n "$MCP_NS" -o json |
+ jq -er --arg name "$NP604_SERVER" 'select(.metadata.labels["clerum.io/mcpserver"]==$name)|.metadata.uid')" || die 'PR A Service ownership missing'
+ np604_kctl delete service "$NP604_SERVER" -n "$MCP_NS" --wait=true --timeout=10s >/dev/null
+ hcc_pr_a_service_absent || die 'PR A Service never became absent'
+}
+
+hcc_pr_a_gate() {
+ local expected=$1 visible=${2:-true} pod
+ pod="$(np604_kctl get pods -n "$HOST_NS" -l "clerum.io/context=${NP604_CONTEXT}" -o json |
+ jq -er '[.items[]|select(.metadata.deletionTimestamp==null)|select(any(.status.conditions[]?;.type=="Ready" and .status=="True"))][0].metadata.name')" || return 1
+ { cat "$HCC_PR_A_LIB/np08-runtime-access.mjs"; cat "$HCC_PR_A_LIB/hcc-watch-gate-probe.mjs"; } |
+ np604_kctl exec -i "$pod" -n "$HOST_NS" -- env "HCC_PR_A_EXPECT_STATUS=$expected" \
+ "HCC_PR_A_AFFECTED=$NP604_SERVER" "HCC_PR_A_CONTROL=$NP604_CONTROL" \
+ "HCC_PR_A_AFFECTED_VISIBLE=$visible" node --input-type=module -
+}
+
+hcc_pr_a_preflight() {
+ local command_name
+ for command_name in node openssl; do
+ command -v "$command_name" >/dev/null || die "PR A requires installed ${command_name}"
+ done
+ [ "$(node -p 'process.versions.node.split(".")[0]')" = 24 ] || die 'PR A requires Node 24'
+ openssl req -help 2>&1 | grep -q -- '-addext' || die 'PR A requires openssl req -addext'
+ HCC_PR_A_CONFIG_SNAPSHOT="$(mktemp "${TMPDIR:-/tmp}/hcc-pr-a-public-config.XXXXXX")"
+ chmod 600 "$HCC_PR_A_CONFIG_SNAPSHOT"
+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |
+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" snapshot > "$HCC_PR_A_CONFIG_SNAPSHOT"
+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || die 'PR A public configuration capture failed'
+ printf 'PR_A_TEST_RESYNC_SECONDS=%s (development test only)\n' "$HCC_PR_A_RESYNC_SEC"
+}
+
+hcc_pr_a_identity_current() {
+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || return 1
+ local current
+ current="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.uid}')" || return 1
+ [ "$current" = "$(jq -r '.uid' "$HCC_PR_A_CONFIG_SNAPSHOT")" ]
+}
+
+hcc_pr_a_enable_proxy() {
+ local patch paths server
+ server="$(truncate_rfc1123 "np604-${RUN_ID}-affected")"
+ paths="$(jq -cn --arg ns "$MCP_NS" --arg server "$server" '[
+ ("/apis/apps/v1/namespaces/"+$ns+"/deployments/"+$server),
+ ("/api/v1/namespaces/"+$ns+"/services/"+$server),
+ ("/apis/networking.k8s.io/v1/namespaces/"+$ns+"/networkpolicies/ext-egress-"+$server+"-1-2-3-4-32-443")]')"
+ # Consumed by the lifecycle cleanup owner.
+ # shellcheck disable=SC2034
+ HCC_PR_A_TLS_CREATED=1
+ # Generated private material stays in this anonymous pipe, never an artifact.
+ node "$HCC_PR_A_LIB/hcc-watch-tls-manifest.mjs" "$PROXY_NAME" "$HCC_NS" "$RUN_ID" \
+ "$HCC_PR_A_LIB/hcc-watch-api-proxy.mjs" | kctl create -f - >/dev/null 2>&1 || die 'PR A TLS fixture creation failed'
+ patch="$(jq -cn --arg name "$PROXY_NAME" --arg paths "$paths" '{spec:{template:{spec:{
+ securityContext:{fsGroup:1000}, containers:[{name:"proxy",command:["node","/fixture/proxy.mjs"],args:[],
+ env:[{name:"PAUSE_PATHS",value:$paths}],volumeMounts:[
+ {name:"fixture-code",mountPath:"/fixture",readOnly:true},
+ {name:"fixture-tls",mountPath:"/fixture-tls",readOnly:true},
+ {name:"upstream-ca",mountPath:"/upstream-ca",readOnly:true}]}],
+ volumes:[{name:"fixture-code",configMap:{name:$name}},
+ {name:"fixture-tls","secret":{secretName:$name,defaultMode:288}},
+ {name:"upstream-ca",configMap:{name:"kube-root-ca.crt"}}]}}}}')"
+ kctl patch deployment "$PROXY_NAME" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null
+ kctl rollout status deployment "$PROXY_NAME" -n "$HCC_NS" --timeout=60s >/dev/null || die 'PR A TLS proxy not Ready'
+}
+
+hcc_pr_a_redirect_patch() {
+ jq -cn --arg uid "$(jq -r '.uid' "$HCC_PR_A_CONFIG_SNAPSHOT")" \
+ --arg name "$PROXY_NAME" --arg cidr "$K8S_API_CIDR" --arg seconds "$HCC_PR_A_RESYNC_SEC" '{metadata:{uid:$uid},spec:{template:{spec:{
+ containers:[{name:"host-context-controller",env:[
+ {name:"KUBECONFIG",valueFrom:null,value:"/hcc-pr-a/config.json"},
+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",valueFrom:null,value:$cidr},
+ {name:"CONTEXT_MAPPER_NETPOL_RESYNC_SEC",valueFrom:null,value:$seconds}],
+ volumeMounts:[{name:"hcc-pr-a-config",mountPath:"/hcc-pr-a",readOnly:true}]}],
+ volumes:[{name:"hcc-pr-a-config",configMap:{name:$name}}]}}}}'
+}
+
+hcc_pr_a_restore() {
+ local patch
+ [ -s "$HCC_PR_A_CONFIG_SNAPSHOT" ] || return 1
+ patch="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |
+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" restore "$HCC_PR_A_CONFIG_SNAPSHOT")" || return 1
+ kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=json -p "$patch" >/dev/null || return 1
+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |
+ node "$HCC_PR_A_LIB/hcc-watch-config.mjs" verify "$HCC_PR_A_CONFIG_SNAPSHOT"
+}
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
index ecb97cbdc..7d813c4f3 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
@@ -167,6 +167,7 @@ EOF
verify_hcc_proxy_network_policy() {
local proxy_dns="${PROXY_NAME}.${HCC_NS}.svc"
local proxy_ip positive_probe negative_probe probe_status
+ local positive_servername='kubernetes.default.svc' proxy_public_ca=''
proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')" ||
die "could not resolve the proxy Service ClusterIP"
@@ -175,12 +176,20 @@ verify_hcc_proxy_network_policy() {
positive_probe="$(cat <<'NODE'
const fs=require('fs'),https=require('https');
const root='/var/run/secrets/kubernetes.io/serviceaccount/';
-const request=https.request({host:process.argv[1],port:443,path:'/version',servername:'kubernetes.default.svc',ca:fs.readFileSync(root+'ca.crt'),headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});
+const request=https.request({host:process.argv[1],port:443,path:'/version',servername:process.argv[2]||'kubernetes.default.svc',ca:process.argv[3]?Buffer.from(process.argv[3],'base64'):fs.readFileSync(root+'ca.crt'),rejectUnauthorized:true,headers:{authorization:'Bearer '+fs.readFileSync(root+'token','utf8')}},response=>{response.resume();response.on('end',()=>process.exit(response.statusCode===200?0:2))});
request.setTimeout(5000,()=>request.destroy(new Error('timeout')));request.on('error',()=>process.exit(3));request.end();
NODE
)"
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ # This is the public certificate HCC's temporary kubeconfig will trust.
+ # Authentication still comes from the original mounted service account.
+ proxy_public_ca="$(kctl get configmap "$PROXY_NAME" -n "$HCC_NS" -o json |
+ jq -er '.data["config.json"]|fromjson|.clusters[0].cluster["certificate-authority-data"]')" ||
+ die 'PR A proxy public trust configuration missing'
+ positive_servername=$proxy_dns
+ fi
kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
- node -e "$positive_probe" "$proxy_dns" >/dev/null ||
+ node -e "$positive_probe" "$proxy_dns" "$positive_servername" "$proxy_public_ca" >/dev/null ||
die "HCC cannot reach the Kubernetes API through the isolated proxy"
PROBE_CREATED=1
diff --git a/scripts/e2e/_lib/hcc-watch-runtime-metric.cjs b/scripts/e2e/_lib/hcc-watch-runtime-metric.cjs
new file mode 100644
index 000000000..cd61ddb7d
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-runtime-metric.cjs
@@ -0,0 +1,29 @@
+// Executed on stdin inside the HCC image's documented working directory.
+// Reuse the deployed config parser; never infer a different listener port.
+const { config } = require('./dist/config')
+const http = require('node:http')
+const threshold = Date.parse(process.argv[2]) / 1000
+if (!Number.isFinite(threshold)) process.exit(1)
+http
+ .get(
+ {
+ hostname: '127.0.0.1',
+ port: config.port,
+ path: '/metrics',
+ signal: AbortSignal.timeout(5000),
+ },
+ response => {
+ let body = ''
+ response.on('data', chunk => {
+ body += chunk
+ })
+ response.on('error', () => process.exit(1))
+ response.on('end', () => {
+ const sample = body.match(
+ /^clerum_hcc_initial_convergence_last_success_timestamp_seconds\{lane="McpServer"\} ([0-9.e+]+)$/m
+ )
+ if (response.statusCode !== 200 || !sample || Number(sample[1]) < threshold) process.exit(1)
+ })
+ }
+ )
+ .on('error', () => process.exit(1))
diff --git a/scripts/e2e/_lib/hcc-watch-tls-manifest.mjs b/scripts/e2e/_lib/hcc-watch-tls-manifest.mjs
new file mode 100644
index 000000000..8af6cda67
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-tls-manifest.mjs
@@ -0,0 +1,94 @@
+// Public configuration and an ephemeral test key go directly to kubectl stdin.
+// Never redirect this program's output to an artifact or log.
+import { spawnSync } from 'node:child_process'
+import { generateKeyPairSync } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+
+const [name, namespace, run, proxySource] = process.argv.slice(2)
+for (const value of [name, namespace, run])
+ if (!/^[a-z0-9][a-z0-9.-]{0,62}$/.test(value ?? '')) throw new Error('invalid_fixture_identity')
+const { privateKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+})
+const signed = spawnSync(
+ 'openssl',
+ [
+ 'req',
+ '-new',
+ '-x509',
+ '-key',
+ '/dev/stdin',
+ '-days',
+ '1',
+ '-subj',
+ `/CN=${name}.${namespace}.svc`,
+ '-addext',
+ `subjectAltName=DNS:${name}.${namespace}.svc`,
+ '-addext',
+ 'basicConstraints=critical,CA:TRUE',
+ '-addext',
+ 'keyUsage=critical,digitalSignature,keyEncipherment,keyCertSign',
+ ],
+ { input: privateKey, encoding: 'utf8', timeout: 15000, maxBuffer: 1048576 }
+)
+if (signed.status !== 0 || !signed.stdout.includes('BEGIN CERTIFICATE'))
+ throw new Error('fixture_certificate_generation_failed')
+const cert = signed.stdout
+const labels = { 'e2e.clerum.io/suite': 'hcc-watch-churn', 'e2e.clerum.io/run': run }
+const metadata = { name, namespace, labels }
+const config = {
+ apiVersion: 'v1',
+ kind: 'Config',
+ clusters: [
+ {
+ name: 'fixture',
+ cluster: {
+ server: `https://${name}.${namespace}.svc:443`,
+ 'certificate-authority-data': Buffer.from(cert).toString('base64'),
+ },
+ },
+ ],
+ users: [
+ {
+ name: 'inClusterUser',
+ user: {
+ 'auth-provider': {
+ name: 'tokenFile',
+ config: {
+ tokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+ },
+ },
+ },
+ },
+ ],
+ contexts: [
+ { name: 'fixture', context: { cluster: 'fixture', user: 'inClusterUser', namespace } },
+ ],
+ 'current-context': 'fixture',
+}
+process.stdout.write(
+ JSON.stringify({
+ apiVersion: 'v1',
+ kind: 'List',
+ items: [
+ {
+ apiVersion: 'v1',
+ kind: 'Secret',
+ metadata,
+ type: 'kubernetes.io/tls',
+ stringData: { 'tls.key': privateKey, 'tls.crt': cert },
+ },
+ {
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ metadata,
+ data: {
+ 'config.json': JSON.stringify(config),
+ 'proxy.mjs': readFileSync(proxySource, 'utf8'),
+ },
+ },
+ ],
+ })
+)
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index 0dbc5f6fa..285ec6e0f 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -50,6 +50,24 @@ case "${E2E_HCC_POLICY_LIFECYCLE:-0}" in
*) echo "E2E_HCC_POLICY_LIFECYCLE must be 0 or 1" >&2; exit 2 ;;
esac
+case "${E2E_HCC_PR_A:-0}" in
+ 0) ;;
+ 1)
+ [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ] || { echo 'PR A requires lifecycle mode' >&2; exit 2; }
+ source "${SCRIPT_DIR}/_lib/hcc-watch-pr-a.sh"
+ # Work600 + cleanup270 leaves30 seconds within the normal900 runner budget.
+ HCC_PR_A_WORK_DEADLINE=600
+ kctl() {
+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))
+ [ "$remaining" -gt 0 ] || return 124
+ [ "$remaining" -le 90 ] || remaining=90
+ node "${SCRIPT_DIR}/_lib/hcc-cleanup-command.mjs" "$remaining" "$KUBECTL_BIN" \
+ --context "$E2E_KUBECONTEXT" --request-timeout=30s "$@"
+ }
+ ;;
+ *) echo 'E2E_HCC_PR_A must be 0 or 1' >&2; exit 2 ;;
+esac
+
# ── Fail-closed guards (identical contract to the sibling HCC gates) ──
[ -n "$E2E_KUBECONTEXT" ] || {
echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2
@@ -163,6 +181,11 @@ wait_until() {
local timeout=$1 description=$2
shift 2
local deadline now
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))
+ [ "$remaining" -gt 0 ] || return 124
+ [ "$timeout" -le "$remaining" ] || timeout=$remaining
+ fi
deadline=$(($(date +%s) + timeout))
while :; do
"$@" && return 0
@@ -211,6 +234,11 @@ hcc_restart_count() {
sample_ready_series() {
local duration=$1 phase=$2 stop_on_ready=$3
local s_deadline t0 now pod status
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ local remaining=$((HCC_PR_A_WORK_DEADLINE - SECONDS))
+ [ "$remaining" -gt 0 ] || return 124
+ [ "$duration" -le "$remaining" ] || duration=$remaining
+ fi
s_deadline=$(($(date +%s) + duration))
while t0="$(date +%s)"; [ "$t0" -lt "$s_deadline" ]; do
status=503
@@ -265,6 +293,11 @@ first_200_epoch() {
}
print_repair_instructions() {
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ printf 'PR A cleanup failed; lock and dependent fixtures retained.\nContext: %s\nHCC: %s/%s\nExact public restoration snapshot: %s\nInspect identity and restore from this snapshot; do not remove environment entries blindly.\n' \
+ "$E2E_KUBECONTEXT" "$HCC_NS" "$HCC_DEPLOY" "${HCC_PR_A_CONFIG_SNAPSHOT:-missing}" >&2
+ return
+ fi
cat >&2 <<EOF
HCC watch-churn gate cleanup could not restore a verified clean state.
Context: ${E2E_KUBECONTEXT}
@@ -351,9 +384,10 @@ trap 'exit 131' QUIT
# ── FASE A: guards + snapshot ──
require_branch_owned_hcc_gate "$HCC_NS"
acquire_hcc_watch_gate_lock
+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then hcc_pr_a_preflight; fi
ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"
[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
-[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||
+[ "${E2E_HCC_PR_A:-0}" = 1 ] || [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||
die "HCC already has hostAliases; refusing a non-restorable injection"
HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
@@ -365,6 +399,7 @@ K8S_API_CIDR="${api_ip}/32"
# ── FASE B: redirect through the self-flapping proxy (HCC still healthy) ──
log "Creating self-flapping API proxy (period=${CHURN_PERIOD_MS}ms, min-age=${CHURN_MIN_AGE_MS}ms)"
create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"
+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then hcc_pr_a_enable_proxy; fi
verify_hcc_proxy_network_policy
proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"
[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"
@@ -388,6 +423,7 @@ patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template
{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},
{name:"KUBERNETES_SERVICE_PORT",value:"443"},
{name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"
+if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then patch="$(hcc_pr_a_redirect_patch)"; fi
HCC_PATCHED=1
kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null
kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
@@ -493,7 +529,7 @@ fi
# consumed READINESS_BUDGET_SEC of churn.
require_hcc_recovery_log_stream ||
warn "HCC log stream ended before the verdict; counting the frozen buffer (fail-closed: an undercount can only turn this gate RED)"
-stop_hcc_recovery_log_stream
+[ "${E2E_HCC_PR_A:-0}" = 1 ] || stop_hcc_recovery_log_stream
recount_churn_evidence
read -r churn_total churn_200 churn_503 churn_maxstreak churn_transitions <<<"$(series_metrics churn)"
read -r hold_total _ hold_503 _ _ <<<"$(series_metrics post-hold)"
@@ -551,6 +587,10 @@ fi
if [ "${E2E_HCC_POLICY_LIFECYCLE:-0}" = 1 ]; then
[ -n "$first_ready" ] || die 'NP604 cannot run without a ready controller'
np604_recover
+ if [ "${E2E_HCC_PR_A:-0}" = 1 ]; then
+ hcc_pr_a_run
+ stop_hcc_recovery_log_stream
+ fi
fi
log "Evidence artifact: ${LOG_ARTIFACT}"
# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.
diff --git a/scripts/tests/hcc-watch-api-proxy.test.mjs b/scripts/tests/hcc-watch-api-proxy.test.mjs
new file mode 100644
index 000000000..2919efd12
--- /dev/null
+++ b/scripts/tests/hcc-watch-api-proxy.test.mjs
@@ -0,0 +1,661 @@
+import assert from 'node:assert/strict'
+import { spawnSync } from 'node:child_process'
+import { X509Certificate, generateKeyPairSync } from 'node:crypto'
+import { once } from 'node:events'
+import { mkdtempSync, readFileSync, rmSync, watch, writeFileSync } from 'node:fs'
+import https from 'node:https'
+import { tmpdir } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+import { before, test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { createProxy, validateCommand } from '../e2e/_lib/hcc-watch-api-proxy.mjs'
+import { changedEnvironment, restorePatch, snapshot } from '../e2e/_lib/hcc-watch-config.mjs'
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..')
+const allowedPath = '/api/v1/namespaces/mcp-server/services/runtime-witness'
+const mcpPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/mcpservers?watch=true'
+const contextPath = '/apis/clerum.io/v1alpha1/namespaces/mcp-server/contexts?watch=true'
+const otherPath = '/api/v1/namespaces/mcp-server/pods?watch=true'
+let front
+let upstreamTls
+let publicConfig
+
+function certificate() {
+ const { privateKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ })
+ const result = spawnSync(
+ 'openssl',
+ [
+ 'req',
+ '-new',
+ '-x509',
+ '-key',
+ '/dev/stdin',
+ '-days',
+ '1',
+ '-subj',
+ '/CN=localhost',
+ '-addext',
+ 'subjectAltName=DNS:localhost,IP:127.0.0.1',
+ '-addext',
+ 'basicConstraints=critical,CA:TRUE',
+ ],
+ {
+ input: privateKey,
+ encoding: 'utf8',
+ timeout: 15000,
+ maxBuffer: 1048576,
+ }
+ )
+ assert.equal(result.status, 0, 'in-memory certificate generation succeeds')
+ return { key: privateKey, cert: result.stdout }
+}
+
+before(() => {
+ const result = spawnSync(
+ process.execPath,
+ [
+ join(root, 'scripts/e2e/_lib/hcc-watch-tls-manifest.mjs'),
+ 'proxy-fixture',
+ 'test-fixture',
+ 'run-fixture',
+ join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'),
+ ],
+ {
+ encoding: 'utf8',
+ timeout: 20000,
+ maxBuffer: 1048576,
+ }
+ )
+ // Captured generator output stays in memory and is never printed in diagnostics.
+ assert.equal(result.status, 0, 'fixture manifest generator succeeds')
+ let manifest
+ try {
+ manifest = JSON.parse(result.stdout)
+ } catch {
+ throw new Error('fixture generator did not emit JSON')
+ }
+ assert.equal(manifest.kind, 'List')
+ assert.equal(manifest.apiVersion, 'v1')
+ assert.equal(manifest.items.length, 2)
+ const tlsResource = manifest.items.find(item => item.kind === 'Secret')
+ const configMap = manifest.items.find(item => item.kind === 'ConfigMap')
+ assert.equal(tlsResource?.type, 'kubernetes.io/tls')
+ assert.equal(tlsResource?.metadata.namespace, 'test-fixture')
+ assert.equal(tlsResource?.metadata.labels['e2e.clerum.io/run'], 'run-fixture')
+ assert.equal(typeof tlsResource?.stringData['tls.key'], 'string')
+ front = { key: tlsResource.stringData['tls.key'], cert: tlsResource.stringData['tls.crt'] }
+ const cert = new X509Certificate(front.cert)
+ assert.equal(cert.checkHost('proxy-fixture.test-fixture.svc'), 'proxy-fixture.test-fixture.svc')
+ assert.equal(cert.ca, true)
+ publicConfig = JSON.parse(configMap.data['config.json'])
+ assert.equal(
+ configMap.data['proxy.mjs'] ===
+ readFileSync(join(root, 'scripts/e2e/_lib/hcc-watch-api-proxy.mjs'), 'utf8'),
+ true
+ )
+ assert.equal(
+ JSON.stringify(configMap).includes(JSON.stringify(front.key).slice(1, -1)),
+ false,
+ 'public config excludes signing material'
+ )
+ upstreamTls = certificate()
+})
+
+function bounded(promise, message, timeoutMs = 4000) {
+ let timer
+ return Promise.race([
+ promise,
+ new Promise((_, reject) => {
+ timer = setTimeout(() => reject(new Error(message)), timeoutMs)
+ }),
+ ]).finally(() => clearTimeout(timer))
+}
+function ack(controlDir, id, state, filename = 'ack.json') {
+ return new Promise((resolveAck, reject) => {
+ let observer
+ const done = (error, value) => {
+ clearTimeout(timer)
+ observer?.close()
+ if (error) reject(error)
+ else resolveAck(value)
+ }
+ const read = () => {
+ let value
+ try {
+ value = JSON.parse(readFileSync(join(controlDir, filename), 'utf8'))
+ } catch {
+ return
+ }
+ if (value.id === id && value.state === state) done(null, value)
+ }
+ const timer = setTimeout(
+ () => done(new Error(`control acknowledgement missing: ${state}`)),
+ 4000
+ )
+ observer = watch(controlDir, read)
+ read()
+ })
+}
+async function fixture(run, { wrongCa = false } = {}) {
+ const controlDir = mkdtempSync(join(tmpdir(), 'hcc-proxy-test-'))
+ const requests = []
+ const responses = new Map()
+ const upstream = https.createServer(upstreamTls, (request, response) => {
+ requests.push({
+ path: request.url,
+ method: request.method,
+ authorization: request.headers.authorization,
+ })
+ if (request.url === '/opaque') {
+ response.destroy()
+ return
+ }
+ if (request.url.includes('watch=true')) {
+ responses.set(request.url, response)
+ response.writeHead(200, { 'content-type': 'application/json' })
+ response.write('watch-start\n')
+ } else {
+ response.writeHead(request.url === '/status' ? 418 : 200, { 'content-type': 'text/plain' })
+ response.end('upstream-response')
+ }
+ })
+ upstream.on('tlsClientError', () => {})
+ await new Promise(resolveListen => upstream.listen(0, '127.0.0.1', resolveListen))
+ const proxy = createProxy({
+ ...front,
+ upstreamCa: wrongCa ? front.cert : upstreamTls.cert,
+ upstreamHost: '127.0.0.1',
+ upstreamPort: upstream.address().port,
+ allowedPaths: [allowedPath],
+ controlDir,
+ periodMs: 60000,
+ minAgeMs: 60000,
+ })
+ await new Promise(resolveListen => proxy.server.listen(0, '127.0.0.1', resolveListen))
+ const clients = new Set()
+ const request = (path, method = 'GET') => {
+ let responseResolve
+ let responseReject
+ const opened = new Promise((resolveResponse, reject) => {
+ responseResolve = resolveResponse
+ responseReject = reject
+ })
+ const req = https.request({
+ hostname: '127.0.0.1',
+ port: proxy.server.address().port,
+ servername: 'proxy-fixture.test-fixture.svc',
+ ca: front.cert,
+ rejectUnauthorized: true,
+ method,
+ path,
+ headers: { authorization: 'Bearer test' },
+ agent: false,
+ })
+ clients.add(req)
+ let body = ''
+ const completed = new Promise((resolveResponse, reject) => {
+ req.on('response', response => {
+ responseResolve(response)
+ response.on('data', chunk => {
+ body += chunk
+ })
+ response.on('end', () => resolveResponse({ status: response.statusCode, body }))
+ response.on('error', reject)
+ })
+ req.on('error', error => {
+ responseReject(error)
+ reject(error)
+ })
+ })
+ opened.catch(() => {})
+ completed.catch(() => {})
+ req.end()
+ return {
+ req,
+ opened,
+ completed,
+ get body() {
+ return body
+ },
+ }
+ }
+ const command = async (value, state) => {
+ const waiting = ack(controlDir, value.id, state)
+ writeFileSync(join(controlDir, 'command.json'), JSON.stringify(value), { mode: 0o600 })
+ return waiting
+ }
+ let closed = false
+ const close = async () => {
+ if (closed) return
+ closed = true
+ const stopped = once(proxy.server, 'close')
+ proxy.close()
+ await bounded(stopped, 'proxy server did not close')
+ }
+ try {
+ await run({ request, command, controlDir, requests, responses, close })
+ } finally {
+ try {
+ await close()
+ } finally {
+ for (const client of clients) client.destroy()
+ upstream.closeAllConnections()
+ await bounded(
+ new Promise(resolveClose => upstream.close(resolveClose)),
+ 'upstream did not close'
+ )
+ rmSync(controlDir, { recursive: true, force: true })
+ }
+ }
+}
+async function watchStarted(stream) {
+ const response = await bounded(stream.opened, 'watch response not opened')
+ if (!stream.body.includes('watch-start'))
+ await bounded(once(response, 'data'), 'watch did not flow')
+ assert.equal(stream.body.includes('watch-start'), true)
+ return response
+}
+
+test('generator emits public CA trust and service-account tokenFile provider', () => {
+ const cluster = publicConfig.clusters[0].cluster
+ assert.equal(cluster.server, 'https://proxy-fixture.test-fixture.svc:443')
+ assert.equal(
+ Buffer.from(cluster['certificate-authority-data'], 'base64').equals(Buffer.from(front.cert)),
+ true
+ )
+ assert.equal('insecure-skip-tls-verify' in cluster, false)
+ const user = publicConfig.users[0].user
+ assert.equal(user['auth-provider'].name, 'tokenFile')
+ assert.equal(
+ user['auth-provider'].config.tokenFile,
+ '/var/run/secrets/kubernetes.io/serviceaccount/token'
+ )
+ assert.deepEqual(Object.keys(user), ['auth-provider'])
+ assert.equal(publicConfig['current-context'], 'fixture')
+})
+test('commands reject invalid id, method, path, duration and cut scope', () => {
+ const valid = { action: 'arm', id: 'valid-1', method: 'GET', path: allowedPath, durationMs: 1000 }
+ assert.equal(validateCommand(valid, [allowedPath]), valid)
+ for (const change of [
+ { id: '../escape' },
+ { id: true },
+ { id: 123 },
+ { method: 'POST' },
+ { path: '/api/v1/pods' },
+ { durationMs: 999 },
+ { durationMs: 25001 },
+ { durationMs: 1000.5 },
+ { action: 'unknown' },
+ ]) {
+ assert.throws(() => validateCommand({ ...valid, ...change }, [allowedPath]))
+ }
+ assert.throws(() => validateCommand({ action: 'cut', id: 'cut-1', kind: 'Pod' }, [allowedPath]))
+})
+test(
+ 'verified HTTPS forwards status and synthetic authorization; upstream errors are opaque',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, requests }) => {
+ const result = await bounded(request('/status').completed, 'short response absent')
+ assert.equal(result.status, 418)
+ assert.equal(result.body, 'upstream-response')
+ assert.equal(requests[0].authorization === 'Bearer test', true)
+ await assert.rejects(bounded(request('/opaque').completed, 'error response absent'), {
+ code: 'ECONNRESET',
+ })
+ })
+ }
+)
+test('wrong upstream CA cannot produce a successful response', { timeout: 10000 }, async () => {
+ await fixture(
+ async ({ request, requests }) => {
+ await assert.rejects(
+ bounded(request('/status').completed, 'untrusted response did not settle'),
+ { code: 'ECONNRESET' }
+ )
+ assert.equal(requests.length, 0)
+ },
+ { wrongCa: true }
+ )
+})
+test(
+ 'exact GET pause acknowledges interception and release while watches keep flowing',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, command, controlDir, requests, responses }) => {
+ const stream = request(mcpPath)
+ const response = await watchStarted(stream)
+ await command(
+ { action: 'arm', id: 'arm-release', path: allowedPath, method: 'GET', durationMs: 5000 },
+ 'armed'
+ )
+ assert.equal(
+ (await bounded(request('/status').completed, 'unrelated short request paused')).status,
+ 418
+ )
+ assert.equal(
+ (await bounded(request(allowedPath, 'POST').completed, 'POST was incorrectly paused'))
+ .status,
+ 200
+ )
+ const intercepted = ack(controlDir, 'arm-release', 'intercepted')
+ const held = request(allowedPath)
+ await intercepted
+ assert.equal(
+ requests.some(value => value.path === allowedPath && value.method === 'GET'),
+ false
+ )
+ const flowing = once(response, 'data')
+ responses.get(mcpPath).write('watch-during-pause\n')
+ await bounded(flowing, 'watch stalled behind short request')
+ assert.equal(stream.body.includes('watch-during-pause'), true)
+ await command({ action: 'release', id: 'release-1', pauseId: 'arm-release' }, 'released')
+ assert.equal((await bounded(held.completed, 'held GET did not resume')).status, 200)
+ assert.equal(
+ requests.filter(value => value.path === allowedPath && value.method === 'GET').length,
+ 1
+ )
+ })
+ }
+)
+test(
+ 'minimum pause expires and resumes the actual intercepted request',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, command, controlDir, requests }) => {
+ await command(
+ { action: 'arm', id: 'arm-expiry', path: allowedPath, method: 'GET', durationMs: 1000 },
+ 'armed'
+ )
+ const intercepted = ack(controlDir, 'arm-expiry', 'intercepted')
+ const held = request(allowedPath)
+ await intercepted
+ assert.equal(
+ requests.some(value => value.path === allowedPath && value.method === 'GET'),
+ false
+ )
+ await ack(controlDir, 'arm-expiry', 'expired', 'pause.json')
+ await command({ action: 'release', id: 'release-expired', pauseId: 'arm-expiry' }, 'rejected')
+ assert.equal(
+ JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state,
+ 'expired'
+ )
+ assert.equal((await bounded(held.completed, 'expired GET did not resume')).status, 200)
+ })
+ }
+)
+test(
+ 'cut targets only the selected watch kind and preserves short requests and other streams',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, command, controlDir, responses }) => {
+ const mcp = request(mcpPath)
+ const context = request(contextPath)
+ const other = request(otherPath)
+ await watchStarted(mcp)
+ const contextResponse = await watchStarted(context)
+ const otherResponse = await watchStarted(other)
+ await command(
+ { action: 'arm', id: 'arm-cut', path: allowedPath, method: 'GET', durationMs: 5000 },
+ 'armed'
+ )
+ const intercepted = ack(controlDir, 'arm-cut', 'intercepted')
+ const held = request(allowedPath)
+ await intercepted
+ const pauseBeforeCut = await ack(controlDir, 'arm-cut', 'intercepted', 'pause.json')
+ const cut = await command({ action: 'cut', id: 'cut-mcp', kind: 'McpServer' }, 'cut')
+ assert.deepEqual(
+ JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')),
+ pauseBeforeCut
+ )
+ assert.equal(cut.count, 1)
+ await assert.rejects(bounded(mcp.completed, 'MCP watch did not close'))
+ for (const [path, response, stream] of [
+ [contextPath, contextResponse, context],
+ [otherPath, otherResponse, other],
+ ]) {
+ const flowing = once(response, 'data')
+ responses.get(path).write('survived-cut\n')
+ await bounded(flowing, 'unselected stream was cut')
+ assert.equal(stream.body.includes('survived-cut'), true)
+ }
+ await command({ action: 'release', id: 'release-cut', pauseId: 'arm-cut' }, 'released')
+ assert.equal((await bounded(held.completed, 'short request was cut')).status, 200)
+ const both = await command({ action: 'cut', id: 'cut-both', kind: 'both' }, 'cut')
+ assert.equal(both.count, 1)
+ await assert.rejects(bounded(context.completed, 'Context watch survived both cut'))
+ const flowing = once(otherResponse, 'data')
+ responses.get(otherPath).write('survived-both\n')
+ await bounded(flowing, 'both cut unrelated watch')
+ assert.equal(other.body.includes('survived-both'), true)
+ })
+ }
+)
+test(
+ 'close tears down active watches and an armed intercepted request',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, command, controlDir, close }) => {
+ const stream = request(mcpPath)
+ await watchStarted(stream)
+ await command(
+ { action: 'arm', id: 'arm-close', path: allowedPath, method: 'GET', durationMs: 25000 },
+ 'armed'
+ )
+ const intercepted = ack(controlDir, 'arm-close', 'intercepted')
+ const held = request(allowedPath)
+ await intercepted
+ await close()
+ await assert.rejects(bounded(stream.completed, 'watch survived shutdown'))
+ await assert.rejects(bounded(held.completed, 'held request survived shutdown'))
+ })
+ }
+)
+
+// Apply the emitted JSON Patch to a cloned in-memory Deployment.
+function applyPatch(deployment, patches) {
+ const result = structuredClone(deployment)
+ for (const patch of patches) {
+ const keys = patch.path.slice(1).split('/')
+ const parent = keys.slice(0, -1).reduce((value, key) => value[key], result)
+ const key = keys.at(-1)
+ if (patch.op === 'test') assert.deepEqual(parent[key], patch.value)
+ else if (patch.op === 'add') parent[key] = structuredClone(patch.value)
+ else if (patch.op === 'remove') delete parent[key]
+ else throw new Error('unexpected restore operation')
+ }
+ return result
+}
+function deploymentFixture() {
+ return {
+ metadata: { uid: 'deployment-fixture-uid', resourceVersion: '1' },
+ spec: {
+ template: {
+ spec: {
+ containers: [
+ { name: 'sidecar', env: [{ name: 'SIDECAR', value: 'untouched' }] },
+ { name: 'host-context-controller' },
+ ],
+ },
+ },
+ },
+ }
+}
+function injectFixture(deployment) {
+ const result = structuredClone(deployment)
+ const pod = result.spec.template.spec
+ const container = pod.containers.find(value => value.name === 'host-context-controller')
+ container.env = [
+ ...(container.env ?? []).filter(value => !changedEnvironment.includes(value.name)),
+ ...changedEnvironment.map(name => ({ name, value: 'fixture-value' })),
+ ]
+ pod.volumes = [
+ ...(pod.volumes ?? []),
+ { name: 'hcc-pr-a-config', configMap: { name: 'fixture-config' } },
+ ]
+ container.volumeMounts = [
+ ...(container.volumeMounts ?? []),
+ { name: 'hcc-pr-a-config', mountPath: '/fixture-config' },
+ ]
+ return result
+}
+test('config restore removes fixture-added fields that were originally absent', () => {
+ const original = deploymentFixture()
+ const saved = snapshot(original)
+ assert.equal(saved.envPresent, false)
+ assert.equal(saved.volumesPresent, false)
+ assert.equal(saved.mountsPresent, false)
+ assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_HOST'), false)
+ assert.equal(changedEnvironment.includes('KUBERNETES_SERVICE_PORT'), false)
+ assert.equal('aliases' in saved, false)
+ assert.equal('aliasesPresent' in saved, false)
+ const restored = applyPatch(injectFixture(original), restorePatch(saved, injectFixture(original)))
+ assert.deepEqual(restored, original)
+})
+test('config snapshot is public-only and restore preserves unrelated environment edits', () => {
+ const original = deploymentFixture()
+ const pod = original.spec.template.spec
+ const container = pod.containers[1]
+ container.env = [
+ { name: 'KUBECONFIG', value: '/previous/config' },
+ { name: 'UNRELATED', value: 'not-in-public-snapshot' },
+ ]
+ container.volumeMounts = [{ name: 'existing', mountPath: '/existing' }]
+ pod.volumes = [{ name: 'existing', emptyDir: {} }]
+ pod.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]
+ const saved = snapshot(original)
+ assert.deepEqual(saved.env, [{ name: 'KUBECONFIG', value: '/previous/config' }])
+ assert.equal(JSON.stringify(saved).includes('not-in-public-snapshot'), false)
+ const changed = injectFixture(original)
+ changed.spec.template.spec.containers[1].env.find(value => value.name === 'UNRELATED').value =
+ 'concurrent-public-edit'
+ const restored = applyPatch(changed, restorePatch(saved, changed))
+ const restoredContainer = restored.spec.template.spec.containers[1]
+ assert.equal(
+ restoredContainer.env.find(value => value.name === 'UNRELATED').value,
+ 'concurrent-public-edit'
+ )
+ assert.equal(
+ restoredContainer.env.find(value => value.name === 'KUBECONFIG').value,
+ '/previous/config'
+ )
+ assert.deepEqual(snapshot(restored), saved)
+ assert.deepEqual(
+ restored.spec.template.spec.containers[0],
+ original.spec.template.spec.containers[0]
+ )
+})
+test('config snapshot and restore reject collisions, identity replacement and unrelated mount changes', () => {
+ const original = deploymentFixture()
+ const saved = snapshot(original)
+ assert.throws(() => snapshot(injectFixture(original)), /fixture_mount_collision/)
+ const replaced = injectFixture(original)
+ replaced.metadata.uid = 'different-deployment'
+ assert.throws(() => restorePatch(saved, replaced), /deployment_uid_changed/)
+ const removed = injectFixture(original)
+ removed.spec.template.spec.containers = []
+ assert.throws(() => restorePatch(saved, removed), /deployment_container_missing/)
+ for (const field of ['volumes', 'volumeMounts']) {
+ const changed = injectFixture(original)
+ const target =
+ field === 'volumes' ? changed.spec.template.spec : changed.spec.template.spec.containers[1]
+ target[field].push({ name: 'unrelated-new-mount' })
+ assert.throws(() => restorePatch(saved, changed), /unrelated_configuration_changed/)
+ }
+})
+
+test('release requires a valid original pause identity', () => {
+ for (const pauseId of [undefined, '', true, 123, '../wrong']) {
+ assert.throws(() =>
+ validateCommand({ action: 'release', id: 'release-invalid', pauseId }, [allowedPath])
+ )
+ }
+ assert.equal(
+ validateCommand({ action: 'release', id: 'release-valid', pauseId: 'arm-valid' }, [allowedPath])
+ .pauseId,
+ 'arm-valid'
+ )
+})
+
+test(
+ 'absent or mismatched pause cannot be released or resume an intercepted GET',
+ { timeout: 10000 },
+ async () => {
+ await fixture(async ({ request, command, controlDir, requests }) => {
+ await command({ action: 'release', id: 'release-absent', pauseId: 'never-armed' }, 'rejected')
+ await command(
+ { action: 'arm', id: 'arm-identity', path: allowedPath, method: 'GET', durationMs: 5000 },
+ 'armed'
+ )
+ await command(
+ { action: 'release', id: 'release-not-intercepted', pauseId: 'arm-identity' },
+ 'rejected'
+ )
+ assert.equal(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')).state, 'armed')
+ const intercepted = ack(controlDir, 'arm-identity', 'intercepted', 'pause.json')
+ const held = request(allowedPath)
+ const ledger = await intercepted
+ await command(
+ { action: 'release', id: 'release-wrong', pauseId: 'different-arm' },
+ 'rejected'
+ )
+ assert.deepEqual(JSON.parse(readFileSync(join(controlDir, 'pause.json'), 'utf8')), ledger)
+ assert.equal(
+ requests.some(value => value.path === allowedPath),
+ false
+ )
+ await command(
+ { action: 'release', id: 'release-correct', pauseId: 'arm-identity' },
+ 'released'
+ )
+ const released = await ack(controlDir, 'arm-identity', 'released', 'pause.json')
+ assert.equal(Number.isFinite(released.at), true)
+ assert.equal(
+ (await bounded(held.completed, 'correctly released GET did not resume')).status,
+ 200
+ )
+ })
+ }
+)
+
+test('config restore compare-and-swap rejects a newer Deployment revision', () => {
+ const original = deploymentFixture()
+ const changed = injectFixture(original)
+ changed.metadata.resourceVersion = '2'
+ const missingRevision = structuredClone(changed)
+ delete missingRevision.metadata.resourceVersion
+ assert.throws(
+ () => restorePatch(snapshot(original), missingRevision),
+ /deployment_resource_version_missing/
+ )
+ const patches = restorePatch(snapshot(original), changed)
+ assert.equal(
+ patches.some(
+ patch =>
+ patch.op === 'test' && patch.path === '/metadata/resourceVersion' && patch.value === '2'
+ ),
+ true
+ )
+ const concurrent = structuredClone(changed)
+ concurrent.metadata.resourceVersion = '3'
+ assert.throws(() => applyPatch(concurrent, patches))
+ assert.equal(applyPatch(changed, patches).metadata.resourceVersion, '2')
+})
+
+test('config restoration preserves new host aliases outside fixture ownership', () => {
+ const original = deploymentFixture()
+ original.spec.template.spec.hostAliases = [{ ip: '127.0.0.3', hostnames: ['original.test'] }]
+ const changed = injectFixture(original)
+ const aliases = [{ ip: '127.0.0.4', hostnames: ['concurrent.test'] }]
+ changed.spec.template.spec.hostAliases = aliases
+ const patches = restorePatch(snapshot(original), changed)
+ assert.equal(
+ patches.some(patch => patch.path.includes('hostAliases')),
+ false
+ )
+ assert.deepEqual(applyPatch(changed, patches).spec.template.spec.hostAliases, aliases)
+})
diff --git a/scripts/tests/test-hcc-watch-api-proxy.sh b/scripts/tests/test-hcc-watch-api-proxy.sh
new file mode 100755
index 000000000..4fea92878
--- /dev/null
+++ b/scripts/tests/test-hcc-watch-api-proxy.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+command -v openssl >/dev/null || { echo 'FAIL: openssl is required' >&2; exit 1; }
+node -e 'if (Number(process.versions.node.split(".")[0]) !== 24) { console.error("FAIL: Node 24 is required"); process.exit(1) }'
+# Test certificates and the generator output stay in Node memory; never tee or
+# redirect the generator itself to any artifact. The reporter prints test results.
+exec node --test --test-timeout=15000 "$SCRIPT_DIR/hcc-watch-api-proxy.test.mjs"
diff --git a/scripts/tests/test-hcc-watch-pr-a.sh b/scripts/tests/test-hcc-watch-pr-a.sh
new file mode 100644
index 000000000..6b704eb1f
--- /dev/null
+++ b/scripts/tests/test-hcc-watch-pr-a.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+tmp="$(mktemp -d)"
+trap 'rm -rf -- "$tmp"' EXIT
+source "$ROOT/scripts/e2e/_lib/hcc-watch-pr-a.sh"
+HCC_LOG_BUFFER="$tmp/logs"
+require_hcc_recovery_log_stream() { return 0; }
+printf '%s\n' '{"event":"networkpolicy-recovery-decision","kind":"McpServer","decision":"skip"}' \
+ '{"event":"networkpolicy-recovery-decision","kind":"Context","decision":"defer"}' > "$HCC_LOG_BUFFER"
+[[ "$(hcc_pr_a_decision_count McpServer skip)" = 1 ]]
+hcc_pr_a_decision_after 0 McpServer skip
+if hcc_pr_a_decision_after 1 McpServer skip; then echo 'FAIL: old recovery reused'; exit 1; fi
+if hcc_pr_a_decision_after 0 Context skip; then echo 'FAIL: deferral counted as optimization'; exit 1; fi
+: > "$HCC_LOG_BUFFER"
+if hcc_pr_a_decision_after 0 McpServer skip; then echo 'FAIL: empty observations passed'; exit 1; fi
+printf 'PASS: recovery observation needs a new explicit omission of the exact kind\n'
+
+NP604_SERVER=fixture-affected MCP_NS=mcp-server HCC_PR_A_SERVICE_UID=old
+np604_kctl() { cat "$tmp/service"; }
+for variation in new old wrong-label wrong-port; do
+ uid=new name=fixture-affected port=3000
+ case "$variation" in old) uid=old;; wrong-label) name=other;; wrong-port) port=3001;; esac
+ jq -cn --arg uid "$uid" --arg name "$name" --argjson port "$port" \
+ '{metadata:{uid:$uid,labels:{"clerum.io/mcpserver":$name}},spec:{ports:[{port:$port}]}}' > "$tmp/service"
+ if hcc_pr_a_service_restored; then
+ [[ "$variation" = new ]] || { echo "FAIL: accepted $variation Service"; exit 1; }
+ else
+ [[ "$variation" != new ]] || { echo 'FAIL: valid repaired Service rejected'; exit 1; }
+ fi
+done
+printf 'service/fixture-affected\n' > "$tmp/service"
+if hcc_pr_a_service_absent; then echo 'FAIL: delete request replaced absence proof'; exit 1; fi
+: > "$tmp/service"
+hcc_pr_a_service_absent
+printf 'PASS: actual absence and new correctly scoped Service are required\n'
+
+# Execute the scenario until its first arm command with harmless boundary
+# doubles. This proves the selected read is before Service repair, rather
+# than searching a comment or passing through an unexecuted branch.
+(
+ NP604_CONTROL=fixture-control
+ hcc_pr_a_gate() { :; }
+ np604_snapshot() { printf '[]'; }
+ hcc_pr_a_periodic_count() { printf 1; }
+ hcc_pr_a_periodic_started_count() { printf 1; }
+ hcc_pr_a_certified_count() { printf 1; }
+ hcc_pr_a_remove_service() { :; }
+ hcc_pr_a_cut() { :; }
+ hcc_pr_a_service_absent() { :; }
+ np604_kctl() { printf '{"metadata":{"uid":"fixture","generation":1},"spec":{}}'; }
+ np604_invoke() { :; }
+ wait_until() { :; }
+ hcc_pr_a_command() { printf '%s\n' "$2" > "$tmp/first-arm"; exit 0; }
+ die() { echo "FAIL: unexpected scenario rejection $*" >&2; exit 1; }
+ hcc_pr_a_run > "$tmp/scenario-output"
+)
+[[ "$(cat "$tmp/first-arm")" = '/api/v1/namespaces/mcp-server/services/fixture-affected' ]]
+printf 'PASS: interrupted runtime holds the Service GET before its missing effect\n'
+
+# Execute the proxy patch builder without generating material or contacting
+# Kubernetes; its allowlist must include the fixture read chosen above.
+(
+ RUN_ID=fixture PROXY_NAME=fixture-proxy HCC_NS=control-plane
+ truncate_rfc1123() { printf '%s' "$1"; }
+ node() { printf '{}'; }
+ kctl() {
+ case "$1" in
+ create) cat >/dev/null;;
+ patch) printf '%s' "${@: -1}" > "$tmp/proxy-patch";;
+ rollout) :;;
+ *) exit 1;;
+ esac
+ }
+ hcc_pr_a_enable_proxy
+)
+jq -e '.spec.template.spec.containers[0].env[0].value|fromjson|index("/api/v1/namespaces/mcp-server/services/np604-fixture-affected")!=null' "$tmp/proxy-patch" >/dev/null
+jq -e '.spec.template.spec.containers[0].env[0].value|fromjson|index("/apis/networking.k8s.io/v1/namespaces/mcp-server/networkpolicies/ext-egress-np604-fixture-affected-1-2-3-4-32-443")!=null' "$tmp/proxy-patch" >/dev/null
+printf 'PASS: proxy allowlist covers only the declared fixture operations\n'
+
+# Run the shared positive probe builder in both modes up to its actual exec
+# boundary. No credentials are read; only the exact public trust arguments
+# supplied to the in-pod HTTPS client are observed here.
+for mode in 0 1; do
+ (
+ source "$ROOT/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh"
+ E2E_HCC_PR_A=$mode PROXY_NAME=fixture-proxy HCC_NS=control-plane HCC_DEPLOY=host-context-controller
+ kctl() {
+ case "$1/$2" in
+ get/service) printf '10.0.0.1';;
+ get/configmap)
+ jq -cn '{data:{"config.json":"{\"clusters\":[{\"cluster\":{\"certificate-authority-data\":\"cHVibGljLXRlc3QtY2E=\"}}]}"}}';;
+ exec/*) printf '%s\n' "$@" > "$tmp/trust-$mode"; exit 0;;
+ *) exit 1;;
+ esac
+ }
+ verify_hcc_proxy_network_policy
+ )
+ grep -q 'rejectUnauthorized:true' "$tmp/trust-$mode"
+ [[ "$(tail -3 "$tmp/trust-$mode" | head -1)" = 'fixture-proxy.control-plane.svc' ]]
+ if [[ "$mode" = 1 ]]; then
+ [[ "$(tail -2 "$tmp/trust-$mode" | head -1)" = 'fixture-proxy.control-plane.svc' ]]
+ [[ "$(tail -1 "$tmp/trust-$mode")" = 'cHVibGljLXRlc3QtY2E=' ]]
+ else
+ [[ "$(tail -2 "$tmp/trust-$mode" | head -1)" = 'kubernetes.default.svc' ]]
+ [[ -z "$(tail -1 "$tmp/trust-$mode")" ]]
+ fi
+done
+printf 'PASS: PR A verifies its public fixture CA/name; PR B preserves Kubernetes trust\n'