Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

chore: merge dev into feat/mcp-host-guardrails (third pass)

Andy Paviab836a3610990
diff --cc host-context-controller/src/hostReconciler.ts
index 94e338245,94008e22f..710eb12a7
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@@ -3732,11 -3972,11 +4018,14 @@@ export class HostReconciler
// surface it via HostRuntimeStatus instead of swallowing it.
const npFailures: string[] = []
try {
+ revalidateHostMutationBoundary()
await this.ensureMcpHostIngressNetworkPolicy(host)
+ revalidateHostMutationBoundary()
await this.ensureMcpHostGfsEgressNetworkPolicy(host)
+ // The mcp-host→llm-hooks egress policy is now owned by LlmHookReconciler
+ // (per-host, scoped to referenced hook pods — N1/N7); host-delete cleanup
+ // of `mcp-host-<host>-egress-llm-hooks` stays in deleteHostNetworkPolicies.
+ revalidateHostMutationBoundary()
await this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure mcp-host NP for "${host.name}":`, err)
diff --cc host-context-controller/src/k8sClient.ts
index 819741eb3,ff4874570..d4ffc430b
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@@ -30,9 -40,19 +40,20 @@@ import
import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
+import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'
- import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'
- import { NetworkPolicyReconciler } from './networkPolicyReconciler'
+ import {
+ confirmAuthoritativeMcpServerAbsence,
+ isMcpServerStatusOnlyUpdate,
+ sameMcpServerDesiredRevision,
+ } from './mcpServerSafety'
+ import {
+ hostDeleteCleanupTotal,
+ hostFleetRequestsTotal,
+ hostWatchRecoverySeconds,
+ initialConvergenceLastSuccessTimestampSeconds,
+ initialConvergenceRetriesTotal,
+ } from './metrics'
+ import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'
import { McpServerReconciler } from './reconciler'
import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'
import {
@@@ -86,32 -105,32 +108,33 @@@ const PLURAL_HOSTS = 'hosts
const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'
const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'
const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'
+const PLURAL_LLMHOOKS = 'llmhooks'
- const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000]
- const EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY = 10
- const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000
+ // Before readiness was decoupled, a failed initial McpServer or NetworkPolicy
+ // sweep made provider.start() fail and Kubernetes restarted HCC. These retries
+ // retain that convergence guarantee now that the sweeps run in the background.
+ const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
- /**
- * External-egress resync delay (ms), H2 (issue #299). The configured interval is
- * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
- * rotating low-TTL host is sampled every rotation. The floor is the hard lower
- * bound to avoid a hot-loop. Pure and unit-testable.
- */
- export function externalEgressResyncDelayMs(
- intervalSec: number,
- floorSec: number,
- minObservedTtlMs = Infinity
- ): number {
- const floorMs = floorSec * 1000
- const configuredMs = intervalSec * 1000
- const ttlAwareMs = Number.isFinite(minObservedTtlMs)
- ? Math.min(configuredMs, minObservedTtlMs / 2)
- : configuredMs
- return Math.max(ttlAwareMs, floorMs)
- }
const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000
- const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
- const HOST_CACHE_RECOVERY_RETRY_MS = 5000
+ // Retry-after-failure pacing for the three hardened inventory lanes
+ // (McpServer/Context/Host) is exponential with full jitter, mirroring the
+ // client-go reflector standard. During a long apiserver outage (a GKE zonal
+ // control-plane upgrade: minutes of every watch down and every re-LIST
+ // failing) a fixed interval hammers the recovering apiserver, and — worse —
+ // the HCC's ~3-6 streams synchronize after a simultaneous cut and hit it in
+ // phase. delay = min(BASE * 2^(failures-1), CAP), then FULL jitter
+ // (random() * delay; AWS-style full jitter maximizes de-correlation across
+ // the streams, which is exactly the goal) floored at MIN so jitter can never
+ // reintroduce a 0-delay busy-loop. A successful recovery resets the ladder.
+ const WATCH_RECOVERY_BACKOFF_BASE_MS = 1000
+ const WATCH_RECOVERY_BACKOFF_CAP_MS = 30000
+ const WATCH_RECOVERY_BACKOFF_MIN_MS = WATCH_RECOVERY_BACKOFF_BASE_MS / 2
+ // Anti-busy-loop floor for IMMEDIATE watch-close recovery. The first re-LIST
+ // after an isolated watch close runs immediately (sub-second readiness blip
+ // instead of the old fixed ~5.5s), but a close arriving within this floor of
+ // the last successful recovery is demoted onto the paced retry timer above so
+ // a degraded apiserver churning its watches is never hammered with
+ // back-to-back LISTs.
+ const WATCH_CLOSE_RECOVERY_FLOOR_MS = 1000
const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]
// Wake-pending Hosts get immediate per-Host admission after watch recovery
// (§10.2 step 7) rather than waiting for the background fleet pass.
@@@ -624,8 -706,47 +758,48 @@@ export class McpServerWatcher implement
private hosts: Map<string, HostCRD> = new Map()
private contexts: Map<string, ContextCRD> = new Map()
private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()
+ private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()
private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()
+ private llmHooks: Map<string, LlmHookCRD> = new Map()
+ private mcpWatchGeneration = 0
+ private contextWatchGeneration = 0
+ private mcpServerDesiredRevision = 0
+ private contextDesiredRevision = 0
+ private hostDesiredRevision = 0
+ // Readiness covers authoritative revocation, not additive fleet completion.
+ // Generations bind the completed safety sweep to the exact LIST -> WATCH
+ // pair that supplied its absence decisions. A recovered watch invalidates
+ // this marker until the new inventory's orphan-allow sweep completes.
+ // Sentinel -1 = "no authoritative revocation has certified yet". A real
+ // desired revision starts at 0, so the readiness gate stays closed until the
+ // first recordNetworkPolicySafetyCertificate writes a real revision here —
+ // the startup role the (now removed) generation equalities used to play.
+ private networkPolicyRevocationContextRevision = -1
+ private networkPolicyRevocationServerRevision = -1
+ private mcpServerCacheSynced = false
+ private contextCacheSynced = false
+ private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
+ private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
+ private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null
+ private contextCacheRecoveryInFlight: Promise<boolean> | null = null
+ // Instant (Date.now) at which the last SUCCESSFUL recovery re-established
+ // each inventory watch. attempt*CacheRecovery compares against these to
+ // apply WATCH_CLOSE_RECOVERY_FLOOR_MS: a close spaced beyond the floor gets
+ // an immediate re-LIST; a burst close within it falls back to the timer.
+ private lastMcpServerWatchRecoveryAt = Number.NEGATIVE_INFINITY
+ private lastContextWatchRecoveryAt = Number.NEGATIVE_INFINITY
+ private lastHostWatchRecoveryAt = Number.NEGATIVE_INFINITY
+ // Consecutive recovery FAILURES per hardened lane, driving the exponential
+ // backoff ladder in schedule*CacheRecovery. Reset to 0 by every successful
+ // recovery so an isolated later close starts again at the base delay. The
+ // paired retryAfterMs (captured from an HTTP 429 Retry-After) overrides the
+ // computed delay for exactly the next scheduled retry.
+ private mcpServerWatchRecoveryFailures = 0
+ private mcpServerWatchRecoveryRetryAfterMs: number | undefined
+ private contextWatchRecoveryFailures = 0
+ private contextWatchRecoveryRetryAfterMs: number | undefined
+ private hostWatchRecoveryFailures = 0
+ private hostWatchRecoveryRetryAfterMs: number | undefined
// A watch callback can still settle after its stream reports completion.
// Host reconciles change the mcp-host pod template, so stale callbacks must
// never replay an older Host spec after a replacement watch is active.
@@@ -1624,45 -2462,47 +2522,72 @@@
* event.
*/
async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
- for (const server of this.servers.values()) {
- if (server.namespace !== secretNamespace) continue
- if (server.spec.envSecret?.name !== secretName) continue
- try {
- console.log(
- `[K8s] Re-reconciling McpServer "${server.name}" after Secret "${secretName}" change`
- )
- await this.reconciler.reconcile(server)
- } catch (err) {
- console.error(`[K8s] Secret-triggered reconcile failed for "${server.name}":`, err)
- }
- }
+ const inventoryGeneration = this.mcpWatchGeneration
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
+ const selected = [...this.servers.values()].filter(
+ server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName
+ )
+ await Promise.all(
+ selected.map(server =>
+ this.enqueueMcpServerReconciliation(server, async () => {
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
+ const current = this.servers.get(server.name)
+ if (
+ !current ||
+ current.namespace !== secretNamespace ||
+ current.spec.envSecret?.name !== secretName
+ ) {
+ return
+ }
+ try {
+ console.log(
+ `[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`
+ )
+ await this.reconciler.reconcile(current, {
+ isCurrent: () => {
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false
+ const latest = this.servers.get(current.name)
+ return (
+ latest !== undefined &&
+ latest.namespace === secretNamespace &&
+ latest.spec.envSecret?.name === secretName &&
+ sameMcpServerDesiredRevision(current, latest)
+ )
+ },
+ })
+ } catch (err) {
+ console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)
+ }
+ })
+ )
+ )
}
+ /**
+ * Trigger a reconcile for any cached image `LlmHook` whose `envSecret` matches
+ * the changed Secret (spec §8.2). A rotation (same name, new contents) leaves
+ * the pod key unchanged but re-stamps the credentials-revision on the pod
+ * template → rolling restart onto the new credential — mirroring
+ * `reconcileByEnvSecret` for McpServer. Driven by the llm-hooks SecretInformer.
+ */
+ async reconcileLlmHookByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
+ if (secretNamespace !== config.llmHooksNamespace) return
+ for (const hook of this.llmHooks.values()) {
+ if (hook.spec.target?.image?.envSecret !== secretName) continue
+ try {
+ console.log(
+ `[K8s] Re-reconciling LlmHook "${hook.name}" after Secret "${secretName}" change`
+ )
+ // Pod key is unchanged by a contents-only rotation, so pass it as the
+ // previous key (no stale-workload teardown) — the reconcile re-stamps the
+ // credentials-revision and rolls the shared pod.
+ await this.llmHookReconciler.reconcile(hook, computePodKey(hook))
+ } catch (err) {
+ console.error(`[K8s] Secret-triggered LlmHook reconcile failed for "${hook.name}":`, err)
+ }
+ }
+ }
+
/**
* Get all cached servers.
*/
@@@ -1889,29 -2704,6 +2789,27 @@@
)
}
+ // ── LlmHook initial load + reconciliation (guardrails phase-4) ──
+ // Runs AFTER the Host cache is populated so the shared hook pods' NetworkPolicy
+ // ingress reflects the current Host→LlmHook reverse index on the first pass.
++ // The GFS lane and the four pre-existing watches moved to dev's per-resource
++ // background lanes (startGlobalFileSystemBackgroundLane and friends); only the
++ // LlmHook lane is new here, so only it survives this merge.
+ try {
+ const initialHooks = await listAllLlmHooks()
+ for (const hook of initialHooks) {
+ this.llmHooks.set(hook.name, hook)
+ }
+ console.log('[K8s] Running initial LlmHook reconciliation...')
+ await this.llmHookReconciler.fullReconcile(initialHooks)
+ } catch (error) {
+ console.error(
+ '[K8s] Skipping initial LlmHook reconciliation because discovery failed:',
+ error
+ )
+ }
-
- await this.startMcpServerWatch()
- await this.startContextWatch()
- await this.startSharedFileSystemWatch()
- await this.startGlobalFileSystemWatch()
+ await this.startLlmHookWatch()
+
const resyncSec = config.hostResyncIntervalSec
if (resyncSec > 0) {
this.resyncTimer = setInterval(() => {
@@@ -2605,147 -3865,36 +3983,120 @@@
}
private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {
- if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return
- if (this.stopped) return
-
- const key = this.externalEgressRetryKey(server)
- if (this.externalEgressRetryTimers.has(key)) return
-
- const attempt = (this.externalEgressRetryAttempts.get(key) ?? 0) + 1
- const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
- if (!delayMs) {
- console.error(
- `[K8s] External egress retry exhausted for McpServer "${server.name}" in namespace "${server.namespace}"`
- )
- return
- }
-
- this.externalEgressRetryAttempts.set(key, attempt)
- console.warn(
- `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length} ` +
- `for McpServer "${server.name}" in ${delayMs}ms`
- )
-
- const timer = setTimeout(() => {
- this.externalEgressRetryTimers.delete(key)
- void this.retryExternalEgress(type, server)
- }, delayMs)
- this.externalEgressRetryTimers.set(key, timer)
+ this.externalEgressCoordinator.scheduleRetry(type, server)
}
- private async retryExternalEgress(type: string, server: McpServerCRD): Promise<void> {
- if (this.stopped) return
- try {
- if (type === 'DELETED') {
- await this.runExternalEgressOnce(type, server)
- } else {
- const current = this.servers.get(server.name)
- if (!current || current.namespace !== server.namespace) {
- this.clearExternalEgressRetry(server)
- return
- }
- await this.runExternalEgressOnce(type, current, { clearRetryOnSuccess: false })
- try {
- await this.reconciler.reconcile(current)
- this.clearExternalEgressRetry(current)
- } catch (error) {
- console.error(
- `[K8s] Runtime reconciliation after external egress retry failed for ${server.name}:`,
- error
- )
- this.scheduleExternalEgressRetry(type, current)
- }
+ private async performExternalEgressMutation(
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ options: {
+ deleteAllowed?: () => Promise<boolean>
+ isCurrent: () => boolean
+ }
+ ): Promise<void> {
+ if (type === 'DELETED') {
+ const deleteAllowed = async (): Promise<boolean> => {
+ if (!options.isCurrent()) return false
+ if (options.deleteAllowed && !(await options.deleteAllowed())) return false
+ return options.isCurrent()
}
- } catch (error) {
- console.error(`[K8s] External egress retry failed for ${server.name}:`, error)
- this.scheduleExternalEgressRetry(type, server)
+ await this.netPolReconciler.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ undefined,
+ deleteAllowed
+ )
+ return
}
+ await this.netPolReconciler.reconcileExternalEgress(server, {
+ isCurrent: options.isCurrent,
+ })
}
+ private getLlmHookWatchCallback(): (
+ type: string,
+ apiObj: {
+ metadata: {
+ name: string
+ namespace?: string
+ uid?: string
+ generation?: number
+ annotations?: Record<string, string>
+ labels?: Record<string, string>
+ }
+ spec: LlmHookSpec
+ status?: LlmHookCRD['status']
+ }
+ ) => Promise<void> {
+ return async (type, apiObj) => {
+ const hook: LlmHookCRD = {
+ name: apiObj.metadata.name,
+ namespace: apiObj.metadata.namespace || config.llmHooksNamespace,
+ uid: apiObj.metadata.uid,
+ generation: apiObj.metadata.generation,
+ annotations: apiObj.metadata.annotations,
+ labels: apiObj.metadata.labels,
+ spec: apiObj.spec,
+ status: apiObj.status,
+ }
+
+ console.log(`[K8s] LlmHook watch event: ${type} for ${hook.name}`)
+
+ // Compute the pod key the CR had BEFORE this event so an image bump can
+ // chain teardown of the old pod key with ensure of the new one (§4), and
+ // a delete can GC the workload the CR was a member of.
+ const previous = this.llmHooks.get(hook.name)
+ const previousPodKey = previous ? computePodKey(previous) : null
+
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ this.llmHooks.set(hook.name, hook)
+ } else if (type === 'DELETED') {
+ this.llmHooks.delete(hook.name)
+ }
+
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.llmHookReconciler.reconcile(hook, previousPodKey)
+ } else if (type === 'DELETED') {
+ await this.llmHookReconciler.reconcileDelete(hook.name, previousPodKey)
+ }
+ } catch (error) {
+ console.error(`[K8s] LlmHook reconciliation failed for ${hook.name}:`, error)
+ }
+ }
+ }
+
+ /**
+ * Start watching LlmHook CRDs in the llm-hooks namespace.
+ */
+ private async startLlmHookWatch(): Promise<void> {
+ const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.llmHooksNamespace}/${PLURAL_LLMHOOKS}`
+ console.log(`[K8s] Starting LlmHook watch`)
+
+ const watchCallback = this.getLlmHookWatchCallback()
+
+ const doneCallback = (err: Error | null) => {
+ if (this.stopped) return
+ if (err) {
+ console.error('[K8s] LlmHook watch error:', err)
+ }
+ console.log('[K8s] LlmHook watch ended, restarting...')
+ setTimeout(() => this.startLlmHookWatch(), err ? 5000 : 1000)
+ }
+
+ this.llmHookWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
+ }
+
+ /** Periodic LlmHook resync: full reconcile drives the orphan sweep (§3). */
+ private async runLlmHookResync(): Promise<void> {
+ if (this.stopped) return
+ try {
+ await this.llmHookReconciler.fullReconcile([...this.llmHooks.values()])
+ } catch (error) {
+ console.error('[K8s] LlmHook periodic resync failed:', error)
+ }
+ }
+
/**
* Start watching Context CRDs for NetworkPolicy reconciliation.
*/
@@@ -2872,15 -4149,19 +4351,23 @@@
this.clearHostWatchRetry(host.name)
const eventRevision = ++this.hostWatchRevision
this.latestHostWatchEventRevisions.set(host.name, eventRevision)
- // Capture the guardrail hook ids the Host referenced BEFORE this event so
- // a removed reference re-reconciles the (now smaller) NetworkPolicy
- // ingress set for the affected hook pod keys (Host→LlmHook reverse index).
- const previousHostForHooks = this.hosts.get(host.name)
+ // Advance hostDesiredRevision only on a real desired-state change (the
+ // content identity the mutation-authority fence reads), separate from the
+ // per-event hostWatchRevision counter above.
+ const previousHost = this.hosts.get(host.name)
+ let hostDesiredStateChanged = false
++ // Same pre-event snapshot, aliased for the Host→LlmHook reverse index: a
++ // removed reference must re-reconcile the (now smaller) NetworkPolicy
++ // ingress set for the affected hook pod keys.
++ const previousHostForHooks = previousHost
if (eventType === 'ADDED' || eventType === 'MODIFIED') {
+ hostDesiredStateChanged =
+ previousHost === undefined || !this.sameHostDesiredRevision(previousHost, host)
this.hosts.set(host.name, host)
} else {
- this.hosts.delete(host.name)
+ hostDesiredStateChanged = this.hosts.delete(host.name)
}
+ if (hostDesiredStateChanged) this.hostDesiredRevision += 1
try {
// Direct dispatch (§10.3): reconcileHostWatchEvent enters the per-Host
diff --cc host-context-controller/src/mcpServerSafety.ts
index 000000000,7f8d8438d..5b15bda16
mode 000000,100644..100644
--- a/host-context-controller/src/mcpServerSafety.ts
+++ b/host-context-controller/src/mcpServerSafety.ts
@@@ -1,0 -1,91 +1,93 @@@
+ import type { McpServerCRD } from './types'
+ import { getErrorCode } from './utils'
+
+ const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'
+
+ /**
+ * This acknowledgement is written by HCC after it has applied a server's
+ * NetworkPolicies so WRC can admit its workload. It is not user intent and it
+ * cannot change any NetworkPolicy decision. Treating that write as desired
+ * state makes HCC invalidate its own authoritative revocation pass under a
+ * large startup fleet, coupling /ready to additive runtime convergence again.
+ */
-function desiredAnnotations(annotations: Record<string, string> | undefined): Record<string, string> {
++function desiredAnnotations(
++ annotations: Record<string, string> | undefined
++): Record<string, string> {
+ if (!annotations?.[NETWORK_READY_ANNOTATION]) return annotations ?? {}
+ const { [NETWORK_READY_ANNOTATION]: _networkReady, ...remaining } = annotations
+ return remaining
+ }
+
+ /**
+ * Kubernetes represents objects such as labels and annotations as maps, whose
+ * wire key order is not meaningful or stable across LIST and WATCH payloads.
+ * Preserve array order (it is semantic) while sorting object keys so a
+ * status-only event cannot masquerade as a desired-state revision.
+ */
+ function stableJson(value: unknown): string {
+ if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`
+ if (value && typeof value === 'object') {
+ const record = value as Record<string, unknown>
+ return `{${Object.keys(record)
+ .sort()
+ .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+ .join(',')}}`
+ }
+ return JSON.stringify(value) ?? 'undefined'
+ }
+
+ export function sameMcpServerDesiredRevision(
+ expected: McpServerCRD,
+ current: McpServerCRD
+ ): boolean {
+ if (expected.name !== current.name || expected.namespace !== current.namespace) return false
+ if (expected.uid && current.uid && expected.uid !== current.uid) return false
+ if (
+ expected.generation !== undefined &&
+ current.generation !== undefined &&
+ expected.generation !== current.generation
+ ) {
+ return false
+ }
+
+ return (
+ stableJson(expected.spec) === stableJson(current.spec) &&
+ stableJson(desiredAnnotations(expected.annotations)) ===
+ stableJson(desiredAnnotations(current.annotations)) &&
+ stableJson(expected.labels ?? {}) === stableJson(current.labels ?? {})
+ )
+ }
+
+ export function isMcpServerStatusOnlyUpdate(
+ previous: McpServerCRD,
+ current: McpServerCRD
+ ): boolean {
+ return (
+ previous.uid === current.uid &&
+ previous.generation === current.generation &&
+ sameMcpServerDesiredRevision(previous, current)
+ )
+ }
+
+ export interface AuthoritativeMcpServerAbsenceCheck {
+ inventoryAuthoritative: () => boolean
+ resolveCurrent: () => McpServerCRD | undefined
+ readCurrent: () => Promise<unknown>
+ }
+
+ export async function confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative,
+ resolveCurrent,
+ readCurrent,
+ }: AuthoritativeMcpServerAbsenceCheck): Promise<boolean> {
+ if (!inventoryAuthoritative() || resolveCurrent()) return false
+
+ try {
+ await readCurrent()
+ return false
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) throw error
+ }
+
+ return inventoryAuthoritative() && !resolveCurrent()
+ }
diff --cc host-context-controller/src/networkPolicyReconciler.test.ts
index 8f3d3f3e1,047ccd88e..463bc546f
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@@ -355,6 -759,82 +759,86 @@@ describe('NetworkPolicyReconciler', ()
// ─── L2: Context-Based Policies ────────────────────────────────────
describe('L2 — reconcileContext', () => {
+ it('refuses to adopt a foreign policy that collides with a live Context policy name', async () => {
+ const server: McpServerCRD = {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: [server.name] },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ctx-dev-mongo',
+ namespace: 'mcp-server',
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'dev',
+ 'clerum.io/mcpserver': 'mongo',
+ },
+ },
+ })
+
- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
++ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(
++ /ownership/i
++ )
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses to replace an HCC Context policy owned by a different Context/server pair', async () => {
+ const server: McpServerCRD = {
+ name: 'b-c',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'a',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
+ const context: ContextCRD = {
+ name: 'a',
+ namespace: 'mcp-server',
+ spec: { contextId: 'a', mcpServers: ['b-c'] },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ctx-a-b-c',
+ namespace: 'mcp-server',
+ uid: 'other-hcc-owner-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'a-b',
+ 'clerum.io/mcpserver': 'c',
+ },
+ },
+ })
+
- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
++ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(
++ /ownership/i
++ )
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('creates context-allow ingress policy per server in mcp-server namespace', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
@@@ -566,6 -1292,118 +1296,122 @@@
// ─── L3: External Egress ───────────────────────────────────────────
describe('L3 — reconcileExternalEgress', () => {
+ it('removes a deterministic safety policy created after its desired-state fence is lost', async () => {
+ const server: McpServerCRD = {
+ name: 'race-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'race:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ const createStarted = deferred()
+ const releaseCreate = deferred()
+ let current = true
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ createStarted.resolve(undefined)
+ await releaseCreate.promise
+ return {
+ metadata: {
+ name: 'ext-egress-race-server-104-18-0-0-16-443',
+ namespace: server.namespace,
+ uid: 'created-race-policy',
+ resourceVersion: '7',
+ },
+ }
+ })
+
+ const safety = (reconciler as any).reconcileExternalEgressSafety(
+ server,
+ [{ metadata: { name: 'ext-egress-race-server-stale' } }],
+ () => current
+ ) as Promise<boolean>
+
+ await createStarted.promise
+ current = false
+ releaseCreate.resolve(undefined)
+
+ expect(await safety).toBe(false)
+ const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: created.metadata.name,
+ namespace: server.namespace,
+ body: { preconditions: { uid: 'created-race-policy', resourceVersion: '7' } },
+ })
+ )
+ })
+
+ it('refuses to adopt a foreign policy that collides with an external-egress name', async () => {
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
+ namespace: 'mcp-server',
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses to replace an HCC external-egress policy owned by a different server', async () => {
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
+ namespace: 'mcp-server',
+ uid: 'other-hcc-owner-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'different-server',
+ },
+ },
+ })
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('creates egress policy for CIDR binding', async () => {
const server: McpServerCRD = {
name: 'openai-mcp',
@@@ -677,15 -1549,202 +1557,208 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('status read denied')
-
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- })
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- it('creates explicit public-web egress with private and special-use ranges excluded', async () => {
- const server: McpServerCRD = {
- name: 'search-mcp',
+ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
+ expect(patch.slice(0, 3)).toEqual([
+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
+ { op: 'test', path: '/metadata/generation', value: 7 },
+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
+ ])
+ expect(patch).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: '/status/conditions',
+ value: expect.arrayContaining([
+ expect.objectContaining({
+ type: 'RuntimeReady',
+ reason: 'DeploymentAvailable',
+ }),
+ expect.objectContaining({
+ type: 'ExternalEgressReady',
+ status: 'True',
+ observedGeneration: 7,
+ }),
+ ]),
+ }),
+ ])
+ )
+ })
+
+ it('fences every identity field read from status when synthetic input omits them', async () => {
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: {
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-17',
+ },
+ status: {},
+ })
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+
+ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
+ expect(patch.slice(0, 3)).toEqual([
+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
+ { op: 'test', path: '/metadata/generation', value: 7 },
+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
+ ])
+ })
+
+ it.each([
+ {
+ name: 'UID',
+ currentMetadata: {
+ uid: 'replacement-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-18',
+ },
+ },
+ {
+ name: 'generation',
+ currentMetadata: {
+ uid: 'openai-mcp-uid',
+ generation: 8,
+ resourceVersion: 'status-rv-18',
+ },
+ },
+ ])(
+ 'rejects a stale $name before patching external egress status',
+ async ({ currentMetadata }) => {
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: currentMetadata,
+ status: {},
+ })
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/stale McpServer/i)
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/stale McpServer/i)
+
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ }
+ )
+
+ it('fails closed when a same-generation status writer wins the resourceVersion race', async () => {
+ const conflict = Object.assign(new Error('JSON patch test failed'), { code: 409 })
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: {
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-17',
+ },
+ status: {
+ conditions: [
+ {
+ type: 'RuntimeReady',
+ status: 'True',
+ reason: 'DeploymentAvailable',
+ lastTransitionTime: '2026-07-29T00:00:00.000Z',
+ },
+ ],
+ },
+ })
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toBe(conflict)
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toBe(conflict)
+
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.arrayContaining([
+ {
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: 'status-rv-17',
+ },
+ ]),
+ })
+ )
+ })
+
+ it('fails closed when ExternalEgressReady status cannot be written', async () => {
+ const statusError = new Error('status update denied')
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
- 'status update denied'
- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow('status update denied')
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
+ })
+
+ it('fails closed when ExternalEgressReady status cannot be read', async () => {
+ const statusError = new Error('status read denied')
+ mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('status read denied')
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow('status read denied')
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
+
+ it('creates explicit public-web egress with private and special-use ranges excluded', async () => {
+ const server: McpServerCRD = {
+ name: 'search-mcp',
namespace: 'mcp-server',
spec: {
contextRef: 'dev',
@@@ -737,9 -1796,9 +1810,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /public-web external egress bindings/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/public-web external egress bindings/)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@@ -755,9 -1814,9 +1828,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /public-web external egress bindings/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/public-web external egress bindings/)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@@ -1219,9 -2278,9 +2292,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /External egress reconciliation failed/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/External egress reconciliation failed/)
}
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@@ -1240,9 -2299,9 +2313,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /resolved disallowed address/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/resolved disallowed address/)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
@@@ -1275,9 -2334,9 +2348,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /private, internal, metadata/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/private, internal, metadata/)
expect(dns.resolve4).not.toHaveBeenCalled()
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@@ -1296,9 -2355,9 +2369,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /resolved disallowed address/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/resolved disallowed address/)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@@ -1351,7 -2412,7 +2426,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('delete denied')
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('delete denied')
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow('delete denied')
})
it('deletes stale policies and fails when DNS resolution for that binding fails', async () => {
@@@ -1374,17 -2435,21 +2451,23 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/dns timeout/)
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/dns timeout/)
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/dns timeout/)
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-api.openai.com-443',
- namespace: 'mcp-server',
- })
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-old-example-com-443',
- namespace: 'mcp-server',
- })
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-openai-mcp-old-example-com-443',
+ namespace: 'mcp-server',
+ })
+ )
expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
@@@ -1415,9 -2480,9 +2498,9 @@@
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
-- /must declare dns or cidr/
-- )
++ await expect(
++ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
++ ).rejects.toThrow(/must declare dns or cidr/)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@@ -1590,5 -5243,467 +5261,474 @@@
reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')
).resolves.toBeUndefined()
})
+
+ it('does not report a revocation when the policy was already absent', async () => {
+ const error = Object.assign(new Error('not found'), { code: 404 })
+ const onDeleted = vi.fn()
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [{ metadata: { name: 'ext-egress-openai-mcp-old-api-443' } }],
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(error)
+
+ await reconciler.cleanupExternalEgress(
+ 'openai-mcp',
+ 'mcp-server',
+ undefined,
+ undefined,
+ onDeleted
+ )
+
+ expect(onDeleted).not.toHaveBeenCalled()
+ })
+ })
+
+ describe('scoped delta certification vs. a stuck safety inventory', () => {
+ const ambiguousReservedPolicy: k8s.V1NetworkPolicy = {
+ metadata: { name: 'ctx-reserved-context-server', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ const deltaContext: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+
+ function stickSafetyInventory(): void {
+ // Only the namespace-wide safety inventory sees the foreign policy. The
+ // label-scoped LISTs the delta lane uses stay clean, which is exactly why
+ // the delta cannot vouch for the namespace.
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [ambiguousReservedPolicy] : [],
+ })
+ )
+ }
+
+ it('refuses to certify a scoped delta while the safety pass is stuck on its inventory', async () => {
+ stickSafetyInventory()
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+
+ // The delta's own revocation still completes — its label-scoped LISTs are
+ // clean. What it may not do is vouch for the namespace-wide inventory,
+ // which the readiness lane checks separately.
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+
+ it('certifies a scoped delta again once a later safety pass completes', async () => {
+ stickSafetyInventory()
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: vi.fn(),
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ const onAuthoritativeRevocationComplete = vi.fn()
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
+ })
+
+ it('does not certify the safety inventory until an authoritative pass has run', async () => {
+ // Fail-closed default: before any authoritative safety pass certifies, the
+ // namespace-wide inventory is NOT certified, even though the scoped delta's
+ // own label-scoped revocation completes (returns true). A delta cannot
+ // vouch for the namespace-wide inventory it never enumerated.
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+ })
+
+ describe('lost delete fence during stale-allow revocation', () => {
+ const staleContext: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+ const stalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-default-alpha',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'default',
+ 'clerum.io/mcpserver': 'alpha',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ const lostFence = Object.assign(new Error('the UID in the precondition does not match'), {
+ code: 409,
+ })
+
+ function listOnlyTheStalePolicy(): void {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: namespace === 'mcp-server' ? [stalePolicy] : [],
+ })
+ )
+ }
+
+ it('refuses to certify a scoped delta instead of throwing the lost fence', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
+
+ // 409 is what Kubernetes returns when the uid/resourceVersion
+ // preconditions no longer match: the object live under that name is not
+ // the one this pass classified. Nothing was revoked, so the delta must
+ // decline to certify — and it must not blow up the caller, which would
+ // only force a spurious full re-convergence.
+ // The scoped delta lane reads this boolean and withholds the certificate,
+ // so it is the one caller allowed to take the lost fence as an outcome
+ // rather than a throw.
- expect(await reconciler.reconcileContext(staleContext, { honorsLostFence: true, isCurrent: () => true })).toBe(false)
++ expect(
++ await reconciler.reconcileContext(staleContext, {
++ honorsLostFence: true,
++ isCurrent: () => true,
++ })
++ ).toBe(false)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'ctx-default-alpha', namespace: 'mcp-server' })
+ )
+ })
+
+ it('withdraws certification when an additive revocation loses the fence after certifying', async () => {
+ // The additive phase re-runs reconcileContext, which carries a second
+ // revocation lane over a fresh LIST. Losing the fence there happens after
+ // the pass already certified, and the outcome recorder is first-wins — so
+ // the duration sample stays 'completed', which is right, but the safety
+ // fence must still degrade. It is an assertion about live allows, and it
+ // may only ever move toward unsafe.
+ const additiveContext: ContextCRD = {
+ name: 'additive',
+ namespace: 'mcp-server',
+ spec: { contextId: 'additive', mcpServers: [] },
+ }
+ const additiveStalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-additive-alpha',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'additive',
+ 'clerum.io/mcpserver': 'alpha',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ // Self-synchronising: the stale policy only becomes visible once the pass
+ // has already certified, so the safety phase is clean by construction and
+ // only the additive re-LIST finds work.
+ let certified = false
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: certified && namespace === 'mcp-server' ? [additiveStalePolicy] : [],
+ })
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ certified = true
+ })
+
+ await expect(
+ reconciler.fullReconcile([additiveContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow()
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+
+ it('marks the inventory uncertified the moment the authoritative pass loses the fence', async () => {
+ // The doom must be visible mid-pass, not only after the unwind. A lost
+ // fence is the one doom cause that does not bump a watch generation, so
+ // it is the only one where the whole certification machinery stays green
+ // while the pass is already condemned. A scoped delta interleaving in
+ // that window would certify readiness over a stale allow.
+ //
+ // Two contexts, each with its own stale policy: the first loses the fence
+ // and the second's delete gives an observation point while the pass is
+ // condemned but still unwinding.
+ const secondContext: ContextCRD = {
+ name: 'second',
+ namespace: 'mcp-server',
+ spec: { contextId: 'second', mcpServers: [] },
+ }
+ const secondStalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-second-beta',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'second',
+ 'clerum.io/mcpserver': 'beta',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: namespace === 'mcp-server' ? [stalePolicy, secondStalePolicy] : [],
+ })
+ )
+ let certifiedDuringUnwind: boolean | undefined
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ if (
+ certifiedDuringUnwind === undefined &&
+ mockApi.deleteNamespacedNetworkPolicy.mock.calls.length > 1
+ ) {
+ certifiedDuringUnwind = reconciler.hasCertifiedSafetyInventory()
+ return {}
+ }
+ throw lostFence
+ })
+
+ await expect(
+ reconciler.fullReconcile([staleContext, secondContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: vi.fn(),
+ })
+ ).rejects.toThrow()
+
+ expect(certifiedDuringUnwind).toBe(false)
+ })
+
+ it('fails the additive phase when it loses the delete fence', async () => {
+ listOnlyTheStalePolicy()
+ // Safety phase revokes cleanly and certifies; the additive phase then
+ // races another writer and loses the fence. The additive caller discards
+ // reconcileContext's boolean, so swallowing the 409 there would record
+ // the pass as a success, disarm the convergence retry, and leave a stale
+ // allow live with no NetworkPolicy resync to recover it.
+ mockApi.deleteNamespacedNetworkPolicy
+ .mockResolvedValueOnce({})
+ .mockRejectedValueOnce(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([staleContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow()
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('refuses to certify the safety pass instead of throwing the lost fence', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([staleContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Desired NetworkPolicy inventory changed/)
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('still fails loud when the delete fails for any other reason', async () => {
+ listOnlyTheStalePolicy()
+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
+
- await expect(reconciler.reconcileContext(staleContext, { isCurrent: () => true })).rejects.toBe(forbidden)
++ await expect(
++ reconciler.reconcileContext(staleContext, { isCurrent: () => true })
++ ).rejects.toBe(forbidden)
+ })
+
+ it('still certifies when the stale policy was already gone', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('not found'), { code: 404 })
+ )
+
+ expect(await reconciler.reconcileContext(staleContext, { isCurrent: () => true })).toBe(true)
+ })
+ })
+
+ describe('same-name conflict creating a safety external-egress policy', () => {
+ const server: McpServerCRD = {
+ name: 'renamed-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'renamed:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ const desiredName = 'ext-egress-renamed-server-104-18-0-0-16-443'
+ const stalePredecessor: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ext-egress-renamed-server-stale',
+ namespace: 'mcp-server',
+ uid: 'stale-uid',
+ resourceVersion: '3',
+ },
+ }
+
+ function reconcileExternalEgressSafety(): Promise<boolean> {
+ return (
+ reconciler as unknown as {
+ reconcileExternalEgressSafety: (
+ server: McpServerCRD,
+ existingPolicies: k8s.V1NetworkPolicy[],
+ isCurrent: () => boolean
+ ) => Promise<boolean>
+ }
+ ).reconcileExternalEgressSafety(server, [stalePredecessor], () => true)
+ }
+
+ it('converges an HCC-owned same-name policy instead of aborting the pass', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: desiredName,
+ namespace: server.namespace,
+ uid: 'own-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
+ // This is the only creation route that bypasses applyNetworkPolicy. A
+ // same-name object HCC already owns is an idempotent re-run, not a
+ // reason to fail the pass and hold readiness down.
+ expect(await reconcileExternalEgressSafety()).toBe(true)
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: desiredName, namespace: server.namespace })
+ )
+ })
+
+ it('still fails closed when the same-name policy is foreign', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: desiredName,
+ namespace: server.namespace,
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
+ await expect(reconcileExternalEgressSafety()).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('still fails loud when the create fails for any other reason', async () => {
+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
+
+ await expect(reconcileExternalEgressSafety()).rejects.toBe(forbidden)
+ })
+ })
+
+ describe('safety-pass duration on the uncertified return path', () => {
+ it('samples a failed outcome when the pass returns without certifying', async () => {
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ // Losing inventory authority ends the pass without an exception and
+ // without certifying. Operators alert on "the pass never completed", so
+ // this path has to produce a `failed` sample — absence is not a signal.
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => false,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).resolves.toBeUndefined()
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'failed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
+
+ it('keeps the completed sample when a certified pass fails an additive effect afterwards', async () => {
+ const context: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+ const additiveFailure = new Error('additive list unavailable')
+ let certified = false
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ certified = true
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {
+ if (certified) throw additiveFailure
+ return { items: [] }
+ })
+
+ await expect(
+ reconciler.fullReconcile([context], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/additive Context NetworkPolicy reconciliations failed/)
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'completed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
})
})
diff --cc host-context-controller/test/__fixtures__/testMocks.ts
index 73a8bae2c,73a8bae2c..8969a0df2
--- a/host-context-controller/test/__fixtures__/testMocks.ts
+++ b/host-context-controller/test/__fixtures__/testMocks.ts
@@@ -220,9 -220,9 +220,18 @@@ export function createMockCustomApi():
export function createMockNetworkingApi(): MockNetworkingApi {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
++ // name + uid + resourceVersion because that is what a real apiserver read
++ // returns, and the NetworkPolicy safety mutations (safetyPolicyIdentity)
++ // refuse to act on a partial identity — a delete keyed on name alone could
++ // hit a policy recreated between the read and the write.
readNamespacedNetworkPolicy: vi.fn(({ name }: { name?: string } = {}) =>
Promise.resolve({
-- metadata: { resourceVersion: '1', labels: hccOwnedLabels(name) },
++ metadata: {
++ name,
++ uid: `uid-${name ?? 'unnamed'}`,
++ resourceVersion: '1',
++ labels: hccOwnedLabels(name),
++ },
} as MockK8sResource)
),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
diff --cc mcp-host/package-lock.json
index b31d53fd6,6353d567d..0d8b61fd5
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@@ -1,12 -1,12 +1,12 @@@
{
"name": "clerum-mcp-host",
- "version": "0.1.268",
- "version": "0.1.243",
++ "version": "0.1.269",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.268",
- "version": "0.1.243",
++ "version": "0.1.269",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --cc mcp-host/package.json
index adb355351,03504a1f5..9639271fd
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@@ -1,6 -1,6 +1,6 @@@
{
"name": "clerum-mcp-host",
- "version": "0.1.268",
- "version": "0.1.243",
++ "version": "0.1.269",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --cc mcp-host/src/main.ts
index b5733f88c,92fefb858..20686d117
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@@ -78,6 -75,6 +78,7 @@@ import
replaceAuthoritativeMcpFleet,
runAuthoritativeMcpInitialization,
} from './mcp/authoritativeFleet'
++import { McpStatusHeartbeat } from './mcp/statusHeartbeat'
import { startMcpInitializationInBackground } from './mcpBackgroundInit'
import { IncomingMessageHandler, PendingTaskEntry } from './messageHandler'
import {
@@@ -168,15 -165,9 +169,15 @@@ let currentPolicy: LlmPolicy | null = n
let failoverEngine: FailoverEngine | null = null
let bootFallbackEntry: FallbackEntry | null = null
let hostWatcher: HostWatcher | null = null
+let llmHookWatcher: LlmHookWatcher | null = null
let contextMapperPollTimer: ReturnType<typeof setInterval> | null = null
+// Periodic guardrail re-resolve — a backstop for dropped LlmHook/Host watch
+// events (the LlmHookWatcher is the primary, immediate path). Host edits also
+// apply immediately via onHostChange.
+let guardrailResolveTimer: ReturnType<typeof setInterval> | null = null
+const GUARDRAIL_RESOLVE_INTERVAL_MS = 300_000
let contextMapperPollRunner: { trigger(): void; stop(): void } | null = null
- let mcpStatusHeartbeatTimer: ReturnType<typeof setInterval> | null = null
+ let mcpStatusHeartbeat: McpStatusHeartbeat | null = null
let lastServerState: Map<string, string> = new Map()
let rpcServer: RPCServer | null = null
let mcpManager: McpManager | null = null
diff --cc tests/e2e/integration/network-policies.test.ts
index aba9302b2,07925e9db..73e9eddce
--- a/tests/e2e/integration/network-policies.test.ts
+++ b/tests/e2e/integration/network-policies.test.ts
@@@ -141,8 -179,32 +179,32 @@@ function testConnectivity
}
describe('NetworkPolicy Enforcement (Calico)', () => {
+ beforeAll(() => {
+ clusterReachable = probeClusterReachable()
+ if (clusterReachable) return
+
+ const msg =
+ '[network-policies] kubernetes cluster not reachable (kubectl get ns default failed)'
+ if (REQUIRE_CLUSTER) {
+ throw new Error(
+ `${msg}. Refusing to skip: E2E_REQUIRE_CLUSTER/CI is set, and a silent skip here ` +
+ 'would report success while asserting nothing about NetworkPolicy enforcement. ' +
+ 'Bring up the cluster (e.g. `make minikube-setup`) or unset the flag.'
+ )
+ }
+ if (SKIP_IF_UNREACHABLE) {
+ console.log(`${msg} — tests will be skipped (E2E_SKIP_IF_CLUSTER_UNREACHABLE=1)`)
+ return
+ }
+ throw new Error(
+ `${msg}. This E2E suite requires a minikube cluster with Calico CNI and all services ` +
+ 'deployed. Run `make minikube-setup` first, or set E2E_SKIP_IF_CLUSTER_UNREACHABLE=1 to skip.'
+ )
+ })
+
// ─── Prerequisite Check ──────────────────────────────────────────────
- it('Calico CNI is active', () => {
- it('Calico CNI is active', (ctx) => {
++ it('Calico CNI is active', ctx => {
+ if (!clusterReachable) ctx.skip()
const pods = execFileSync(
'kubectl',
['get', 'pods', '-n', 'kube-system', '-l', 'k8s-app=calico-node', '--no-headers'],
@@@ -151,7 -213,8 +213,8 @@@
expect(pods).toContain('Running')
})
- it('deny-all policies exist in secured namespaces', () => {
- it('deny-all policies exist in secured namespaces', (ctx) => {
++ it('deny-all policies exist in secured namespaces', ctx => {
+ if (!clusterReachable) ctx.skip()
const namespaces = ['mcp-host', 'mcp-server', 'rpc-proxy', 'sandbox-recipes']
for (const ns of namespaces) {
let nps: string
@@@ -184,7 -247,8 +247,8 @@@
// ─── POSITIVE TESTS (Allowed Traffic) ────────────────────────────────
describe('Allowed traffic (positive)', () => {
- it('control-api → registry-api:8085 (cross-namespace, registry access)', () => {
- it('control-api → registry-api:8085 (cross-namespace, registry access)', (ctx) => {
++ it('control-api → registry-api:8085 (cross-namespace, registry access)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-api',
@@@ -193,7 -257,8 +257,8 @@@
expect(result.reachable).toBe(true)
})
- it('control-ui → control-api:8090 (UI backend path)', () => {
- it('control-ui → control-api:8090 (UI backend path)', (ctx) => {
++ it('control-ui → control-api:8090 (UI backend path)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-ui',
@@@ -202,7 -267,8 +267,8 @@@
expect(result.reachable).toBe(true)
})
- it('external-rest-api → profile-control-funnel:8080 (internal control path)', () => {
- it('external-rest-api → profile-control-funnel:8080 (internal control path)', (ctx) => {
++ it('external-rest-api → profile-control-funnel:8080 (internal control path)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'profiles',
'external-rest-api',
@@@ -211,12 -277,14 +277,14 @@@
expect(result.reachable).toBe(true)
})
- it('rpc-proxy → mcp-host:8080 (message relay)', () => {
- it('rpc-proxy → mcp-host:8080 (message relay)', (ctx) => {
++ it('rpc-proxy → mcp-host:8080 (message relay)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('rpc-proxy', 'rpc-proxy', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(true)
})
- it('mcp-host can reach the HCC gateway readiness endpoint', () => {
- it('mcp-host can reach the HCC gateway readiness endpoint', (ctx) => {
++ it('mcp-host can reach the HCC gateway readiness endpoint', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@@ -230,7 -298,8 +298,8 @@@
// ─── NEGATIVE TESTS (Blocked by Design) ──────────────────────────────
describe('Blocked traffic (negative — deny-all enforced)', () => {
- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', () => {
- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', (ctx) => {
++ it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@@ -240,7 -309,8 +309,8 @@@
expect(result.error).toContain('timeout')
})
- it('mcp-host → control-api:8090 (no direct access to control-plane)', () => {
- it('mcp-host → control-api:8090 (no direct access to control-plane)', (ctx) => {
++ it('mcp-host → control-api:8090 (no direct access to control-plane)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@@ -250,7 -320,8 +320,8 @@@
expect(result.error).toContain('timeout')
})
- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', () => {
- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', (ctx) => {
++ it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', ctx => {
+ if (!clusterReachable) ctx.skip()
const podName = getChannelReaderPodName()
if (!podName) {
console.warn('SKIPPED: no running channel-reader pod in channels')
@@@ -266,7 -337,8 +337,8 @@@
expect(result.error).toContain('timeout')
})
- it('mcp-host → registry-api:8085 (no direct registry access)', () => {
- it('mcp-host → registry-api:8085 (no direct registry access)', (ctx) => {
++ it('mcp-host → registry-api:8085 (no direct registry access)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@@ -276,7 -348,8 +348,8 @@@
expect(result.error).toContain('timeout')
})
- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', () => {
- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', (ctx) => {
++ it('control-api → rpc-proxy:8094 (no direct control-plane hop)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-api',
@@@ -286,7 -359,8 +359,8 @@@
expect(result.error).toContain('timeout')
})
- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', () => {
- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', (ctx) => {
++ it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'profiles',
'external-rest-api',
@@@ -296,19 -370,22 +370,22 @@@
expect(result.error).toContain('timeout')
})
- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
++ it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('control-plane', 'control-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
++ it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('profiles', 'external-rest-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('mcp-proxy → external-rest-api:3000 (no reverse path)', () => {
- it('mcp-proxy → external-rest-api:3000 (no reverse path)', (ctx) => {
++ it('mcp-proxy → external-rest-api:3000 (no reverse path)', ctx => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-server',
'mcp-proxy',
@@@ -348,7 -425,8 +425,8 @@@
}
}
- it('sandbox-recipes → control-api:8090 (isolated sandbox)', () => {
- it('sandbox-recipes → control-api:8090 (isolated sandbox)', (ctx) => {
++ it('sandbox-recipes → control-api:8090 (isolated sandbox)', ctx => {
+ if (!clusterReachable) ctx.skip()
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
@@@ -364,7 -442,8 +442,8 @@@
expect(result.error).toContain('timeout')
})
- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', () => {
- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', (ctx) => {
++ it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', ctx => {
+ if (!clusterReachable) ctx.skip()
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
diff --cc tests/e2e/package-lock.json
index 558c31760,558c31760..5ace9758b
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@@ -1,12 -1,12 +1,12 @@@
{
"name": "clerum-e2e-tests",
-- "version": "0.1.66",
++ "version": "0.1.67",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
-- "version": "0.1.66",
++ "version": "0.1.67",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --cc tests/e2e/package.json
index 19145d619,19145d619..6770e4a49
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@@ -1,6 -1,6 +1,6 @@@
{
"name": "clerum-e2e-tests",
-- "version": "0.1.66",
++ "version": "0.1.67",
"private": true,
"type": "module",
"scripts": {