Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
Reapply PR #205 (HCC readiness decoupling)
Alfredo Lopez5d2ea4c78619
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 18daf933c..08541e5bc 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -390,6 +390,21 @@ jobs:- name: Test HCC watch-recovery gate coordinationrun: bash scripts/tests/test-hcc-watch-recovery-gate.sh+ - name: Test HCC API gateway readiness separation+ run: bash scripts/tests/test-hcc-api-gateway-readiness.sh++ - name: Test host-storm metric adjudication+ run: bash scripts/tests/test-host-storm-metric-gate.sh++ - name: Test WRC-to-HCC contracts+ run: bash scripts/tests/test-wrc-hcc-contracts-gate.sh++ - name: Test WRC internal dependency NetworkPolicy contracts+ run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh++ - name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)+ run: bash scripts/tests/test-lint-networkpolicies.sh+- name: Test Plugin Workload SDK auth bootstrap policyrun: bash scripts/tests/test-plugin-workload-sdk-auth-bootstrap.sh@@ -399,6 +414,24 @@ jobs:- name: Test minikube detect-k8s-api-ip template renderingrun: bash scripts/tests/test-minikube-detect-k8s-api-ip.sh+ - name: Validate single-writer rollout invariant on rendered overlays+ run: |+ # The k8s-api-ip patch is rendered from a gitignored template at setup+ # time; render it with a dummy IP so `kubectl kustomize` can build the+ # minikube overlay here. The gcp-* overlays live in evenfire-infra, so+ # this repo validates only the minikube overlay it actually ships.+ sed 's#__K8S_API_IP__#10.96.0.1#g' \+ deploy/overlays/minikube/patches/k8s-api-ip.yaml.template \+ > deploy/overlays/minikube/patches/k8s-api-ip.yaml+ SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" \+ bash deploy/scripts/validate-postgres-single-writer-strategy.sh++ - name: Test single-writer validator fails loud on an absent local deploy+ run: bash scripts/tests/test-postgres-single-writer-strategy.sh++ - name: Test post-reset convergence isolates DB-dependent controllers+ run: bash scripts/tests/test-converge-control-db-after-reset.sh+- name: Test release coordinates (writer, checker, floor decoupling)run: bash scripts/tests/test-release-coordinates.shdiff --git a/Makefile b/Makefileindex 57f47ec5a..107633d70 100644--- a/Makefile+++ b/Makefile@@ -934,7 +934,20 @@ test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R.PHONY: test-e2e-hcc-communicationchannel-watch-recoverytest-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC watch-recovery fault-injection gate@echo "Running HCC CommunicationChannel watch-recovery gate..."- E2E_HCC_WATCH_FAULT_INJECTION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh++.PHONY: test-e2e-hcc-readiness-bootstrap+test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active+ @echo "Running HCC initial-fleet readiness gate..."+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh++.PHONY: test-e2e-hcc-mcp-context-readiness+test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence+ @echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh.PHONY: test-e2e-stateless-multinodetest-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)diff --git a/deploy/base/control-plane/host-context-controller-api-gateway.yaml b/deploy/base/control-plane/host-context-controller-api-gateway.yamlindex 961cebe48..1c30ccfbc 100644--- a/deploy/base/control-plane/host-context-controller-api-gateway.yaml+++ b/deploy/base/control-plane/host-context-controller-api-gateway.yaml@@ -43,7 +43,7 @@ spec:timeoutSeconds: 5readinessProbe:httpGet:- path: /ready+ path: /healthport: httpscheme: HTTPinitialDelaySeconds: 3diff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yamlindex df8126a53..24a4c21dd 100644--- a/deploy/base/control-plane/host-context-controller.yaml+++ b/deploy/base/control-plane/host-context-controller.yaml@@ -13,21 +13,33 @@ metadata:app: host-context-controllerspec:replicas: 1- # HCC's /ready stays 503 until its initial fleet reconciliation finishes, so time-to-Ready- # scales with the McpServer/Context fleet rather than with process start. clerum-dev took- # 651s on 2026-08-05 at 108 McpServers / 22 Contexts.+ # HCC's safety model assumes a single NetworkPolicy writer: replicas:1 with no+ # leader election (see statelessLifecycleExecutor.ts). RollingUpdate (the+ # default) starts the new pod before terminating the old one, so every rollout+ # would overlap two writers — during which the outgoing replica can win a+ # conflict-retry replace and land its wider spec back over a policy the new+ # replica just hardened and certified, with no NetworkPolicy resync to heal it+ # and /ready still 200. Recreate terminates the old pod first, honouring the+ # single-writer assumption through the rollout too.+ strategy:+ type: Recreate+ # Recreate + replicas:1 means a botched rollout (new pod never Ready) leaves+ # the control plane down until a manual `kubectl rollout undo` — the single-+ # writer invariant forbids any overlap-based auto-recovery. This deadline is+ # DETECTION ONLY: it flips Progressing=False (ProgressDeadlineExceeded) and+ # fails `kubectl rollout status` for callers without their own --timeout; it+ # does not roll back.#- # Unset, this defaults to 600s, and that default is what actually governed the deploy:- # `kubectl rollout status` aborts the instant the controller sets ProgressDeadlineExceeded,- # so provision-gfs-runtime.sh's --timeout could never exceed the deadline remaining. The- # clock also starts several steps earlier, at apply-inter-service-tokens.sh's `rollout- # restart` of HCC, so the wait inherits only what is left. On 2026-08-05 that was 600-160- # = 440s against a 480s timeout: the timeout raised in #202 never bound, and the deploy- # failed 50s before HCC came Ready.- #- # 1200s covers the 900s wait plus a 300s head-start allowance. This is a mitigation, not- # the fix — evenfire-ai/evenfire#205 decouples readiness from fleet convergence, after- # which both numbers should come back down.+ # Value (origin/dev, 2026-08-05): HCC's /ready stays 503 until its initial fleet+ # reconciliation finishes, so time-to-Ready scales with the McpServer/Context+ # fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default+ # (600s) governed the deploy because `kubectl rollout status` aborts the instant+ # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's+ # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s+ # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s+ # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the+ # fix — #205 decouples readiness from fleet convergence, after which this should+ # come back down.progressDeadlineSeconds: 1200selector:matchLabels:@@ -91,6 +103,8 @@ spec:# now fails closed at startup on any interval > overlap/2.- name: HCC_EXTERNAL_EGRESS_RESYNC_SECvalue: "60"+ - name: HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS+ value: "5000"- name: HCC_HOST_K8S_REQUEST_TIMEOUT_MSvalue: "30000"# Bounded cross-Host concurrency for a full/lifecycle Host fleet pass.diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.shindex 99f50d8bf..404d61076 100755--- a/deploy/scripts/converge-control-db-after-reset.sh+++ b/deploy/scripts/converge-control-db-after-reset.sh@@ -183,7 +183,7 @@ if [ "$STATE_PHASE" = converged ]; thenfikc -n control-plane scale deployment/host-context-controller --replicas="$HCC_REPLICAS" >/dev/nullif [ "$HCC_REPLICAS" -gt 0 ]; then- kc -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null+ kc -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/nullfiassert_replacement_uiddelete_reset_state@@ -264,7 +264,7 @@ kc -n control-plane scale deployment/host-context-controller \--replicas="$HCC_REPLICAS" >/dev/nullif [ "$HCC_REPLICAS" -gt 0 ]; thenkc -n control-plane rollout status \- deployment/host-context-controller --timeout=180s >/dev/null+ deployment/host-context-controller --timeout=900s >/dev/nullfiassert_replacement_uidread_reset_state || fail "cannot reload reset state before completion"diff --git a/deploy/scripts/validate-postgres-single-writer-strategy.sh b/deploy/scripts/validate-postgres-single-writer-strategy.shindex f1a77091c..ee90b96be 100755--- a/deploy/scripts/validate-postgres-single-writer-strategy.sh+++ b/deploy/scripts/validate-postgres-single-writer-strategy.sh@@ -2,40 +2,86 @@set -euo pipefailROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-OVERLAYS=(- "deploy/overlays/minikube"- "deploy/overlays/gcp-dev"- "deploy/overlays/gcp-prod"-)+# Overlays to render. Defaults to the full production set; the gcp-* overlays+# live in keyper-labs/evenfire-infra, so this script's full coverage runs there.+# Override (space-separated) to validate only the overlays present in a given+# checkout — e.g. SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" in this repo.+read -r -a OVERLAYS <<<"${SINGLE_WRITER_OVERLAYS:-deploy/overlays/minikube deploy/overlays/gcp-dev deploy/overlays/gcp-prod}"DATABASE_DEPLOYS=("control-plane/control-postgres""registry/registry-postgres")+# Deployments whose safety model assumes a single writer with NO leader election+# (see host-context-controller/src/statelessLifecycleExecutor.ts). For these the+# RENDERED manifest — not just deploy/base — must keep replicas<=1 and+# strategy=Recreate with no rollingUpdate residue, or a rollout reopens the+# two-writer NetworkPolicy overlap closed in PR #205. Unlike DATABASE_DEPLOYS+# these mount no PVC, so they need their own list rather than the PVC heuristic.+SINGLE_WRITER_DEPLOYS=(+ "control-plane/host-context-controller"+)fail() {echo "ERROR: $*" >&2exit 1}+# A Deployment id ("namespace/name") is DEFINED in this repo when its namespace+# has a base manifest tree here (deploy/base/<namespace>/). For such a LOCAL+# deploy the rendered overlay MUST contain it: a silent disappearance would let+# the loops below validate the single-writer/database rollout invariant over an+# absent deploy (a fail-open). Deploys whose base lives only in the sibling+# evenfire-infra repo (e.g. registry/*) are FOREIGN here and are validated where+# they are defined, so for them an absence is a NOTICE-and-skip, not a failure.+deploy_defined_in_this_repo() {+ local id="$1"+ local namespace="${id%%/*}"+ [ -d "${ROOT_DIR}/deploy/base/${namespace}" ]+}+command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"command -v jq >/dev/null 2>&1 || fail "jq is required"+command -v yq >/dev/null 2>&1 || fail "yq is required"++# Zero overlays means the loop below runs zero iterations and the script exits 0+# having validated nothing — a fail-open. An empty or whitespace-only+# SINGLE_WRITER_OVERLAYS must fail loudly, not pass vacuously.+[ "${#OVERLAYS[@]}" -gt 0 ] ||+ fail "SINGLE_WRITER_OVERLAYS resolved to zero overlays — refusing to pass vacuously"for overlay in "${OVERLAYS[@]}"; do+ # Render OFFLINE: `kubectl kustomize` builds the overlay client-side, and yq+ # converts the multi-document YAML into a single JSON array for jq. The+ # previous `kubectl create --dry-run=client` performed API discovery+ # (kind -> resource), which needs a live API server and fails in CI with+ # "unable to recognize STDIN: connection refused". This validation is a pure+ # rendered-manifest lint and must never touch a cluster.rendered_json="$(cd "${ROOT_DIR}" &&kubectl kustomize "${overlay}" |- kubectl create --dry-run=client --validate=false -f - -o json |- jq -s '.'+ yq ea -o=json '[.]')"for id in "${DATABASE_DEPLOYS[@]}"; donamespace="${id%%/*}"name="${id#*/}"- deployment="$(+ if ! deployment="$(jq -e --arg namespace "${namespace}" --arg name "${name}" \'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \<<<"${rendered_json}"- )" || fail "${overlay}: missing Deployment ${id}"+ )"; then+ # This overlay does not render ${id}. A deploy this repo DEFINES+ # (deploy/base/<namespace>/ exists here) MUST be rendered by a local+ # overlay: a silent disappearance would validate the rollout invariant over+ # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and+ # the registry Deployment live in the sibling evenfire-infra repo) are+ # validated where they are defined, so for them a NOTICE-and-skip is right.+ if deploy_defined_in_this_repo "${id}"; then+ fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"+ fi+ echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2+ continue+ fipvc_count="$(jq '[.spec.template.spec.volumes[]? | select(.persistentVolumeClaim.claimName != null)] | length' \@@ -54,6 +100,42 @@ for overlay in "${OVERLAYS[@]}"; dofail "${overlay}: Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"fidone++ for id in "${SINGLE_WRITER_DEPLOYS[@]}"; do+ namespace="${id%%/*}"+ name="${id#*/}"+ if ! deployment="$(+ jq -e --arg namespace "${namespace}" --arg name "${name}" \+ 'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \+ <<<"${rendered_json}"+ )"; then+ # This overlay does not render ${id}. A deploy this repo DEFINES+ # (deploy/base/<namespace>/ exists here) MUST be rendered by a local+ # overlay: a silent disappearance would validate the rollout invariant over+ # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and+ # the registry Deployment live in the sibling evenfire-infra repo) are+ # validated where they are defined, so for them a NOTICE-and-skip is right.+ if deploy_defined_in_this_repo "${id}"; then+ fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"+ fi+ echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2+ continue+ fi++ replicas="$(jq -r '.spec.replicas // 1' <<<"${deployment}")"+ if [ "${replicas}" -gt 1 ]; then+ fail "${overlay}: single-writer Deployment ${id} must keep replicas<=1 (no leader election), got ${replicas}"+ fi++ strategy="$(jq -r '.spec.strategy.type // "RollingUpdate"' <<<"${deployment}")"+ if [ "${strategy}" != "Recreate" ]; then+ fail "${overlay}: single-writer Deployment ${id} must use strategy.type=Recreate, got ${strategy}"+ fi++ if jq -e '.spec.strategy.rollingUpdate != null' >/dev/null <<<"${deployment}"; then+ fail "${overlay}: single-writer Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"+ fi+ donedone-echo "Postgres single-writer rollout strategy validation passed"+echo "Single-writer rollout strategy validation passed"diff --git a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsxindex 0b1007986..ff8667aa0 100644--- a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx+++ b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx@@ -52,9 +52,7 @@ function createDeferred<T>(): Deferred<T> {}function createSessionState(authenticated: boolean, me = HARNESS_ME) {- return authenticated- ? { authenticated: true, me }- : { authenticated: false, me: null }+ return authenticated ? { authenticated: true, me } : { authenticated: false, me: null }}function createCatalog(agentNames: string[]) {diff --git a/docs/architecture/hcc-readiness-flows.md b/docs/architecture/hcc-readiness-flows.mdnew file mode 100644index 000000000..6a4e89ca9--- /dev/null+++ b/docs/architecture/hcc-readiness-flows.md@@ -0,0 +1,90 @@+# HCC readiness & external-egress — design note (PR #205)++> Design note for the readiness-decoupling and single-writer NetworkPolicy work in+> `host-context-controller` (HCC). Addresses the "no design/decision doc for the+> 500+-line external-egress coordinator and the certification barrier" review gap.++++## 1. Readiness lifecycle — decoupled from initial fleet convergence++Before this work, `McpServerWatcher.start()` awaited full initial fleet+reconciliation, so a large MCP/Context/Host fleet could keep `provider.start()`+and `/ready` unavailable until every individual reconcile finished.++Now readiness is gated on an **authoritative, fail-closed safety boundary**, not on+full convergence:++1. **HCC start** — four watch lanes (Context ingress, mcp-host egress, rpc-proxy+ egress, external egress) start. A transient LIST/WATCH failure on any lane enters+ in-process recovery and **does not kill the process**; HCC simply stays unready.+2. **4-LIST inventory** — bounded inventory across the four lanes.+3. **Certify safety inventory** — the authoritative pass revokes any stale /+ unprovable allow **before** recording a certificate (fail-closed).+4. **`/ready = 200`** — served **only** when the inventory is certified and no stale+ allow survives.+5. **Additive convergence** — the remaining fleet converges **after** ready,+ asynchronously, without blocking it.++## 2. Fail-closed fences++| Fence | Rule | Source |+|-------|------|--------|+| Readiness gate | `/ready = ready && providerAuthoritativeFn()`; constructor default `() => false`; a throwing gate → 503 | `server.ts`, `readinessGate.ts` |+| G1 — unconfigured authority | `currentWhenUnconfigured()` returns `false` + warns once; prod always wires the source, dev never constructs the reconciler | `reconciler.ts` |+| F1 — readiness-poll rebind | rebinds `isCurrent`/`server` on a superseding reconcile so the poll window does not self-destruct on a stale fence | `reconciler.ts` |+| Authority fence — `isCurrent` | **required** (not an optional/defaulted arg) on all three allow-granting reconcilers — `bindingPolicyReconciler`, `reconcileContext` (L2 context-allow), `reconcileExternalEgress` (L3 external-egress ALLOW). No `?? (() => true)` fail-open default: a caller cannot compile without supplying the fence (TS2554/TS2345) | `networkPolicyReconciler.ts`, `bindingPolicyReconciler.ts` |++The affirmative literal (`() => true`) is reachable only from the dev/test wiring;+every production path resolves the real authority function, and — because the fence+is a **required argument** on every allow-granting reconciler rather than a defaulted+option — no production caller can silently fall back to it.++## 3. DNS external-egress contract (B3) — retain / revoke decision table++A DNS-derived external-egress allow is compared against the desired policy+reconstructed from the **live** resolved CIDRs ("modulo cidr"). The decision:++| Observed change | Classification | Action | Rationale |+|-----------------|----------------|--------|-----------|+| identity unchanged (same protocol, port, selector, ownership; cidrs only) | identity-stable | **RETAIN** while Ready; refresh cidrs in the additive lane | revoking a healthy binding every pass created a guaranteed deny window |+| protocol drift (TCP→UDP) | identity-changed | **REVOKE before readiness** | old allow is unprovable/stale |+| spec-level port drift | identity-changed | **REVOKE before readiness** | " |+| ownership drift | identity-changed | **REVOKE before readiness** | " |+| zero-cidr (deny-in-disguise) | never retained | **REVOKE** | an empty cidr set is never a valid retained allow |++Revocation of an unprovable policy runs immediately (`cleanupExternalEgress`) under+the `isCurrent()` fence, **before** the safety inventory is certified — so `/ready`+never reports 200 with a stale divergent policy present. The unit suite pins both+directions (retain identity-stable; revoke on each drift), and the branch-owned+Minikube gate `e2e-hcc-mcp-context-readiness.sh` exercises both phases live.++## 4. ExternalEgressConvergenceCoordinator++The coordinator is the **single owner** of external-egress reconcile cadence+(replacing an inline timer in `k8sClient`):++- **Per-server FIFO + concurrency limiter** — a slow or never-resolving binding+ cannot head-of-line block the control-plane event loop.+- **Sliding-window CIDR accumulation (#299)** — resolved IPs live for `TTL + overlap`;+ large provider pools are covered by provider CIDRs rather than per-IP `/32`s. A+ failed refresh deletes the allow (fail-closed), never widens it.+- **TTL-aware periodic resync (H2)** — `startPeriodicResync` re-arms after each pass+ with a delay of `min(interval, TTL/2)` bounded below by a floor, so low-TTL hosts+ refresh promptly. The timer is `.unref()`'d and re-programs **only after a pass+ completes** (no overlap).+- **Fence** — startup rejects `interval > overlap/2`, so the accumulated window can+ never lapse regardless of the configured cadence.+- **Observability** — `clerum_hcc_external_egress_retries_at_cap` gauges how many+ servers are stuck retrying at the delay cap (a never-resolving DNS binding is+ otherwise invisible: denied with `/ready` = 200 and no signal).++## 5. Deploy invariant — single writer++The HCC Deployment ships `strategy: Recreate`, `replicas: 1`, and+`progressDeadlineSeconds: 1200`: the stateless single-writer safety model has no+leader election, so two concurrently-running controllers must be impossible during a+rollout. The trade-off (full HCC control-plane downtime per rollout; a rollout whose+new pod never becomes Ready needs a manual `kubectl rollout undo`) is declared in the+manifest and asserted by the bootstrap gate. `validate-postgres-single-writer-strategy.sh`+enforces the invariant on the rendered manifest in CI.diff --git a/docs/architecture/pr205-before-after.png b/docs/architecture/pr205-before-after.pngnew file mode 100644index 000000000..524481e1fBinary files /dev/null and b/docs/architecture/pr205-before-after.png differdiff --git a/docs/architecture/pr205-before-after.svg b/docs/architecture/pr205-before-after.svgnew file mode 100644index 000000000..471f74cc3--- /dev/null+++ b/docs/architecture/pr205-before-after.svg@@ -0,0 +1,71 @@+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1620 880" font-family="Helvetica, Arial, sans-serif">+<defs>+<marker id="arr_2563eb" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#2563eb"/></marker>+<marker id="arr_dc2626" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#dc2626"/></marker>+<marker id="arr_16a34a" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#16a34a"/></marker>+<marker id="arr_d97706" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#d97706"/></marker>+<marker id="arr_64748b" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#64748b"/></marker>+</defs>+<style>text{font-family:Helvetica,Arial,sans-serif;}</style>+<rect x="0" y="0" width="1620" height="880" rx="0" fill="#ffffff" stroke="#ffffff" stroke-width="0" opacity="1.0"/>+<text x="810.0" y="46" font-size="27" fill="#0f172a" font-weight="700" text-anchor="middle">PR #205 — Decoupling HCC readiness: before vs after</text>+<text x="810.0" y="76" font-size="15" fill="#475569" font-weight="400" text-anchor="middle">The Host Context Controller's /ready endpoint stops waiting for full fleet convergence</text>+<rect x="30" y="100" width="760" height="660" rx="18" fill="#fff7f6" stroke="#fecaca" stroke-width="2" opacity="1.0"/>+<rect x="830" y="100" width="760" height="660" rx="18" fill="#f5fbf7" stroke="#bbf7d0" stroke-width="2" opacity="1.0"/>+<text x="410" y="136" font-size="19" fill="#dc2626" font-weight="700" text-anchor="middle">BEFORE · origin/dev (issue #204)</text>+<text x="1210" y="136" font-size="19" fill="#16a34a" font-weight="700" text-anchor="middle">AFTER · PR #205</text>+<rect x="100.0" y="168" width="600" height="58" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>+<text x="400" y="201.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>+<line x1="400" y1="226" x2="400" y2="279" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>+<rect x="100.0" y="282" width="600" height="128" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>+<text x="400" y="330.05" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">Waits to reconcile the ENTIRE initial fleet</text>+<text x="400" y="357.35" font-size="14" fill="#475569" font-weight="400" text-anchor="middle">108 McpServers · 22 Contexts · Hosts</text>+<text x="400" y="383.94" font-size="15" fill="#dc2626" font-weight="700" text-anchor="middle">/ready = 503 for the entire wait</text>+<rect x="636" y="300" width="132" height="92" rx="12" fill="#dc2626" stroke="#dc2626" stroke-width="0" opacity="1.0"/>+<text x="702" y="336" font-size="15" fill="#ffffff" font-weight="700" text-anchor="middle">BLOCKED</text>+<text x="702" y="362" font-size="22" fill="#ffffff" font-weight="700" text-anchor="middle">~651 s</text>+<line x1="400" y1="410" x2="400" y2="467" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>+<rect x="100.0" y="470" width="600" height="58" rx="12" fill="#f1f5f9" stroke="#64748b" stroke-width="2" opacity="1.0"/>+<text x="400" y="503.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">/ready = 200 (only after the ENTIRE fleet converges)</text>+<line x1="400" y1="528" x2="400" y2="585" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>+<rect x="100.0" y="588" width="600" height="76" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>+<text x="400" y="621.7" font-size="17" fill="#dc2626" font-weight="700" text-anchor="middle">rollout status TIMEOUT → deploy FAILS</text>+<text x="400" y="649.0" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">even though the HCC is healthy and progressing</text>+<text x="410" y="700" font-size="14" fill="#dc2626" font-weight="700" text-anchor="middle" font-style="italic">Readiness COUPLED to full fleet convergence</text>+<rect x="890.0" y="168" width="520" height="54" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>+<text x="1150" y="199.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>+<line x1="1150" y1="222" x2="1150" y2="265" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>+<rect x="890.0" y="268" width="520" height="50" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>+<text x="1150" y="297.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">BOUNDED inventory (bounded 4-LIST)</text>+<line x1="1150" y1="318" x2="1150" y2="357" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>+<rect x="890.0" y="360" width="520" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>+<text x="1150" y="391.89" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">Certifies authoritative fail-closed BARRIER</text>+<text x="1150" y="417.77" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">revokes stale / unprovable NetworkPolicies</text>+<text x="1150" y="442.22999999999996" font-size="13" fill="#d97706" font-weight="700" text-anchor="middle">BEFORE declaring Ready</text>+<line x1="1150" y1="452" x2="1150" y2="493" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>+<rect x="890.0" y="496" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>+<text x="1150" y="528.9300000000001" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">/ready = 200 (seconds)</text>+<line x1="1150" y1="552" x2="1150" y2="593" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>+<rect x="890.0" y="596" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>+<text x="1150" y="628.93" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">rollout OK → deploy OK</text>+<path d="M 1410.0 524 H 1452 V 596" fill="none" stroke="#d97706" stroke-width="3" stroke-dasharray="7 5" marker-end="url(#arr_d97706)"/>+<rect x="1336.0" y="596" width="232" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>+<text x="1452" y="629.44" font-size="13" fill="#0f172a" font-weight="700" text-anchor="middle">ADDITIVE fleet convergence</text>+<text x="1452" y="653.19" font-size="12" fill="#475569" font-weight="400" text-anchor="middle">async, AFTER Ready</text>+<text x="1452" y="675.5200000000001" font-size="11" fill="#475569" font-weight="400" text-anchor="middle">(the rest of the policies converge)</text>+<rect x="1362.0" y="357" width="200.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>+<text x="1462" y="373" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">isCurrent required (J3)</text>+<rect x="1382.0" y="389" width="160.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>+<text x="1462" y="405" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">B3 retain / revoke</text>+<rect x="1358.0" y="421" width="208.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>+<text x="1462" y="437" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">single-writer Recreate·1</text>+<text x="1150" y="700" font-size="14" fill="#16a34a" font-weight="700" text-anchor="middle" font-style="italic">Readiness DECOUPLED: gated on the safety barrier, not the fleet</text>+<rect x="300" y="812" width="26" height="20" rx="5" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>+<text x="336" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Startup step</text>+<rect x="506" y="812" width="26" height="20" rx="5" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>+<text x="542" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Block / failure</text>+<rect x="742" y="812" width="26" height="20" rx="5" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>+<text x="778" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Fast Ready / OK</text>+<rect x="978" y="812" width="26" height="20" rx="5" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>+<text x="1014" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Barrier / async branch</text>+</svg>\ No newline at end of filediff --git a/docs/architecture/pr205-hcc-flows.png b/docs/architecture/pr205-hcc-flows.pngnew file mode 100644index 000000000..d0bbfa3d7Binary files /dev/null and b/docs/architecture/pr205-hcc-flows.png differdiff --git a/docs/architecture/pr205-hcc-flows.svg b/docs/architecture/pr205-hcc-flows.svgnew file mode 100644index 000000000..adb22120f--- /dev/null+++ b/docs/architecture/pr205-hcc-flows.svg@@ -0,0 +1,126 @@+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 880" font-family="Inter, Segoe UI, Helvetica, Arial, sans-serif">+<defs>+<marker id="aBlue" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#2563eb"/></marker>+<marker id="aRose" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#dc2626"/></marker>+<marker id="aViolet" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#7c3aed"/></marker>+<marker id="aAmber" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#d97706"/></marker>+<marker id="aGreen" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#059669"/></marker>+<marker id="aGray" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#64748b"/></marker>+<filter id="sh" x="-5%" y="-6%" width="110%" height="122%"><feDropShadow dx="0" dy="1.5" stdDeviation="1.8" flood-color="#0f172a" flood-opacity="0.16"/></filter>+</defs>+<rect width="1280" height="880" fill="#ffffff"/>+<text x="40" y="36" font-size="20" font-weight="800" fill="#0f172a">host-context-controller — flujos que resuelve el PR #205</text>+<text x="40" y="57" font-size="12.5" fill="#64748b">decouple readiness from initial fleet convergence · seguridad fail-closed · gobernanza single-writer de NetworkPolicy</text>+<text x="40" y="92" font-size="13" font-weight="700" fill="#1d4ed8">1 · Lifecycle de readiness — DESACOPLADO de la convergencia de la flota</text>+<rect x="40.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>+<text x="147.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">HCC start</text>+<text x="147.0" y="146" text-anchor="middle" font-size="11" fill="#475569">4 watch lanes arrancan;</text>+<text x="147.0" y="162" text-anchor="middle" font-size="11" fill="#475569">una lane que falla → recovery</text>+<text x="147.0" y="178" text-anchor="middle" font-size="11" fill="#475569">in-process, NO mata el proceso</text>+<rect x="286.5" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>+<text x="393.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">4-LIST inventory</text>+<text x="393.5" y="146" text-anchor="middle" font-size="11" fill="#475569">Context ingress · mcp-host</text>+<text x="393.5" y="162" text-anchor="middle" font-size="11" fill="#475569">egress · rpc-proxy egress</text>+<text x="393.5" y="178" text-anchor="middle" font-size="11" fill="#475569">· external egress</text>+<rect x="533.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>+<text x="640.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Certificar safety inventory</text>+<text x="640.0" y="146" text-anchor="middle" font-size="11" fill="#475569">revoca allows stale/</text>+<text x="640.0" y="162" text-anchor="middle" font-size="11" fill="#475569">improbables ANTES de</text>+<text x="640.0" y="178" text-anchor="middle" font-size="11" fill="#475569">certificar (fail-closed)</text>+<rect x="779.5" y="104" width="214" height="92" rx="11" fill="#dbeafe" stroke="#1d4ed8" stroke-width="1.8" filter="url(#sh)"/>+<text x="886.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">/ready = 200</text>+<text x="886.5" y="146" text-anchor="middle" font-size="11" fill="#475569">solo si el inventory está</text>+<text x="886.5" y="162" text-anchor="middle" font-size="11" fill="#475569">certificado y ningún allow</text>+<text x="886.5" y="178" text-anchor="middle" font-size="11" fill="#475569">stale sobrevive</text>+<rect x="1026.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>+<text x="1133.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Additive convergence</text>+<text x="1133.0" y="146" text-anchor="middle" font-size="11" fill="#475569">la flota converge DESPUÉS</text>+<text x="1133.0" y="162" text-anchor="middle" font-size="11" fill="#475569">del ready — asíncrono,</text>+<text x="1133.0" y="178" text-anchor="middle" font-size="11" fill="#475569">sin bloquearlo</text>+<line x1="254.0" y1="150.0" x2="286.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>+<rect x="229.05" y="130.0" width="82.4" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="270.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">inventariar</text>+<line x1="500.5" y1="150.0" x2="533.0" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>+<rect x="478.75" y="130.0" width="76.0" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="516.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">certificar</text>+<line x1="747.0" y1="150.0" x2="779.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>+<rect x="738.05" y="130.0" width="50.400000000000006" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="763.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">/ready</text>+<line x1="993.5" y1="150.0" x2="1026.0" y2="150.0" stroke="#64748b" stroke-width="2.4" stroke-dasharray="6,4" marker-end="url(#aGray)" stroke-linecap="round"/>+<rect x="946.15" y="130.0" width="127.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="1009.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#475569">async · post-ready</text>+<text x="40" y="275" font-size="13" font-weight="700" fill="#b91c1c">2 · Fences fail-closed (R3 · R4 · J3)</text>+<rect x="40.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>+<text x="230.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Readiness gate — default () => false</text>+<text x="230.0" y="326" text-anchor="middle" font-size="11" fill="#475569">/ready = ready && providerAuthoritativeFn()</text>+<text x="230.0" y="342" text-anchor="middle" font-size="11" fill="#475569">throw → 503 · nunca un 200 equivocado</text>+<rect x="450.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>+<text x="640.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">G1 — deny sin inventory-authority</text>+<text x="640.0" y="326" text-anchor="middle" font-size="11" fill="#475569">currentWhenUnconfigured() → false + warn</text>+<text x="640.0" y="342" text-anchor="middle" font-size="11" fill="#475569">dev nunca construye el reconciler</text>+<rect x="860.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>+<text x="1050.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">F1 — rebind del readiness-poll</text>+<text x="1050.0" y="326" text-anchor="middle" font-size="11" fill="#475569">re-liga isCurrent/server en reconciles</text>+<text x="1050.0" y="342" text-anchor="middle" font-size="11" fill="#475569">superpuestos → la ventana no se autodestruye</text>+<text x="40" y="421" font-size="13" font-weight="700" fill="#6d28d9">3 · Egress externo DNS — contrato B3 (retener/revocar) + sliding-window #299</text>+<rect x="40" y="438" width="224" height="80" rx="11" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>+<text x="152.0" y="461" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">DNS binding observado</text>+<text x="152.0" y="480" text-anchor="middle" font-size="11" fill="#475569">resolveExternalEgressDns</text>+<text x="152.0" y="496" text-anchor="middle" font-size="11" fill="#475569">timeout 5s + fence isCurrent</text>+<polygon points="372,436 472,478 372,520 272,478" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>+<text x="372" y="475" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">¿identity</text>+<text x="372" y="491" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">unchanged?</text>+<line x1="264" y1="478" x2="272" y2="478" stroke="#7c3aed" stroke-width="2.6" marker-end="url(#aViolet)" stroke-linecap="round"/>+<rect x="540" y="414" width="306" height="60" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>+<text x="693.0" y="437" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">RETAIN — allow identity-stable</text>+<text x="693.0" y="456" text-anchor="middle" font-size="11" fill="#475569">mismo protocolo/puerto/ownership/cidrs</text>+<text x="693.0" y="472" text-anchor="middle" font-size="11" fill="#475569">· HCC sigue Ready (módulo cidr)</text>+<rect x="540" y="496" width="306" height="60" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>+<text x="693.0" y="519" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">REVOKE antes de readiness</text>+<text x="693.0" y="538" text-anchor="middle" font-size="11" fill="#475569">drift TCP→UDP · port · ownership · zero-cidr</text>+<text x="693.0" y="554" text-anchor="middle" font-size="11" fill="#475569">→ unprovable → cleanup fail-closed</text>+<path d="M 472,470 H 506.0 V 444 H 540" fill="none" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linejoin="round"/>+<rect x="423.6" y="415" width="88.80000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="468" y="427" text-anchor="middle" font-size="10" font-weight="700" fill="#047857">sí · retiene</text>+<path d="M 472,486 H 506.0 V 526 H 540" fill="none" stroke="#dc2626" stroke-width="2.6" marker-end="url(#aRose)" stroke-linejoin="round"/>+<rect x="417.2" y="523" width="101.60000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="468" y="535" text-anchor="middle" font-size="10" font-weight="700" fill="#b91c1c">drift · revoca</text>+<rect x="892" y="430" width="348" height="88" rx="11" fill="#fffbeb" stroke="#d97706" stroke-width="1.8" filter="url(#sh)"/>+<text x="1066.0" y="453" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ExternalEgressConvergenceCoordinator</text>+<text x="1066.0" y="472" text-anchor="middle" font-size="11" fill="#475569">resync periódico TTL-aware (≤ TTL/2, floor)</text>+<text x="1066.0" y="488" text-anchor="middle" font-size="11" fill="#475569">sliding-window accumulate · provider CIDRs</text>+<text x="1066.0" y="504" text-anchor="middle" font-size="11" fill="#475569">fence interval ≤ overlap/2 · limiter</text>+<line x1="846" y1="444" x2="892" y2="460" stroke="#d97706" stroke-width="2.6" marker-end="url(#aAmber)" stroke-linecap="round"/>+<rect x="821.4" y="425" width="95.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>+<text x="869" y="437" text-anchor="middle" font-size="10" font-weight="700" fill="#b45309">add (aditiva)</text>+<text x="40" y="575" font-size="13" font-weight="700" fill="#047857">4 · Seam de rotación #223 (preservado) & invariante de deploy single-writer</text>+<rect x="40" y="584" width="236" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>+<text x="158.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">validateSecret()</text>+<text x="158.0" y="626" text-anchor="middle" font-size="11" fill="#475569">sha256(canonicalJSON(secret.data))</text>+<rect x="306" y="584" width="306" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>+<text x="459.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ensureDeployment(rev, isCurrent)</text>+<text x="459.0" y="626" text-anchor="middle" font-size="11" fill="#475569">el fence de #205 se propaga aquí</text>+<rect x="642" y="584" width="300" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>+<text x="792.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">pod annotation</text>+<text x="792.0" y="626" text-anchor="middle" font-size="11" fill="#475569">clerum.io/credentials-revision → rueda 1 vez</text>+<line x1="276" y1="617" x2="306" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>+<line x1="612" y1="617" x2="642" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>+<rect x="972" y="584" width="268" height="66" rx="11" fill="#f8fafc" stroke="#475569" stroke-width="1.8" filter="url(#sh)"/>+<text x="1106.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Deploy single-writer</text>+<text x="1106.0" y="626" text-anchor="middle" font-size="11" fill="#475569">strategy: Recreate · replicas 1</text>+<text x="1106.0" y="642" text-anchor="middle" font-size="11" fill="#475569">· progressDeadlineSeconds 1200</text>+<text x="40" y="700" font-size="11.5" font-weight="700" fill="#334155">Leyenda</text>+<rect x="118" y="688" width="14" height="14" rx="3" fill="#2563eb" opacity="0.9"/>+<text x="137" y="700" font-size="11" fill="#475569">lifecycle readiness</text>+<rect x="282.6" y="688" width="14" height="14" rx="3" fill="#dc2626" opacity="0.9"/>+<text x="301.6" y="700" font-size="11" fill="#475569">fence fail-closed / revoke</text>+<rect x="492.0" y="688" width="14" height="14" rx="3" fill="#7c3aed" opacity="0.9"/>+<text x="511.0" y="700" font-size="11" fill="#475569">egress DNS (B3)</text>+<rect x="631.0" y="688" width="14" height="14" rx="3" fill="#d97706" opacity="0.9"/>+<text x="650.0" y="700" font-size="11" fill="#475569">coordinator #299</text>+<rect x="776.4" y="688" width="14" height="14" rx="3" fill="#059669" opacity="0.9"/>+<text x="795.4" y="700" font-size="11" fill="#475569">retain / rotación #223</text>+<rect x="960.2" y="688" width="14" height="14" rx="3" fill="#475569" opacity="0.9"/>+<text x="979.2" y="700" font-size="11" fill="#475569">invariante deploy</text>+<text x="40" y="858" font-size="10.5" fill="#94a3b8">PR #205 · evenfire-ai/evenfire · el /ready certifica una frontera de seguridad autoritativa ANTES de que la flota converja — la convergencia restante es aditiva y asíncrona</text>+</svg>\ No newline at end of filediff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex e1227184a..0ae6125bb 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.7.0',externalRestApiVersion: '0.1.80',- rpcProxyVersion: '0.1.64',+ rpcProxyVersion: '0.1.65',desktopVersion: '0.7.0',minimumDesktopVersion: '0.1.252',}diff --git a/host-context-controller/README.md b/host-context-controller/README.mdindex 8ee40e981..169458690 100644--- a/host-context-controller/README.md+++ b/host-context-controller/README.md@@ -39,6 +39,8 @@ When an `McpServer` CRD is created, modified, or deleted, the host-context-contr- **MODIFIED** — Updates the Deployment and Service to match the CRD spec (env vars, image, ports, resources, etc.).- **DELETED** — Removes the Deployment and Service.+Remote egress-proxy McpServers keep the author's `spec.image` as desired-state input; HCC no longer rewrites it to the platform proxy image. The former `canonicalizeRemoteEgressProxyImage` step and its `ImageCanonicalized` status condition were removed (PR #205 readiness-safety cleanup): canonicalization mutated the observed CRD cache object and bumped the resource generation, retiring the in-flight reconcile before it could create the runtime resources. The platform-owned proxy image is instead selected at Deployment build time (`buildDeployment`), leaving the CRD spec untouched.+The reconciler builds Deployments with labels used for pod selection and network policy targeting:| Label | Value | Purpose |@@ -97,7 +99,21 @@ Servers removed from a Context lose their policies on the next reconcile; deleti- **`exact-host`** (default) — a CIDR binding, or a public DNS hostname resolved to `/32` ipBlocks, on one declared port/protocol. CIDRs and resolved IPs are rejected if they overlap `PUBLIC_EGRESS_EXCEPT_CIDRS` (RFC1918, CGNAT, loopback, link-local incl. cloud metadata, documentation, benchmarking, multicast, reserved ranges). Hostnames like `localhost`, `metadata.goog`, `*.internal`, `*.svc`, `*.cluster.local` are rejected outright.- **`public-web`** — `0.0.0.0/0` on TCP 80/443 with `PUBLIC_EGRESS_EXCEPT_CIDRS` carved out via `except`.-Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS answers are re-resolved periodically (`HCC_EXTERNAL_EGRESS_RESYNC_SEC`, default 300s).+Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS-derived allows are never retained across a failed refresh: HCC revokes the unprovable policy before certifying global readiness, while only the affected runtime remains blocked until DNS recovers. Each lookup has a bounded deadline.++HCC readiness certifies authoritative inventory and revocation safety, not+completion of positive policy creation. A renamed or newly requested allow may+therefore remain unavailable after HCC becomes Ready until its additive+reconciliation succeeds. HCC never keeps an obsolete allow merely to avoid+that availability gap: when old and new ownership cannot be proven equivalent,+revocation wins and only the affected runtime remains fail-closed.++During an in-place release, HCC is also the controller that stamps+`CONTEXT_MAPPER_HOST_IMAGE` onto managed mcp-host workloads. The new HCC+producer is consequently running before it replaces those consumers with the+new mcp-host image. For compatibility, an omitted `status.authoritative` field+retains the legacy fail-closed meaning; only an explicit `authoritative: false`+asks a new mcp-host to preserve an unchanged live connection.### Recipe binding policies@@ -105,7 +121,8 @@ McpServers carrying a `clerum.io/recipe-bindings` annotation (set for WorkflowRe### Reconciliation Lifecycle-- **Startup** — Ensures L0/L1 defaults, reconciles all Contexts and all McpServer egress bindings, then deletes orphaned context-allow policies in the MCP server namespace, plus orphaned rpc-proxy-egress and external-egress policies, whose owning CRD no longer exists (`fullReconcile`).+- **Startup safety** — Ensures L0/L1 defaults, revokes stale or orphaned context-allow, rpc-proxy-egress, and external-egress policies from authoritative Context/McpServer inventories, then certifies readiness safety.+- **Startup additive convergence** — Reconciles Context allows after certification. The dedicated external-egress coordinator is the sole owner of McpServer egress creation, per-server startup gating, retry, and periodic DNS refresh; a failed binding blocks only that server's runtime.- **Context ADDED/MODIFIED/DELETED** — Creates, updates, or removes the L2 policy set for that context.- **McpServer ADDED/MODIFIED/DELETED** — Reconciles L3 external egress (before the workload Deployment is created) and recipe binding policies.@@ -267,16 +284,17 @@ a permissive policy — see the NetworkPolicy and GFS tables:### MCP server provisioning-| Variable | Default | Description |-| -------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------- |-| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |-| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |-| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |-| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync (seconds), so DNS changes converge without a watch event. `0` disables. |+| Variable | Default | Description |+| -------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------- |+| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |+| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |+| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |+| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync interval (seconds). `0` disables periodic refresh; watch events still reconcile. |+| `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS` | `5000` | Per-hostname DNS lookup deadline in milliseconds. |### mcp-host provisioningdiff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex e9b28c157..9bcfc9c0f 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.223","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.223","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 17c98b81a..42fe8f398 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.223","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.tsindex f8d6857cb..f4e15bf35 100644--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts@@ -119,6 +119,7 @@ describe('connector credentials revision (issue #223)', () => {data: { 'api-key': stored('old-key') },})reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -220,6 +221,7 @@ describe('generation-aware rollout readiness (issue #223)', () => {vi.clearAllMocks()coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -322,6 +324,7 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',vi.useFakeTimers()coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -548,3 +551,111 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)})})++describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {+ const appsApi = createMockAppsApi()+ const coreApi = createMockCoreApi()+ const customApi = createMockCustomApi()+ let reconciler: McpServerReconciler+ // The inventory-authority generation IS mcpWatchGeneration in production+ // (k8sClient wiring): a routine McpServer watch recovery bumps it.+ let authorityGeneration: number+ let currentServer: McpServerCRD++ // A rollout mid-flight (total 2 vs updated 1) that never converges on its own.+ const midRollout = {+ metadata: { generation: 2, labels: {} },+ spec: { replicas: 1 },+ status: {+ observedGeneration: 2,+ replicas: 2,+ updatedReplicas: 1,+ readyReplicas: 1,+ availableReplicas: 1,+ unavailableReplicas: 0,+ },+ }++ beforeEach(() => {+ vi.clearAllMocks()+ vi.useFakeTimers()+ authorityGeneration = 1+ currentServer = makeServer()+ coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(customApi),+ })+ reconciler.setInventoryAuthority(() => ({ known: true, generation: authorityGeneration }))+ reconciler.setResolveCurrentServer(name => (name === 'linear' ? currentServer : undefined))+ })++ afterEach(() => {+ vi.useRealTimers()+ })++ it('rebinds the fence so a converged rollout is recorded after a watch-generation bump (T1)', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)+ // R1 arms the window under authority generation 1 (timer pending).+ await reconciler.reconcile(makeServer())+ // A routine McpServer watch recovery bumps the inventory-authority generation.+ authorityGeneration = 2+ // R2, current under generation 2, finds R1's pending window and rebinds it.+ await reconciler.reconcile(makeServer())+ // The rollout converges before the (single) pending tick fires.+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)++ // Without the rebind, R1's tick evaluates its stale generation-1 fence,+ // self-deletes, and DeploymentReady=True is never written.+ expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')+ expect(+ (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls.size+ ).toBe(0)+ })++ it('control: without the generation bump the rollout converges the same way (T2)', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)+ await reconciler.reconcile(makeServer())+ await reconciler.reconcile(makeServer()) // no bump+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)++ expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')+ })++ it('rebind does NOT reset the attempt budget across the bump (E2 guard, T3)', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout) // never converges+ await reconciler.reconcile(makeServer())+ await vi.advanceTimersByTimeAsync(2 * 5000) // 2 attempts consumed+ authorityGeneration = 2+ await reconciler.reconcile(makeServer()) // rebind — must NOT reset attempts+ await vi.advanceTimersByTimeAsync(21 * 5000) // 2 + 21 = 23 attempts, still short of 24++ expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)+ await vi.advanceTimersByTimeAsync(5000) // the 24th tick+ expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(true)+ })++ it('does NOT adopt a same-name recreate — a new uid gets a fresh window (T4)', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)+ await reconciler.reconcile(makeServer())+ const polls = (reconciler as unknown as { readinessPolls: Map<string, { attempts: number }> })+ .readinessPolls+ const firstWindow = polls.get('linear')+ await vi.advanceTimersByTimeAsync(12 * 5000)++ // Same name, NEW uid: identity retirement (beginStatusTracking), not a+ // rebind, must own this — a rebind adopting the recreate would carry the+ // old window's spent budget.+ const recreated = { ...makeServer(), uid: 'uid-linear-9999' }+ currentServer = recreated+ await reconciler.reconcile(recreated)++ const secondWindow = polls.get('linear')+ expect(secondWindow).not.toBe(firstWindow)+ expect(secondWindow?.attempts).toBe(0)+ })+})diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.tsindex a4b425ae7..fa30ec7a5 100644--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts@@ -89,6 +89,7 @@ describe('McpServer generated pod hardening', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.tsindex 09c0b5b84..044452efd 100644--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts@@ -87,6 +87,7 @@ function buildReconciler(customApi: MockCustomApi): McpServerReconciler {return new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -171,7 +172,14 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {const calls = customApi.patchNamespacedCustomObjectStatus.mock.callsconst lastCall = calls[calls.length - 1]?.[0]const patch = lastCall?.body- const value = Array.isArray(patch) ? patch[0]?.value : undefined+ const statusAdd = Array.isArray(patch)+ ? patch.find(+ candidate =>+ candidate?.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ : undefined+ const value = statusAdd?.valuereturn Array.isArray(value) ? value : (value?.conditions ?? [])}@@ -668,105 +676,6 @@ describe('McpServerReconciler remote egress proxy', () => {})})- describe('canonicalizeRemoteEgressProxyImage', () => {- it('patches stale remote McpServer spec.image to the platform egress proxy image', async () => {- const staleRemote = cloneServer(REMOTE_SERVER, {- image: LEGACY_REMOTE_EGRESS_IMAGE,- })-- await (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)-- expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(- expect.objectContaining({- group: 'clerum.io',- version: 'v1alpha1',- namespace: 'mcp-server',- plural: 'mcpservers',- name: 'mcp-sentry-remote',- body: {- spec: {- image: 'clerum/nginx-egress-proxy:0.1.0',- },- },- }),- expect.objectContaining({- middleware: expect.any(Array),- })- )- expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')- expect(lastPatchedStatusConditions(customApi)).toEqual(- expect.arrayContaining([- expect.objectContaining({- type: 'ImageCanonicalized',- status: 'True',- reason: 'RemoteEgressProxyImageMatches',- message: 'Remote McpServer spec.image matches the platform egress proxy image',- }),- ])- )- })-- it('does not patch spec for a remote McpServer already using the platform image', async () => {- const currentRemote = cloneServer(REMOTE_SERVER)-- await (reconciler as any).canonicalizeRemoteEgressProxyImage(currentRemote)-- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()- expect(lastPatchedStatusConditions(customApi)).toEqual(- expect.arrayContaining([- expect.objectContaining({- type: 'ImageCanonicalized',- status: 'True',- reason: 'RemoteEgressProxyImageMatches',- message: 'Remote McpServer spec.image matches the platform egress proxy image',- }),- ])- )- })-- it('does not patch local McpServers even when their image differs', async () => {- const localServer = cloneServer(LOCAL_SERVER, {- image: 'clerum/custom-local-mcp:1.2.3',- })-- await (reconciler as any).canonicalizeRemoteEgressProxyImage(localServer)-- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- expect(localServer.spec.image).toBe('clerum/custom-local-mcp:1.2.3')- })-- it('does not throw and records status when declarative image patching fails', async () => {- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- const staleRemote = cloneServer(REMOTE_SERVER, {- image: LEGACY_REMOTE_EGRESS_IMAGE,- })- customApi.patchNamespacedCustomObject.mockRejectedValueOnce(new Error('rbac denied'))-- await expect(- (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)- ).resolves.toBeUndefined()-- expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)- expect(warn).toHaveBeenCalledWith(- expect.stringContaining('Failed to canonicalize remote McpServer'),- expect.any(Error)- )- expect(lastPatchedStatusConditions(customApi)).toEqual(- expect.arrayContaining([- expect.objectContaining({- type: 'ImageCanonicalized',- status: 'False',- reason: 'RemoteEgressProxyImagePatchFailed',- message:- 'Failed to patch remote McpServer spec.image to the platform egress proxy image',- }),- ])- )- warn.mockRestore()- })- })-// ─── Test 3: buildDeployment for local servers ─────────────────────────describe('buildDeployment for local servers', () => {@@ -835,16 +744,12 @@ describe('McpServerReconciler remote egress proxy', () => {expect(cmIndex).toBeLessThan(deployIndex)})- it('should canonicalize stale remote image before syncing ConfigMap and Deployment', async () => {+ it('should use the platform image without rewriting a divergent remote desired image', async () => {const callOrder: string[] = []const staleRemote = cloneServer(REMOTE_SERVER, {image: LEGACY_REMOTE_EGRESS_IMAGE,})- customApi.patchNamespacedCustomObject.mockImplementation(async () => {- callOrder.push('canonicalizeImage')- return {}- })coreApi.createNamespacedConfigMap.mockImplementation(async () => {callOrder.push('createConfigMap')return {}@@ -860,23 +765,28 @@ describe('McpServerReconciler remote egress proxy', () => {await reconciler.reconcile(staleRemote)- expect(callOrder).toEqual([- 'canonicalizeImage',- 'createConfigMap',- 'createService',- 'createDeployment',- ])- expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')- expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(- expect.objectContaining({- body: {- spec: {- image: 'clerum/nginx-egress-proxy:0.1.0',- },- },- }),+ expect(callOrder).toEqual(['createConfigMap', 'createService', 'createDeployment'])+ expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(+ expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),+ expect.anything()+ )+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(expect.objectContaining({- middleware: expect.any(Array),+ body: expect.objectContaining({+ spec: expect.objectContaining({+ template: expect.objectContaining({+ spec: expect.objectContaining({+ containers: [+ expect.objectContaining({+ name: 'egress-proxy',+ image: 'clerum/nginx-egress-proxy:0.1.0',+ }),+ ],+ }),+ }),+ }),+ }),}))})@@ -1115,6 +1025,12 @@ describe('McpServerReconciler remote egress proxy', () => {})describe('reconcileDelete for remote servers', () => {+ beforeEach(() => {+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 1 }))+ reconciler.setResolveCurrentServer(() => undefined)+ customApi.getNamespacedCustomObject.mockRejectedValue(make404Error())+ })+it('should clean up ConfigMap via deleteResources', async () => {mockHccOwnedRuntimeReads(appsApi, coreApi)@@ -1136,11 +1052,14 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should clear status tracking after delete', async () => {+ let current: McpServerCRD | undefined = REMOTE_SERVER+ reconciler.setResolveCurrentServer(() => current)// First reconcile to populate statusawait reconciler.reconcile(REMOTE_SERVER)expect(reconciler.getStatus('mcp-sentry-remote').deployed).toBe(true)// Then delete+ current = undefinedawait reconciler.reconcileDelete('mcp-sentry-remote', 'mcp-server')const status = reconciler.getStatus('mcp-sentry-remote')expect(status.deployed).toBe(false)diff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.tsindex b3d979815..38b7c571f 100644--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts@@ -91,6 +91,7 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -122,6 +123,19 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {const container = capturedContainer(appsApi)expect(container.imagePullPolicy).toBe('IfNotPresent')})++ it('does not mutate a watched desired spec while building the sanitized Deployment', async () => {+ const server = makeServer({+ imagePullPolicy: 'Always',+ security: { runAsUser: 0, addCapabilities: ['CHOWN', 'SYS_ADMIN'] },+ })+ const originalSpec = structuredClone(server.spec)++ await reconciler.reconcile(server)++ expect(capturedContainer(appsApi).imagePullPolicy).toBe('IfNotPresent')+ expect(server.spec).toEqual(originalSpec)+ })})// ════════════════════════════════════════════════════════════════════diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.tsindex b2fd2bc8f..48d6fe191 100644--- a/host-context-controller/src/bindingPolicyReconciler.test.ts+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts@@ -2,6 +2,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'+function deferred<T = void>() {+ let resolve!: (value: T | PromiseLike<T>) => void+ const promise = new Promise<T>(res => {+ resolve = res+ })+ return { promise, resolve }+}+function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -40,7 +48,9 @@ describe('BindingPolicyReconciler', () => {const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]it('creates egress + ingress policies for a binding', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {+ isCurrent: () => true,+ })// 2 calls: 1 egress + 1 ingressexpect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)@@ -75,7 +85,9 @@ describe('BindingPolicyReconciler', () => {})it('applies binding-allow labels to all policies', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {+ isCurrent: () => true,+ })for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body@@ -89,12 +101,43 @@ describe('BindingPolicyReconciler', () => {it('handles 409 conflict with create-or-replace pattern', async () => {mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {+ isCurrent: () => true,+ })expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()})+ it('does not replace or start another apply after its authority lease is retired', async () => {+ const readStarted = deferred()+ const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ readStarted.resolve()+ return releaseRead.promise+ })+ let current = true++ const reconcile = reconciler.reconcileBindings(+ 'my-recipe',+ singleBinding,+ 'redis-mcp',+ 'redis-mcp',+ { isCurrent: () => current }+ )+ await readStarted.promise++ current = false+ releaseRead.resolve({ metadata: { resourceVersion: '1' } })+ await reconcile++ // The first create was authorized and returned 409 without mutating. Once+ // the lease is retired, no replacement or second policy apply may start.+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+it('cleanupBindings deletes policies in both namespaces', async () => {mockApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],@@ -108,8 +151,65 @@ describe('BindingPolicyReconciler', () => {expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalled()})+ it('does not delete listed policies when a same-name recipe is recreated during LIST', async () => {+ const bothListsStarted = deferred()+ const releaseLists = deferred<{+ items: Array<{ metadata: { name: string } }>+ }>()+ let listCalls = 0+ mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {+ listCalls += 1+ if (listCalls === 2) bothListsStarted.resolve()+ return releaseLists.promise+ })+ let recipeRecreated = false+ const deleteAllowed = vi.fn(async () => !recipeRecreated)++ const cleanup = reconciler.cleanupBindings('my-recipe', { deleteAllowed })+ await bothListsStarted.promise++ recipeRecreated = true+ releaseLists.resolve({+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],+ })+ await cleanup++ expect(deleteAllowed).toHaveBeenCalledTimes(2)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('rejects cleanup when listing binding policies fails', async () => {+ mockApi.listNamespacedNetworkPolicy.mockRejectedValue(new Error('transient LIST failure'))++ await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(+ 'Failed to clean up binding policies for recipe "my-recipe"'+ )+ })++ it('rejects cleanup when deleting a binding policy fails with a non-404 error', async () => {+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],+ })+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('transient DELETE failure'))++ await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(+ 'Failed to clean up binding policies for recipe "my-recipe"'+ )+ })++ it('treats a 404 while deleting binding policies as successful cleanup', async () => {+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],+ })+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })++ await expect(reconciler.cleanupBindings('my-recipe')).resolves.toBeUndefined()+ })+it('uses _from for ingress rules (K8s client ObjectSerializer maps _from → from)', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {+ isCurrent: () => true,+ })const calls = mockApi.createNamespacedNetworkPolicy.mock.callsconst ingressCall = calls.find(@@ -126,7 +226,9 @@ describe('BindingPolicyReconciler', () => {})it('uses correct pod selectors and namespace-aware placement', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {+ isCurrent: () => true,+ })const calls = mockApi.createNamespacedNetworkPolicy.mock.callsconst egressCall = calls.find(@@ -196,7 +298,8 @@ describe('BindingPolicyReconciler', () => {'my-recipe',singleBinding,'redis-mcp',- 'my-recipe-redis-mcp'+ 'my-recipe-redis-mcp',+ { isCurrent: () => true })const calls = mockApi.createNamespacedNetworkPolicy.mock.callsdiff --git a/host-context-controller/src/bindingPolicyReconciler.ts b/host-context-controller/src/bindingPolicyReconciler.tsindex b0486d63c..0281e69f6 100644--- a/host-context-controller/src/bindingPolicyReconciler.ts+++ b/host-context-controller/src/bindingPolicyReconciler.ts@@ -37,6 +37,18 @@ export interface BindingDef {protocol?: 'TCP' | 'UDP'}+export interface BindingPolicyReconcileOptions {+ // Required (fail-closed): every caller must supply the authority predicate so+ // a retired reconcile pass cannot keep mutating binding NetworkPolicies. There+ // is no assume-current default — an omitted predicate is a compile error, not+ // a silent fail-open (the old `?? (() => true)`) nor a silent no-op.+ isCurrent: () => boolean+}++export interface BindingPolicyCleanupOptions {+ deleteAllowed?: () => Promise<boolean>+}+export class BindingPolicyReconciler {private networkingApi: k8s.NetworkingV1Apiprivate mcpServerNamespace: string@@ -60,70 +72,86 @@ export class BindingPolicyReconciler {recipeName: string,bindings: BindingDef[],mcpWorkloadName: string,- mcpServerName = mcpWorkloadName+ mcpServerName = mcpWorkloadName,+ options: BindingPolicyReconcileOptions): Promise<void> {+ const isCurrent = options.isCurrent+ if (!isCurrent()) returnconsole.log(`[BindingNP] Reconciling ${bindings.length} binding(s) for recipe "${recipeName}"`)- await Promise.all(- bindings.map(async binding => {- const protocol = binding.protocol ?? 'TCP'+ // Sequence writes so authority can be rechecked between every Kubernetes+ // effect. A retired pass must not start the next binding policy mutation.+ for (const binding of bindings) {+ if (!isCurrent()) return+ const protocol = binding.protocol ?? 'TCP'- // Determine namespaces based on which workload is the MCP server.- // The MCP workload (with transport) lives in mcp-server namespace,- // the other workload lives in sandbox-recipes namespace.- const fromIsMcp = binding.from === mcpWorkloadName- const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE- const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace- const fromPodSelector: Record<string, string> = fromIsMcp- ? { [MCPSERVER_LABEL]: mcpServerName }- : { app: binding.from }- const toPodSelector: Record<string, string> = fromIsMcp- ? { app: binding.to }- : { [MCPSERVER_LABEL]: mcpServerName }+ // Determine namespaces based on which workload is the MCP server.+ // The MCP workload (with transport) lives in mcp-server namespace,+ // the other workload lives in sandbox-recipes namespace.+ const fromIsMcp = binding.from === mcpWorkloadName+ const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE+ const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace+ const fromPodSelector: Record<string, string> = fromIsMcp+ ? { [MCPSERVER_LABEL]: mcpServerName }+ : { app: binding.from }+ const toPodSelector: Record<string, string> = fromIsMcp+ ? { app: binding.to }+ : { [MCPSERVER_LABEL]: mcpServerName }- const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')- const egressPolicy = this.buildEgressPolicy(- egressName,- recipeName,- fromPodSelector,- toPodSelector,- fromNs,- toNs,- binding.port,- protocol- )- const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')- const ingressPolicy = this.buildIngressPolicy(- ingressName,- recipeName,- fromPodSelector,- toPodSelector,- fromNs,- toNs,- binding.port,- protocol- )+ const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')+ const egressPolicy = this.buildEgressPolicy(+ egressName,+ recipeName,+ fromPodSelector,+ toPodSelector,+ fromNs,+ toNs,+ binding.port,+ protocol+ )+ const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')+ const ingressPolicy = this.buildIngressPolicy(+ ingressName,+ recipeName,+ fromPodSelector,+ toPodSelector,+ fromNs,+ toNs,+ binding.port,+ protocol+ )- // Egress and ingress target different namespaces — safe to parallelize- await Promise.all([- this.applyPolicy(egressName, egressPolicy, fromNs),- this.applyPolicy(ingressName, ingressPolicy, toNs),- ])- })- )+ if (!isCurrent()) return+ await this.applyPolicy(egressName, egressPolicy, fromNs, isCurrent)+ if (!isCurrent()) return+ await this.applyPolicy(ingressName, ingressPolicy, toNs, isCurrent)+ }}/*** Delete all binding-scoped NetworkPolicies for a recipe.*/- async cleanupBindings(recipeName: string): Promise<void> {+ async cleanupBindings(+ recipeName: string,+ options: BindingPolicyCleanupOptions = {}+ ): Promise<void> {console.log(`[BindingNP] Cleaning up binding policies for recipe "${recipeName}"`)- // Delete from both namespaces in parallel (independent operations)- await Promise.all([- this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE),- this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace),+ // LIST operations are read-only and namespace-independent. Each resulting+ // delete is separately authorized after its LIST completes.+ const results = await Promise.allSettled([+ this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE, options.deleteAllowed),+ this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace, options.deleteAllowed),])+ const failures = results.flatMap(result =>+ result.status === 'rejected' ? [result.reason] : []+ )+ if (failures.length > 0) {+ throw new AggregateError(+ failures,+ `Failed to clean up binding policies for recipe "${recipeName}"`+ )+ }}// ─── Policy Builders ────────────────────────────────────────────@@ -242,43 +270,49 @@ export class BindingPolicyReconciler {private async applyPolicy(name: string,policy: k8s.V1NetworkPolicy,- namespace: string+ namespace: string,+ isCurrent: () => boolean): Promise<void> {- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]')+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]', isCurrent)}- private async deletePoliciesForRecipe(recipeName: string, namespace: string): Promise<void> {+ private async deletePoliciesForRecipe(+ recipeName: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {+ let response: k8s.V1NetworkPolicyListtry {- const response = await this.networkingApi.listNamespacedNetworkPolicy({+ response = await this.networkingApi.listNamespacedNetworkPolicy({namespace,labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=binding-allow,${RECIPE_LABEL}=${recipeName}`,})-- await Promise.all(- (response.items || []).map(async policy => {- const policyName = policy.metadata?.name- if (!policyName) {- console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)- return- }- try {- await this.networkingApi.deleteNamespacedNetworkPolicy({- name: policyName,- namespace,- })- console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)- } catch (error: unknown) {- if (getErrorCode(error) !== 404) {- console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)- }- }- })- )} catch (error) {console.error(`[BindingNP] Failed to list policies for recipe "${recipeName}" in ${namespace}:`,error)+ throw error+ }++ for (const policy of response.items || []) {+ const policyName = policy.metadata?.name+ if (!policyName) {+ console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)+ continue+ }+ try {+ if (deleteAllowed && !(await deleteAllowed())) return+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) continue+ console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)+ throw error+ }}}}diff --git a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts b/host-context-controller/src/config.externalEgressDnsTimeout.test.tsnew file mode 100644index 000000000..7396924b9--- /dev/null+++ b/host-context-controller/src/config.externalEgressDnsTimeout.test.ts@@ -0,0 +1,46 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++const originalEnv = process.env++async function loadConfig(raw: string | undefined) {+ vi.resetModules()+ process.env = { ...originalEnv }+ if (raw === undefined) {+ delete process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS+ } else {+ process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS = raw+ }+ return import('./config')+}++afterEach(() => {+ process.env = originalEnv+ vi.resetModules()+})++describe('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS', () => {+ it.each([undefined, '', ' '])('defaults to 5000 for %s', async raw => {+ const { config } = await loadConfig(raw)++ expect(config.externalEgressDnsResolveTimeoutMs).toBe(5_000)+ })++ it.each([+ ['1', 1],+ ['5000', 5_000],+ ['2147483647', 2_147_483_647],+ ])('accepts the positive bounded integer %s', async (raw, expected) => {+ const { config } = await loadConfig(raw)++ expect(config.externalEgressDnsResolveTimeoutMs).toBe(expected)+ })++ it.each(['0', '-1', '1.5', '5000junk', 'NaN', 'Infinity', '2147483648'])(+ "fails configuration validation for '%s'",+ async raw => {+ await expect(loadConfig(raw)).rejects.toThrow(+ 'HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer'+ )+ }+ )+})diff --git a/host-context-controller/src/config.ts b/host-context-controller/src/config.tsindex 7b7632b75..162ccd06e 100644--- a/host-context-controller/src/config.ts+++ b/host-context-controller/src/config.ts@@ -197,6 +197,10 @@ export interface Config {// so accumulated IPs never expire between refreshes (issue #299).externalEgressResyncIntervalSec: number+ // Deadline for one DNS resolution attempt. A silent resolver cannot block+ // safety convergence indefinitely.+ externalEgressDnsResolveTimeoutMs: number+// Grace kept after a DNS TTL before an accumulated /32 may expire (seconds).// The sliding window that fixes #299: entries live for TTL + overlap, so a// provider that rotates a single A record still resolves through the union of@@ -245,6 +249,17 @@ function getEnvInt(key: string, defaultValue: number): number {return isNaN(parsed) ? defaultValue : parsed}+export function parseExternalEgressDnsTimeoutMs(raw: string | undefined): number {+ if (raw === undefined || raw.trim() === '') return 5_000+ const parsed = Number(raw)+ if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 2_147_483_647) {+ throw new Error(+ `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer no greater than 2147483647, got '${raw}'`+ )+ }+ return parsed+}+/*** Loud integer env parser for the external-egress knobs (audit R1-L1). Unlike* getEnvInt (parseInt), this rejects non-canonical values instead of silently@@ -714,6 +729,9 @@ export const config: Config = {// they expire, closing the #299 rotation gap. 0 disables (reintroduces the// stale-snapshot risk; only for operators who explicitly opt out).externalEgressResyncIntervalSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_RESYNC_SEC', 60),+ externalEgressDnsResolveTimeoutMs: parseExternalEgressDnsTimeoutMs(+ getEnv('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS')+ ),externalEgressOverlapSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_OVERLAP_SEC', 300),externalEgressRefreshFloorSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_REFRESH_FLOOR_SEC',diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.tsnew file mode 100644index 000000000..7a9258139--- /dev/null+++ b/host-context-controller/src/deploymentStrategy.test.ts@@ -0,0 +1,66 @@+import { describe, expect, it } from 'vitest'+import { loadAll } from 'js-yaml'+import { readFileSync } from 'node:fs'+import path from 'node:path'++/**+ * HCC's safety model assumes a single writer: replicas:1 with NO leader+ * election (see statelessLifecycleExecutor.ts — "Running >1 replica risks two+ * controllers racing"). The default Deployment strategy is RollingUpdate with+ * maxSurge>0, which starts the new pod before the old one exits — a guaranteed+ * two-replica overlap on every rollout. During that window the outgoing replica+ * can win a conflict-retry replace (applyNetworkPolicy → replaceWithConflictRetry)+ * and land its wider spec back over a policy the new replica just hardened and+ * certified. There is no NetworkPolicy watch or periodic resync to heal it, and+ * the new replica's safety fence only sees failures in its own process — so+ * /ready stays 200 over a stale allow. Recreate closes the overlap, which is the+ * only way to honour the documented single-writer assumption during a rollout.+ */+describe('host-context-controller Deployment strategy', () => {+ const docs = loadAll(+ readFileSync(+ path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),+ 'utf8'+ )+ ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>++ const deployment = docs.find(+ d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'+ )++ it('defines the host-context-controller Deployment', () => {+ expect(deployment).toBeDefined()+ })++ it('runs a single replica', () => {+ expect(deployment?.spec?.replicas).toBe(1)+ })++ it('uses Recreate so a rollout never overlaps two writers', () => {+ // RollingUpdate (the default) would start the new pod before terminating the+ // old one; with no leader election that races two NetworkPolicy writers.+ const strategy = deployment?.spec?.strategy as { type?: string } | undefined+ expect(strategy?.type).toBe('Recreate')+ })++ it('carries no rollingUpdate block under Recreate', () => {+ // A leftover spec.strategy.rollingUpdate is invalid under Recreate and, if a+ // future edit flipped only `type` back, would quietly reintroduce surge.+ const strategy = deployment?.spec?.strategy as { rollingUpdate?: unknown } | undefined+ expect(strategy?.rollingUpdate).toBeUndefined()+ })++ it('bounds rollout-stall detection above the worst legitimate startup', () => {+ // Detection only, not recovery: Recreate + replicas:1 means a botched+ // rollout is healed operationally (kubectl rollout undo), never by K8s.+ // This deadline just flips Progressing=False / fails `kubectl rollout+ // status` deterministically for observers without their own --timeout. It+ // must exceed the worst legitimate startup, which — until #205 fully+ // decouples readiness from fleet convergence — scales with the McpServer/+ // Context fleet: origin/dev observed 651s on clerum-dev (108 McpServers /+ // 22 Contexts) on 2026-08-05, so 1200s (900s wait + 300s head-start) is the+ // merged mitigation. Comes back down once readiness no longer waits on the+ // fleet.+ expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)+ })+})diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.tsnew file mode 100644index 000000000..ac8a5d926--- /dev/null+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts@@ -0,0 +1,968 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import {+ ExternalEgressConvergenceCoordinator,+ type ExternalEgressRetryHandle,+ type ExternalEgressWatchEventType,+} from './externalEgressConvergenceCoordinator'+import { externalEgressRetriesAtCap } from './metrics'+import type { McpServerCRD } from './types'++function deferred<T = void>() {+ let resolve!: (value: T | PromiseLike<T>) => void+ let reject!: (reason?: unknown) => void+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {+ resolve = resolvePromise+ reject = rejectPromise+ })+ return { promise, resolve, reject }+}++function server(+ name: string,+ generation = 1,+ options: { bindings?: boolean; namespace?: string } = {}+): McpServerCRD {+ return {+ name,+ namespace: options.namespace ?? 'mcp-server',+ uid: `${name}-uid`,+ generation,+ spec: {+ contextRef: 'default',+ image: `clerum/${name}:v${generation}`,+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings:+ options.bindings === false ? undefined : [{ dns: `${name}.example`, port: 443 }],+ },+ }+}++type CoordinatorOverrides = Partial<+ ConstructorParameters<typeof ExternalEgressConvergenceCoordinator>[0]+>++function coordinator(overrides: CoordinatorOverrides = {}) {+ const servers = new Map<string, McpServerCRD>()+ const mutate = vi.fn(async () => undefined)+ const replay = vi.fn(async () => undefined)+ const instance = new ExternalEgressConvergenceCoordinator({+ listServers: () => [...servers.values()],+ getCurrentServer: name => servers.get(name),+ inventoryAuthoritative: () => true,+ sameDesiredRevision: (left, right) =>+ left.uid === right.uid &&+ left.generation === right.generation &&+ JSON.stringify(left.spec) === JSON.stringify(right.spec),+ enqueue: async (_selected, work) => work(),+ mutate,+ replay,+ externalEgressRefreshMinTtlMs: () => Infinity,+ ...overrides,+ })+ return { instance, mutate, replay, servers }+}++async function flushMicrotasks(): Promise<void> {+ await Promise.resolve()+ await Promise.resolve()+ await Promise.resolve()+}++describe('ExternalEgressConvergenceCoordinator', () => {+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it.each([+ ['presence', 'DELETED' as const, undefined],+ ['revision', 'MODIFIED' as const, server('redis-tools', 2)],+ ])(+ 'lets an existing timer consume the latest %s intent and resets its attempt ladder',+ async (_change, nextType, nextServer) => {+ vi.useFakeTimers()+ const initial = server('redis-tools', 1)+ const state = new Map<string, McpServerCRD>([[initial.name, initial]])+ const replayed: Array<{ type: ExternalEgressWatchEventType; server: McpServerCRD }> = []+ let instance!: ExternalEgressConvergenceCoordinator+ instance = coordinator({+ getCurrentServer: name => state.get(name),+ replay: async (type, selected) => {+ replayed.push({ type, server: selected })+ instance.scheduleRetry(type, selected)+ },+ }).instance++ instance.scheduleRetry('ADDED', initial)+ if (nextServer) state.set(nextServer.name, nextServer)+ else state.delete(initial.name)+ instance.scheduleRetry(nextType, nextServer ?? initial)++ await vi.advanceTimersByTimeAsync(5000)+ expect(replayed).toEqual([+ {+ type: nextType,+ server: expect.objectContaining({+ generation: nextServer?.generation ?? initial.generation,+ }),+ },+ ])++ await vi.advanceTimersByTimeAsync(4999)+ expect(replayed).toHaveLength(1)+ await vi.advanceTimersByTimeAsync(1)+ expect(replayed).toHaveLength(2)+ instance.stop()+ }+ )++ it('continues retrying at the capped delay until the replay succeeds', async () => {+ vi.useFakeTimers()+ const selected = server('web-search')+ let instance!: ExternalEgressConvergenceCoordinator+ let attempts = 0+ instance = coordinator({+ getCurrentServer: () => selected,+ replay: async (type, current, retry) => {+ attempts += 1+ if (attempts < 5) {+ instance.scheduleRetry(type, current)+ return+ }+ retry.complete()+ },+ }).instance++ instance.scheduleRetry('ADDED', selected)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(15000)+ await vi.advanceTimersByTimeAsync(30000)+ expect(attempts).toBe(3)+ await vi.advanceTimersByTimeAsync(29999)+ expect(attempts).toBe(3)+ await vi.advanceTimersByTimeAsync(1)+ expect(attempts).toBe(4)+ await vi.advanceTimersByTimeAsync(29999)+ expect(attempts).toBe(4)+ await vi.advanceTimersByTimeAsync(1)++ expect(attempts).toBe(5)+ await vi.advanceTimersByTimeAsync(30000)+ expect(attempts).toBe(5)+ instance.stop()+ })++ it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {+ vi.useFakeTimers()+ const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')+ const selected = server('web-search')+ let instance!: ExternalEgressConvergenceCoordinator+ let succeed = false+ instance = coordinator({+ getCurrentServer: () => selected,+ replay: async (type, current, retry) => {+ if (succeed) {+ retry.complete()+ return+ }+ instance.scheduleRetry(type, current)+ },+ }).instance++ instance.scheduleRetry('ADDED', selected)+ // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.+ await vi.advanceTimersByTimeAsync(5000)+ expect(setSpy).not.toHaveBeenCalled()++ // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.+ await vi.advanceTimersByTimeAsync(15000)+ expect(setSpy).toHaveBeenLastCalledWith(1)++ // Once the replay finally succeeds, the capped server is cleared back to 0.+ succeed = true+ await vi.advanceTimersByTimeAsync(30000)+ expect(setSpy).toHaveBeenLastCalledWith(0)++ instance.stop()+ setSpy.mockRestore()+ })++ it('never clears a pending full replay after an egress-only success', async () => {+ vi.useFakeTimers()+ const selected = server('web-search')+ const { instance, replay, servers } = coordinator()+ servers.set(selected.name, selected)++ instance.scheduleRetry('ADDED', selected)+ const retry = await instance.reconcile('MODIFIED', selected)+ expect(retry).toBeDefined()++ await vi.advanceTimersByTimeAsync(5000)+ expect(replay).toHaveBeenCalledOnce()+ instance.stop()+ })++ it('clears a retry only after its complete full-replay handle finishes', async () => {+ vi.useFakeTimers()+ const selected = server('web-search')+ let instance!: ExternalEgressConvergenceCoordinator+ const replays: ExternalEgressRetryHandle[] = []+ instance = coordinator({+ getCurrentServer: () => selected,+ replay: async (type, current, retry) => {+ replays.push(retry)+ const completion = await instance.reconcile(type, current, { retry })+ expect(completion).toBe(retry)+ retry.complete()+ },+ }).instance++ instance.scheduleRetry('ADDED', selected)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(30000)++ expect(replays).toHaveLength(1)+ instance.stop()+ })++ it('prevents a stale retry handle from clearing a newer intent', async () => {+ vi.useFakeTimers()+ const first = server('web-search', 1)+ const second = server('web-search', 2)+ const state = new Map<string, McpServerCRD>([[first.name, first]])+ const handles: ExternalEgressRetryHandle[] = []+ const replayed: McpServerCRD[] = []+ const firstReplayStarted = deferred()+ const releaseFirstReplay = deferred()+ const { instance } = coordinator({+ getCurrentServer: name => state.get(name),+ replay: async (_type, current, retry) => {+ replayed.push(current)+ handles.push(retry)+ if (replayed.length === 1) {+ firstReplayStarted.resolve()+ await releaseFirstReplay.promise+ }+ },+ })++ instance.scheduleRetry('ADDED', first)+ await vi.advanceTimersByTimeAsync(5000)+ await firstReplayStarted.promise+ expect(handles).toHaveLength(1)++ state.set(second.name, second)+ instance.scheduleRetry('MODIFIED', second)+ handles[0].complete()+ releaseFirstReplay.resolve()+ await flushMicrotasks()+ await vi.advanceTimersByTimeAsync(5000)++ expect(replayed).toEqual([+ expect.objectContaining({ generation: 1 }),+ expect.objectContaining({ generation: 2 }),+ ])+ instance.stop()+ })++ it('keeps retry, resync, and startup concurrency lanes independent', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const retryRelease = deferred()+ const activeRetries = deferred()+ let retryCount = 0+ const resyncServer = server('resync-server')+ const startupServer = server('startup-server')+ const canonical = new Map<string, McpServerCRD>([+ [resyncServer.name, resyncServer],+ [startupServer.name, startupServer],+ ])+ const mutations: string[] = []+ const { instance } = coordinator({+ listServers: () => [resyncServer],+ getCurrentServer: name => canonical.get(name),+ mutate: async (_type, selected) => {+ mutations.push(selected.name)+ },+ replay: async () => {+ retryCount += 1+ if (retryCount === 10) activeRetries.resolve()+ await retryRelease.promise+ },+ })++ for (let index = 0; index < 10; index += 1) {+ const selected = server(`retry-${index}`, 1, { bindings: false })+ instance.scheduleRetry('ADDED', selected)+ }+ await vi.advanceTimersByTimeAsync(5000)+ await activeRetries.promise++ const resync = instance.runResync()+ const startup = instance.prepareStartupGates([startupServer])+ await Promise.all([resync, startup.waitFor(startupServer)])++ expect(mutations).toEqual(expect.arrayContaining(['resync-server', 'startup-server']))+ retryRelease.resolve()+ instance.stop()+ })++ it('queues at most one latest-wins retry job per key behind a saturated limiter', async () => {+ vi.useFakeTimers()+ const blockersRelease = deferred()+ const tenBlockersStarted = deferred()+ const finalReplayStarted = deferred()+ const initial = server('churning-server', 1)+ const second = server('churning-server', 2)+ const final = server('churning-server', 3)+ const state = new Map<string, McpServerCRD>([[initial.name, initial]])+ const targetEnqueues: McpServerCRD[] = []+ const targetReplays: McpServerCRD[] = []+ let blockerCount = 0+ const { instance } = coordinator({+ getCurrentServer: name => state.get(name),+ enqueue: async (selected, work) => {+ if (selected.name === initial.name) targetEnqueues.push(selected)+ await work()+ },+ replay: async (_type, selected, retry) => {+ if (selected.name.startsWith('limiter-blocker-')) {+ blockerCount += 1+ if (blockerCount === 10) tenBlockersStarted.resolve()+ await blockersRelease.promise+ return+ }+ targetReplays.push(selected)+ retry.complete()+ finalReplayStarted.resolve()+ },+ })++ for (let index = 0; index < 10; index += 1) {+ instance.scheduleRetry('DELETED', server(`limiter-blocker-${index}`))+ }+ await vi.advanceTimersByTimeAsync(5000)+ await tenBlockersStarted.promise++ instance.scheduleRetry('ADDED', initial)+ await vi.advanceTimersByTimeAsync(5000)+ state.set(second.name, second)+ instance.scheduleRetry('MODIFIED', second)+ await vi.advanceTimersByTimeAsync(5000)+ state.set(final.name, final)+ instance.scheduleRetry('MODIFIED', final)+ await vi.advanceTimersByTimeAsync(5000)++ expect(targetEnqueues).toHaveLength(0)+ blockersRelease.resolve()+ await finalReplayStarted.promise+ await flushMicrotasks()+ await flushMicrotasks()++ expect(targetEnqueues).toHaveLength(1)+ expect(targetReplays).toEqual([expect.objectContaining({ generation: 3 })])+ instance.stop()+ })++ it('re-arms a retry scheduled by an active replay only after that key leaves its slot', async () => {+ vi.useFakeTimers()+ const selected = server('slow-retry')+ const firstReplayStarted = deferred()+ const releaseFirstReplay = deferred()+ let instance!: ExternalEgressConvergenceCoordinator+ let replayCount = 0+ instance = coordinator({+ getCurrentServer: () => selected,+ replay: async (type, current, retry) => {+ replayCount += 1+ if (replayCount === 1) {+ instance.scheduleRetry(type, current)+ firstReplayStarted.resolve()+ await releaseFirstReplay.promise+ return+ }+ retry.complete()+ },+ }).instance++ instance.scheduleRetry('ADDED', selected)+ await vi.advanceTimersByTimeAsync(5000)+ await firstReplayStarted.promise++ await vi.advanceTimersByTimeAsync(15000)+ expect(replayCount).toBe(1)++ releaseFirstReplay.resolve()+ await flushMicrotasks()+ await vi.advanceTimersByTimeAsync(14999)+ expect(replayCount).toBe(1)+ await vi.advanceTimersByTimeAsync(1)+ expect(replayCount).toBe(2)+ instance.stop()+ })++ it('cancels a retry waiter on stop before limiter capacity becomes available', async () => {+ vi.useFakeTimers()+ const blockersRelease = deferred()+ const tenBlockersStarted = deferred()+ const queued = server('queued-before-stop')+ let blockerCount = 0+ const enqueuedTargets: string[] = []+ const { instance } = coordinator({+ enqueue: async (selected, work) => {+ if (selected.name === queued.name) enqueuedTargets.push(selected.name)+ await work()+ },+ replay: async (_type, selected) => {+ if (!selected.name.startsWith('stop-blocker-')) return+ blockerCount += 1+ if (blockerCount === 10) tenBlockersStarted.resolve()+ await blockersRelease.promise+ },+ })++ for (let index = 0; index < 10; index += 1) {+ instance.scheduleRetry('DELETED', server(`stop-blocker-${index}`))+ }+ await vi.advanceTimersByTimeAsync(5000)+ await tenBlockersStarted.promise+ instance.scheduleRetry('DELETED', queued)+ await vi.advanceTimersByTimeAsync(5000)++ instance.stop()+ blockersRelease.resolve()+ await flushMicrotasks()+ await flushMicrotasks()++ expect(enqueuedTargets).toEqual([])+ })++ it('resolves the current canonical server after periodic jitter', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0.5)+ const stale = server('web-search', 1)+ const current = server('web-search', 2)+ const state = new Map<string, McpServerCRD>([[stale.name, stale]])+ const reconciled: McpServerCRD[] = []+ const { instance } = coordinator({+ listServers: () => [stale],+ getCurrentServer: name => state.get(name),+ mutate: async (_type, selected) => {+ reconciled.push(selected)+ },+ })++ const resync = instance.runResync()+ await flushMicrotasks()+ state.set(current.name, current)+ await vi.advanceTimersByTimeAsync(2500)+ await resync++ expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])+ instance.stop()+ })++ it('returns one global single-flight promise for overlapping resync requests', async () => {+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const selected = server('web-search')+ const release = deferred()+ const mutationStarted = deferred()+ const { instance, servers } = coordinator({+ mutate: async () => {+ mutationStarted.resolve()+ await release.promise+ },+ })+ servers.set(selected.name, selected)++ const first = instance.runResync()+ const overlapping = instance.runResync()+ expect(overlapping).toBe(first)+ await mutationStarted.promise++ release.resolve()+ await first+ instance.stop()+ })++ it('skips a resync key whose external-egress mutation is already in flight', async () => {+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const selected = server('web-search')+ const release = deferred()+ const mutationStarted = deferred()+ const mutate = vi.fn(async () => {+ mutationStarted.resolve()+ await release.promise+ })+ const { instance, servers } = coordinator({ mutate })+ servers.set(selected.name, selected)++ const active = instance.reconcile('MODIFIED', selected)+ await mutationStarted.promise+ await instance.runResync()++ expect(mutate).toHaveBeenCalledOnce()+ release.resolve()+ await active+ instance.stop()+ })++ it('resolves canonical state after entering the keyed resync queue', async () => {+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const stale = server('web-search', 1)+ const current = server('web-search', 2)+ const state = new Map<string, McpServerCRD>([[stale.name, stale]])+ const queueEntered = deferred()+ const releaseQueue = deferred()+ const reconciled: McpServerCRD[] = []+ const { instance } = coordinator({+ listServers: () => [stale],+ getCurrentServer: name => state.get(name),+ enqueue: async (_selected, work) => {+ queueEntered.resolve()+ await releaseQueue.promise+ await work()+ },+ mutate: async (_type, selected) => {+ reconciled.push(selected)+ },+ })++ const resync = instance.runResync()+ await queueEntered.promise+ state.set(current.name, current)+ releaseQueue.resolve()+ await resync++ expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])+ instance.stop()+ })++ it('keeps a startup gate false when desired revision changes during egress', async () => {+ const selected = server('web-search', 1)+ const current = server('web-search', 2)+ const state = new Map<string, McpServerCRD>([[selected.name, selected]])+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let selectedLease: (() => boolean) | undefined+ const { instance } = coordinator({+ getCurrentServer: name => state.get(name),+ mutate: async (_type, _server, options) => {+ selectedLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ },+ })++ const gates = instance.prepareStartupGates([selected])+ await mutationStarted.promise+ expect(selectedLease?.()).toBe(true)+ state.set(current.name, current)+ expect(selectedLease?.()).toBe(false)+ releaseMutation.resolve()++ await expect(gates.waitFor(selected)).resolves.toBe(false)+ instance.stop()+ })++ it('keeps a failed startup DNS gate isolated and recovers it through the dedicated retry', async () => {+ vi.useFakeTimers()+ const selected = server('dns-held-server')+ const dnsFailure = new Error('dns resolution failed')+ const mutate = vi.fn().mockRejectedValueOnce(dnsFailure).mockResolvedValue(undefined)+ const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ let instance!: ExternalEgressConvergenceCoordinator+ const built = coordinator({+ mutate,+ replay: async (type, current, retry) => {+ const completion = await instance.reconcile(type, current, { retry })+ completion?.complete()+ },+ })+ instance = built.instance+ built.servers.set(selected.name, selected)++ const gates = instance.prepareStartupGates([selected])+ await expect(gates.waitFor(selected)).resolves.toBe(false)+ expect(mutate).toHaveBeenCalledOnce()+ expect(error).toHaveBeenCalledWith(+ `[K8s] Initial external egress reconciliation failed for ${selected.namespace}/${selected.name}; ` ++ 'runtime reconciliation will stay blocked until retry succeeds:',+ dnsFailure+ )++ await vi.advanceTimersByTimeAsync(5000)+ expect(mutate).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(30000)+ expect(mutate).toHaveBeenCalledTimes(2)+ instance.stop()+ })++ it('retires a direct mutation lease when desired revision changes in place', async () => {+ const selected = server('web-search', 1)+ const replacement = server('web-search', 2)+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let selectedLease: (() => boolean) | undefined+ const { instance, servers } = coordinator({+ mutate: async (_type, _server, options) => {+ selectedLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ },+ })+ servers.set(selected.name, selected)++ const mutation = instance.reconcile('MODIFIED', selected)+ await mutationStarted.promise+ expect(selectedLease?.()).toBe(true)++ servers.set(replacement.name, replacement)+ expect(selectedLease?.()).toBe(false)+ releaseMutation.resolve()+ await mutation+ instance.stop()+ })++ it('does not start a second mutation after stop releases an in-flight wait', async () => {+ const firstMutation = deferred()+ const selected = server('web-search')+ const mutate = vi.fn(async () => firstMutation.promise)+ const { instance, servers } = coordinator({ mutate })+ servers.set(selected.name, selected)++ const first = instance.reconcile('MODIFIED', selected)+ await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())+ const waiting = instance.reconcile('MODIFIED', selected)+ instance.stop()+ firstMutation.resolve()+ await Promise.all([first, waiting])++ expect(mutate).toHaveBeenCalledOnce()+ })++ it('rechecks per-key in-flight ownership when multiple waiters resume together', async () => {+ const selected = server('web-search')+ const releases = [deferred(), deferred(), deferred()]+ const starts = [deferred(), deferred(), deferred()]+ let active = 0+ let maxActive = 0+ let call = 0+ const mutate = vi.fn(async () => {+ const index = call+++ active += 1+ maxActive = Math.max(maxActive, active)+ starts[index].resolve()+ await releases[index].promise+ active -= 1+ })+ const { instance, servers } = coordinator({ mutate })+ servers.set(selected.name, selected)++ const first = instance.reconcile('MODIFIED', selected)+ await starts[0].promise+ const second = instance.reconcile('MODIFIED', selected)+ const third = instance.reconcile('MODIFIED', selected)++ try {+ releases[0].resolve()+ await starts[1].promise+ await flushMicrotasks()+ expect(maxActive).toBe(1)+ } finally {+ for (const release of releases) release.resolve()+ await Promise.all([first, second, third])+ instance.stop()+ }+ })++ it('rechecks a caller lease after waiting for an in-flight mutation', async () => {+ const firstMutation = deferred()+ const selected = server('web-search')+ let current = true+ const mutate = vi.fn(async () => firstMutation.promise)+ const { instance, servers } = coordinator({ mutate })+ servers.set(selected.name, selected)++ const first = instance.reconcile('MODIFIED', selected)+ await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())+ const waiting = instance.reconcile('MODIFIED', selected, {+ isCurrent: () => current,+ })+ current = false+ firstMutation.resolve()+ await Promise.all([first, waiting])++ expect(mutate).toHaveBeenCalledOnce()+ instance.stop()+ })++ it('keeps a retry pending while inventory authority is unavailable', async () => {+ vi.useFakeTimers()+ const selected = server('web-search')+ let authoritative = false+ const { instance, replay, servers } = coordinator({+ inventoryAuthoritative: () => authoritative,+ })+ servers.set(selected.name, selected)++ instance.scheduleRetry('ADDED', selected)+ await vi.advanceTimersByTimeAsync(5000)+ expect(replay).not.toHaveBeenCalled()++ authoritative = true+ await vi.advanceTimersByTimeAsync(14_999)+ expect(replay).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1)+ expect(replay).toHaveBeenCalledOnce()+ instance.stop()+ })++ it('skips periodic resync while inventory authority is unavailable', async () => {+ const selected = server('web-search')+ let authoritative = false+ const { instance, mutate, servers } = coordinator({+ inventoryAuthoritative: () => authoritative,+ })+ servers.set(selected.name, selected)++ await instance.runResync()+ expect(mutate).not.toHaveBeenCalled()++ authoritative = true+ await instance.runResync()+ expect(mutate).toHaveBeenCalledOnce()+ instance.stop()+ })++ it('does not resume periodic work from jitter after stop', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0.5)+ const selected = server('web-search')+ const { instance, mutate, servers } = coordinator()+ servers.set(selected.name, selected)++ const resync = instance.runResync()+ await flushMicrotasks()+ instance.stop()+ await vi.advanceTimersByTimeAsync(2500)+ await resync++ expect(mutate).not.toHaveBeenCalled()+ })++ it('does not start an eleventh startup mutation after stop releases the limiter', async () => {+ const release = deferred()+ const tenStarted = deferred()+ let active = 0+ const selected = Array.from({ length: 11 }, (_, index) => server(`startup-${index}`))+ const state = new Map(selected.map(item => [item.name, item]))+ const mutate = vi.fn(async () => {+ active += 1+ if (active === 10) tenStarted.resolve()+ await release.promise+ })+ const { instance } = coordinator({+ getCurrentServer: name => state.get(name),+ mutate,+ })++ const gates = instance.prepareStartupGates(selected)+ await tenStarted.promise+ instance.stop()+ release.resolve()+ await Promise.all(selected.map(item => gates.waitFor(item)))++ expect(mutate).toHaveBeenCalledTimes(10)+ })++ it('passes DELETE authority through to the watcher mutation without evaluating it', async () => {+ const selected = server('web-search')+ const deleteAllowed = vi.fn(async () => false)+ const mutate = vi.fn(async () => undefined)+ const { instance } = coordinator({ mutate })++ await instance.reconcile('DELETED', selected, { deleteAllowed })++ expect(mutate).toHaveBeenCalledWith('DELETED', selected, {+ deleteAllowed,+ isCurrent: expect.any(Function),+ })+ expect(deleteAllowed).not.toHaveBeenCalled()+ instance.stop()+ })++ it('owns periodic resync scheduling and stops it immediately', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const selected = server('web-search')+ const { instance, mutate, servers } = coordinator()+ servers.set(selected.name, selected)++ instance.startPeriodicResync(1, 1)+ await vi.advanceTimersByTimeAsync(1000)+ expect(mutate).toHaveBeenCalledOnce()++ instance.stop()+ await vi.advanceTimersByTimeAsync(5000)+ expect(mutate).toHaveBeenCalledOnce()+ })++ it('warns once when periodic resync is disabled (intervalSec <= 0) and arms no timer', async () => {+ vi.useFakeTimers()+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const selected = server('web-search')+ const { instance, mutate, servers } = coordinator()+ servers.set(selected.name, selected)++ instance.startPeriodicResync(0, 5)+ // No timer armed: advancing time never triggers a resync pass.+ await vi.advanceTimersByTimeAsync(60_000)+ expect(mutate).not.toHaveBeenCalled()++ expect(warnSpy).toHaveBeenCalledTimes(1)+ expect(warnSpy).toHaveBeenCalledWith(+ expect.stringContaining('External egress periodic DNS resync disabled')+ )++ // Called again while still disabled: no duplicate warning (once only).+ instance.startPeriodicResync(-1, 5)+ expect(warnSpy).toHaveBeenCalledTimes(1)++ instance.stop()+ })++ it('does not warn about a disabled resync when it is enabled (intervalSec > 0)', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const selected = server('web-search')+ const { instance, servers } = coordinator()+ servers.set(selected.name, selected)++ instance.startPeriodicResync(1, 1)+ expect(warnSpy).not.toHaveBeenCalledWith(+ expect.stringContaining('External egress periodic DNS resync disabled')+ )++ instance.stop()+ })++ it('arms one resync at a time, rescheduling only after each pass completes (H2)', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const selected = server('web-search')+ const release: Array<() => void> = []+ const mutate = vi.fn(+ () =>+ new Promise<void>(resolve => {+ release.push(resolve)+ })+ )+ const { instance, servers } = coordinator({ mutate })+ servers.set(selected.name, selected)++ instance.startPeriodicResync(10, 5)+ // First pass fires at the configured interval (10s) and then blocks.+ await vi.advanceTimersByTimeAsync(10_000)+ expect(mutate).toHaveBeenCalledOnce()++ // Complete the pass off the fixed grid (at t=17s). The successor is armed+ // relative to completion (t=27s), NOT on a fixed 20s/30s interval grid.+ await vi.advanceTimersByTimeAsync(7_000)+ release[0]()+ await flushMicrotasks()++ // A fixed setInterval would fire a second pass at t=20s; the self-+ // rescheduling timer must not, because it only re-armed at completion.+ await vi.advanceTimersByTimeAsync(3_000)+ expect(mutate).toHaveBeenCalledOnce()++ // The successor fires 10s after completion (t=27s).+ await vi.advanceTimersByTimeAsync(7_000)+ expect(mutate).toHaveBeenCalledTimes(2)++ instance.stop()+ release.forEach(resolve => resolve())+ })++ it('advances the next periodic resync to <= observed TTL/2 (H2, issue #299)', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const selected = server('web-search')+ // The mutation observes a low DNS TTL, ratcheting the coordinator's source+ // down; scheduleNext reads it AFTER the pass and must advance to <= TTL/2.+ let minObservedTtlMs = Infinity+ const mutate = vi.fn(async () => {+ minObservedTtlMs = 20_000+ })+ const { instance, servers } = coordinator({+ mutate,+ externalEgressRefreshMinTtlMs: () => minObservedTtlMs,+ })+ servers.set(selected.name, selected)++ // Interval 60s, floor 5s. No TTL observed yet, so the first pass fires at 60s.+ instance.startPeriodicResync(60, 5)+ await vi.advanceTimersByTimeAsync(59_999)+ expect(mutate).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1)+ expect(mutate).toHaveBeenCalledOnce()++ // TTL/2 = 10s now governs the next delay, far below the 60s backstop.+ await vi.advanceTimersByTimeAsync(9_999)+ expect(mutate).toHaveBeenCalledOnce()+ await vi.advanceTimersByTimeAsync(1)+ expect(mutate).toHaveBeenCalledTimes(2)++ instance.stop()+ })++ it('logs and absorbs a rejected periodic resync pass, then re-arms (no unhandledRejection)', async () => {+ vi.useFakeTimers()+ vi.spyOn(Math, 'random').mockReturnValue(0)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ // Capture any process-level unhandledRejection: the pre-fix code left the+ // rejected `void runResync()` promise unhandled, which crashes modern Node.+ const unhandled: unknown[] = []+ const onUnhandled = (reason: unknown): void => {+ unhandled.push(reason)+ }+ process.on('unhandledRejection', onUnhandled)++ const selected = server('web-search')+ // Reject OUTSIDE the per-server try/catch in runResyncCore (that catch only+ // wraps the mutation). A rejecting `enqueue` propagates through the worker to+ // Promise.all -> runResyncCore -> runResync, so the whole pass rejects.+ const enqueue = vi.fn(async () => {+ throw new Error('resync enqueue boom')+ })+ const { instance, servers } = coordinator({ enqueue })+ servers.set(selected.name, selected)++ try {+ instance.startPeriodicResync(10, 5)+ // First pass fires at 10s and rejects.+ await vi.advanceTimersByTimeAsync(10_000)+ await flushMicrotasks()++ // The rejection is logged by the fix's .catch (RED without it: no such log).+ expect(errorSpy).toHaveBeenCalledWith(+ '[K8s] External egress periodic resync pass failed:',+ expect.any(Error)+ )+ expect(enqueue).toHaveBeenCalledTimes(1)++ // The loop RE-ARMS despite the failure: the successor fires 10s later.+ await vi.advanceTimersByTimeAsync(10_000)+ await flushMicrotasks()+ expect(enqueue).toHaveBeenCalledTimes(2)++ // The fix handles the rejection, so nothing escaped as unhandled.+ expect(unhandled).toEqual([])+ } finally {+ process.off('unhandledRejection', onUnhandled)+ instance.stop()+ }+ })+})diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.tsnew file mode 100644index 000000000..6a8a19db2--- /dev/null+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.ts@@ -0,0 +1,607 @@+import { externalEgressRetriesAtCap } from './metrics'+import type { McpServerCRD } from './types'++const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const+const EXTERNAL_EGRESS_MAX_CONCURRENCY = 10+const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000++export type ExternalEgressWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'++export type ExternalEgressRetryHandle = {+ isCurrent: () => boolean+ complete: () => void+}++type ExternalEgressRetryIntent = {+ sequence: number+ type: ExternalEgressWatchEventType+ server: McpServerCRD+}++type ExternalEgressMutationOptions = {+ deleteAllowed?: () => Promise<boolean>+ isCurrent: () => boolean+}++type ExternalEgressReconcileOptions = {+ deleteAllowed?: () => Promise<boolean>+ retry?: ExternalEgressRetryHandle+ isCurrent?: () => boolean+}++type ExternalEgressCoordinatorDependencies = {+ listServers: () => McpServerCRD[]+ getCurrentServer: (name: string) => McpServerCRD | undefined+ inventoryAuthoritative: () => boolean+ sameDesiredRevision: (left: McpServerCRD, right: McpServerCRD) => boolean+ enqueue: (server: McpServerCRD, work: () => Promise<void>) => Promise<void>+ mutate: (+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ options: ExternalEgressMutationOptions+ ) => Promise<void>+ replay: (+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ retry: ExternalEgressRetryHandle+ ) => Promise<void>+ // Smallest DNS TTL (ms) observed across external-egress resolutions, or+ // Infinity if none. startPeriodicResync reads it to advance the next resync to+ // <= TTL/2 (H2, issue #299).+ externalEgressRefreshMinTtlMs: () => number+}++export type ExternalEgressStartupGates = {+ waitFor: (server: Pick<McpServerCRD, 'name' | 'namespace'>) => Promise<boolean>+}++class AsyncConcurrencyLimiter {+ private active = 0+ private readonly waiters: Array<(admitted: boolean) => void> = []+ private closed = false++ constructor(private readonly limit: number) {}++ async run<T>(work: () => Promise<T>): Promise<T | undefined> {+ if (this.closed) return undefined+ if (this.active >= this.limit) {+ const admitted = await new Promise<boolean>(resolve => this.waiters.push(resolve))+ if (!admitted) return undefined+ } else {+ this.active += 1+ }+ try {+ return await work()+ } finally {+ this.release()+ }+ }++ cancelPending(): void {+ if (this.closed) return+ this.closed = true+ for (const resume of this.waiters.splice(0)) {+ resume(false)+ }+ }++ private release(): void {+ this.active -= 1+ if (this.closed) return+ const resume = this.waiters.shift()+ if (!resume) return+ // Transfer the released permit before waking the waiter. Reserving it here+ // prevents a new caller from entering between wake-up and its microtask.+ this.active += 1+ resume(true)+ }+}++function delay(ms: number): Promise<void> {+ return new Promise(resolve => setTimeout(resolve, ms))+}++/**+ * External-egress resync delay (ms), H2 (issue #299). The configured interval is+ * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a+ * rotating low-TTL host is sampled every rotation. The floor is the hard lower+ * bound to avoid a hot-loop. Pure and unit-testable; consumed by+ * {@link ExternalEgressConvergenceCoordinator.startPeriodicResync}.+ */+export function externalEgressResyncDelayMs(+ intervalSec: number,+ floorSec: number,+ minObservedTtlMs = Infinity+): number {+ const floorMs = floorSec * 1000+ const configuredMs = intervalSec * 1000+ const ttlAwareMs = Number.isFinite(minObservedTtlMs)+ ? Math.min(configuredMs, minObservedTtlMs / 2)+ : configuredMs+ return Math.max(ttlAwareMs, floorMs)+}++/**+ * Owns temporal coordination for external-egress convergence. Kubernetes+ * authority and mutations remain behind callbacks supplied by the watcher.+ */+export class ExternalEgressConvergenceCoordinator {+ private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>()+ private readonly retryAttempts = new Map<string, number>()+ private readonly retryIntents = new Map<string, ExternalEgressRetryIntent>()+ private retrySequence = 0+ private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)+ private readonly retrySlots = new Map<string, object>()+ // Keys whose retry backoff is currently pinned at the capped (maximum) delay.+ // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a+ // binding stuck retrying forever is observable, not silently denied.+ private readonly retriesAtCap = new Set<string>()+ private readonly inFlight = new Map<string, Promise<void>>()+ private resyncInFlight: Promise<void> | null = null+ private resyncTimer: ReturnType<typeof setTimeout> | null = null+ private resyncDisabledWarned = false+ private stopped = false++ constructor(private readonly dependencies: ExternalEgressCoordinatorDependencies) {}++ startPeriodicResync(intervalSec: number, floorSec: number): void {+ if (this.stopped || this.resyncTimer) return+ if (intervalSec <= 0) {+ // Disabled. Warn once (mirrors the Host/SFS/GFS resync lanes) so the+ // operational consequence is visible: without periodic resync the+ // accumulated external-egress DNS allows are never refreshed, so a+ // binding whose upstream DNS rotates (issue #299) can drift until an+ // unrelated McpServer event triggers reconciliation.+ if (!this.resyncDisabledWarned) {+ this.resyncDisabledWarned = true+ console.warn(+ '[K8s] External egress periodic DNS resync disabled; accumulated DNS allows will not ' ++ 'be refreshed, so a binding whose upstream DNS rotates (issue #299) can drift until ' ++ 'another McpServer event triggers reconciliation.'+ )+ }+ return+ }+ // Self-rescheduling (not a fixed setInterval) so the delay can advance to+ // <= observed TTL/2 (H2, issue #299), and rescheduling AFTER each pass+ // completes guarantees no overlapping resyncs. runResync still dedupes any+ // in-flight pass; the floor is the hard lower bound against a hot-loop.+ const scheduleNext = (): void => {+ if (this.stopped) return+ const delayMs = externalEgressResyncDelayMs(+ intervalSec,+ floorSec,+ this.dependencies.externalEgressRefreshMinTtlMs()+ )+ this.resyncTimer = setTimeout(() => {+ // Re-arm FIRST (.finally) so a rejected pass still schedules its+ // successor — the "re-arms after completion" property must survive a+ // failure. Then .catch logs and absorbs the rejection so a rejecting+ // pass (or a throwing scheduleNext) never becomes an unhandledRejection,+ // which crashes the process on modern Node and would silently kill the+ // periodic resync loop (reopening the #299 DNS-drift window).+ void this.runResync()+ .finally(scheduleNext)+ .catch(error =>+ console.error('[K8s] External egress periodic resync pass failed:', error)+ )+ }, delayMs)+ this.resyncTimer.unref?.()+ }+ scheduleNext()+ console.log(+ `[K8s] External egress periodic DNS resync enabled (every ${intervalSec}s, advancing to <= TTL/2)`+ )+ }++ runResync(): Promise<void> {+ if (this.resyncInFlight) return this.resyncInFlight+ const run = this.runResyncCore()+ this.resyncInFlight = run+ const settle = () => {+ if (this.resyncInFlight === run) this.resyncInFlight = null+ }+ void run.then(settle, settle)+ return run+ }++ prepareStartupGates(+ servers: McpServerCRD[],+ isCurrent: () => boolean = this.dependencies.inventoryAuthoritative+ ): ExternalEgressStartupGates {+ const gates = new Map<string, Promise<boolean>>()+ const selectedServers = servers+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)+ .sort((left, right) => this.key(left).localeCompare(this.key(right)))++ if (selectedServers.length > 0) {+ console.log(+ `[K8s] Reconciling external egress before startup runtime reconciliation for ` ++ `${selectedServers.length} McpServer(s)`+ )+ }++ // Startup admission is intentionally independent from retry and periodic+ // resync capacity, so a saturated recovery lane cannot head-of-line block+ // initial runtime gates for unrelated servers.+ const limiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)+ for (const selected of selectedServers) {+ const key = this.key(selected)+ const gate = limiter+ .run(async () => {+ if (this.stopped || !isCurrent()) return false+ let ready = false+ await this.dependencies.enqueue(selected, async () => {+ if (this.stopped || !isCurrent()) return+ const current = this.dependencies.getCurrentServer(selected.name)+ if (+ !current ||+ current.namespace !== selected.namespace ||+ !this.dependencies.sameDesiredRevision(selected, current)+ ) {+ return+ }+ try {+ // Startup is an egress-only gate. Any pre-existing retry intent is+ // completed only by its full replay, never by this mutation.+ await this.reconcile('MODIFIED', current, { isCurrent })+ const latest = this.dependencies.getCurrentServer(selected.name)+ ready =+ !this.stopped &&+ isCurrent() &&+ latest !== undefined &&+ latest.namespace === selected.namespace &&+ this.dependencies.sameDesiredRevision(selected, latest)+ } catch (error) {+ console.error(+ `[K8s] Initial external egress reconciliation failed for ${key}; ` ++ 'runtime reconciliation will stay blocked until retry succeeds:',+ error+ )+ this.scheduleRetry('MODIFIED', current)+ }+ })+ return ready+ })+ .then(result => result ?? false)+ // Side-attach a logger so a rejection is never *unhandled* (which crashes+ // the process on modern Node) if fullReconcile aborts before waitFor+ // awaits this gate. This does NOT swallow the failure: gates.set stores+ // the original `gate`, so the awaited path in fullReconcile still rejects+ // and fails the pass loudly — this only covers the unawaited case.+ gate.catch(error => {+ console.error(+ `[K8s] External egress startup gate for ${key} rejected before it was awaited:`,+ error+ )+ })+ gates.set(key, gate)+ }++ return {+ waitFor: server => gates.get(this.key(server)) ?? Promise.resolve(true),+ }+ }++ async reconcile(+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ options: ExternalEgressReconcileOptions = {}+ ): Promise<ExternalEgressRetryHandle | undefined> {+ if (!this.reconcileIsCurrent(type, server, options)) return undefined+ const key = this.key(server)+ let existing = this.inFlight.get(key)+ while (existing) {+ console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)+ await existing+ if (!this.reconcileIsCurrent(type, server, options)) return undefined+ existing = this.inFlight.get(key)+ }++ if (!this.reconcileIsCurrent(type, server, options)) return undefined+ const completion = options.retry ?? this.currentRetryHandle(key)+ const run = this.dependencies.mutate(type, server, {+ deleteAllowed: options.deleteAllowed,+ isCurrent: () => this.reconcileIsCurrent(type, server, options),+ })+ this.inFlight.set(key, run)+ try {+ await run+ return completion+ } finally {+ if (this.inFlight.get(key) === run) {+ this.inFlight.delete(key)+ }+ }+ }++ scheduleRetry(type: string, server: McpServerCRD): void {+ if (!this.isWatchEventType(type) || this.stopped) return++ const cached = this.dependencies.getCurrentServer(server.name)+ const retryType: ExternalEgressWatchEventType =+ cached && cached.namespace === server.namespace+ ? type === 'ADDED' && this.dependencies.sameDesiredRevision(server, cached)+ ? 'ADDED'+ : 'MODIFIED'+ : 'DELETED'+ const retryServer = cached && cached.namespace === server.namespace ? cached : server+ const key = this.key(server)+ const previousIntent = this.retryIntents.get(key)+ if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {+ this.retryAttempts.delete(key)+ // A new target restarts the backoff ladder, so the key is no longer capped.+ this.clearRetryAtCap(key)+ }++ const intent: ExternalEgressRetryIntent = {+ sequence: ++this.retrySequence,+ type: retryType,+ server: retryServer,+ }+ this.retryIntents.set(key, intent)+ // A timer or queued/active slot already owning the key consumes the newest+ // intent when it next enters the retry and per-server lanes.+ if (this.retryTimers.has(key) || this.retrySlots.has(key)) return+ this.armRetryTimer(key)+ }++ private armRetryTimer(key: string): void {+ if (+ this.stopped ||+ this.retryTimers.has(key) ||+ this.retrySlots.has(key) ||+ !this.retryIntents.has(key)+ ) {+ return+ }+ const attempt = Math.min(+ (this.retryAttempts.get(key) ?? 0) + 1,+ EXTERNAL_EGRESS_RETRY_DELAYS_MS.length+ )+ const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]+ this.retryAttempts.set(key, attempt)+ const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length+ if (capped) this.markRetryAtCap(key)+ console.warn(+ `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` ++ `${capped ? ' (capped)' : ''} ` ++ `for McpServer "${this.retryIntents.get(key)!.server.name}" in ${delayMs}ms`+ )++ const timer = setTimeout(() => {+ if (this.retryTimers.get(key) !== timer) return+ this.retryTimers.delete(key)+ this.queueRetry(key)+ }, delayMs)+ this.retryTimers.set(key, timer)+ }++ private queueRetry(key: string): void {+ if (this.stopped || !this.retryIntents.has(key)) return+ if (this.retrySlots.has(key)) {+ // A timer can overlap work admitted through another lane. The durable+ // slot owns the latest intent and rearms it on exit if it remains live.+ return+ }++ const slot = {}+ this.retrySlots.set(key, slot)+ void this.retryLimiter+ .run(async () => {+ if (+ this.stopped ||+ this.retrySlots.get(key) !== slot ||+ !this.dependencies.inventoryAuthoritative()+ ) {+ return+ }+ // Do not capture intent before the limiter. Churn while capacity is+ // saturated is represented by one slot and resolved latest-wins here.+ const admittedIntent = this.retryIntents.get(key)+ if (!admittedIntent) return+ await this.dependencies.enqueue(admittedIntent.server, async () => {+ if (+ this.stopped ||+ this.retrySlots.get(key) !== slot ||+ !this.dependencies.inventoryAuthoritative()+ ) {+ return+ }+ // The keyed queue may also wait. Resolve the intent again so only the+ // latest desired revision or absence is replayed.+ const currentIntent = this.retryIntents.get(key)+ if (!currentIntent) return+ await this.replayIntent(currentIntent)+ })+ })+ .catch(error => {+ if (!this.stopped) {+ console.error(`[K8s] External egress retry execution failed for ${key}:`, error)+ }+ })+ .finally(() => {+ if (this.retrySlots.get(key) !== slot) return+ this.retrySlots.delete(key)+ // A replay may schedule its successor while still active, or a newer+ // intent may arrive after this slot selected work. Rearm only after the+ // key leaves the slot so one key never accumulates limiter backlog.+ if (!this.stopped && this.retryIntents.has(key) && !this.retryTimers.has(key)) {+ this.armRetryTimer(key)+ }+ })+ }++ stop(): void {+ if (this.stopped) return+ this.stopped = true+ this.retryLimiter.cancelPending()+ if (this.resyncTimer) {+ clearTimeout(this.resyncTimer)+ this.resyncTimer = null+ }+ for (const timer of this.retryTimers.values()) {+ clearTimeout(timer)+ }+ this.retryTimers.clear()+ this.retryAttempts.clear()+ this.retryIntents.clear()+ this.retrySlots.clear()+ if (this.retriesAtCap.size > 0) {+ this.retriesAtCap.clear()+ externalEgressRetriesAtCap.set(0)+ }+ this.inFlight.clear()+ }++ private async runResyncCore(): Promise<void> {+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return+ const servers = this.dependencies+ .listServers()+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)+ .sort((left, right) => this.key(left).localeCompare(this.key(right)))+ if (servers.length === 0) return++ console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)+ let index = 0+ const workers = Array.from(+ { length: Math.min(EXTERNAL_EGRESS_MAX_CONCURRENCY, servers.length) },+ async () => {+ while (!this.stopped && this.dependencies.inventoryAuthoritative()) {+ const server = servers[index++]+ if (!server) return+ const key = this.key(server)+ if (this.inFlight.has(key)) {+ console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)+ continue+ }+ const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)+ if (jitterMs > 0) await delay(jitterMs)+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return++ await this.dependencies.enqueue(server, async () => {+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return+ // The fleet snapshot selects work only. Desired state is resolved+ // after jitter and after entering the per-server FIFO lane.+ const current = this.dependencies.getCurrentServer(server.name)+ if (+ !current ||+ current.namespace !== server.namespace ||+ (current.spec.egressBindings?.length ?? 0) === 0+ ) {+ return+ }+ try {+ const retryIntent = this.retryIntents.get(key)+ if (retryIntent) {+ await this.replayIntent(retryIntent)+ return+ }+ // Periodic refresh is egress-only and therefore cannot complete+ // a retry intent that may appear while the mutation is running.+ await this.reconcile('MODIFIED', current, {+ isCurrent: this.dependencies.inventoryAuthoritative,+ })+ } catch (error) {+ console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)+ this.scheduleRetry('MODIFIED', current)+ }+ })+ }+ }+ )+ await Promise.all(workers)+ }++ private async replayIntent(intent: ExternalEgressRetryIntent): Promise<void> {+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return+ const handle = this.retryHandle(intent)+ if (!handle.isCurrent()) return++ const current = this.dependencies.getCurrentServer(intent.server.name)+ const type: ExternalEgressWatchEventType =+ current && current.namespace === intent.server.namespace ? 'MODIFIED' : 'DELETED'+ const server = type === 'DELETED' ? intent.server : current!+ await this.dependencies.replay(type, server, handle)+ }++ private reconcileIsCurrent(+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ options: ExternalEgressReconcileOptions+ ): boolean {+ const callerIsCurrent =+ !this.stopped &&+ (!options.retry || options.retry.isCurrent()) &&+ (!options.isCurrent || options.isCurrent())+ if (!callerIsCurrent || type === 'DELETED') return callerIsCurrent++ const current = this.dependencies.getCurrentServer(server.name)+ return (+ current !== undefined &&+ current.namespace === server.namespace &&+ this.dependencies.sameDesiredRevision(server, current)+ )+ }++ private currentRetryHandle(key: string): ExternalEgressRetryHandle | undefined {+ const intent = this.retryIntents.get(key)+ return intent ? this.retryHandle(intent) : undefined+ }++ private retryHandle(intent: ExternalEgressRetryIntent): ExternalEgressRetryHandle {+ return {+ isCurrent: () =>+ !this.stopped &&+ this.retryIntents.get(this.key(intent.server))?.sequence === intent.sequence,+ complete: () => {+ if (this.retryIntents.get(this.key(intent.server))?.sequence !== intent.sequence) return+ this.clearRetry(intent.server)+ },+ }+ }++ private clearRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {+ const key = this.key(server)+ const timer = this.retryTimers.get(key)+ if (timer) {+ clearTimeout(timer)+ this.retryTimers.delete(key)+ }+ this.retryAttempts.delete(key)+ this.retryIntents.delete(key)+ this.clearRetryAtCap(key)+ }++ private markRetryAtCap(key: string): void {+ if (this.retriesAtCap.has(key)) return+ this.retriesAtCap.add(key)+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)+ }++ private clearRetryAtCap(key: string): void {+ if (!this.retriesAtCap.delete(key)) return+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)+ }++ private sameRetryTarget(+ previous: ExternalEgressRetryIntent,+ type: ExternalEgressWatchEventType,+ server: McpServerCRD+ ): boolean {+ const previousPresent = previous.type !== 'DELETED'+ const nextPresent = type !== 'DELETED'+ return (+ previousPresent === nextPresent &&+ this.dependencies.sameDesiredRevision(previous.server, server)+ )+ }++ private key(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {+ return `${server.namespace}/${server.name}`+ }++ private isWatchEventType(type: string): type is ExternalEgressWatchEventType {+ return type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED'+ }+}diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.tsindex c9cde6bcb..d2fd7965b 100644--- a/host-context-controller/src/gfsReconciler.test.ts+++ b/host-context-controller/src/gfsReconciler.test.ts@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'import type * as k8s from '@kubernetes/client-node'import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'import { GfsFactoryConfig } from './k8s/gfsFactory'-import type { GlobalFileSystemCRD } from './types'+import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'const config: GfsFactoryConfig = {gfsNamespace: 'gfs',@@ -42,11 +42,12 @@ class FakeApi implements GfsK8sApi {netpols: string[] = []pvcs: string[] = []deleted: string[] = []- statuses: { phase?: string; serviceUrl?: string }[] = []+ statuses: GlobalFileSystemStatus[] = []writerNeedsUpdate = truewriterAvailable = truewriterAvailabilitySequence: boolean[] = []failPvc = false+ statusFailuresRemaining = 0async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {if (this.failPvc) throw new Error('simulated PVC apply failure')@@ -101,11 +102,11 @@ class FakeApi implements GfsK8sApi {async deletePvc(name: string): Promise<void> {this.deleted.push(`pvc/${name}`)}- async patchStatus(- _name: string,- _ns: string,- status: { phase?: string; serviceUrl?: string }- ): Promise<void> {+ async patchStatus(_name: string, _ns: string, status: GlobalFileSystemStatus): Promise<void> {+ if (this.statusFailuresRemaining > 0) {+ this.statusFailuresRemaining--+ throw new Error('simulated status patch failure')+ }this.statuses.push(status)}}@@ -218,6 +219,59 @@ describe('GfsReconciler.reconcile', () => {expect(seed.calls).toBe(1)})+ it('does not rewrite unchanged status but still publishes a later phase transition', async () => {+ const api = new FakeApi()+ api.writerNeedsUpdate = false+ const reconciler = new GfsReconciler(api, config, undefined, 0, 0)++ await reconciler.reconcile(gfs)+ const observedReady = { ...gfs, status: api.statuses[0] }+ await reconciler.reconcile(observedReady)++ expect(api.statuses).toHaveLength(1)+ expect(api.statuses[0]?.phase).toBe('Ready')++ api.writerAvailable = false+ await reconciler.reconcile(observedReady)++ expect(api.statuses.map(status => status.phase)).toEqual(['Ready', 'Initializing'])+ })++ it('publishes status for a same-name recreation even when the DELETE watch was missed', async () => {+ const api = new FakeApi()+ const reconciler = new GfsReconciler(api, config)++ await reconciler.reconcile(gfs)+ await reconciler.reconcile({ ...gfs, status: api.statuses[0] })+ await reconciler.reconcile({ ...gfs, status: undefined })++ expect(api.statuses).toHaveLength(2)+ })++ it('repairs status that was removed externally after HCC observed its prior write', async () => {+ const api = new FakeApi()+ const reconciler = new GfsReconciler(api, config)++ await reconciler.reconcile(gfs)+ await reconciler.reconcile({ ...gfs, status: api.statuses[0] })+ await reconciler.reconcile({ ...gfs, status: undefined })++ expect(api.statuses).toHaveLength(2)+ expect(api.statuses.at(-1)?.phase).toBe('Ready')+ })++ it('retries an unchanged status when the previous patch failed', async () => {+ const api = new FakeApi()+ api.statusFailuresRemaining = 1+ const reconciler = new GfsReconciler(api, config)++ await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')+ await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()++ expect(api.statuses).toHaveLength(1)+ expect(api.statuses[0]?.phase).toBe('Ready')+ })+it('reports Initializing and does NOT seed when the writer is not yet available', async () => {const api = new FakeApi()api.writerAvailable = false@@ -265,4 +319,15 @@ describe('GfsReconciler.reconcileDelete', () => {expect(api.deleted).toContain('pvc/gfs-drive')expect(api.deleted.at(-1)).toBe('pvc/gfs-drive') // bytes removed LAST})++ it('publishes status again after delete clears the prior write suppression', async () => {+ const api = new FakeApi()+ const reconciler = new GfsReconciler(api, config)++ await reconciler.reconcile(gfs)+ await reconciler.reconcileDelete(gfs)+ await reconciler.reconcile(gfs)++ expect(api.statuses).toHaveLength(2)+ })})diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.tsindex de5e29ea2..0462a12e6 100644--- a/host-context-controller/src/gfsReconciler.ts+++ b/host-context-controller/src/gfsReconciler.ts@@ -95,6 +95,27 @@ export class GfsReconciler {} while (true)}+ private async writeStatus(+ gfs: GlobalFileSystemCRD,+ status: GlobalFileSystemStatus+ ): Promise<void> {+ // Compare against status observed on this exact LIST/watch object, not+ // process memory keyed only by name. A local name cache could suppress the+ // repair of a recreated object when its DELETE event was missed, or of+ // status that another actor removed. Conversely, an identical observed+ // status proves this is our own status-only MODIFIED event and closes the+ // patch -> watch -> reconcile loop without another API write.+ if (+ gfs.status?.phase === status.phase &&+ gfs.status?.pvcName === status.pvcName &&+ gfs.status?.serviceName === status.serviceName &&+ gfs.status?.serviceUrl === status.serviceUrl+ ) {+ return+ }+ await this.api.patchStatus(gfs.name, gfs.namespace, status)+ }+async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {const ns = this.config.gfsNamespaceconst writerDeployment = buildDeployment(gfs, this.config, 'writer')@@ -135,7 +156,7 @@ export class GfsReconciler {}const phase = writerReady ? 'Ready' : 'Initializing'- await this.api.patchStatus(gfs.name, gfs.namespace, {+ await this.writeStatus(gfs, {phase,pvcName: pvcName(gfs),serviceName: serviceName(gfs),diff --git a/host-context-controller/src/hostFleetScheduler.test.ts b/host-context-controller/src/hostFleetScheduler.test.tsnew file mode 100644index 000000000..0f6817cfa--- /dev/null+++ b/host-context-controller/src/hostFleetScheduler.test.ts@@ -0,0 +1,284 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import {+ type HostFleetReconcileRequest,+ type HostFleetRequestResult,+ HostFleetScheduler,+} from './hostFleetScheduler'++function deferred<T = void>() {+ let resolve!: (value: T | PromiseLike<T>) => void+ let reject!: (reason?: unknown) => void+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {+ resolve = resolvePromise+ reject = rejectPromise+ })+ return { promise, resolve, reject }+}++describe('HostFleetScheduler', () => {+ afterEach(() => {+ vi.useRealTimers()+ })++ it('merges newer work into one trailing pass with full coverage and latest inputs', async () => {+ const activePass = deferred()+ const requests: HostFleetReconcileRequest[] = []+ const results: HostFleetRequestResult[] = []+ const scheduler = new HostFleetScheduler({+ perform: async request => {+ requests.push({ ...request })+ if (requests.length === 1) await activePass.promise+ },+ recordRequest: result => results.push(result),+ })++ const firstGeneration = scheduler.beginLifecycleTransition()+ scheduler.bumpInputRevision()+ const active = scheduler.request('active lifecycle', firstGeneration, 'lifecycle')+ await vi.waitFor(() => expect(requests).toHaveLength(1))++ scheduler.bumpInputRevision()+ const pendingFull = scheduler.request('pending full', firstGeneration, 'full')+ const latestGeneration = scheduler.beginLifecycleTransition()+ scheduler.bumpInputRevision()+ const latestLifecycle = scheduler.request('latest lifecycle', latestGeneration, 'lifecycle')++ activePass.resolve()+ await Promise.all([active, pendingFull, latestLifecycle])++ expect(requests).toEqual([+ {+ reason: 'active lifecycle',+ mode: 'lifecycle',+ ccLifecycleGeneration: firstGeneration,+ inputRevision: 1,+ },+ {+ reason: 'latest lifecycle',+ mode: 'full',+ ccLifecycleGeneration: latestGeneration,+ inputRevision: 3,+ },+ ])+ expect(results).toEqual(['started', 'coalesced', 'coalesced', 'trailing'])+ })++ it('carries the known lifecycle generation when merging a generation-less full request', async () => {+ const activePass = deferred()+ const requests: HostFleetReconcileRequest[] = []+ const scheduler = new HostFleetScheduler({+ perform: async request => {+ requests.push({ ...request })+ if (requests.length === 1) await activePass.promise+ },+ recordRequest: vi.fn(),+ })+ const generation = scheduler.beginLifecycleTransition()+ const active = scheduler.request('active')+ await vi.waitFor(() => expect(requests).toHaveLength(1))+ scheduler.bumpInputRevision()+ const pendingLifecycle = scheduler.request('lifecycle', generation, 'lifecycle')+ scheduler.bumpInputRevision()+ const pendingFull = scheduler.request('full')++ activePass.resolve()+ await Promise.all([active, pendingLifecycle, pendingFull])++ expect(requests[1]).toMatchObject({+ reason: 'full',+ mode: 'full',+ ccLifecycleGeneration: generation,+ inputRevision: 2,+ })+ })++ it('reuses active work only when its mode, revision, and lifecycle generation cover the request', async () => {+ const activePass = deferred()+ const requests: HostFleetReconcileRequest[] = []+ const scheduler = new HostFleetScheduler({+ perform: async request => {+ requests.push({ ...request })+ if (requests.length === 1) await activePass.promise+ },+ recordRequest: vi.fn(),+ })+ const generation = scheduler.beginLifecycleTransition()+ scheduler.bumpInputRevision()++ const active = scheduler.request('full', generation, 'full')+ await vi.waitFor(() => expect(requests).toHaveLength(1))+ const covered = scheduler.request('covered lifecycle', generation, 'lifecycle')++ activePass.resolve()+ await Promise.all([active, covered])++ expect(requests).toHaveLength(1)+ })++ it('rejects stale or already-applied lifecycle work without suppressing a current full pass', async () => {+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(+ async () => undefined+ )+ const scheduler = new HostFleetScheduler({+ perform,+ recordRequest: vi.fn(),+ })+ const firstGeneration = scheduler.beginLifecycleTransition()+ scheduler.markLifecycleApplied(firstGeneration)++ await scheduler.request('already applied', firstGeneration, 'lifecycle')+ await scheduler.request('current full', firstGeneration, 'full')+ scheduler.beginLifecycleTransition()+ await scheduler.request('stale full', firstGeneration, 'full')++ expect(perform).toHaveBeenCalledOnce()+ expect(perform).toHaveBeenCalledWith(+ expect.objectContaining({ reason: 'current full', mode: 'full' })+ )+ })++ it('does not let a rejected pass poison later work', async () => {+ const failure = new Error('transient failure')+ const perform = vi+ .fn<(request: HostFleetReconcileRequest) => Promise<void>>()+ .mockRejectedValueOnce(failure)+ .mockResolvedValueOnce(undefined)+ const scheduler = new HostFleetScheduler({+ perform,+ recordRequest: vi.fn(),+ })++ await expect(scheduler.request('fails')).rejects.toThrow(failure)+ await expect(scheduler.request('recovers')).resolves.toBeUndefined()++ expect(perform).toHaveBeenCalledTimes(2)+ })++ it('releases active and queued waiters on stop without starting queued work', async () => {+ const activePass = deferred()+ const requests: HostFleetReconcileRequest[] = []+ const scheduler = new HostFleetScheduler({+ perform: async request => {+ requests.push({ ...request })+ await activePass.promise+ },+ recordRequest: vi.fn(),+ })++ const active = scheduler.request('active')+ await vi.waitFor(() => expect(requests).toHaveLength(1))+ scheduler.bumpInputRevision()+ const queued = scheduler.request('queued')++ scheduler.stop()+ await Promise.all([active, queued, scheduler.request('after stop')])+ activePass.resolve()+ await Promise.resolve()++ expect(requests).toHaveLength(1)+ })++ it('tracks lifecycle state and cancels obsolete retries', async () => {+ vi.useFakeTimers()+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(+ async () => undefined+ )+ const scheduler = new HostFleetScheduler({+ perform,+ recordRequest: vi.fn(),+ })+ const firstGeneration = scheduler.beginLifecycleTransition()++ expect(scheduler.currentLifecycleGeneration).toBe(firstGeneration)+ expect(scheduler.appliedLifecycleGeneration).toBe(-1)++ scheduler.scheduleLifecycleRetry({+ reason: 'failed lifecycle',+ mode: 'lifecycle',+ ccLifecycleGeneration: firstGeneration,+ inputRevision: 0,+ })+ await vi.advanceTimersByTimeAsync(4999)+ expect(perform).not.toHaveBeenCalled()++ const secondGeneration = scheduler.beginLifecycleTransition()+ await vi.advanceTimersByTimeAsync(1)+ expect(perform).not.toHaveBeenCalled()++ scheduler.scheduleLifecycleRetry({+ reason: 'current lifecycle',+ mode: 'lifecycle',+ ccLifecycleGeneration: secondGeneration,+ inputRevision: 0,+ })+ scheduler.markLifecycleApplied(secondGeneration)+ await vi.advanceTimersByTimeAsync(5000)++ expect(scheduler.appliedLifecycleGeneration).toBe(secondGeneration)+ expect(perform).not.toHaveBeenCalled()+ })++ it('does not let a stale applied marker clear the current generation retry', async () => {+ vi.useFakeTimers()+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(+ async () => undefined+ )+ const scheduler = new HostFleetScheduler({+ perform,+ recordRequest: vi.fn(),+ })+ const staleGeneration = scheduler.beginLifecycleTransition()+ const currentGeneration = scheduler.beginLifecycleTransition()+ scheduler.scheduleLifecycleRetry({+ reason: 'current lifecycle',+ mode: 'lifecycle',+ ccLifecycleGeneration: currentGeneration,+ inputRevision: 0,+ })++ scheduler.markLifecycleApplied(staleGeneration)+ await vi.advanceTimersByTimeAsync(5000)++ expect(perform).toHaveBeenCalledWith(+ expect.objectContaining({ ccLifecycleGeneration: currentGeneration })+ )+ })++ it('retries an unapplied current lifecycle generation with bounded backoff', async () => {+ vi.useFakeTimers()+ const requests: HostFleetReconcileRequest[] = []+ const scheduler = new HostFleetScheduler({+ perform: async request => {+ requests.push({ ...request })+ },+ recordRequest: vi.fn(),+ })+ const generation = scheduler.beginLifecycleTransition()+ const failedRequest: HostFleetReconcileRequest = {+ reason: 'failed lifecycle',+ mode: 'lifecycle',+ ccLifecycleGeneration: generation,+ inputRevision: 0,+ }++ scheduler.scheduleLifecycleRetry(failedRequest)+ await vi.advanceTimersByTimeAsync(5000)+ scheduler.scheduleLifecycleRetry(failedRequest)+ await vi.advanceTimersByTimeAsync(14999)+ expect(requests).toHaveLength(1)+ await vi.advanceTimersByTimeAsync(1)++ expect(requests).toEqual([+ expect.objectContaining({+ reason: 'CommunicationChannel lifecycle convergence retry',+ mode: 'lifecycle',+ ccLifecycleGeneration: generation,+ }),+ expect.objectContaining({+ reason: 'CommunicationChannel lifecycle convergence retry',+ mode: 'lifecycle',+ ccLifecycleGeneration: generation,+ }),+ ])+ })+})diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.tsnew file mode 100644index 000000000..d872d6cad--- /dev/null+++ b/host-context-controller/src/hostFleetScheduler.ts@@ -0,0 +1,246 @@+const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000] as const++export type HostFleetReconcileMode = 'full' | 'lifecycle'++export type HostFleetReconcileRequest = {+ reason: string+ mode: HostFleetReconcileMode+ ccLifecycleGeneration?: number+ inputRevision: number+}++type PendingHostFleetReconcile = HostFleetReconcileRequest & {+ promise: Promise<void>+ resolve: () => void+}++type ActiveHostFleetReconcile = HostFleetReconcileRequest & {+ promise: Promise<void>+}++export type HostFleetRequestResult = 'started' | 'coalesced' | 'trailing'++type HostFleetSchedulerDependencies = {+ perform: (request: HostFleetReconcileRequest) => Promise<void>+ recordRequest: (result: HostFleetRequestResult) => void+}++function hostFleetRequestCovers(+ active: HostFleetReconcileRequest,+ requested: HostFleetReconcileRequest+): boolean {+ const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'+ if (!modeCovered) return false+ if (active.inputRevision !== requested.inputRevision) return false+ if (requested.ccLifecycleGeneration === undefined) return true+ return active.ccLifecycleGeneration === requested.ccLifecycleGeneration+}++function mergeHostFleetRequests(+ pending: HostFleetReconcileRequest,+ requested: HostFleetReconcileRequest+): HostFleetReconcileRequest {+ const pendingGeneration = pending.ccLifecycleGeneration+ const requestedGeneration = requested.ccLifecycleGeneration+ return {+ reason: requested.reason,+ mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',+ inputRevision: Math.max(pending.inputRevision, requested.inputRevision),+ ccLifecycleGeneration:+ pendingGeneration === undefined+ ? requestedGeneration+ : requestedGeneration === undefined+ ? pendingGeneration+ : Math.max(pendingGeneration, requestedGeneration),+ }+}++/**+ * Owns temporal coordination for Host fleet convergence. Kubernetes reads and+ * mutations stay behind `perform`, so this scheduler only serializes requests,+ * coalesces a single trailing pass, and manages lifecycle retry/shutdown state.+ */+export class HostFleetScheduler {+ private ccLifecycleGeneration = 0+ private ccAppliedLifecycleGeneration = -1+ private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null+ private ccFleetRetryAttempt = 0+ private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null+ private hostFleetReconcilePending: PendingHostFleetReconcile | null = null+ private hostFleetInputRevision = 0+ private resolveHostFleetShutdown: () => void = () => {}+ private readonly hostFleetShutdown = new Promise<void>(resolve => {+ this.resolveHostFleetShutdown = resolve+ })+ private stopped = false++ constructor(private readonly dependencies: HostFleetSchedulerDependencies) {}++ get currentLifecycleGeneration(): number {+ return this.ccLifecycleGeneration+ }++ get appliedLifecycleGeneration(): number {+ return this.ccAppliedLifecycleGeneration+ }++ beginLifecycleTransition(): number {+ this.ccLifecycleGeneration += 1+ this.ccFleetRetryAttempt = 0+ this.clearLifecycleRetry()+ return this.ccLifecycleGeneration+ }++ bumpInputRevision(): number {+ this.hostFleetInputRevision += 1+ return this.hostFleetInputRevision+ }++ markLifecycleApplied(generation: number): void {+ if (generation !== this.ccLifecycleGeneration) return+ this.ccAppliedLifecycleGeneration = generation+ this.ccFleetRetryAttempt = 0+ this.clearLifecycleRetry()+ }++ scheduleLifecycleRetry(request: HostFleetReconcileRequest): void {+ const generation = request.ccLifecycleGeneration+ if (generation === undefined) return+ if (+ this.stopped ||+ generation !== this.ccLifecycleGeneration ||+ generation === this.ccAppliedLifecycleGeneration ||+ this.ccFleetRetryTimer+ ) {+ return+ }++ const delayIndex = Math.min(+ this.ccFleetRetryAttempt,+ COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1+ )+ const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]+ this.ccFleetRetryAttempt += 1+ this.ccFleetRetryTimer = setTimeout(() => {+ this.ccFleetRetryTimer = null+ if (+ this.stopped ||+ generation !== this.ccLifecycleGeneration ||+ generation === this.ccAppliedLifecycleGeneration+ ) {+ return+ }+ void this.request(+ 'CommunicationChannel lifecycle convergence retry',+ generation,+ request.mode+ )+ }, retryDelay)+ }++ /**+ * Cache transitions, periodic resyncs, and retries share one active pass and+ * at most one merged trailing pass. Each caller waits only for work covering+ * its request, keeping startup bounded under a sustained retry stream.+ */+ request(+ reason: string,+ ccLifecycleGeneration?: number,+ mode?: HostFleetReconcileMode+ ): Promise<void> {+ if (this.stopped) return Promise.resolve()+ const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')+ if (+ ccLifecycleGeneration !== undefined &&+ (ccLifecycleGeneration !== this.ccLifecycleGeneration ||+ (requestMode === 'lifecycle' &&+ ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))+ ) {+ return Promise.resolve()+ }++ const request: HostFleetReconcileRequest = {+ reason,+ mode: requestMode,+ ccLifecycleGeneration,+ inputRevision: this.hostFleetInputRevision,+ }+ const active = this.hostFleetReconcileInFlight+ if (!active) {+ this.dependencies.recordRequest('started')+ return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))+ }+ if (hostFleetRequestCovers(active, request)) {+ this.dependencies.recordRequest('coalesced')+ return this.waitForHostFleetOrShutdown(active.promise)+ }++ if (!this.hostFleetReconcilePending) {+ let resolve!: () => void+ const promise = new Promise<void>(resolvePromise => {+ resolve = resolvePromise+ })+ this.hostFleetReconcilePending = {+ ...request,+ promise,+ resolve,+ }+ } else {+ const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)+ this.hostFleetReconcilePending.reason = merged.reason+ this.hostFleetReconcilePending.mode = merged.mode+ this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration+ this.hostFleetReconcilePending.inputRevision = merged.inputRevision+ }+ this.dependencies.recordRequest('coalesced')+ return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)+ }++ stop(): void {+ if (this.stopped) return+ this.stopped = true+ this.resolveHostFleetShutdown()+ this.hostFleetReconcilePending?.resolve()+ this.hostFleetReconcilePending = null+ this.clearLifecycleRetry()+ }++ private clearLifecycleRetry(): void {+ if (!this.ccFleetRetryTimer) return+ clearTimeout(this.ccFleetRetryTimer)+ this.ccFleetRetryTimer = null+ }++ private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {+ return Promise.race([coverage, this.hostFleetShutdown])+ }++ private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {+ const reconcileRequest: HostFleetReconcileRequest = {+ reason: request.reason,+ mode: request.mode,+ ccLifecycleGeneration: request.ccLifecycleGeneration,+ inputRevision: request.inputRevision,+ }+ const promise = this.dependencies.perform(reconcileRequest)+ const active: ActiveHostFleetReconcile = { ...reconcileRequest, promise }+ this.hostFleetReconcileInFlight = active++ const settle = () => {+ if (this.hostFleetReconcileInFlight !== active) return+ this.hostFleetReconcileInFlight = null+ const pending = this.hostFleetReconcilePending+ this.hostFleetReconcilePending = null+ if (!pending) return+ if (this.stopped) {+ pending.resolve()+ return+ }+ this.dependencies.recordRequest('trailing')+ const trailing = this.startHostFleetReconcile(pending)+ void trailing.then(pending.resolve, pending.resolve)+ }+ void promise.then(settle, settle)+ return promise+ }+}diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.tsindex 495a4bc38..53625ff43 100644--- a/host-context-controller/src/hostReconciler.test.ts+++ b/host-context-controller/src/hostReconciler.test.ts@@ -943,6 +943,42 @@ describe('HostReconciler cleanup authority gate', () => {expect(del).not.toHaveBeenCalled()})+ it('defers orphan deletion when authority changes before serializer admission', async () => {+ const getObj = vi.fn().mockRejectedValue({ code: 404 })+ const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })+ const authority = { known: true, generation: 3 }+ reconciler.setHostWatchAuthority(() => ({ ...authority }))+ reconciler.setResolveCurrentHost(() => undefined)+ const internals = reconciler as unknown as {+ lifecycle: {+ serializeByHost(name: string, task: () => Promise<void>): Promise<void>+ }+ listManagedHostDeployments(): Promise<k8s.V1Deployment[]>+ deleteHostRuntimeResources(name: string, namespace: string): Promise<void>+ }+ vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([+ orphanDeployment('authority-race-host'),+ ])+ const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = internals.lifecycle.serializeByHost('authority-race-host', async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ })+ await blockerStarted.promise++ const cleanup = reconciler.fullReconcile([])+ await vi.waitFor(() => expect(getObj).toHaveBeenCalledOnce())+ authority.known = false+ authority.generation = 4+ releaseBlocker.resolve(undefined)+ await occupied+ await cleanup++ expect(del).not.toHaveBeenCalled()+ })+it('defers an orphan whose fresh authoritative read fails', async () => {const getObj = vi.fn().mockRejectedValue({ code: 503 })const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })@@ -996,3 +1032,687 @@ describe('HostReconciler cleanup authority gate', () => {expect(del).not.toHaveBeenCalled()})})++describe('HostReconciler Host inventory mutation authority', () => {+ function wireAuthoritativeHost(+ reconciler: HostReconciler,+ host: HostCRD,+ authorityRef: { current: { known: boolean; generation: number } }+ ): void {+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => authorityRef.current)+ }++ function stubPreDeploymentReconcileEffects(reconciler: HostReconciler): void {+ vi.spyOn(reconciler as any, 'validateHostSecret').mockResolvedValue({ ok: true })+ for (const method of [+ 'ensureHostServiceAccount',+ 'ensureHostRole',+ 'ensureHostRoleBinding',+ 'ensurePvc',+ 'ensureService',+ 'ensureMcpHostIngressNetworkPolicy',+ 'ensureMcpHostGfsEgressNetworkPolicy',+ 'ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy',+ 'ensureRpcProxyMcpHostIngressNetworkPolicy',+ 'ensureRpcProxyHostEgressNetworkPolicy',+ 'ensureDesktopNetworkPolicy',+ 'ensureChannelReaderEgressNetworkPolicy',+ 'ensureWorkflowApprovalReaderHostEgressNetworkPolicy',+ ]) {+ vi.spyOn(reconciler as any, method).mockResolvedValue(undefined)+ }+ }++ function runtimeTokenScopeWithoutChannels(host: HostCRD): string {+ return (+ HostReconciler as unknown as {+ runtimeTokenScopeHash(selected: HostCRD, hasChannelIngress: boolean): string+ }+ ).runtimeTokenScopeHash(host, false)+ }++ it('rejects a queued reconcile when Host authority changes before admission', async () => {+ const reconciler = makeBasicReconciler()+ const host = makeHost({ name: 'authority-race' })+ const authority = { current: { known: true, generation: 7 } }+ wireAuthoritativeHost(reconciler, host, authority)+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const pending = reconciler.reconcile(host)+ authority.current = { known: false, generation: 8 }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(pending).rejects.toThrow(/Host inventory authority/)+ expect(reconcileCore).not.toHaveBeenCalled()+ })++ it('rejects a queued fleet worker after Context authority retires and recovers', async () => {+ const reconciler = makeBasicReconciler()+ const hostA = makeHost({ name: 'context-authority-a' })+ const hostB = makeHost({ name: 'context-authority-b' })+ const hosts = new Map([+ [hostA.name, hostA],+ [hostB.name, hostB],+ ])+ reconciler.setResolveCurrentHost(name => hosts.get(name))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 7 }))+ const mutationAuthority = {+ current: {+ known: true,+ hostGeneration: 7,+ contextGeneration: 11,+ },+ }+ ;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ vi.spyOn(reconciler as any, 'collectHostCleanupFailures').mockResolvedValue([])+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = (reconciler as any).lifecycle.serializeByHost(hostB.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const fleet = reconciler.fullReconcile([hostA, hostB])+ await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))++ // Context retired and recovered to a fresh LIST -> WATCH generation while+ // Host authority remained continuously known: a host-only lease would miss+ // this ABA transition and admit Host B with stale derived mounts.+ mutationAuthority.current = {+ known: true,+ hostGeneration: 7,+ contextGeneration: 13,+ }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(fleet).rejects.toBeInstanceOf(HostFleetReconcileError)+ expect(reconcileCore).toHaveBeenCalledTimes(1)+ expect(reconcileCore).not.toHaveBeenCalledWith(hostB, expect.any(Function))+ })++ it('retires a Host reconcile when its Context is superseded within the same watch', async () => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'context-revision-race', contextRef: 'context-a' }),+ uid: 'context-revision-race-uid',+ generation: 5,+ }+ const contextV1 = { name: 'context-a', spec: { sharedFileSystems: [] } }+ const contextV2 = {+ name: 'context-a',+ spec: {+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],+ },+ }+ let selectedContext: unknown = contextV1+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostMutationAuthority(() => ({+ known: true,+ hostGeneration: 7,+ contextGeneration: 11,+ }))+ reconciler.setResolveHostMutationDependencies(() => [selectedContext])++ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return { ok: true }+ })+ const ensureServiceAccount = vi+ .spyOn(reconciler as any, 'ensureHostServiceAccount')+ .mockResolvedValue(undefined)++ // R1-M2: a benign mid-pass supersession must NOT be reported as a hard+ // reconcile failure. The single reconcile() catch used to fire+ // enqueueControllerError('reconcile_exception') → controller_error telemetry+ // (status:'failed') AND an administrative 'failed' outcome to control-api for+ // these retirements. Spy (observe-only) to prove the benign branch emits+ // neither, while still re-throwing so callers treat it as a retire.+ const enqueueControllerError = vi.spyOn(reconciler as any, 'enqueueControllerError')+ const enqueueAdministrativeOutcome = vi.spyOn(reconciler as any, 'enqueueAdministrativeOutcome')++ const pending = reconciler.reconcile(host)+ await secretReadStarted.promise+ selectedContext = contextV2+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toThrow(/Host mutation dependency changed/)+ expect(ensureServiceAccount).not.toHaveBeenCalled()+ expect(enqueueControllerError).not.toHaveBeenCalled()+ expect(enqueueAdministrativeOutcome).not.toHaveBeenCalledWith(+ expect.anything(),+ 'failed',+ expect.anything()+ )+ })++ it('keeps a Host reconcile admitted when a watch replaces dependencies with deep-equal values', async () => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'deep-equal-dependency', contextRef: 'context-a' }),+ uid: 'deep-equal-dependency-uid',+ generation: 5,+ }+ let selectedContext: unknown = {+ name: 'context-a',+ spec: {+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],+ },+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostMutationAuthority(() => ({+ known: true,+ hostGeneration: 7,+ contextGeneration: 11,+ }))+ reconciler.setResolveHostMutationDependencies(() => [selectedContext])++ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return { ok: true }+ })+ const admitted = new Error('deep-equal dependency admitted')+ const ensureServiceAccount = vi+ .spyOn(reconciler as any, 'ensureHostServiceAccount')+ .mockRejectedValue(admitted)++ const pending = reconciler.reconcile(host)+ await secretReadStarted.promise+ selectedContext = structuredClone(selectedContext)+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toBe(admitted)+ expect(ensureServiceAccount).toHaveBeenCalled()+ })++ it.each([+ [+ 'referenced SharedFileSystem revision',+ { name: 'documents', spec: { storageClassName: 'fast-v1' } },+ { name: 'documents', spec: { storageClassName: 'fast-v2' } },+ ],+ ['referenced SharedFileSystem mountability', true, false],+ [+ 'Host-scoped CommunicationChannel revision',+ { name: 'telegram', spec: { hostRef: 'dependency-race' } },+ {+ name: 'telegram',+ spec: {+ hostRef: 'dependency-race',+ credentialsSecretRef: { name: 'telegram-v2' },+ },+ },+ ],+ ])('retires a Host reconcile when its %s changes', async (_label, original, replacement) => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'dependency-race', contextRef: 'context-a' }),+ uid: 'dependency-race-uid',+ generation: 5,+ }+ const context = { name: 'context-a', spec: { sharedFileSystems: [] } }+ let selectedDependency: unknown = original+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostMutationAuthority(() => ({+ known: true,+ hostGeneration: 7,+ contextGeneration: 11,+ }))+ reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])++ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return { ok: true }+ })+ const ensureServiceAccount = vi+ .spyOn(reconciler as any, 'ensureHostServiceAccount')+ .mockResolvedValue(undefined)++ const pending = reconciler.reconcile(host)+ await secretReadStarted.promise+ selectedDependency = replacement+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toThrow(/Host mutation dependency changed/)+ expect(ensureServiceAccount).not.toHaveBeenCalled()+ })++ it('does not delete runtime after a Secret rejection when Host authority changes in flight', async () => {+ const reconciler = makeBasicReconciler()+ const host = { ...makeHost({ name: 'secret-authority-race' }), uid: 'host-uid', generation: 3 }+ const authority = { current: { known: true, generation: 7 } }+ wireAuthoritativeHost(reconciler, host, authority)+ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return {+ ok: false,+ reason: 'SecretNotFound',+ message: 'secret missing in stale spec',+ }+ })+ const deleteRuntime = vi+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')+ .mockResolvedValue(undefined)++ const pending = reconciler.reconcile(host)+ await secretReadStarted.promise+ authority.current = { known: false, generation: 8 }+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toThrow(/Host inventory authority/)+ expect(deleteRuntime).not.toHaveBeenCalled()+ })++ it.each([+ [+ 'UID',+ (host: HostCRD) => ({ ...host, uid: 'replacement-host-uid', generation: 1 }),+ /Host identity/,+ ],+ [+ 'spec generation',+ (host: HostCRD) => ({+ ...host,+ generation: (host.generation ?? 0) + 1,+ spec: { ...host.spec, secretRef: 'replacement-secret' },+ }),+ /Host spec generation/,+ ],+ ] as const)(+ 'does not delete runtime after a Secret rejection when the Host %s changes in flight',+ async (_, replaceCurrent, expectedError) => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'secret-identity-race' }),+ uid: 'host-uid',+ generation: 3,+ }+ let current: HostCRD = host+ const authority = { current: { known: true, generation: 7 } }+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return {+ ok: false,+ reason: 'SecretNotFound',+ message: 'secret missing in stale spec',+ }+ })+ const deleteRuntime = vi+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')+ .mockResolvedValue(undefined)++ const pending = reconciler.reconcile(host)+ await secretReadStarted.promise+ current = replaceCurrent(host)+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toThrow(expectedError)+ expect(deleteRuntime).not.toHaveBeenCalled()+ }+ )++ it('revalidates the Host spec after token provisioning and before Deployment mutation', async () => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'deployment-generation-race' }),+ uid: 'host-uid',+ generation: 3,+ }+ let current = host+ const authority = { current: { known: true, generation: 11 } }+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ stubPreDeploymentReconcileEffects(reconciler)+ const tokenProvisionStarted = deferred()+ const releaseTokenProvision = deferred()+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {+ tokenProvisionStarted.resolve(undefined)+ await releaseTokenProvision.promise+ return { revision: 'stale-revision', scopeHash: 'stale-scope' }+ })+ const ensureDeployment = vi+ .spyOn(reconciler as any, 'ensureDeployment')+ .mockResolvedValue(undefined)++ const pending = reconciler.reconcile(host)+ await tokenProvisionStarted.promise+ current = {+ ...host,+ generation: 4,+ spec: { ...host.spec, contextRef: 'replacement-context' },+ }+ releaseTokenProvision.resolve(undefined)++ await expect(pending).rejects.toThrow(/Host spec generation/)+ expect(ensureDeployment).not.toHaveBeenCalled()+ })++ it('revalidates the Host spec at the final Deployment create-state boundary', async () => {+ const appsApi = createMockAppsApi()+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(createMockCoreApi()),+ networkingApi: asNetworkingApi(createMockNetworkingApi()),+ rbacApi: asRbacApi(createMockRbacApi()),+ })+ const host = {+ ...makeHost({ name: 'deployment-create-race' }),+ uid: 'host-uid',+ generation: 3,+ }+ let current = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))+ stubPreDeploymentReconcileEffects(reconciler)+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({+ revision: 'runtime-revision',+ scopeHash: runtimeTokenScopeWithoutChannels(host),+ })+ let channelIngressReads = 0+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockImplementation(() => {+ channelIngressReads += 1+ if (channelIngressReads === 2) {+ current = {+ ...host,+ generation: 4,+ spec: { ...host.spec, contextRef: 'replacement-context' },+ }+ }+ return false+ })++ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)++ expect(channelIngressReads).toBe(2)+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()+ })++ it('revalidates the Host spec after a Deployment create conflict and before replace', async () => {+ const appsApi = createMockAppsApi()+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(createMockCoreApi()),+ networkingApi: asNetworkingApi(createMockNetworkingApi()),+ rbacApi: asRbacApi(createMockRbacApi()),+ })+ const host = {+ ...makeHost({ name: 'deployment-replace-race' }),+ uid: 'host-uid',+ generation: 3,+ }+ let current = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))+ stubPreDeploymentReconcileEffects(reconciler)+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({+ revision: 'runtime-revision',+ scopeHash: runtimeTokenScopeWithoutChannels(host),+ })+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)+ appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {+ current = {+ ...host,+ generation: 4,+ spec: { ...host.spec, contextRef: 'replacement-context' },+ }+ return { metadata: { resourceVersion: 'rv-existing' } }+ })++ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)++ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()+ })++ it('revalidates the Host spec before retrying a conflicted Deployment replace', async () => {+ const appsApi = createMockAppsApi()+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(createMockCoreApi()),+ networkingApi: asNetworkingApi(createMockNetworkingApi()),+ rbacApi: asRbacApi(createMockRbacApi()),+ })+ const host = {+ ...makeHost({ name: 'deployment-retry-race' }),+ uid: 'host-uid',+ generation: 3,+ }+ let current = host+ let deploymentReads = 0+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))+ stubPreDeploymentReconcileEffects(reconciler)+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({+ revision: 'runtime-revision',+ scopeHash: runtimeTokenScopeWithoutChannels(host),+ })+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)+ appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ appsApi.readNamespacedDeployment.mockImplementation(async () => {+ deploymentReads += 1+ if (deploymentReads === 2) {+ current = {+ ...host,+ generation: 4,+ spec: { ...host.spec, contextRef: 'replacement-context' },+ }+ }+ return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }+ })+ appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ const random = vi.spyOn(Math, 'random').mockReturnValue(0)++ try {+ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)+ } finally {+ random.mockRestore()+ }++ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ })++ it('resolves the latest authoritative Host object at reconcile admission', async () => {+ const reconciler = makeBasicReconciler()+ const stale = { ...makeHost({ name: 'latest-host' }), generation: 1 }+ const current = {+ ...stale,+ generation: 2,+ spec: { ...stale.spec, contextRef: 'current-context' },+ }+ const authority = { current: { known: true, generation: 7 } }+ wireAuthoritativeHost(reconciler, current, authority)+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)++ await reconciler.reconcile(stale)++ expect(reconcileCore).toHaveBeenCalledOnce()+ expect(reconcileCore).toHaveBeenCalledWith(+ expect.objectContaining({+ generation: 2,+ spec: expect.objectContaining({ contextRef: 'current-context' }),+ }),+ expect.any(Function)+ )+ })++ it('does not patch channel-reader after Host authority changes during revision resolution', async () => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'channel-authority-race' }),+ uid: 'channel-authority-race-uid',+ generation: 3,+ }+ const authority = { current: { known: true, generation: 9 } }+ wireAuthoritativeHost(reconciler, host, authority)+ const revisionStarted = deferred()+ const releaseRevision = deferred<string>()+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(+ async () => {+ revisionStarted.resolve(undefined)+ return releaseRevision.promise+ }+ )+ const patchDeployment = vi+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')+ .mockResolvedValue(undefined)++ const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)+ await revisionStarted.promise+ authority.current = { known: false, generation: 10 }+ releaseRevision.resolve('resolved-revision')+ await pending++ expect(patchDeployment).not.toHaveBeenCalled()+ })++ it.each([+ ['UID', (host: HostCRD) => ({ ...host, uid: 'replacement-channel-reader-uid' })],+ [+ 'spec generation',+ (host: HostCRD) => ({+ ...host,+ generation: (host.generation ?? 0) + 1,+ spec: { ...host.spec, contextRef: 'replacement-context' },+ }),+ ],+ ] as const)(+ 'does not patch channel-reader when the Host %s changes during revision resolution',+ async (_, replaceCurrent) => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'channel-host-race' }),+ uid: 'channel-host-race-uid',+ generation: 3,+ }+ let current: HostCRD = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 9 }))+ const revisionStarted = deferred()+ const releaseRevision = deferred<string>()+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(+ async () => {+ revisionStarted.resolve(undefined)+ return releaseRevision.promise+ }+ )+ const patchDeployment = vi+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')+ .mockResolvedValue(undefined)++ const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)+ await revisionStarted.promise+ current = replaceCurrent(host)+ releaseRevision.resolve('resolved-revision')+ await pending++ expect(patchDeployment).not.toHaveBeenCalled()+ }+ )++ it('serializes a fresh channel-reader revision behind an older in-flight Host reconcile', async () => {+ const reconciler = makeBasicReconciler()+ const host = {+ ...makeHost({ name: 'channel-revision-order' }),+ uid: 'channel-revision-order-uid',+ generation: 4,+ }+ const authority = { current: { known: true, generation: 9 } }+ wireAuthoritativeHost(reconciler, host, authority)+ const staleReconcileStarted = deferred()+ const releaseStaleReconcile = deferred()+ const order: string[] = []+ let liveRevision = ''++ vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async () => {+ order.push('reconcile-a:start')+ staleReconcileStarted.resolve(undefined)+ await releaseStaleReconcile.promise+ liveRevision = 'revision-a'+ order.push('reconcile-a:write')+ })+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue(+ 'revision-b'+ )+ const patchDeployment = vi+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')+ .mockImplementation(async (request: any) => {+ const { body } = request+ liveRevision = body.spec.template.metadata.annotations['clerum.io/credentials-revision']+ order.push('revision-b:patch')+ })++ const staleReconcile = reconciler.reconcile(host)+ await staleReconcileStarted.promise+ const freshRevision = reconciler.patchChannelReaderRevisionAnnotation(host.name)++ // Give B the opportunity to patch before A is released. Without a shared+ // Host lane, B writes first and A deterministically overwrites it.+ await Promise.resolve()+ await Promise.resolve()+ releaseStaleReconcile.resolve(undefined)+ await Promise.all([staleReconcile, freshRevision])++ expect(liveRevision).toBe('revision-b')+ expect(patchDeployment).toHaveBeenCalledOnce()+ expect(order).toEqual(['reconcile-a:start', 'reconcile-a:write', 'revision-b:patch'])+ })++ it('rejects a queued delete when Host authority changes before admission', async () => {+ const reconciler = makeBasicReconciler()+ const host = makeHost({ name: 'delete-authority-race' })+ const authority = { current: { known: true, generation: 11 } }+ reconciler.setResolveCurrentHost(() => undefined)+ reconciler.setHostWatchAuthority(() => authority.current)+ const deleteRuntime = vi+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')+ .mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const pending = reconciler.reconcileDelete(host.name, host.namespace)+ authority.current = { known: false, generation: 12 }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(pending).rejects.toThrow(/Host inventory authority/)+ expect(deleteRuntime).not.toHaveBeenCalled()+ })+})diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 5730e5553..9e2f236d5 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -1,6 +1,7 @@import * as k8s from '@kubernetes/client-node'import { IntOrString } from '@kubernetes/client-node/dist/types.js'import { createHash, randomUUID } from 'crypto'+import { isDeepStrictEqual } from 'node:util'import * as path from 'path'import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'import { config } from './config'@@ -43,7 +44,7 @@ import {resolveStatelessImagePullPolicy,} from './statelessDeployment'import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'-import { StatelessLifecycleExecutor } from './statelessLifecycleExecutor'+import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'import {CommunicationChannelCRD,HostCRD,@@ -78,6 +79,74 @@ export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'/** Immutable Host watch authority snapshot captured at a full-pass boundary. */export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }+/**+ * Immutable lease for every inventory that contributes to a Host mutation.+ * Host resources come from the Host watch; mounted SharedFileSystems come from+ * the Context watch. Keeping both generations prevents a false-stable+ * Host-only lease when Context retires and recovers during queued work.+ */+export type HostMutationAuthoritySnapshot = {+ known: boolean+ hostGeneration: number+ contextGeneration: number+}++class HostInventoryAuthorityUnavailableError extends Error {+ constructor(+ action: string,+ captured: HostMutationAuthoritySnapshot,+ current: HostMutationAuthoritySnapshot+ ) {+ super(+ `Host inventory authority changed before ${action} admission ` ++ `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` ++ `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` ++ `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`+ )+ this.name = 'HostInventoryAuthorityUnavailableError'+ }+}++class HostMutationIdentityChangedError extends Error {+ constructor(action: string, hostName: string) {+ super(`Host identity changed before ${action} admission for "${hostName}"`)+ this.name = 'HostMutationIdentityChangedError'+ }+}++class HostMutationSpecRevisionChangedError extends Error {+ constructor(action: string, hostName: string) {+ super(`Host spec generation changed before ${action} admission for "${hostName}"`)+ this.name = 'HostMutationSpecRevisionChangedError'+ }+}++class HostMutationDependencyChangedError extends Error {+ constructor(action: string, hostName: string) {+ super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)+ this.name = 'HostMutationDependencyChangedError'+ }+}++/**+ * The four benign mid-pass supersession signals. revalidateHostMutationBoundary()+ * throws these when the watch observes a newer spec/uid/authority-generation while+ * a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass+ * is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT+ * `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts+ * synthesizes name-equivalent plain Errors for the same races.+ */+const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([+ 'HostInventoryAuthorityUnavailableError',+ 'HostMutationIdentityChangedError',+ 'HostMutationSpecRevisionChangedError',+ 'HostMutationDependencyChangedError',+])++function isBenignSupersessionError(error: unknown): boolean {+ return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)+}+/*** The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's* owned bundle is permitted ONLY when watch authority is known, the watch@@ -378,6 +447,12 @@ export class HostReconciler {* snapshot for reconcile, and cleanup stays fail-closed (authority unknown).*/private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null+ /**+ * H2: late-bound cache reflector. Wired by McpServerWatcher next to+ * setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect+ * only onto the caller's own object (legacy contract).+ */+ private reflectHostOutcome: ReflectHostOutcomeFn | null = null/*** Snapshot the current Host watch authority + generation. Wired by* McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never@@ -387,6 +462,27 @@ export class HostReconciler {known: false,generation: 0,})+ /**+ * Positive Host mutations depend on Host plus Context authority. The default+ * preserves the standalone HostReconciler contract by deriving a Host-only+ * lease; McpServerWatcher wires the complete composite lease.+ */+ private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {+ const host = this.hostWatchAuthority()+ return {+ known: host.known,+ hostGeneration: host.generation,+ contextGeneration: 0,+ }+ }+ /**+ * Exact revisions of the cross-resource inputs selected by a Host. Watch+ * generations fence LIST -> WATCH authority, but normal MODIFIED events stay+ * in the same generation. The watcher supplies the selected Context,+ * referenced SharedFileSystems plus mountability, and Host-scoped+ * CommunicationChannels so unrelated fleet changes do not retire this lease.+ */+ private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = nullconstructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {this.appsApi =@@ -420,7 +516,10 @@ export class HostReconciler {now: this.now,countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),- reconcileCore: host => this.reconcileCore(host),+ reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),+ prepareHostMutationAdmission: (action, host) =>+ this.prepareHostMutationAdmission(action, host),+ reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),onLifecycleStatusCommitted: (host, lifecycle) => {const occurredAt = this.now().toISOString()this.infrastructureTelemetryReporter?.enqueueHealthTransition({@@ -486,6 +585,15 @@ export class HostReconciler {this.resolveCurrentHost = fn}+ /**+ * H2: late-bound setter for the guarded cache reflector. Wired by+ * McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern+ * mirrors setResolveCurrentHost.+ */+ setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {+ this.reflectHostOutcome = fn+ }+/*** Late-bound setter for the Host watch authority/generation snapshot. Wired by* McpServerWatcher so orphan cleanup can require known authority and a stable@@ -495,6 +603,103 @@ export class HostReconciler {this.hostWatchAuthority = fn}+ setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {+ this.hostMutationAuthority = fn+ }++ setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {+ this.resolveHostMutationDependencies = fn+ }++ /**+ * A standalone HostReconciler receives an explicit Host object and has no+ * watch-backed cache to fence. Once a live-cache resolver is wired (the+ * production McpServerWatcher path), every mutation captures and revalidates+ * Host LIST -> WATCH authority at its actual serializer admission boundary.+ */+ private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {+ return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()+ }++ private requireHostMutationAuthority(+ action: string,+ captured: HostMutationAuthoritySnapshot | null+ ): void {+ if (captured === null) return+ const current = this.hostMutationAuthority()+ if (+ !captured.known ||+ !current.known ||+ captured.hostGeneration !== current.hostGeneration ||+ captured.contextGeneration !== current.contextGeneration+ ) {+ throw new HostInventoryAuthorityUnavailableError(action, captured, current)+ }+ }++ /**+ * Heartbeat writers own their serializer inside StatelessLifecycleExecutor.+ * Capture authority here at dispatch, then let the returned closure re-check+ * it and resolve the current same-UID Host when that serializer admits the+ * work. This preserves one queue per Host without a nested-lock deadlock.+ */+ private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {+ const capturedAuthority = this.captureHostMutationAuthority()+ const capturedDependencies = this.resolveHostMutationDependencies?.(requested)+ return this.makeHostMutationAdmission(+ action,+ requested,+ capturedAuthority,+ capturedDependencies+ )+ }++ private makeHostMutationAdmission(+ action: string,+ requested: HostCRD,+ capturedAuthority: HostMutationAuthoritySnapshot | null,+ capturedDependencies: readonly unknown[] | undefined+ ): () => HostCRD {+ return () => {+ this.requireHostMutationAuthority(action, capturedAuthority)+ if (this.resolveCurrentHost === null) return requested+ const current = this.resolveCurrentHost(requested.name)+ if (+ current === undefined ||+ requested.uid === undefined ||+ current.uid === undefined ||+ requested.uid !== current.uid+ ) {+ throw new HostMutationIdentityChangedError(action, requested.name)+ }+ if (+ requested.generation === undefined ||+ current.generation === undefined ||+ requested.generation !== current.generation+ ) {+ throw new HostMutationSpecRevisionChangedError(action, requested.name)+ }+ if (this.resolveHostMutationDependencies !== null) {+ const currentDependencies = this.resolveHostMutationDependencies(current)+ const sameDependencies =+ capturedDependencies !== undefined &&+ capturedDependencies.length === currentDependencies.length &&+ capturedDependencies.every((value, index) =>+ isDeepStrictEqual(value, currentDependencies[index])+ )+ if (!sameDependencies) {+ throw new HostMutationDependencyChangedError(action, requested.name)+ }+ }+ // McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation+ // bodies historically reflect status into their admitted object for+ // same-pass decisions, so detach the working snapshot whenever a live+ // cache resolver is wired. Standalone callers without a resolver retain+ // the legacy caller-visible reflection contract.+ return structuredClone(current)+ }+ }+/*** Late-bound setter for the host-ref lookup used by* `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced@@ -2092,29 +2297,52 @@ export class HostReconciler {* Host CRD is present).*/async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {+ const capturedAuthority = this.captureHostMutationAuthority()const depName = `channel-reader-${hostName}`try {- const revision = await this.computeChannelReaderRevisionForHost(hostName)- const patchBody = {- spec: {- template: {- metadata: {- annotations: { 'clerum.io/credentials-revision': revision },+ await this.lifecycle.serializeByHost(hostName, async () => {+ const action = `channel-reader "${hostName}" revision patch`+ this.requireHostMutationAuthority(action, capturedAuthority)++ const resolveCurrentHost = this.resolveCurrentHost+ const admittedHost = resolveCurrentHost?.(hostName)+ if (resolveCurrentHost !== null && admittedHost === undefined) return+ const revalidate = admittedHost+ ? this.makeHostMutationAdmission(+ action,+ admittedHost,+ capturedAuthority,+ this.resolveHostMutationDependencies?.(admittedHost)+ )+ : () => this.requireHostMutationAuthority(action, capturedAuthority)++ revalidate()+ const revision = await this.computeChannelReaderRevisionForHost(hostName)+ // The revision calculation can await multiple Secret reads. Revalidate+ // Host LIST/WATCH authority plus UID and generation immediately before+ // entering the Kubernetes mutation, while still holding this Host lane.+ revalidate()+ const patchBody = {+ spec: {+ template: {+ metadata: {+ annotations: { 'clerum.io/credentials-revision': revision },+ },},},- },- }- await this.appsApi.patchNamespacedDeployment(- { name: depName, namespace: config.channelsNamespace, body: patchBody },- {- middleware: [- k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),- ],}- )- console.log(- `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`- )+ await this.appsApi.patchNamespacedDeployment(+ { name: depName, namespace: config.channelsNamespace, body: patchBody },+ {+ middleware: [+ k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),+ ],+ }+ )+ console.log(+ `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`+ )+ })} catch (err) {if (getErrorCode(err) === 404) returnconsole.error(`[HostReconciler] Failed to reconcile ${depName} revision:`, err)@@ -3549,16 +3777,37 @@ export class HostReconciler {async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {const dispatchedAt = Date.now()+ const capturedAuthority = this.captureHostMutationAuthority()+ const capturedDependencies = this.resolveHostMutationDependencies?.(host)return this.lifecycle.serializeByHost(host.name, async () => {+ this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)+ const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host+ if (!admittedHost) return+ const revalidate = this.makeHostMutationAdmission(+ `Host "${host.name}" reconcile`,+ admittedHost,+ capturedAuthority,+ capturedDependencies+ )const admittedAt = Date.now()hostReconcileInFlight.inc({ lane: source })try {- await this.reconcileCore(host)- this.enqueueReconcileOutcome(host)+ await this.reconcileCore(admittedHost, revalidate)+ this.enqueueReconcileOutcome(admittedHost)this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)} catch (error) {- this.enqueueControllerError(host, 'reconcile_exception', error)- this.enqueueReconcileOutcome(host)+ if (isBenignSupersessionError(error)) {+ // A newer spec/uid/authority-generation superseded this pass mid-flight;+ // a fresh reconcile runs after. Model it as a withdrawal (superseded),+ // NOT a hard failure — do NOT emit controller_error/reconcile_exception+ // or an administrative 'failed' outcome. Callers already treat the+ // rethrow as a retire (reconcileDelete → 'superseded', watch callers+ // only console.error), so the rethrow is preserved.+ this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)+ throw error+ }+ this.enqueueControllerError(admittedHost, 'reconcile_exception', error)+ this.enqueueReconcileOutcome(admittedHost)this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)throw error} finally {@@ -3569,7 +3818,7 @@ export class HostReconciler {private observeReconcileLatency(source: HostReconcileSource,- outcome: 'success' | 'error',+ outcome: 'success' | 'error' | 'superseded',dispatchedAt: number,admittedAt: number): void {@@ -3583,7 +3832,19 @@ export class HostReconciler {)}- private async reconcileCore(host: HostCRD): Promise<void> {+ private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {+ // A per-Host serializer orders work, but it does not invalidate an older+ // reconcile when the watch cache observes a newer spec while this body is+ // awaiting Kubernetes or issuer I/O. Re-check the captured authority,+ // identity, and spec generation at every material mutation boundary.+ // Individual API writes remain atomic at the Kubernetes resource level;+ // these checks prevent subsequent stale writes from spanning an observed+ // generation change.+ const revalidateHostMutationBoundary = (): void => {+ revalidate?.()+ }++ revalidateHostMutationBoundary()// Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles// are serialized PER HOST (serializeByHost), so a pending wake for THIS// Host must not wait behind token issuance, NetworkPolicies or the@@ -3592,6 +3853,7 @@ export class HostReconciler {// funnels through this same method, so a watch event dropped on disconnect// is recovered here.const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true+ revalidateHostMutationBoundary()// Track whether this host has desktop enabledconst isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)@@ -3602,11 +3864,18 @@ export class HostReconciler {}const secretResult = await this.validateHostSecret(host)+ revalidateHostMutationBoundary()if (!secretResult.ok) {if (secretResult.reason === 'SecretNotFound' ||secretResult.reason === 'SecretAccessDenied') {+ // This branch deletes every ephemeral runtime resource. The Secret+ // decision may have blocked on the API while Host watch authority or+ // the Host UID/spec generation changed. Revalidate at the destructive+ // commit boundary so a stale secretRef can never tear down the current+ // runtime; the recovered/current Host reconcile will decide again.+ revalidateHostMutationBoundary()await this.deleteHostRuntimeResources(host.name, host.namespace, {deleteWorkspacePvc: false,})@@ -3627,8 +3896,11 @@ export class HostReconciler {// Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,// otherwise the kubelet can't mount the SA token and the pod will// crash-loop until RBAC is created.+ revalidateHostMutationBoundary()await this.ensureHostServiceAccount(host)+ revalidateHostMutationBoundary()await this.ensureHostRole(host)+ revalidateHostMutationBoundary()await this.ensureHostRoleBinding(host)let mounts: ResolvedSfsMount[] = []@@ -3641,10 +3913,12 @@ export class HostReconciler {// will inject them.console.error(`[HostReconciler] Failed to resolve context mounts for "${host.name}":`, err)}+ revalidateHostMutationBoundary()// Stateless lifecycle (Stage 2): assess enable/reject and persist the// durable state to the Host status subresource BEFORE building the// Deployment, so replicas derive from the same assessment.let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)+ revalidateHostMutationBoundary()// Drained-pre-scale guard (Stage 4.3): a pending wake must abort the// suspension IMMEDIATELY before replicas:0 derives from this assessment// — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null@@ -3652,14 +3926,17 @@ export class HostReconciler {// pass (the periodic resync retries) rather than scaling to 0 on stale// data.const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)+ revalidateHostMutationBoundary()if (guardedLifecycle === null) {return}lifecycle = guardedLifecycle+ revalidateHostMutationBoundary()const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(host,lifecycle)+ revalidateHostMutationBoundary()if (lifecycleStatusCommitted) {this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {status: 'committed',@@ -3670,7 +3947,9 @@ export class HostReconciler {})}+ revalidateHostMutationBoundary()await this.ensurePvc(host)+ revalidateHostMutationBoundary()await this.ensureService(host)// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and@@ -3686,23 +3965,31 @@ export class HostReconciler {// surface it via HostRuntimeStatus instead of swallowing it.const npFailures: string[] = []try {+ revalidateHostMutationBoundary()await this.ensureMcpHostIngressNetworkPolicy(host)+ revalidateHostMutationBoundary()await this.ensureMcpHostGfsEgressNetworkPolicy(host)+ revalidateHostMutationBoundary()await this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure mcp-host NP for "${host.name}":`, err)npFailures.push(`mcp-host NP: ${(err as Error).message}`)}try {+ revalidateHostMutationBoundary()await this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)+ revalidateHostMutationBoundary()await this.ensureRpcProxyHostEgressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure rpc-proxy host NP for "${host.name}":`, err)npFailures.push(`rpc-proxy NP: ${(err as Error).message}`)}+ revalidateHostMutationBoundary()await this.ensureDesktopNetworkPolicy(host)try {+ revalidateHostMutationBoundary()await this.ensureChannelReaderEgressNetworkPolicy(host)+ revalidateHostMutationBoundary()await this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure channels egress NP for "${host.name}":`, err)@@ -3717,8 +4004,10 @@ export class HostReconciler {)if (bootstrapLifecycle !== lifecycle) {lifecycle = bootstrapLifecycle+ revalidateHostMutationBoundary()await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)}+ revalidateHostMutationBoundary()// Bootstrap captures the scope contract used for issuance. The Deployment// guard below compares that contract with the live channel cache after this@@ -3727,23 +4016,28 @@ export class HostReconciler {forceFreshForWake,targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})+ revalidateHostMutationBoundary()// replaceWithConflictRetry may wait and re-read after a 409, so each body is// rebuilt from a stable lifecycle plus token-scope pair before mutation.const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {+ revalidateHostMutationBoundary()const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(host.name,lifecycle)if (deploymentLifecycle !== lifecycle) {lifecycle = deploymentLifecycle+ revalidateHostMutationBoundary()await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)}+ revalidateHostMutationBoundary()return lifecycle.effective}const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {for (let attempt = 1; attempt <= 3; attempt++) {+ revalidateHostMutationBoundary()const currentScopeHash = HostReconciler.runtimeTokenScopeHash(host,this.hasChannelIngress(host)@@ -3755,6 +4049,7 @@ export class HostReconciler {targetSuspended:lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})+ revalidateHostMutationBoundary()const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(host,this.hasChannelIngress(host)@@ -3774,6 +4069,7 @@ export class HostReconciler {const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {for (let attempt = 1; attempt <= 3; attempt++) {+ revalidateHostMutationBoundary()await resolveDeploymentLifecycle()await ensureCurrentRuntimeTokenScope()const effective = await resolveDeploymentLifecycle()@@ -3782,6 +4078,7 @@ export class HostReconciler {this.hasChannelIngress(host))if (runtimeTokenProvision.scopeHash === currentScopeHash) {+ revalidateHostMutationBoundary()return {lifecycle: effective,runtimeTokenRevision: runtimeTokenProvision.revision,@@ -3798,6 +4095,7 @@ export class HostReconciler {)}+ revalidateHostMutationBoundary()await this.ensureDeployment(host,mounts,@@ -3805,6 +4103,7 @@ export class HostReconciler {lifecycle.effective,resolveDeploymentState)+ revalidateHostMutationBoundary()const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'if (suspended) {@@ -3813,6 +4112,7 @@ export class HostReconciler {this.lifecycle.markHostNotSuspended(host.name)}const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)+ revalidateHostMutationBoundary()// If any per-Host NP failed to apply, mark the host as degraded:// deployed remains true (the pod is in place), but ready=false +// a message that names the missing security boundary. Operators see@@ -3848,6 +4148,7 @@ export class HostReconciler {() => this.reconcileChannelReaderDeployment(host),]) {try {+ revalidateHostMutationBoundary()await reconcileResource()} catch (err) {channelReaderFailures.push(err)@@ -3868,6 +4169,7 @@ export class HostReconciler {// Best-effort — checkChannelReaderStatus never throws.const ccCount = this.countCommunicationChannels(host.name)const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)+ revalidateHostMutationBoundary()if (channelReaderFailures.length > 0) {// Overlay the reconcile error onto the status message so both signals// are visible (the deploy error may explain why the Deployment is missing).@@ -3902,18 +4204,21 @@ export class HostReconciler {* that path, so cache presence at admission is the correct fence. This* mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.*- * Callers that omit `opts` keep the previous unconditional behavior exactly.+ * Callers that omit `opts` drop only the extra `skipIf` term; the+ * resolveCurrentHost supersede fence still gates the delete.*/async reconcileDelete(name: string,namespace: string,opts?: { skipIf?: () => boolean }): Promise<void> {+ const capturedAuthority = this.captureHostMutationAuthority()// A DELETED event must not race an in-flight create/update for the same// Host: route it through the same per-Host serializer as reconcile(). The// global convergence tail no longer provides this ordering.await this.lifecycle.serializeByHost(name, async () => {- if (opts?.skipIf?.()) {+ this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)+ if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {// Same vocabulary the watch path already reports for this condition// (reconcileHostWatchEvent), so the skip is observable, not silent.hostDeleteCleanupTotal.inc({ outcome: 'superseded' })@@ -4060,6 +4365,15 @@ export class HostReconciler {}try {await this.lifecycle.serializeByHost(hostName, async () => {+ // The fresh 404 and the cache omission are only destructive+ // authority while the same LIST -> WATCH generation remains live.+ // Revalidate inside the serializer: queued Host work can otherwise+ // outlive that generation and delete a same-name recreation whose+ // ADDED event has not reached the retired cache.+ if (!authorityValid()) {+ hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })+ return+ }// F2 / #827 TOCTOU: a same-name Host recreated between the fresh read// above and admission into this serializer updates the current cache// (its ADDED event sets the cache before its reconcile runs, and thatdiff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 2aa0ccea4..e45095c41 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -1,13 +1,19 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as http from 'http'import { HostFleetReconcileError } from './hostReconciler'import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'+import { externalEgressResyncDelayMs } from './externalEgressConvergenceCoordinator'import {McpServerWatcher,- externalEgressResyncDelayMs,+ getContext,listAllCommunicationChannels,+ listAllGlobalFileSystems,listAllHosts,+ listAllSharedFileSystems,} from './k8sClient'-import type { HostCRD } from './types'+import { registry } from './metrics'+import { ContextMapperServer } from './server'+import type { HostCRD, McpServerCRD } from './types'function deferred<T = void>(): {promise: Promise<T>@@ -23,19 +29,116 @@ function deferred<T = void>(): {return { promise, resolve, reject }}+async function flushMicrotasks(turns = 8): Promise<void> {+ for (let turn = 0; turn < turns; turn += 1) {+ await Promise.resolve()+ }+}++async function readInitialConvergenceMetric(+ name:+ | 'clerum_hcc_initial_convergence_retries_total'+ | 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ lane: 'McpServer' | 'NetworkPolicy'+): Promise<number> {+ const metric = registry.getSingleMetric(name)+ if (!metric) throw new Error(`${name} is not registered`)+ const snapshot = await metric.get()+ return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0+}++async function requestReadyOverHttp(server: ContextMapperServer): Promise<{+ statusCode: number | undefined+ body: string+}> {+ const listener = (server as unknown as { server: http.Server | null }).server+ const address = listener?.address()+ if (!address || typeof address === 'string') {+ throw new Error('HCC readiness listener did not bind a TCP port')+ }++ return new Promise((resolve, reject) => {+ const request = http.get(+ { host: '127.0.0.1', port: address.port, path: '/ready' },+ response => {+ let body = ''+ response.setEncoding('utf8')+ response.on('data', chunk => {+ body += chunk+ })+ response.on('end', () => resolve({ statusCode: response.statusCode, body }))+ }+ )+ request.once('error', reject)+ })+}++function stubAuthoritativeInventoryWatch(+ watcher: McpServerWatcher,+ kind: 'McpServer' | 'Context',+ effect?: () => void | Promise<void>+) {+ const method = kind === 'McpServer' ? 'startMcpServerWatch' : 'startContextWatch'+ const generationField = kind === 'McpServer' ? 'mcpWatchGeneration' : 'contextWatchGeneration'+ const requestField = kind === 'McpServer' ? 'mcpWatchRequest' : 'ctxWatchRequest'+ return vi.spyOn(watcher as any, method).mockImplementation(async () => {+ const generation = ((watcher as any)[generationField] as number) + 1+ ;(watcher as any)[generationField] = generation+ ;(watcher as any)[requestField] = { abort: vi.fn() }+ await effect?.()+ return generation+ })+}++function markHostInventoryAuthoritative(watcher: McpServerWatcher): void {+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+}++function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {+ ;(watcher as any).mcpServerCacheSynced = true+}++function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {+ ;(watcher as any).networkPolicyRevocationContextGeneration = (+ watcher as any+ ).contextWatchGeneration+ ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration+ ;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision+ ;(watcher as any).networkPolicyRevocationServerRevision = (+ watcher as any+ ).mcpServerDesiredRevision+}++function newContextAuthoritativeWatcher(): McpServerWatcher {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ return watcher+}+const mocks = vi.hoisted(() => {const listNamespacedCustomObject = vi.fn()+ const getNamespacedCustomObject = vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)- const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)+ const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)+ const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ })const serverFullReconcile = vi.fn().mockResolvedValue(undefined)const hostFullReconcile = vi.fn().mockResolvedValue(undefined)const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })const hostListCallOptions = vi.fn()+ const sharedFileSystemListCallOptions = vi.fn()const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)return {+ hasCertifiedSafetyInventory,listNamespacedCustomObject,+ getNamespacedCustomObject,ensureDefaultPolicies,netPolFullReconcile,serverFullReconcile,@@ -44,12 +147,15 @@ const mocks = vi.hoisted(() => {sfsFullReconcile,watch,hostListCallOptions,+ sharedFileSystemListCallOptions,createAdministrativeOutcomeReporter,}})-// Mutable so a test can enable the external-egress resync timer (R1-M4); every-// other test keeps externalEgressResyncIntervalSec: 0 (reset in afterEach).+// Kept at externalEgressResyncIntervalSec: 0 in every test (reset in afterEach):+// the external-egress periodic resync timer now lives in the convergence+// coordinator and is unit-tested in externalEgressConvergenceCoordinator.test.ts,+// so the k8sClient integration tests leave it disabled to avoid a stray timer.const mockConfig = vi.hoisted(() => ({devMode: false,namespace: 'mcp-server',@@ -90,9 +196,12 @@ vi.mock('@kubernetes/client-node', () => {return {listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {if (request.plural === 'hosts') mocks.hostListCallOptions(options)+ if (request.plural === 'sharedfilesystems')+ mocks.sharedFileSystemListCallOptions(options)const response = await mocks.listNamespacedCustomObject(request)if (- request.plural === 'hosts' &&+ request.plural &&+ ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&response &&typeof response === 'object' &&!('metadata' in response)@@ -102,11 +211,17 @@ vi.mock('@kubernetes/client-node', () => {// production missing-resourceVersion failure path.return {...response,- metadata: { resourceVersion: 'test-host-collection-rv' },+ metadata: {+ resourceVersion:+ request.plural === 'hosts'+ ? 'test-host-collection-rv'+ : `test-${request.plural}-collection-rv`,+ },}}return response},+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,}}return {}@@ -131,6 +246,8 @@ vi.mock('./reconciler', () => ({reconcile = vi.fn()reconcileDelete = vi.fn()getStatus = vi.fn()+ setInventoryAuthority = vi.fn()+ setResolveCurrentServer = vi.fn()},}))@@ -153,6 +270,28 @@ vi.mock('./hostReconciler', () => ({// and the watch-authority snapshot into HostReconciler at construction.setResolveCurrentHost = vi.fn()setHostWatchAuthority = vi.fn()+ setHostMutationAuthority = vi.fn()+ // H2: capture the injected uid-guarded cache reflector so Test C can invoke it.+ _reflectHostOutcomeFn:+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)+ | null = null+ setReflectHostOutcome = vi.fn().mockImplementation(function (+ this: {+ _reflectHostOutcomeFn:+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)+ | null+ },+ fn: (name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void+ ) {+ this._reflectHostOutcomeFn = fn+ })+ _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null = null+ setResolveHostMutationDependencies = vi.fn().mockImplementation(function (+ this: { _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null },+ fn: (host: HostCRD) => readonly unknown[]+ ) {+ this._hostMutationDependenciesFn = fn+ })// Stores the injected callback so tests can invoke it directly._countCommunicationChannelsFn: ((host: string) => number) | null = nullsetCountCommunicationChannels = vi.fn().mockImplementation(function (@@ -172,16 +311,26 @@ vi.mock('./hostReconciler', () => ({// B2: mock the CC cache-sync setter so McpServerWatcher constructor succeeds.setIsCommunicationChannelCacheSynced = vi.fn()patchChannelReaderRevisionAnnotation = vi.fn().mockResolvedValue(undefined)+ reconcileChannelReaderRevision = vi.fn().mockResolvedValue(undefined)},}))vi.mock('./networkPolicyReconciler', () => ({+ sameContextDesiredRevision: (+ expected: { name: string; namespace: string; spec: unknown },+ current: { name: string; namespace: string; spec: unknown }+ ) =>+ expected.name === current.name &&+ expected.namespace === current.namespace &&+ JSON.stringify(expected.spec) === JSON.stringify(current.spec),NetworkPolicyReconciler: class {ensureDefaultPolicies = mocks.ensureDefaultPoliciesfullReconcile = mocks.netPolFullReconcilereconcileExternalEgress = vi.fn()reconcileContext = vi.fn()+ reconcileDeleteContext = vi.fn()cleanupExternalEgress = vi.fn()+ hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory},}))@@ -199,6 +348,7 @@ vi.mock('./sharedFileSystemReconciler', () => ({reconcileDelete = vi.fn()getStatus = vi.fn()setListContextRefs = vi.fn()+ isMountable = vi.fn().mockReturnValue(false)},}))@@ -213,7 +363,7 @@ describe('McpServerWatcher startup', () => {mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'mcpservers') return { items: [] }- if (plural === 'contexts') throw new Error('context discovery failed')+ if (plural === 'contexts') return { items: [] }if (plural === 'hosts') return { items: [] }if (plural === 'sharedfilesystems') return { items: [] }if (plural === 'communicationchannels')@@ -224,60 +374,24 @@ describe('McpServerWatcher startup', () => {afterEach(() => {vi.useRealTimers()- mockConfig.externalEgressResyncIntervalSec = 0 // R1-M4: keep the timer off by default+ vi.restoreAllMocks()+ // Certified is the resting state. vi.clearAllMocks() in beforeEach clears+ // calls but not implementations, so a test that degrades this getter and+ // fails before its own restore would silently re-gate readiness for every+ // later test in the file. Restore it structurally rather than by+ // per-test discipline.+ mocks.hasCertifiedSafetyInventory.mockReset().mockReturnValue(true)+ mockConfig.externalEgressResyncIntervalSec = 0 // keep periodic resync off by defaultmocks.ensureDefaultPolicies.mockReset().mockResolvedValue(undefined)- mocks.netPolFullReconcile.mockReset().mockResolvedValue(undefined)+ mocks.netPolFullReconcile.mockReset().mockImplementation(async (...args: unknown[]) => {+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ })mocks.serverFullReconcile.mockReset().mockResolvedValue(undefined)mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)mocks.sfsFullReconcile.mockReset().mockResolvedValue(undefined)})- // R1-M4 (zach88): the self-rescheduling resync timer (only the pure delay- // helper was covered). Assert the behavioral invariants the comments promise:- // arms once, reschedules only AFTER a pass resolves, and stop() ends the chain.- it('R1-M4: external-egress resync timer arms once, reschedules after each pass, and stops', async () => {- vi.useFakeTimers()- mockConfig.externalEgressResyncIntervalSec = 60- const watcher = new McpServerWatcher()- for (const m of [- 'startMcpServerWatch',- 'startContextWatch',- 'startSharedFileSystemWatch',- 'startGlobalFileSystemWatch',- 'startCommunicationChannelWatch',- ]) {- vi.spyOn(watcher as any, m).mockImplementation(async () => undefined)- }- // Control each resync pass's completion so we can prove the reschedule waits.- let pending = deferred()- const runSpy = vi- .spyOn(watcher as any, 'runExternalEgressResync')- .mockImplementation(() => pending.promise)-- await watcher.start()- expect(runSpy).not.toHaveBeenCalled() // not until the first delay elapses-- await vi.advanceTimersByTimeAsync(60_000)- expect(runSpy).toHaveBeenCalledTimes(1) // armed once, fired once-- // The NEXT pass must not be scheduled until this one resolves.- await vi.advanceTimersByTimeAsync(60_000)- expect(runSpy).toHaveBeenCalledTimes(1) // still 1 — no overlap while pending-- pending.resolve()- await Promise.resolve()- pending = deferred()- await vi.advanceTimersByTimeAsync(60_000)- expect(runSpy).toHaveBeenCalledTimes(2) // rescheduled after the pass resolved-- // stop() ends the chain — no further fires.- watcher.stop()- pending.resolve()- await Promise.resolve()- await vi.advanceTimersByTimeAsync(60_000 * 5)- expect(runSpy).toHaveBeenCalledTimes(2)- })-it('passes the governed tracing switch to the administrative reporter factory', () => {new McpServerWatcher()@@ -286,35 +400,83 @@ describe('McpServerWatcher startup', () => {})})- it('ensures default policies even when initial Context discovery fails', async () => {+ it('ensures baseline policies before opening any inventory watch', async () => {const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)- // #281 (Task 9): startCommunicationChannelWatch is wired into start() — stub to prevent the real watch from firing.- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(- async () => undefined- )await watcher.start()expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(1)- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])+ expect(mocks.ensureDefaultPolicies.mock.invocationCallOrder[0]).toBeLessThan(+ mocks.watch.mock.invocationCallOrder[0]+ )+ await vi.waitFor(() =>+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {+ runEffect: expect.any(Function),+ })+ )expect(mocks.hostFullReconcile).toHaveBeenCalledWith([])expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([])+ await watcher.stop()})- it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {+ it('loads the SharedFileSystem inventory before scheduling the first Host fleet pass', async () => {+ const sharedFileSystemsListed = deferred<{ items: unknown[] }>()mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return sharedFileSystemsListed.promise+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'sfs-ordering-cc-rv' }, items: [] }+ }return { items: [] }})+ const watcher = new McpServerWatcher()+ // Sampling "not yet called" after a fixed number of microtask turns cannot+ // distinguish "correctly waiting" from "scheduled but not drained yet".+ // Record what the first pass actually observed instead: the invariant is+ // that it never runs against an unpopulated SharedFileSystem cache, because+ // that is what writes mount-less templates and rerolls the fleet.+ let sfsCacheSizeAtFirstPass: number | undefined+ mocks.hostFullReconcile.mockImplementation(async () => {+ if (sfsCacheSizeAtFirstPass === undefined) {+ sfsCacheSizeAtFirstPass = (watcher as any).sharedFileSystems.size+ }+ })+ const start = watcher.start()- await expect(listAllHosts()).resolves.toEqual([])+ try {+ // Await the whole startup first. That is the positive signal that the+ // cold-start Host recovery already reached the point where it decides+ // whether to schedule the fleet pass, so the absence assertion below+ // cannot pass merely because nothing has been drained yet.+ await start+ await flushMicrotasks()+ expect(mocks.hostFullReconcile).not.toHaveBeenCalled()- expect(mocks.hostListCallOptions).toHaveBeenCalledWith(+ sharedFileSystemsListed.resolve({+ items: [{ metadata: { name: 'initial-sfs', namespace: 'mcp-host' }, spec: {} }],+ })+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())+ expect(sfsCacheSizeAtFirstPass).toBe(1)+ expect((watcher as any).sharedFileSystems.get('initial-sfs')).toEqual(+ expect.objectContaining({ name: 'initial-sfs' })+ )+ } finally {+ sharedFileSystemsListed.resolve({ items: [] })+ await start.catch(() => undefined)+ await watcher.stop()+ }+ })++ it('lists the SharedFileSystem inventory through the deadline-bearing client', async () => {+ // The cold-start Host fleet pass waits on this inventory. Bounding the wait+ // is not enough: an unbounded request stays hung, so the socket leaks and+ // `startSharedFileSystemWatch()` — which runs after this await — never+ // starts. Only the deadline-bearing client aborts the request itself, and+ // it is observable because it appends its deadline middleware to the call.+ mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })++ await listAllSharedFileSystems()++ expect(mocks.sharedFileSystemListCallOptions).toHaveBeenCalledWith(expect.objectContaining({middleware: [expect.objectContaining({ pre: expect.any(Function) })],middlewareMergeStrategy: 'append',@@ -322,125 +484,4142 @@ describe('McpServerWatcher startup', () => {)})- it('preserves Host UID from the Kubernetes inventory boundary', async () => {+ it('schedules the first Host fleet pass when the SharedFileSystem watch never starts', async () => {+ // `k8s.Watch` carries no transport deadline — client-node overwrites its own+ // AbortSignal.timeout with a bare controller signal — so awaiting the watch+ // start inside the inventory promise re-strands the cold-start fleet pass+ // that bounding the LIST was meant to protect.mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'sfs-watch-stall-cc-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.watch.mockImplementation(async (path: string) => {+ if (path.endsWith('/sharedfilesystems')) return new Promise(() => undefined)+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const start = watcher.start()++ try {+ await start+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())+ } finally {+ await watcher.stop()+ await start.catch(() => undefined)+ }+ })++ it('schedules the first Host fleet pass when the SharedFileSystem inventory fails', async () => {+ // The inventory LIST runs on the deadline-bearing client, so an apiserver+ // that never answers is aborted and surfaces here as a rejection. The Host+ // fleet must still converge: stranding it would leave every Host+ // unreconciled behind an already-certified readiness.+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'sharedfilesystems') throw new Error('sharedfilesystems LIST aborted')+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'sfs-abort-cc-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const start = watcher.start()++ try {+ await start+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())+ } finally {+ await watcher.stop()+ await start.catch(() => undefined)+ }+ })++ it('retains Context uid and generation from the authoritative initial snapshot', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {return {items: [{metadata: {- name: 'uid-host',- namespace: 'mcp-host',- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',- generation: 4,+ name: 'identity-context',+ namespace: 'mcp-server',+ uid: 'context-uid-1',+ generation: 7,},- spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },+ spec: { contextId: 'identity-context', mcpServers: [] },},],}}+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'context-identity-cc-rv' }, items: [] }+ }return { items: [] }})+ const watcher = new McpServerWatcher()- await expect(listAllHosts()).resolves.toEqual([- expect.objectContaining({- name: 'uid-host',- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',- generation: 4,- }),- ])+ try {+ await watcher.start()+ expect((watcher as any).contexts.get('identity-context')).toEqual(+ expect.objectContaining({ uid: 'context-uid-1', generation: 7 })+ )+ } finally {+ await watcher.stop()+ }})- it('reconciles startup external egress before runtime full reconciliation', async () => {- const eventLog: string[] = []- const server = {- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },- spec: {- contextRef: 'wf-research',- image: 'clerum/web-search:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],- },+ it('fails safe bootstrap and keeps readiness unavailable when baseline NetworkPolicies fail', async () => {+ const policyError = new Error('baseline NetworkPolicies unavailable')+ mocks.ensureDefaultPolicies.mockRejectedValueOnce(policyError)+ const watcher = new McpServerWatcher()+ const server = new ContextMapperServer(watcher, 0)+ await server.start()+ const bootstrap = watcher.start().then(() => server.setReady(true))++ try {+ await expect(bootstrap).rejects.toThrow(policyError)+ const ready = await requestReadyOverHttp(server)+ expect(ready.statusCode).toBe(503)+ expect(JSON.parse(ready.body)).toEqual({ status: 'starting', ready: false })+ expect(mocks.watch).not.toHaveBeenCalled()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ } finally {+ await server.stop()+ await watcher.stop()+ }+ })++ it.each(['LIST', 'WATCH'] as const)(+ 'stays live and recovers readiness after a transient Host %s bootstrap failure',+ async failureBoundary => {+ vi.useFakeTimers()+ let hostListAttempts = 0+ let hostWatchAttempts = 0+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListAttempts += 1+ if (failureBoundary === 'LIST' && hostListAttempts === 1) {+ throw new Error('host discovery temporarily unavailable')+ }+ return { metadata: { resourceVersion: 'host-recovery-rv' }, items: [] }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ }+ )+ mocks.watch.mockImplementation(async path => {+ if (path.endsWith('/hosts')) {+ hostWatchAttempts += 1+ if (failureBoundary === 'WATCH' && hostWatchAttempts === 1) {+ throw new Error('host watch temporarily unavailable')+ }+ }+ return { abort: vi.fn() }+ })+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()++ try {+ await expect(watcher.start()).resolves.toBeUndefined()+ server.setReady(true)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()++ await vi.advanceTimersByTimeAsync(5000)+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())++ expect(hostListAttempts).toBe(2)+ expect(hostWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)+ expect((watcher as any).ccAppliedLifecycleGeneration).toBe(1)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ } finally {+ await server.stop()+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it.each(['LIST', 'WATCH'] as const)(+ 'stays live and recovers readiness after a transient McpServer %s bootstrap failure',+ async failureBoundary => {+ vi.useFakeTimers()+ let mcpServerListAttempts = 0+ let mcpServerWatchAttempts = 0+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') {+ mcpServerListAttempts += 1+ if (failureBoundary === 'LIST' && mcpServerListAttempts === 1) {+ throw new Error('mcpserver discovery temporarily unavailable')+ }+ return { metadata: { resourceVersion: 'mcpserver-recovery-rv' }, items: [] }+ }+ if (plural === 'hosts') {+ return { metadata: { resourceVersion: 'host-rv' }, items: [] }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ }+ )+ mocks.watch.mockImplementation(async path => {+ if (path.endsWith('/mcpservers')) {+ mcpServerWatchAttempts += 1+ if (failureBoundary === 'WATCH' && mcpServerWatchAttempts === 1) {+ throw new Error('mcpserver watch temporarily unavailable')+ }+ }+ return { abort: vi.fn() }+ })+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ // The McpServer lane runs for real (LIST + WATCH) so its bootstrap+ // failure and recovery exercise production code. Context stays+ // authoritative so readiness can be promoted once the lane recovers.+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()++ try {+ await expect(watcher.start()).resolves.toBeUndefined()+ server.setReady(true)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ expect((watcher as any).mcpServerCacheRecoveryTimer).not.toBeNull()++ await vi.advanceTimersByTimeAsync(5000)+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())++ expect(mcpServerListAttempts).toBe(2)+ expect(mcpServerWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ } finally {+ await server.stop()+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it.each(['LIST', 'WATCH'] as const)(+ 'stays live and recovers readiness after a transient Context %s bootstrap failure',+ async failureBoundary => {+ vi.useFakeTimers()+ let contextListAttempts = 0+ let contextWatchAttempts = 0+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ contextListAttempts += 1+ if (failureBoundary === 'LIST' && contextListAttempts === 1) {+ throw new Error('context discovery temporarily unavailable')+ }+ return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }+ }+ if (plural === 'hosts') {+ return { metadata: { resourceVersion: 'host-rv' }, items: [] }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ }+ )+ mocks.watch.mockImplementation(async path => {+ if (path.endsWith('/contexts')) {+ contextWatchAttempts += 1+ if (failureBoundary === 'WATCH' && contextWatchAttempts === 1) {+ throw new Error('context watch temporarily unavailable')+ }+ }+ return { abort: vi.fn() }+ })+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ // The Context lane runs for real (LIST + WATCH) so its bootstrap+ // failure and recovery exercise production code. McpServer stays+ // authoritative so readiness can be promoted once the lane recovers.+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()++ try {+ await expect(watcher.start()).resolves.toBeUndefined()+ server.setReady(true)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ expect((watcher as any).contextCacheRecoveryTimer).not.toBeNull()++ await vi.advanceTimersByTimeAsync(5000)+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())++ expect(contextListAttempts).toBe(2)+ expect(contextWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ } finally {+ await server.stop()+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }}+ )+ it('cancels a pending Host bootstrap recovery when the watcher stops', async () => {+ vi.useFakeTimers()+ let hostListAttempts = 0mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [server] }- if (plural === 'contexts') return { items: [] }- if (plural === 'hosts') return { items: [] }- if (plural === 'sharedfilesystems') return { items: [] }- if (plural === 'communicationchannels')- return { metadata: { resourceVersion: '1' }, items: [] }+ if (plural === 'hosts') {+ hostListAttempts += 1+ throw new Error('host inventory remains unavailable')+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }return { items: [] }})- mocks.ensureDefaultPolicies.mockImplementation(async () => {- eventLog.push('defaults')- })- mocks.serverFullReconcile.mockImplementation(async () => {- eventLog.push('runtime')- })-+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementation(async () => {- eventLog.push('egress')- })- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(- async () => undefined- )+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- await watcher.start()+ try {+ await watcher.start()+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()- expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'web-search' }),- ])+ await watcher.stop()+ await vi.advanceTimersByTimeAsync(5000)- watcher.stop()+ expect(hostListAttempts).toBe(1)+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }})- it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {+ it('defers every Host-dependent effect while a bootstrap LIST lacks WATCH authority', async () => {vi.useFakeTimers()- const server = {- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ const staleHost = {+ metadata: {+ name: 'stale-bootstrap-host',+ namespace: 'mcp-host',+ generation: 1,+ uid: 'stale-bootstrap-host-uid',+ },spec: {- contextRef: 'wf-research',- image: 'clerum/web-search:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ host: 'stale-bootstrap-host',+ contextRef: 'bootstrap-context',+ secretRef: 'bootstrap-secret',+ model: { provider: 'openai', name: 'gpt-4o-mini' },},}-mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [server] }- if (plural === 'contexts') return { items: [] }- if (plural === 'hosts') return { items: [] }- if (plural === 'sharedfilesystems') return { items: [] }- if (plural === 'communicationchannels')- return { metadata: { resourceVersion: '1' }, items: [] }+ if (plural === 'hosts') {+ return {+ metadata: { resourceVersion: 'stale-bootstrap-rv' },+ items: [staleHost],+ }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }return { items: [] }})-- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('dns resolution failed'))+ mocks.watch.mockImplementation(async path => {+ if (path.endsWith('/hosts')) {+ throw new Error('host watch remains unavailable')+ }+ return { abort: vi.fn() }+ })const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(- async () => undefined- )-- await watcher.start()+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const hostReconciler = (watcher as any).hostReconciler- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ try {+ await watcher.start()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ expect(watcher.getHost('stale-bootstrap-host')).toBeUndefined()+ const ccWatchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await ccWatchCallback('ADDED', {+ metadata: { name: 'bootstrap-channel', namespace: 'channels' },+ spec: { hostRef: 'stale-bootstrap-host' },+ })+ ;(watcher as any).contexts.set('bootstrap-context', {+ name: 'bootstrap-context',+ namespace: 'mcp-server',+ spec: {+ contextId: 'bootstrap-context',+ sharedFileSystems: [{ name: 'bootstrap-sfs', mountPath: '/shared' }],+ },+ })+ await (watcher as any).reconcileHostsReferencingSfs('bootstrap-sfs')+ await (watcher as any).reconcileHostsReferencingContext('bootstrap-context')+ ;(watcher as any).ccCacheSynced = false+ await (watcher as any).reconcileChannelReaderRevision('bootstrap-channel-secret', 'channels')++ expect(hostReconciler.reconcile).not.toHaveBeenCalled()+ expect(hostReconciler.patchChannelReaderRevisionAnnotation).not.toHaveBeenCalled()+ expect(hostReconciler.reconcileChannelReaderRevision).not.toHaveBeenCalled()+ expect(warnSpy).toHaveBeenCalledWith(+ '[K8s] Deferring CommunicationChannel Secret "channels/bootstrap-channel-secret" Host convergence; CommunicationChannel cache is not authoritative'+ )+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ })++ it('defers Host fleet convergence until both Host and Context inventories are authoritative', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = false+ ;(watcher as any).hosts.set('context-dependent-host', {+ name: 'context-dependent-host',+ namespace: 'mcp-host',+ spec: {+ host: 'context-dependent-host',+ contextRef: 'current-context',+ secretRef: 'host-secret',+ },+ })++ await (watcher as any).performHostFleetReconcileOnce({+ reason: 'Context authority test',+ mode: 'full',+ })++ expect(mocks.hostFullReconcile).not.toHaveBeenCalled()+ ;(watcher as any).contextCacheSynced = true+ await (watcher as any).performHostFleetReconcileOnce({+ reason: 'Context authority recovered',+ mode: 'full',+ })++ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'context-dependent-host' }),+ ])+ await watcher.stop()+ })++ it('applies a pending CommunicationChannel lifecycle generation during Context recovery', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = false+ const pendingGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()++ await (watcher as any).recoverContextInventoryAndWatch()+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())++ expect((watcher as any).ccAppliedLifecycleGeneration).toBe(pendingGeneration)+ await watcher.stop()+ })++ it('starts McpServer and Context watches from their inventory resourceVersions', async () => {+ const watcher = new McpServerWatcher()++ await (watcher as any).startMcpServerWatch('mcp-inventory-rv')+ await (watcher as any).startContextWatch('context-inventory-rv')++ expect(mocks.watch).toHaveBeenNthCalledWith(+ 1,+ expect.stringContaining('/mcpservers'),+ { resourceVersion: 'mcp-inventory-rv' },+ expect.any(Function),+ expect.any(Function)+ )+ expect(mocks.watch).toHaveBeenNthCalledWith(+ 2,+ expect.stringContaining('/contexts'),+ { resourceVersion: 'context-inventory-rv' },+ expect.any(Function),+ expect.any(Function)+ )+ watcher.stop()+ })++ it.each([+ ['Context', 'startContextWatch', 'contextWatchGeneration'],+ ['Host', 'startHostWatch', 'hostWatchGeneration'],+ ] as const)(+ 'assigns a fresh generation to every replacement %s watch',+ async (_kind, method, generationField) => {+ const watcher = new McpServerWatcher()+ const before = (watcher as any)[generationField] as number++ const first = await (watcher as any)[method]('first-rv')+ const second = await (watcher as any)[method]('second-rv')++ expect(first).toBe(before + 1)+ expect(second).toBe(before + 2)+ expect((watcher as any)[generationField]).toBe(before + 2)+ await watcher.stop()+ }+ )++ it.each([+ [+ 'McpServer',+ 'restartMcpServerWatch',+ 'startMcpServerWatch',+ 'mcpWatchGeneration',+ 'mcpServerCacheSynced',+ { resourceVersion: 'mcp-rv', servers: [] },+ ],+ [+ 'Context',+ 'restartContextWatch',+ 'startContextWatch',+ 'contextWatchGeneration',+ 'contextCacheSynced',+ { resourceVersion: 'context-rv', contexts: [] },+ ],+ ] as const)(+ 'rejects a %s LIST snapshot that is not paired with a live watch request',+ async (_kind, restartMethod, startMethod, generationField, cacheField, snapshot) => {+ const watcher = new McpServerWatcher()+ vi.spyOn(watcher as any, startMethod).mockImplementation(async () => {+ ;(watcher as any)[generationField] += 1+ return (watcher as any)[generationField]+ })++ await expect((watcher as any)[restartMethod](snapshot)).rejects.toThrow(+ 'snapshot could not be paired with an active watch'+ )+ expect((watcher as any)[cacheField]).toBe(false)+ await watcher.stop()+ }+ )++ it.each([+ ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],+ ['Context', 'contexts', 'contextCacheRecoveryTimer'],+ ] as const)(+ 'stays live and arms in-process recovery when the %s inventory LIST is unavailable',+ async (kind, failingPlural, recoveryTimerField) => {+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === failingPlural) throw new Error(`${plural} inventory unavailable`)+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()++ try {+ await expect(watcher.start()).resolves.toBeUndefined()++ expect(errorSpy).toHaveBeenCalledWith(+ `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,+ expect.objectContaining({ message: `${failingPlural} inventory unavailable` })+ )+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it.each([+ ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],+ ['Context', 'contexts', 'contextCacheRecoveryTimer'],+ ] as const)(+ 'stays live and arms in-process recovery when the %s collection resourceVersion is missing',+ async (kind, missingPlural, recoveryTimerField) => {+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === missingPlural) return { metadata: {}, items: [] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()++ try {+ await expect(watcher.start()).resolves.toBeUndefined()++ expect(errorSpy).toHaveBeenCalledWith(+ `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,+ expect.objectContaining({ message: `${kind} snapshot missing resourceVersion` })+ )+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it('replays McpServer and Context changes from the inventory resourceVersion before convergence', async () => {+ const staleServer = {+ metadata: {+ name: 'gap-server',+ namespace: 'mcp-server',+ uid: 'gap-server-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/gap-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const staleContext = {+ metadata: { name: 'gap-context', namespace: 'mcp-server' },+ spec: { contextId: 'gap-context', mcpServers: [] },+ }+ const currentContext = {+ metadata: { name: 'gap-context', namespace: 'mcp-server' },+ spec: { contextId: 'gap-context', mcpServers: ['current-server'] },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') {+ return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }+ }+ if (plural === 'contexts') {+ return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.watch.mockImplementation(async (path, options, callback) => {+ if (path.endsWith('/mcpservers')) {+ expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })+ await callback('DELETED', staleServer)+ }+ if (path.endsWith('/contexts')) {+ expect(options).toEqual({ resourceVersion: 'opaque/context:202' })+ await callback('MODIFIED', currentContext)+ }+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()++ await watcher.start()++ await vi.waitFor(() => {+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {+ runEffect: expect.any(Function),+ })+ expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(+ [expect.objectContaining({ spec: currentContext.spec })],+ [],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ contextDesiredRevision: expect.any(Function),+ serverDesiredRevision: expect.any(Function),+ })+ )+ })+ expect(watcher.getAllServers()).toEqual([])+ await watcher.stop()+ })++ it('recovers a closed McpServer watch through a fresh authoritative snapshot', async () => {+ vi.useFakeTimers()+ const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []+ const doneCallbacks: Array<(error: Error | null) => void> = []+ const watchQueries: unknown[] = []+ mocks.watch.mockImplementation(async (path, options, callback, done) => {+ if (path.endsWith('/mcpservers')) {+ callbacks.push(callback)+ doneCallbacks.push(done)+ watchQueries.push(options)+ }+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') {+ return {+ metadata: { resourceVersion: 'mcp-recovery-rv' },+ items: [+ {+ metadata: {+ name: 'recovered-server',+ namespace: 'mcp-server',+ uid: 'recovered-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/recovered:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ },+ ],+ }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ await (watcher as any).restartMcpServerWatch({+ resourceVersion: 'mcp-start-rv',+ servers: [],+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)++ doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.waitFor(() => expect(watchQueries).toHaveLength(2))++ expect(watchQueries).toEqual([+ { resourceVersion: 'mcp-start-rv' },+ { resourceVersion: 'mcp-recovery-rv' },+ ])+ expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])+ expect(callbacks).toHaveLength(2)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ await watcher.stop()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ })++ it('withdraws readiness when the safety pass leaves its inventory uncertified', async () => {+ // Losing a delete fence is the one doom cause that bumps no watch+ // generation and moves no desired revision, so every equality below stays+ // satisfied while an allow this pass classified as stale is still live.+ // The reconciler already degrades its fence at the point of loss; readiness+ // has to read it, or /ready keeps answering 200 over that allow until a+ // retry lands (>=5s, and indefinitely against a persistent second writer —+ // e.g. the two-replica overlap every rolling update opens, since the+ // Deployment carries no leader election).+ const watcher = newContextAuthoritativeWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)++ try {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ } finally {+ // vi.clearAllMocks() clears calls, not implementations, so an escaped+ // mockReturnValue would silently re-gate every later test in this file.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ }+ })++ it('restores readiness once a later safety pass certifies its inventory again', async () => {+ // The withdrawal above must not be a one-way latch: a fence raised by a+ // transient loss has to clear on the next completed pass, or the retry+ // machinery would leave the controller permanently unready.+ const watcher = newContextAuthoritativeWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ try {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ } finally {+ // An assertion failure above must not escape a false implementation into+ // the rest of the file — vi.clearAllMocks() would not undo it.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ }+ })++ it('ignores a late McpServer callback from a retired watch generation', async () => {+ const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []+ mocks.watch.mockImplementation(async (path, _options, callback) => {+ if (path.endsWith('/mcpservers')) callbacks.push(callback)+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ await (watcher as any).restartMcpServerWatch({+ resourceVersion: 'mcp-old-rv',+ servers: [],+ })+ await (watcher as any).restartMcpServerWatch({+ resourceVersion: 'mcp-new-rv',+ servers: [+ {+ name: 'current-server',+ namespace: 'mcp-server',+ uid: 'current-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/current:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ },+ ],+ })++ await callbacks[0]('ADDED', {+ metadata: {+ name: 'stale-server',+ namespace: 'mcp-server',+ uid: 'stale-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/stale:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })++ expect(watcher.getAllServers().map(server => server.name)).toEqual(['current-server'])+ await watcher.stop()+ })++ it('recovers a closed Context watch through a fresh authoritative snapshot', async () => {+ vi.useFakeTimers()+ const doneCallbacks: Array<(error: Error | null) => void> = []+ const watchQueries: unknown[] = []+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {+ if (path.endsWith('/contexts')) {+ doneCallbacks.push(done)+ watchQueries.push(options)+ }+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return {+ metadata: { resourceVersion: 'context-recovery-rv' },+ items: [+ {+ metadata: { name: 'recovered-context', namespace: 'mcp-server' },+ spec: { contextId: 'recovered-context', mcpServers: [] },+ },+ ],+ }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ await (watcher as any).restartContextWatch({+ resourceVersion: 'context-start-rv',+ contexts: [],+ })++ doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))+ await vi.advanceTimersByTimeAsync(5000)+ await vi.waitFor(() => expect(watchQueries).toHaveLength(2))++ expect(watchQueries).toEqual([+ { resourceVersion: 'context-start-rv' },+ { resourceVersion: 'context-recovery-rv' },+ ])+ expect((watcher as any).contexts.get('recovered-context')).toEqual(+ expect.objectContaining({ name: 'recovered-context' })+ )+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())+ await watcher.stop()+ })++ it('becomes ready after authoritative revocation while every additive convergence lane remains pending', async () => {+ const initialServerFleet = deferred()+ const initialNetworkPolicies = deferred()+ const initialHostFleet = deferred()+ const initialSfsFleet = deferred()+ const initialGfsFleet = deferred()+ let completeAuthoritativeRevocation: (() => void) | undefined+ mocks.serverFullReconcile.mockImplementationOnce(() => initialServerFleet.promise)+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete+ await initialNetworkPolicies.promise+ }+ )+ mocks.hostFullReconcile.mockImplementationOnce(() => initialHostFleet.promise)+ mocks.sfsFullReconcile.mockImplementationOnce(() => initialSfsFleet.promise)+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'safe-bootstrap-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const startMcpServerWatch = stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ const startContextWatch = stubAuthoritativeInventoryWatch(watcher, 'Context')+ const startSharedFileSystemWatch = vi+ .spyOn(watcher as any, 'startSharedFileSystemWatch')+ .mockResolvedValue(undefined)+ const startGlobalFileSystemWatch = vi+ .spyOn(watcher as any, 'startGlobalFileSystemWatch')+ .mockResolvedValue(undefined)+ const gfsFullReconcile = vi+ .spyOn((watcher as any).gfsReconciler, 'fullReconcile')+ .mockImplementationOnce(() => initialGfsFleet.promise)++ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ const bootstrap = watcher.start()++ try {+ await vi.waitFor(() => {+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.sfsFullReconcile).toHaveBeenCalledOnce()+ expect(gfsFullReconcile).toHaveBeenCalledOnce()+ })+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()++ await expect(bootstrap).resolves.toBeUndefined()+ server.setReady(true)+ const beforeRevocation = await requestReadyOverHttp(server)+ expect(beforeRevocation.statusCode).toBe(503)+ expect(completeAuthoritativeRevocation).toBeTypeOf('function')++ completeAuthoritativeRevocation?.()+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())+ const ready = await requestReadyOverHttp(server)+ expect(ready.statusCode).toBe(200)+ expect(JSON.parse(ready.body)).toEqual({ status: 'ready', ready: true })+ expect(startMcpServerWatch).toHaveBeenCalledOnce()+ expect(startContextWatch).toHaveBeenCalledOnce()+ expect(startSharedFileSystemWatch).toHaveBeenCalledOnce()+ expect(startGlobalFileSystemWatch).toHaveBeenCalledOnce()+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)++ // Runtime convergence starts at the safety-certification boundary, not+ // after the NetworkPolicy additive fleet completes. This prevents the+ // startup egress lane from racing the policy safety pass while retaining+ // readiness decoupling from both additive fleets.+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ } finally {+ initialServerFleet.resolve(undefined)+ initialNetworkPolicies.resolve(undefined)+ initialHostFleet.resolve(undefined)+ initialSfsFleet.resolve(undefined)+ initialGfsFleet.resolve(undefined)+ await bootstrap.catch(() => undefined)+ await server.stop()+ watcher.stop()+ }+ })++ it('keeps readiness after a current Context safety delta while an older additive pass is pending', async () => {+ const olderAdditivePass = deferred()+ const currentDeltaSafety = deferred<boolean>()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ let networkPolicyPasses = 0+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {+ if (path.endsWith('/contexts')) contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {+ networkPolicyPasses += 1+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ if (networkPolicyPasses === 1) await olderAdditivePass.promise+ })++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('context-delta-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('delta-context', {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ spec: { contextId: 'delta-context', mcpServers: [] },+ })+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(+ () => currentDeltaSafety.promise+ )++ try {+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)++ const currentDelta = contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ },+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },+ })++ await vi.waitFor(() =>+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()+ )+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ // The scoped revocation completed, which is what authorises the delta+ // certificate; see the aborted-pass twin above for the negative side.+ currentDeltaSafety.resolve(true)+ await currentDelta+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ } finally {+ olderAdditivePass.resolve(undefined)+ await initialPass+ await server.stop()+ await watcher.stop()+ }+ })++ it('does not certify a Context delta whose stale-allow revocation aborted mid-pass', async () => {+ const olderAdditivePass = deferred()+ const abortedDeltaSafety = deferred<boolean>()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ let networkPolicyPasses = 0+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {+ if (path.endsWith('/contexts')) contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {+ networkPolicyPasses += 1+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ if (networkPolicyPasses === 1) await olderAdditivePass.promise+ })++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('aborted-delta-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('delta-context', {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ spec: { contextId: 'delta-context', mcpServers: [] },+ })+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ // `reconcileContext` reports that its authority fence broke, so the scoped+ // revocation never finished deleting this Context's stale allows.+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(+ () => abortedDeltaSafety.promise+ )++ try {+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)++ const abortedDelta = contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ },+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },+ })++ await vi.waitFor(() =>+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()+ )+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ abortedDeltaSafety.resolve(false)+ await abortedDelta++ // An incomplete revocation must leave readiness withheld until the+ // authoritative full pass re-certifies. Certifying here would report+ // Ready while a stale allow is still live.+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ olderAdditivePass.resolve(undefined)+ await initialPass+ await server.stop()+ await watcher.stop()+ }+ })++ it('does not certify a Context delta while the namespace-wide inventory is uncertified', async () => {+ const olderAdditivePass = deferred()+ const uncertifiedDelta = deferred<boolean>()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ let networkPolicyPasses = 0+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {+ if (path.endsWith('/contexts')) contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {+ networkPolicyPasses += 1+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ if (networkPolicyPasses === 1) await olderAdditivePass.promise+ })++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('uncertified-inventory-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('delta-context', {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ spec: { contextId: 'delta-context', mcpServers: [] },+ })+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(+ () => uncertifiedDelta.promise+ )+ try {+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())+ // Baseline: the inventory is still certified here, so readiness is+ // genuinely green. Degrading the fence before this point would assert a+ // 200 that only the missing readiness gate could produce.+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)++ // The delta's own revocation completes, but the last authoritative pass left+ // the namespace-wide inventory uncertified. A label-scoped delta cannot+ // vouch for that namespace, so readiness must stay withheld.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)++ const delta = contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'delta-context',+ namespace: 'mcp-server',+ uid: 'delta-context-uid',+ },+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },+ })++ await vi.waitFor(() =>+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()+ )+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ uncertifiedDelta.resolve(true)+ await delta++ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ olderAdditivePass.resolve(undefined)+ await initialPass+ await server.stop()+ await watcher.stop()+ }+ })++ it('recovers when a Context delta loses the delete fence with no certificate to withhold', async () => {+ // A MODIFIED that does not move the desired revision — a status or label+ // write by another controller — still runs a scoped revocation, but builds+ // no delta certificate. Nothing reads the returned boolean there, so a lost+ // delete fence must stay loud and reach the convergence retry instead of+ // leaving a stale allow live with no NetworkPolicy resync to recover it.+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string; generation?: number }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {+ if (path.endsWith('/contexts')) contextWatchCallback = callback+ return { abort: vi.fn() }+ })++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('lost-fence-no-certificate-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('steady-context', {+ name: 'steady-context',+ namespace: 'mcp-server',+ uid: 'steady-context-uid',+ generation: 4,+ spec: { contextId: 'steady-context', mcpServers: [] },+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ const passesBefore = mocks.netPolFullReconcile.mock.calls.length+ // With honorsLostFence false the real reconciler throws the 409 rather than+ // reporting it, which is the whole point: the throw is what reaches the+ // convergence retry below.+ ;(watcher as any).netPolReconciler.reconcileContext.mockRejectedValueOnce(+ Object.assign(new Error('the UID in the precondition does not match'), { code: 409 })+ )++ try {+ await contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'steady-context',+ namespace: 'mcp-server',+ uid: 'steady-context-uid',+ generation: 4,+ },+ spec: { contextId: 'steady-context', mcpServers: [] },+ })++ // This lane cannot act on the boolean, so it must not have opted into the+ // reported outcome.+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'steady-context' }),+ expect.objectContaining({ honorsLostFence: false })+ )+ await vi.waitFor(() =>+ expect(mocks.netPolFullReconcile.mock.calls.length).toBeGreaterThan(passesBefore)+ )+ } finally {+ await watcher.stop()+ }+ })++ it('does not issue a Context delta certificate before any authoritative safety certificate', async () => {+ const pendingFullSafetyPass = deferred()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('uncertified-context-delta-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('uncertified-context', {+ name: 'uncertified-context',+ namespace: 'mcp-server',+ uid: 'uncertified-context-uid',+ spec: { contextId: 'uncertified-context', mcpServers: [] },+ })+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)++ try {+ await contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'uncertified-context',+ namespace: 'mcp-server',+ uid: 'uncertified-context-uid',+ },+ spec: { contextId: 'uncertified-context', mcpServers: ['delta-server'] },+ })++ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ pendingFullSafetyPass.resolve(undefined)+ await watcher.stop()+ await server.stop()+ }+ })++ it('does not let a Context delta certify an outstanding McpServer revision', async () => {+ const pendingFullSafetyPass = deferred()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('server-revision-context-delta-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('unrelated-context', {+ name: 'unrelated-context',+ namespace: 'mcp-server',+ uid: 'unrelated-context-uid',+ spec: { contextId: 'unrelated-context', mcpServers: [] },+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).mcpServerDesiredRevision += 1+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)++ try {+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ await contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'unrelated-context',+ namespace: 'mcp-server',+ uid: 'unrelated-context-uid',+ },+ spec: { contextId: 'unrelated-context', mcpServers: ['delta-server'] },+ })++ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ pendingFullSafetyPass.resolve(undefined)+ await watcher.stop()+ await server.stop()+ }+ })++ it('does not let one Context delta certify another Context revision still in flight', async () => {+ const pendingFullSafetyPass = deferred()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string; uid: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)++ const watcher = new McpServerWatcher()+ await (watcher as any).startContextWatch('parallel-context-delta-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 1+ ;(watcher as any).contexts.set('first-context', {+ name: 'first-context',+ namespace: 'mcp-server',+ uid: 'first-context-uid',+ spec: { contextId: 'first-context', mcpServers: [] },+ })+ ;(watcher as any).contexts.set('second-context', {+ name: 'second-context',+ namespace: 'mcp-server',+ uid: 'second-context-uid',+ spec: { contextId: 'second-context', mcpServers: [] },+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ // This represents a prior Context change whose global safety sweep is+ // pending. It invalidates the old certificate before the scoped delta.+ ;(watcher as any).contextDesiredRevision += 1+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)++ try {+ await contextWatchCallback!('MODIFIED', {+ metadata: {+ name: 'second-context',+ namespace: 'mcp-server',+ uid: 'second-context-uid',+ },+ spec: { contextId: 'second-context', mcpServers: ['delta-server'] },+ })++ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ pendingFullSafetyPass.resolve(undefined)+ await watcher.stop()+ await server.stop()+ }+ })++ it.each([+ ['Context', 'contextDesiredRevision'],+ ['McpServer', 'mcpServerDesiredRevision'],+ ] as const)(+ 'does not restore readiness from a stale %s delta safety certificate',+ async (_kind, revisionField) => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ const certificate = {+ contextGeneration: (watcher as any).contextWatchGeneration,+ serverGeneration: (watcher as any).mcpWatchGeneration,+ contextRevision: (watcher as any).contextDesiredRevision,+ serverRevision: (watcher as any).mcpServerDesiredRevision,+ }++ ;(watcher as any)[revisionField] += 1++ expect((watcher as any).recordNetworkPolicySafetyCertificate(certificate)).toBe(false)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ await watcher.stop()+ }+ )++ it.each([+ ['SharedFileSystem', 'LIST', 'sharedfilesystems', 'startSharedFileSystemWatch'],+ ['SharedFileSystem', 'WATCH', 'sharedfilesystems', 'startSharedFileSystemWatch'],+ ['GlobalFileSystem', 'LIST', 'globalfilesystems', 'startGlobalFileSystemWatch'],+ ['GlobalFileSystem', 'WATCH', 'globalfilesystems', 'startGlobalFileSystemWatch'],+ ] as const)(+ 'does not let a pending %s %s delay NetworkPolicy safety, bootstrap, or readiness certification',+ async (_kind, boundary, plural, watchMethod) => {+ const pendingBoundary = deferred<unknown>()+ const pendingAdditivePolicies = deferred()+ let completeAuthoritativeRevocation: (() => void) | undefined+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural: requestedPlural }: { plural: string }) => {+ if (requestedPlural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'background-lane-rv' }, items: [] }+ }+ if (boundary === 'LIST' && requestedPlural === plural) {+ return pendingBoundary.promise+ }+ return { items: [] }+ }+ )+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete+ await pendingAdditivePolicies.promise+ }+ )++ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ const sfsWatch = vi+ .spyOn(watcher as any, 'startSharedFileSystemWatch')+ .mockResolvedValue(undefined)+ const gfsWatch = vi+ .spyOn(watcher as any, 'startGlobalFileSystemWatch')+ .mockResolvedValue(undefined)+ const targetWatch = watchMethod === 'startSharedFileSystemWatch' ? sfsWatch : gfsWatch+ if (boundary === 'WATCH') {+ targetWatch.mockImplementationOnce(() => pendingBoundary.promise as Promise<void>)+ }+ const bootstrap = watcher.start()++ try {+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())+ await expect(bootstrap).resolves.toBeUndefined()++ expect(completeAuthoritativeRevocation).toBeTypeOf('function')+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ if (boundary === 'LIST') {+ expect(targetWatch).not.toHaveBeenCalled()+ } else {+ expect(targetWatch).toHaveBeenCalledOnce()+ }++ completeAuthoritativeRevocation?.()+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))+ } finally {+ pendingBoundary.resolve({ items: [] })+ pendingAdditivePolicies.resolve(undefined)+ await flushMicrotasks()+ await watcher.stop()+ }+ }+ )++ it('starts every initial background convergence lane from the caches current after watches', async () => {+ const staleServer = {+ metadata: { name: 'stale-server', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const currentServer = {+ name: 'current-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/current-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const staleContext = {+ metadata: { name: 'stale-context', namespace: 'mcp-server' },+ spec: { contextId: 'stale-context', mcpServers: [] },+ }+ const currentContext = {+ name: 'current-context',+ namespace: 'mcp-server',+ spec: { contextId: 'current-context', mcpServers: [] },+ }+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }+ const currentSfs = { name: 'current-sfs', namespace: 'mcp-host', spec: {} }+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }+ const currentGfs = { name: 'current-gfs', namespace: 'mcp-host', spec: {} }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [staleServer] }+ if (plural === 'contexts') return { items: [staleContext] }+ if (plural === 'sharedfilesystems') return { items: [staleSfs] }+ if (plural === 'globalfilesystems') return { items: [staleGfs] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'current-cache-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer', () => {+ ;(watcher as any).servers.clear()+ ;(watcher as any).servers.set(currentServer.name, currentServer)+ })+ stubAuthoritativeInventoryWatch(watcher, 'Context', () => {+ ;(watcher as any).contexts.clear()+ ;(watcher as any).contexts.set(currentContext.name, currentContext)+ })+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => {+ ;(watcher as any).sharedFileSystems.clear()+ ;(watcher as any).sharedFileSystems.set(currentSfs.name, currentSfs)+ })+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => {+ ;(watcher as any).globalFileSystems.clear()+ ;(watcher as any).globalFileSystems.set(currentGfs.name, currentGfs)+ })+ const gfsFullReconcile = vi.spyOn((watcher as any).gfsReconciler, 'fullReconcile')++ await watcher.start()++ await vi.waitFor(() => {+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([currentServer], {+ runEffect: expect.any(Function),+ })+ expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(+ [currentContext],+ [currentServer],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ })+ )+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([currentSfs])+ expect(gfsFullReconcile).toHaveBeenCalledWith([currentGfs])+ })++ watcher.stop()+ })++ it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {+ const previousServer = {+ name: 'status-server',+ namespace: 'mcp-server',+ uid: 'status-server-uid',+ generation: 7,+ annotations: { 'clerum.io/recipe': 'status-recipe' },+ labels: { 'clerum.io/workload': 'status-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/status-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(previousServer.name, previousServer)+ const changed = vi.fn()+ watcher.onChange(changed)+ const reconciler = (watcher as any).reconciler+ const netPol = (watcher as any).netPolReconciler+ const callback = (watcher as any).getMcpServerWatchCallback()+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)++ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: previousServer.generation,+ annotations: previousServer.annotations,+ labels: previousServer.labels,+ },+ spec: previousServer.spec,+ status: { conditions: [{ type: 'Ready', status: 'True' }] },+ })++ expect(watcher.getAllServers()[0]?.status).toEqual({+ conditions: [{ type: 'Ready', status: 'True' }],+ })+ expect(changed).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)++ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: 8,+ annotations: previousServer.annotations,+ labels: previousServer.labels,+ },+ spec: { ...previousServer.spec, image: 'clerum/status-server:next' },+ })+ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)+ await watcher.stop()+ })++ it('does not advance desired revision for the controller-owned network-ready handshake', async () => {+ const previousServer = {+ name: 'network-ready-server',+ namespace: 'mcp-server',+ uid: 'network-ready-server-uid',+ generation: 7,+ annotations: { 'clerum.io/pre-deploy': 'true' },+ labels: { 'clerum.io/workload': 'network-ready-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/network-ready-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(previousServer.name, previousServer)+ const callback = (watcher as any).getMcpServerWatchCallback()++ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: previousServer.generation,+ annotations: {+ ...previousServer.annotations,+ 'clerum.io/network-ready': 'true',+ },+ labels: previousServer.labels,+ },+ spec: previousServer.spec,+ })++ expect((watcher as any).mcpServerDesiredRevision).toBe(0)+ expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('advances McpServer desired revision for owner addition and deletion', async () => {+ const serverObject = {+ metadata: {+ name: 'revision-server',+ namespace: 'mcp-server',+ uid: 'revision-server-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/revision-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const watcher = new McpServerWatcher()+ const callback = (watcher as any).getMcpServerWatchCallback()++ expect((watcher as any).mcpServerDesiredRevision).toBe(0)+ await callback('ADDED', serverObject)+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)+ await callback('DELETED', serverObject)+ expect((watcher as any).mcpServerDesiredRevision).toBe(2)++ await watcher.stop()+ })++ it('advances Context desired revision only when the authoritative desired cache changes', async () => {+ const watcher = new McpServerWatcher()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('context-desired-revision-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true++ const added = {+ metadata: { name: 'revision-context', namespace: 'mcp-server' },+ spec: { contextId: 'revision-context', mcpServers: [] as string[] },+ }+ expect((watcher as any).contextDesiredRevision).toBe(0)+ await contextWatchCallback!('ADDED', added)+ expect((watcher as any).contextDesiredRevision).toBe(1)++ await contextWatchCallback!('MODIFIED', added)+ expect((watcher as any).contextDesiredRevision).toBe(1)++ const modified = {+ ...added,+ spec: { ...added.spec, mcpServers: ['revision-server'] },+ }+ await contextWatchCallback!('MODIFIED', modified)+ expect((watcher as any).contextDesiredRevision).toBe(2)++ await contextWatchCallback!('DELETED', modified)+ expect((watcher as any).contextDesiredRevision).toBe(3)++ await watcher.stop()+ })++ it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {+ const initialHosts = Array.from({ length: 50 }, (_, index) => ({+ metadata: {+ name: `initial-host-${index}`,+ namespace: 'mcp-host',+ uid: `initial-host-uid-${index}`,+ generation: 1,+ },+ spec: {+ host: `initial-host-${index}`,+ contextRef: 'default',+ secretRef: 'host-secret',+ },+ }))+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ return {+ metadata: { resourceVersion: 'initial-host-rv' },+ items: initialHosts,+ }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')++ await watcher.start()+ await vi.waitFor(() =>+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith(+ initialHosts.map(host =>+ expect.objectContaining({+ name: host.metadata.name,+ generation: host.metadata.generation,+ })+ )+ )+ )+ await flushMicrotasks()++ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()+ expect(urgentReconcile).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {+ const staleServer = {+ name: 'stale-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'stale.example', port: 443 }],+ },+ }+ const currentServer = {+ metadata: { name: 'current-server', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/current-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const reconciliationOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(staleServer.name, staleServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, async () => {+ reconciliationOrder.push('initial')+ })+ )+ )+ })+ const reconciler = (watcher as any).reconciler+ reconciler.reconcile.mockImplementationOnce(async () => {+ reconciliationOrder.push('watch')+ })++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)++ await flushMicrotasks()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'current-server' }),+ { isCurrent: expect.any(Function) }+ )++ releaseEgress.resolve(undefined)+ await Promise.all([convergence, watchConvergence])++ expect(reconciliationOrder).toEqual(['watch'])+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'current-server' }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('continues a same-server reconciliation lane after a rejected operation', async () => {+ const watcher = new McpServerWatcher()+ const server = { name: 'retry-safe-server', namespace: 'mcp-server' }+ const order: string[] = []+ const firstStarted = deferred()+ const rejectFirst = deferred()++ const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {+ order.push('rejected')+ firstStarted.resolve(undefined)+ await rejectFirst.promise+ throw new Error('transient lane failure')+ }) as Promise<void>+ await firstStarted.promise+ const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {+ order.push('recovered')+ }) as Promise<void>+ rejectFirst.resolve(undefined)++ await expect(rejected).rejects.toThrow('transient lane failure')+ await expect(recovered).resolves.toBeUndefined()+ expect(order).toEqual(['rejected', 'recovered'])+ expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)+ await watcher.stop()+ })++ it('revalidates env-Secret matches inside each server lane before reconciling', async () => {+ const currentServer = {+ name: 'current-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/current-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const staleServer = {+ name: 'stale-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const unrelatedServer = {+ name: 'unrelated-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/unrelated-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'other-env' },+ },+ }+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const reconciler = (watcher as any).reconciler+ for (const server of [currentServer, staleServer, unrelatedServer]) {+ ;(watcher as any).servers.set(server.name, server)+ }++ const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ })+ await blockerStarted.promise++ const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')+ ;(watcher as any).servers.set(staleServer.name, {+ ...staleServer,+ spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },+ })+ await vi.waitFor(() =>+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: currentServer.name }),+ { isCurrent: expect.any(Function) }+ )+ )+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, convergence])++ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ expect(+ reconciler.reconcile.mock.calls.some(+ ([server]: [McpServerCRD]) => server.name === staleServer.name+ )+ ).toBe(false)+ expect(+ reconciler.reconcile.mock.calls.some(+ ([server]: [McpServerCRD]) => server.name === unrelatedServer.name+ )+ ).toBe(false)+ await watcher.stop()+ })++ it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {+ const selected = {+ name: 'authority-fenced-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/authority-fenced-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).mcpServerCacheSynced = false++ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ markMcpServerInventoryAuthoritative(watcher)+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: selected.name }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {+ const context = {+ name: 'sfs-context',+ namespace: 'mcp-server',+ spec: {+ contextId: 'sfs-context',+ sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],+ },+ }+ const sharedFileSystem = {+ name: 'shared-data',+ namespace: 'mcp-host',+ spec: {},+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).sharedFileSystemReconciler+ ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)++ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(+ undefined,+ context,+ () => false+ )++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(+ undefined,+ context,+ () => true+ )++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: sharedFileSystem.name })+ )+ await watcher.stop()+ })++ it('fences Context-triggered Host convergence to the current Context lease', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ ;(watcher as any).hosts.set('context-host', {+ name: 'context-host',+ namespace: 'mcp-host',+ spec: {+ host: 'context-host',+ contextRef: 'context-a',+ secretRef: 'host-secret',+ },+ })+ const hostReconciler = (watcher as any).hostReconciler++ await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)++ expect(hostReconciler.reconcile).not.toHaveBeenCalled()++ await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)++ expect(hostReconciler.reconcile).toHaveBeenCalledOnce()+ expect(hostReconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'context-host' })+ )+ await watcher.stop()+ })++ it('keeps env-Secret reconciliation a no-op after stop', async () => {+ const selected = {+ name: 'stopped-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stopped-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ ;(watcher as any).servers.set(selected.name, selected)++ await watcher.stop()+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ })++ it('admits an initial server without egress while another initial server waits on DNS', async () => {+ const dnsBlockedServer = {+ name: 'dns-blocked-server',+ namespace: 'mcp-server',+ uid: 'dns-blocked-server-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/dns-blocked-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'blocked.example', port: 443 }],+ },+ }+ const independentServer = {+ name: 'independent-server',+ namespace: 'mcp-server',+ uid: 'independent-server-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/independent-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)+ ;(watcher as any).servers.set(independentServer.name, independentServer)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(+ async (+ servers: Array<typeof dnsBlockedServer | typeof independentServer>,+ options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }+ ) => {+ await Promise.all(+ servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))+ )+ }+ )++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ await flushMicrotasks()++ expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)+ expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)++ releaseEgress.resolve(undefined)+ await convergence++ expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)+ await watcher.stop()+ })++ it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {+ const staleContext = {+ name: 'stale-context',+ namespace: 'mcp-server',+ spec: { contextId: 'stale-context', mcpServers: [] },+ }+ const currentContext = {+ metadata: { name: 'current-context', namespace: 'mcp-server' },+ spec: { contextId: 'current-context', mcpServers: [] },+ }+ const fullReconcileStarted = deferred()+ const releaseFullReconcile = deferred()+ const reconciliationOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).contexts.set(staleContext.name, staleContext)+ mocks.netPolFullReconcile.mockImplementationOnce(async () => {+ fullReconcileStarted.resolve(undefined)+ await releaseFullReconcile.promise+ reconciliationOrder.push('initial')+ })+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(async () => {+ reconciliationOrder.push('watch')+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof currentContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('context-queue-rv')++ const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)+ await fullReconcileStarted.promise+ const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)++ await flushMicrotasks()+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'current-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ releaseFullReconcile.resolve(undefined)+ await Promise.all([convergence, watchConvergence])++ expect(reconciliationOrder).toEqual(['watch', 'initial'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'current-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {+ const contextName = 'recreated-context-resource'+ const contextId = 'recreated-context'+ const recreatedContext = {+ metadata: { name: contextName, namespace: 'mcp-server' },+ spec: {+ contextId,+ description: 'recreated',+ mcpServers: [],+ },+ }+ const orphanCleanupStarted = deferred()+ const releaseOrphanCleanup = deferred()+ const order: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown[],+ _servers: unknown[],+ options: {+ runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>+ }+ ) => {+ await options.runContextEffect(contextId, async () => {+ orphanCleanupStarted.resolve(undefined)+ await releaseOrphanCleanup.promise+ order.push('orphan-delete')+ })+ }+ )+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(async () => {+ order.push('live-recreate')+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('recreated-context-rv')++ const convergence = (watcher as any).runInitialNetworkPolicyConvergence()+ await orphanCleanupStarted.promise+ const recreation = contextWatchCallback!('ADDED', recreatedContext)+ await flushMicrotasks()++ expect(netPol.reconcileContext).not.toHaveBeenCalled()++ releaseOrphanCleanup.resolve(undefined)+ await Promise.all([convergence, recreation])++ expect(order).toEqual(['orphan-delete', 'live-recreate'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({+ name: contextName,+ spec: expect.objectContaining({ contextId, description: 'recreated' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('retires old contextId policies before reconciling a mutable Context identity', async () => {+ const oldContext = {+ name: 'mutable-context-resource',+ namespace: 'mcp-server',+ spec: {+ contextId: 'old-context-id',+ mcpServers: ['redis-tools'],+ },+ }+ const newContext = {+ metadata: { name: oldContext.name, namespace: oldContext.namespace },+ spec: {+ contextId: 'new-context-id',+ mcpServers: ['redis-tools'],+ },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return {+ metadata: { resourceVersion: 'mutable-context-rv' },+ items: [newContext],+ }+ }+ return { items: [] }+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof newContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contexts.set(oldContext.name, oldContext)+ ;(watcher as any).contextCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const order: string[] = []+ netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {+ order.push(`delete:${contextId}`)+ })+ netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {+ order.push(`reconcile:${context.spec.contextId}`)+ })+ await (watcher as any).startContextWatch('mutable-context-rv')++ await contextWatchCallback!('MODIFIED', newContext)++ expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])+ expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(+ 'old-context-id',+ expect.any(Function)+ )+ await watcher.stop()+ })++ it('retires queued Context policy work immediately when its watch ends', async () => {+ const context = {+ metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },+ spec: { contextId: 'retired-context-id', mcpServers: [] },+ }+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined+ let contextWatchDone: ((error: Error | null) => void) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {+ contextWatchCallback = callback+ contextWatchDone = done+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ await (watcher as any).startContextWatch('retired-context-rv')+ const blocker = (watcher as any).enqueueContextReconciliation(+ context.spec.contextId,+ async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }+ )+ await blockerStarted.promise++ const event = contextWatchCallback!('MODIFIED', context)+ contextWatchDone!(null)+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, event])++ expect(netPol.reconcileContext).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {+ const server = {+ metadata: {+ name: 'context-policy-server',+ namespace: 'mcp-server',+ uid: 'context-policy-server-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'context-policy',+ image: 'clerum/context-policy-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const oldContext = {+ name: 'context-policy-resource',+ namespace: 'mcp-server',+ spec: {+ contextId: 'context-policy',+ description: 'old',+ mcpServers: [server.metadata.name],+ },+ }+ const newContext = {+ metadata: { name: oldContext.name, namespace: oldContext.namespace },+ spec: {+ ...oldContext.spec,+ description: 'new',+ },+ }+ const oldStarted = deferred()+ const releaseOld = deferred()+ const order: string[] = []+ let oldLease: (() => boolean) | undefined+ let contextWatchCallback:+ | ((type: string, apiObj: typeof newContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contexts.set(oldContext.name, oldContext)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementation(+ async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {+ if (context.spec.description === 'old') {+ oldLease = options.isCurrent+ oldStarted.resolve(undefined)+ await releaseOld.promise+ order.push('old')+ return+ }+ order.push('new')+ }+ )+ await (watcher as any).startContextWatch('context-policy-rv')++ const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)+ await oldStarted.promise+ const contextEvent = contextWatchCallback!('MODIFIED', newContext)+ await flushMicrotasks()++ expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)+ expect(oldLease?.()).toBe(false)+ releaseOld.resolve(undefined)+ await Promise.all([serverEvent, contextEvent])++ expect(order).toEqual(['old', 'new'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({+ spec: expect.objectContaining({ description: 'new' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {+ const oldContext = {+ metadata: { name: 'same-stream-context', namespace: 'mcp-server' },+ spec: {+ contextId: 'same-stream-context',+ description: 'old',+ mcpServers: ['old-server'],+ },+ }+ const newContext = {+ metadata: oldContext.metadata,+ spec: {+ ...oldContext.spec,+ description: 'new',+ mcpServers: ['new-server'],+ },+ }+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let oldLease: (() => boolean) | undefined+ let contextWatchCallback:+ | ((type: string, apiObj: typeof oldContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(+ async (_context: unknown, options: { isCurrent: () => boolean }) => {+ oldLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ }+ )+ await (watcher as any).startContextWatch('same-stream-context-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true++ const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)+ await mutationStarted.promise+ expect(oldLease?.()).toBe(true)++ const newReconcile = contextWatchCallback!('MODIFIED', newContext)+ expect(oldLease?.()).toBe(false)++ releaseMutation.resolve()+ await Promise.all([oldReconcile, newReconcile])+ await watcher.stop()+ })++ it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {+ const server = {+ name: 'periodic-egress-server',+ namespace: 'mcp-server',+ uid: 'periodic-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/periodic-egress:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'periodic.example', port: 443 }],+ },+ }+ const context = {+ metadata: { name: 'live-context', namespace: 'mcp-server' },+ spec: { contextId: 'live-context', mcpServers: [] },+ }+ const resyncStarted = deferred()+ const releaseResync = deferred()+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ ;(watcher as any).servers.set(server.name, server)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ resyncStarted.resolve(undefined)+ await releaseResync.promise+ })+ await (watcher as any).startContextWatch('live-context-rv')++ const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ const overlappingResync = (+ watcher as any+ ).externalEgressCoordinator.runResync() as Promise<void>+ expect(overlappingResync).toBe(firstResync)+ await resyncStarted.promise++ const liveEvent = contextWatchCallback!('MODIFIED', context)+ await flushMicrotasks()+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'live-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ releaseResync.resolve(undefined)+ await Promise.all([firstResync, liveEvent])+ await watcher.stop()+ randomSpy.mockRestore()+ })++ it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {+ const oldServer = {+ name: 'same-stream-egress',+ namespace: 'mcp-server',+ uid: 'same-stream-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/same-stream-egress:old',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'old.example', port: 443 }],+ },+ }+ const newServer = {+ metadata: {+ name: oldServer.name,+ namespace: oldServer.namespace,+ uid: oldServer.uid,+ generation: 2,+ },+ spec: {+ ...oldServer.spec,+ image: 'clerum/same-stream-egress:new',+ egressBindings: [{ dns: 'new.example', port: 443 }],+ },+ }+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let oldLease: (() => boolean) | undefined+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ ;(watcher as any).servers.set(oldServer.name, oldServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(+ async (_server: unknown, options: { isCurrent: () => boolean }) => {+ oldLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ }+ )+ vi.spyOn(Math, 'random').mockReturnValue(0)++ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ await mutationStarted.promise+ expect(oldLease?.()).toBe(true)++ const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)+ expect(oldLease?.()).toBe(false)++ releaseMutation.resolve()+ await Promise.all([periodic, replacement])+ await watcher.stop()+ })++ it('uses the latest cached McpServer after periodic external-egress jitter', async () => {+ vi.useFakeTimers()+ const staleServer = {+ name: 'periodic-egress-server',+ namespace: 'mcp-server',+ uid: 'periodic-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/periodic-egress:old',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'old.example', port: 443 }],+ },+ }+ const currentServer = {+ metadata: {+ name: staleServer.name,+ namespace: staleServer.namespace,+ uid: staleServer.uid,+ generation: 2,+ },+ spec: {+ ...staleServer.spec,+ image: 'clerum/periodic-egress:new',+ egressBindings: [{ dns: 'new.example', port: 443 }],+ },+ }+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)+ const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(staleServer.name, staleServer)+ const netPol = (watcher as any).netPolReconciler++ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ await flushMicrotasks()+ await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)+ await vi.advanceTimersByTimeAsync(2500)+ await periodic++ expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(+ expect.objectContaining({+ name: staleServer.name,+ generation: 2,+ spec: expect.objectContaining({+ egressBindings: [{ dns: 'new.example', port: 443 }],+ }),+ }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ randomSpy.mockRestore()+ })++ it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {+ const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }+ const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }+ const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }+ const currentGfs = {+ metadata: { name: 'stale-gfs', namespace: 'mcp-host' },+ spec: {},+ status: {+ phase: 'Ready',+ pvcName: 'gfs-drive',+ serviceName: 'gfsc',+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',+ },+ }+ const sfsInitialStarted = deferred()+ const releaseSfsInitial = deferred()+ const gfsInitialStarted = deferred()+ const releaseGfsInitial = deferred()+ const sfsOrder: string[] = []+ const gfsOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)+ ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)++ mocks.sfsFullReconcile.mockImplementationOnce(async () => {+ sfsInitialStarted.resolve(undefined)+ await releaseSfsInitial.promise+ sfsOrder.push('initial')+ })+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler+ sfsReconciler.reconcile.mockImplementationOnce(async () => {+ sfsOrder.push('watch')+ })+ const gfsReconciler = (watcher as any).gfsReconciler+ vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {+ gfsInitialStarted.resolve(undefined)+ await releaseGfsInitial.promise+ gfsOrder.push('initial')+ })+ const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {+ gfsOrder.push('watch')+ })++ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined+ mocks.watch+ .mockImplementationOnce(async (_path, _options, callback) => {+ sfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ .mockImplementationOnce(async (_path, _options, callback) => {+ gfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startSharedFileSystemWatch()+ await (watcher as any).startGlobalFileSystemWatch()++ const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()+ const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()+ await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)++ await flushMicrotasks()+ expect(sfsReconciler.reconcile).not.toHaveBeenCalled()+ expect(gfsReconcile).not.toHaveBeenCalled()++ releaseSfsInitial.resolve(undefined)+ releaseGfsInitial.resolve(undefined)+ await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])++ expect(sfsOrder).toEqual(['initial', 'watch'])+ expect(gfsOrder).toEqual(['initial', 'watch'])+ expect(gfsReconcile).toHaveBeenCalledWith(+ expect.objectContaining({ status: currentGfs.status })+ )+ await watcher.stop()+ })++ it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }+ const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }+ const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }+ const sfsListStarted = deferred()+ const releaseSfsList = deferred<unknown>()+ const gfsListStarted = deferred()+ const releaseGfsList = deferred<unknown>()+ const watcher = new McpServerWatcher()+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'sharedfilesystems') {+ sfsListStarted.resolve(undefined)+ return releaseSfsList.promise+ }+ if (plural === 'globalfilesystems') {+ gfsListStarted.resolve(undefined)+ return releaseGfsList.promise+ }+ return { items: [] }+ })+ const gfsReconciler = (watcher as any).gfsReconciler+ vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)++ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined+ mocks.watch+ .mockImplementationOnce(async (_path, _options, callback) => {+ sfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ .mockImplementationOnce(async (_path, _options, callback) => {+ gfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startSharedFileSystemWatch()+ await (watcher as any).startGlobalFileSystemWatch()++ const sfsResync = (watcher as any).runSfsResync()+ const gfsResync = (watcher as any).runGfsResync()+ await Promise.all([sfsListStarted.promise, gfsListStarted.promise])+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)+ releaseSfsList.resolve({ items: [staleSfs] })+ releaseGfsList.resolve({ items: [staleGfs] })+ await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])++ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'current-sfs' }),+ ])+ expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'current-gfs' }),+ ])+ expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(+ expect.objectContaining({ name: 'current-sfs' })+ )+ expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)+ expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(+ expect.objectContaining({ name: 'current-gfs' })+ )+ expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)+ await watcher.stop()+ })++ it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {+ const listedSfs = {+ metadata: { name: 'listed-sfs', namespace: 'mcp-host' },+ spec: { capacity: '1Gi' },+ }+ const listedGfs = {+ metadata: { name: 'listed-gfs', namespace: 'gfs' },+ spec: { capacity: '10Gi' },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'sharedfilesystems') return { items: [listedSfs] }+ if (plural === 'globalfilesystems') return { items: [listedGfs] }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const gfsReconciler = (watcher as any).gfsReconciler+ const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)++ await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])++ expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })+ )+ expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })+ )+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),+ ])+ expect(gfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),+ ])+ await watcher.stop()+ })++ it('does not run queued fleet effects after the watcher stops', async () => {+ const initialServer = {+ name: 'initial-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/initial-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'initial.example', port: 443 }],+ },+ }+ const queuedServer = {+ metadata: { name: 'queued-server', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/queued-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(initialServer.name, initialServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })++ const initial = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)++ await watcher.stop()+ releaseEgress.resolve(undefined)+ await Promise.all([initial, queuedWatch])++ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ })++ it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const snapshots: string[][] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set('first-server', {+ name: 'first-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/first-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ mocks.serverFullReconcile+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {+ snapshots.push(servers.map(server => server.name))+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ })+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {+ snapshots.push(servers.map(server => server.name))+ })++ const first = (watcher as any).runInitialMcpServerConvergence()+ await firstPassStarted.promise+ ;(watcher as any).servers.set('second-server', {+ name: 'second-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/second-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ const second = (watcher as any).runInitialMcpServerConvergence()+ const third = (watcher as any).runInitialMcpServerConvergence()++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ releaseFirstPass.resolve(undefined)+ await Promise.all([first, second, third])++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)+ expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])+ await watcher.stop()+ })++ it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const snapshots: string[][] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contexts.set('first-context', {+ name: 'first-context',+ namespace: 'mcp-server',+ spec: { contextId: 'first-context', mcpServers: [] },+ })+ mocks.netPolFullReconcile+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {+ snapshots.push(contexts.map(context => context.name))+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ })+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {+ snapshots.push(contexts.map(context => context.name))+ })++ const first = (watcher as any).runInitialNetworkPolicyConvergence()+ await firstPassStarted.promise+ ;(watcher as any).contexts.set('second-context', {+ name: 'second-context',+ namespace: 'mcp-server',+ spec: { contextId: 'second-context', mcpServers: [] },+ })+ const second = (watcher as any).runInitialNetworkPolicyConvergence()+ const third = (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)+ releaseFirstPass.resolve(undefined)+ await Promise.all([first, second, third])++ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)+ expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])+ await watcher.stop()+ })++ it('does not lose an initial convergence request queued in the settlement microtask', async () => {+ const watcher = new McpServerWatcher()+ let trailing: Promise<void> | undefined+ const convergenceCore = vi+ .fn()+ .mockImplementationOnce(+ () =>+ ({+ then: (resolve: () => void) => {+ resolve()+ queueMicrotask(() => {+ trailing = (watcher as any).runInitialMcpServerConvergence()+ })+ },+ }) as PromiseLike<void>+ )+ .mockResolvedValueOnce(undefined)+ ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore++ const first = (watcher as any).runInitialMcpServerConvergence()+ await first+ await vi.waitFor(() => expect(trailing).toBeDefined())+ await trailing++ expect(convergenceCore).toHaveBeenCalledTimes(2)+ await watcher.stop()+ })++ it('retains one keyed queue until every same-key waiter has drained', async () => {+ const watcher = new McpServerWatcher()+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const effects: string[] = []+ const queues = (watcher as any).contextReconciliationQueues as Map<+ string,+ { references: number }+ >++ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {+ effects.push('first')+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ })+ await firstStarted.promise+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {+ effects.push('second')+ })++ expect(queues.get('shared')?.references).toBe(2)+ expect(effects).toEqual(['first'])+ releaseFirst.resolve(undefined)+ await Promise.all([first, second])+ expect(effects).toEqual(['first', 'second'])+ expect(queues.size).toBe(0)+ await watcher.stop()+ })++ it('does not execute newly enqueued keyed work after stop', async () => {+ const watcher = new McpServerWatcher()+ const work = vi.fn()+ ;(watcher as any).stopped = true++ await (watcher as any).enqueueContextReconciliation('stopped', work)++ expect(work).not.toHaveBeenCalled()+ expect((watcher as any).contextReconciliationQueues.size).toBe(0)+ })++ it('acquires multi-context identity keys in one deterministic sorted order', async () => {+ const watcher = new McpServerWatcher()+ const acquired: string[] = []+ vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(+ async (...args: unknown[]) => {+ const [key, work] = args as [string, () => Promise<void>]+ acquired.push(key)+ await work()+ }+ )++ await (watcher as any).enqueueContextIdentityReconciliation(+ ['z-context', 'a-context', 'm-context', 'z-context'],+ async () => undefined+ )++ expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])+ await watcher.stop()+ })++ it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:v1',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'stale.example', port: 443 }],+ },+ })+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })++ const stalePass = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).mcpWatchGeneration = 8+ const trailing = (watcher as any).runInitialMcpServerConvergence()+ releaseEgress.resolve(undefined)+ await Promise.all([stalePass, trailing])++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ generation: 2,+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:v2',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'current.example', port: 443 }],+ },+ })+ ;(watcher as any).mcpServerCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ await (watcher as any).runInitialMcpServerConvergence()++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ generation: 2,+ spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),+ })+ )+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)+ await watcher.stop()+ })++ it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const appliedContexts: string[] = []+ const appliedServers: string[] = []+ let pass = 0+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 11+ ;(watcher as any).mcpWatchGeneration = 13+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)+ ;(watcher as any).contexts.set('stale-context', {+ name: 'stale-context',+ namespace: 'mcp-server',+ spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },+ })+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'stale-context',+ image: 'clerum/stale-server:v1',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {+ pass += 1+ if (pass === 1) {+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ }+ await Promise.all(+ contexts.map((context: { spec: { contextId: string } }) =>+ options.runContextEffect(context.spec.contextId, async () => {+ appliedContexts.push(context.spec.contextId)+ })+ )+ )+ await Promise.all(+ servers.map((server: { name: string }) =>+ options.runServerEffect(server.name, async () => {+ appliedServers.push(server.name)+ })+ )+ )+ })++ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()+ await firstPassStarted.promise+ ;(watcher as any).contextCacheSynced = false+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).contextWatchGeneration = 12+ ;(watcher as any).mcpWatchGeneration = 14+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence()+ releaseFirstPass.resolve(undefined)+ await Promise.all([stalePass, trailing])++ expect(appliedContexts).toEqual([])+ expect(appliedServers).toEqual([])+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)+ ;(watcher as any).contexts.clear()+ ;(watcher as any).servers.clear()+ ;(watcher as any).contexts.set('current-context', {+ name: 'current-context',+ namespace: 'mcp-server',+ spec: { contextId: 'current-context', mcpServers: ['current-server'] },+ })+ ;(watcher as any).servers.set('current-server', {+ name: 'current-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'current-context',+ image: 'clerum/current-server:v2',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(appliedContexts).toEqual(['current-context'])+ expect(appliedServers).toEqual(['current-server'])+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ await watcher.stop()+ })++ it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 21+ ;(watcher as any).mcpWatchGeneration = 34+ mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {+ ;(watcher as any).contextCacheSynced = false+ options.onAuthoritativeRevocationComplete()+ })++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)+ expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)+ await watcher.stop()+ })++ it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 21+ ;(watcher as any).mcpWatchGeneration = 34+ const server = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }+ const retrySpy = vi+ .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')+ .mockImplementation(() => undefined)+ let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {+ captured = options+ }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')+ captured!.onExternalEgressRevoked!(server)+ expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)+ await watcher.stop()+ })++ it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7++ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()++ await (watcher as any).runInitialNetworkPolicyConvergence()+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())++ await watcher.stop()+ })++ it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {+ // M1: with the revocation counters aligned, the ONLY thing that can null the+ // certificate is the lost safety fence. Fence lost -> cert null -> the fleet+ // lane must not run; fence restored -> it runs exactly once.+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ try {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()++ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()+ } finally {+ // vi.clearAllMocks() clears calls, not implementations — restore explicitly+ // or every later test would re-gate on a stale false.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ await watcher.stop()+ }+ })++ it('invalidates readiness when either desired policy revision advances', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ ;(watcher as any).mcpServerDesiredRevision += 1+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ ;(watcher as any).contextDesiredRevision += 1+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ await watcher.stop()+ })++ it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {+ const previous = {+ metadata: {+ name: 'restrictive-change',+ namespace: 'mcp-server',+ uid: 'restrictive-change-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/restrictive-change:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],+ },+ }+ const current = {+ ...previous,+ metadata: { ...previous.metadata, generation: 2 },+ spec: {+ ...previous.spec,+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],+ },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ ;(watcher as any).servers.set(previous.metadata.name, {+ name: previous.metadata.name,+ namespace: previous.metadata.namespace,+ uid: previous.metadata.uid,+ generation: previous.metadata.generation,+ spec: previous.spec,+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ const safetyPass = vi+ .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')+ .mockResolvedValue(undefined)+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(+ new Error('replacement policy unavailable')+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect(safetyPass).toHaveBeenCalled()+ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('fences runtime effects when either generation retires the safety certificate', async () => {+ const selected = {+ name: 'certificate-fenced-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/certificate-fenced:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const runtimeEffect = vi.fn()+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 11+ ;(watcher as any).mcpWatchGeneration = 13+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))+ )+ })++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ ;(watcher as any).contextWatchGeneration = 12+ releaseEgress.resolve(undefined)+ await convergence++ expect(runtimeEffect).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it.each([+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],+ ] as const)(+ 'does not admit queued %s effects after the watcher stops',+ async (_, enqueueMethod) => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const watcher = new McpServerWatcher()+ let queuedEffectRan = false++ const first = (watcher as any)[enqueueMethod]('same-object', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ }) as Promise<void>+ await firstStarted.promise+ const queued = (watcher as any)[enqueueMethod]('same-object', async () => {+ queuedEffectRan = true+ }) as Promise<void>++ await watcher.stop()+ releaseFirst.resolve(undefined)+ await Promise.all([first, queued])++ expect(queuedEffectRan).toBe(false)+ }+ )++ it.each([+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],+ ] as const)(+ 'serializes the same %s identity without blocking an unrelated fleet member',+ async (_, enqueueMethod) => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const secondStarted = deferred()+ const unrelatedStarted = deferred()+ const watcher = new McpServerWatcher()+ let secondRan = false++ const first = (watcher as any)[enqueueMethod]('slow', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ }) as Promise<void>+ await firstStarted.promise+ const second = (watcher as any)[enqueueMethod]('slow', async () => {+ secondRan = true+ secondStarted.resolve(undefined)+ }) as Promise<void>+ const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {+ unrelatedStarted.resolve(undefined)+ }) as Promise<void>++ await unrelatedStarted.promise+ expect(secondRan).toBe(false)+ releaseFirst.resolve(undefined)+ await Promise.all([first, second, unrelated])+ await expect(secondStarted.promise).resolves.toBeUndefined()+ await watcher.stop()+ }+ )++ it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const secondStarted = deferred()+ const releaseSecond = deferred()+ const thirdStarted = deferred()+ let thirdRan = false+ const watcher = new McpServerWatcher()++ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ })+ await firstStarted.promise+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {+ secondStarted.resolve(undefined)+ await releaseSecond.promise+ })+ const third = (watcher as any).enqueueContextReconciliation('shared', async () => {+ thirdRan = true+ thirdStarted.resolve(undefined)+ })++ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)+ releaseFirst.resolve(undefined)+ await secondStarted.promise+ expect(thirdRan).toBe(false)+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)+ releaseSecond.resolve(undefined)+ await Promise.all([first, second, third])+ expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)+ await watcher.stop()+ })++ it('caps initial convergence retry delay after attempts exceed the schedule', async () => {+ vi.useFakeTimers()+ const watcher = new McpServerWatcher()+ const convergence = vi+ .spyOn(watcher as any, 'runInitialMcpServerConvergence')+ .mockResolvedValue(undefined)+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)+ ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')+ await vi.advanceTimersByTimeAsync(299_999)+ expect(convergence).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1)+ expect(convergence).toHaveBeenCalledOnce()+ await watcher.stop()+ })++ it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)+ ;(watcher as any).mcpServerCacheSynced = true+ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(+ [],+ [],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ })+ )+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ await watcher.stop()+ })++ it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 17+ ;(watcher as any).mcpWatchGeneration = 23+ let options:+ | {+ contextInventoryAuthoritative: () => boolean+ serverInventoryAuthoritative: () => boolean+ }+ | undefined+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown[],+ _servers: unknown[],+ received: {+ contextInventoryAuthoritative: () => boolean+ serverInventoryAuthoritative: () => boolean+ }+ ) => {+ options = received+ }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(options?.contextInventoryAuthoritative()).toBe(true)+ expect(options?.serverInventoryAuthoritative()).toBe(true)++ // A recovered watch may already be authoritative again, but it is a+ // different snapshot generation and cannot authorize cleanup by this pass.+ ;(watcher as any).contextWatchGeneration = 18+ ;(watcher as any).mcpWatchGeneration = 24++ expect(options?.contextInventoryAuthoritative()).toBe(false)+ expect(options?.serverInventoryAuthoritative()).toBe(false)+ await watcher.stop()+ })++ it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {+ vi.useFakeTimers()+ const server = {+ metadata: { name: 'initial-runtime', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/initial-runtime:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.serverFullReconcile+ .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))+ .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))+ .mockResolvedValueOnce(undefined)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const retryMetricBefore = await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_retries_total',+ 'McpServer'+ )+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)++ await vi.advanceTimersByTimeAsync(14_999)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(1)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_retries_total',+ 'McpServer'+ )+ ).toBe(retryMetricBefore + 2)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'McpServer'+ )+ ).toBeGreaterThan(0)++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('cancels a pending initial convergence retry when the watcher stops', async () => {+ vi.useFakeTimers()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ mocks.serverFullReconcile.mockRejectedValueOnce(+ new Error('initial runtime reconciliation unavailable')+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await (watcher as any).runInitialMcpServerConvergence()++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)++ await watcher.stop()+ await vi.advanceTimersByTimeAsync(300_000)++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)+ expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)+ errorSpy.mockRestore()+ })++ it('retries failed initial NetworkPolicy convergence from the current caches', async () => {+ vi.useFakeTimers()+ const context = {+ metadata: { name: 'initial-context', namespace: 'mcp-server' },+ spec: { contextId: 'initial-context', mcpServers: [] },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [] }+ if (plural === 'contexts') return { items: [context] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.netPolFullReconcile+ .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))+ .mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ options?.onAuthoritativeRevocationComplete?.()+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ await vi.advanceTimersByTimeAsync(5000)+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(+ [expect.objectContaining({ name: 'initial-context' })],+ [],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ })+ )++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('does not report a post-certification additive failure as a readiness safety failure', async () => {+ vi.useFakeTimers()+ const additiveFailure = new AggregateError(+ [new Error('Context policy API unavailable')],+ 'One or more additive Context NetworkPolicy reconciliations failed'+ )+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ options?.onAuthoritativeRevocationComplete?.()+ throw additiveFailure+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(errorSpy).toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',+ additiveFailure+ )+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy background reconciliation failed:',+ additiveFailure+ )+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it.each([+ ['McpServer', 'startMcpServerWatch'],+ ['Context', 'startContextWatch'],+ ] as const)(+ 'keeps safe bootstrap authority-gated when the %s watch cannot be established',+ async (_kind, method) => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ if (method === 'startMcpServerWatch') {+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))+ } else {+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ }+ if (method === 'startContextWatch') {+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))+ } else if (method !== 'startMcpServerWatch') {+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ }+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)+ const recoveryTimerField =+ method === 'startMcpServerWatch'+ ? 'mcpServerCacheRecoveryTimer'+ : 'contextCacheRecoveryTimer'++ try {+ await expect(watcher.start()).resolves.toBeUndefined()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it.each([+ ['SharedFileSystem', 'startSharedFileSystemWatch'],+ ['GlobalFileSystem', 'startGlobalFileSystemWatch'],+ ] as const)(+ 'isolates, logs, and retries an initial %s background watch rejection',+ async (kind, method) => {+ vi.useFakeTimers()+ const failure = new Error(`${method} unavailable`)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ const rejectedWatch = vi+ .spyOn(watcher as any, method)+ .mockRejectedValueOnce(failure)+ .mockResolvedValueOnce(undefined)+ const otherMethod =+ method === 'startSharedFileSystemWatch'+ ? 'startGlobalFileSystemWatch'+ : 'startSharedFileSystemWatch'+ vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await expect(watcher.start()).resolves.toBeUndefined()+ await vi.advanceTimersByTimeAsync(0)++ expect(errorSpy).toHaveBeenCalledWith(+ `[K8s] ${kind} background watch failed to start:`,+ failure+ )+ expect(rejectedWatch).toHaveBeenCalledOnce()+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()++ await vi.advanceTimersByTimeAsync(4_999)+ expect(rejectedWatch).toHaveBeenCalledOnce()+ await vi.advanceTimersByTimeAsync(1)+ expect(rejectedWatch).toHaveBeenCalledTimes(2)++ await watcher.stop()+ errorSpy.mockRestore()+ }+ )++ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') return { items: [] }+ return { items: [] }+ })++ await expect(listAllHosts()).resolves.toEqual([])++ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(+ expect.objectContaining({+ middleware: [expect.objectContaining({ pre: expect.any(Function) })],+ middlewareMergeStrategy: 'append',+ })+ )+ })++ it('preserves Host UID from the Kubernetes inventory boundary', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ return {+ items: [+ {+ metadata: {+ name: 'uid-host',+ namespace: 'mcp-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,+ },+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },+ },+ ],+ }+ }+ return { items: [] }+ })++ await expect(listAllHosts()).resolves.toEqual([+ expect.objectContaining({+ name: 'uid-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,+ }),+ ])+ })++ it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {+ const status = {+ phase: 'Ready',+ pvcName: 'gfs-drive',+ serviceName: 'gfsc',+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'globalfilesystems') {+ return {+ items: [+ {+ metadata: { name: 'gfs', namespace: 'gfs' },+ spec: {},+ status,+ },+ ],+ }+ }+ return { items: [] }+ })++ await expect(listAllGlobalFileSystems()).resolves.toEqual([+ expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),+ ])+ })++ it('reconciles startup external egress before runtime full reconciliation', async () => {+ const eventLog: string[] = []+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ spec: {+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ },+ }++ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }+ return { items: [] }+ })+ mocks.ensureDefaultPolicies.mockImplementation(async () => {+ eventLog.push('defaults')+ })+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, async () => {+ eventLog.push('runtime')+ await reconciler.reconcile(selected)+ })+ )+ )+ })+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementation(async () => {+ eventLog.push('egress')+ })+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )++ await watcher.start()++ await vi.waitFor(() => expect(eventLog).toContain('runtime'))+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(+ [expect.objectContaining({ name: 'web-search' })],+ { runEffect: expect.any(Function) }+ )++ watcher.stop()+ })++ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {+ vi.useFakeTimers()+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ spec: {+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ },+ }++ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }+ return { items: [] }+ })++ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })+ const netPol = (watcher as any).netPolReconciler+ const dnsFailure = new Error('dns resolution failed')+ netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )++ await watcher.start()++ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(+ [expect.objectContaining({ name: 'web-search' })],+ { runEffect: expect.any(Function) }+ )+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy background reconciliation failed:',+ dnsFailure+ )watcher.stop()errorSpy.mockRestore()@@ -448,6 +4627,29 @@ describe('McpServerWatcher startup', () => {})})+describe('getContext error semantics', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ })++ it('preserves not-found semantics only for an authoritative 404', async () => {+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(+ Object.assign(new Error('context absent'), { code: 404 })+ )++ await expect(getContext('missing-context')).resolves.toBeNull()+ })++ it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {+ const unavailable = Object.assign(new Error('apiserver unavailable'), {+ response: { statusCode: 503 },+ })+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)++ await expect(getContext('production-context')).rejects.toBe(unavailable)+ })+})+describe('McpServerWatcher external egress retries', () => {const serverObject = {metadata: { name: 'redis-tools', namespace: 'mcp-server' },@@ -459,42 +4661,195 @@ describe('McpServerWatcher external egress retries', () => {},}- beforeEach(() => {- vi.useFakeTimers()- })+ beforeEach(() => {+ vi.clearAllMocks()+ vi.useFakeTimers()+ })++ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it('retries failed external egress reconciliation from an ADDED event', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ const serverWithBindings = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ annotations: {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { source: 'redis-tools', target: 'downstream' },+ ]),+ },+ },+ }+ ;(watcher as any).contexts.set('context1', {+ name: 'context1',+ namespace: 'mcp-server',+ spec: { contextId: 'context1', mcpServers: ['redis-tools'] },+ })+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverWithBindings)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()+ expect(netPol.reconcileContext).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) }+ )+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({+ spec: expect.objectContaining({ contextId: 'context1' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ watcher.stop()+ })++ it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const bindingReconciler = (watcher as any).bindingReconciler+ const bindingStarted = deferred()+ const releaseBinding = deferred()+ bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {+ bindingStarted.resolve(undefined)+ await releaseBinding.promise+ })+ const annotations = {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { from: 'redis-tools', to: 'downstream', port: 443 },+ ]),+ }+ const original = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-uid',+ generation: 1,+ annotations,+ },+ }+ const replacement = {+ ...original,+ metadata: { ...original.metadata, generation: 2 },+ spec: { ...original.spec, image: 'redis-tools:v2' },+ }+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const originalEffect = watchCallback('ADDED', original)+ await bindingStarted.promise+ const replacementEffect = watchCallback('MODIFIED', replacement)++ const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]+ expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })+ expect(originalOptions.isCurrent()).toBe(false)++ releaseBinding.resolve(undefined)+ await Promise.all([originalEffect, replacementEffect])++ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)+ expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(+ 'redis-tools',+ expect.any(Array),+ 'redis-tools',+ 'redis-tools',+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const bindingReconciler = (watcher as any).bindingReconciler+ const cleanupStarted = deferred()+ const releaseCleanup = deferred()+ bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {+ cleanupStarted.resolve(undefined)+ await releaseCleanup.promise+ })+ const deleted = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-old-uid',+ generation: 1,+ },+ }+ const recreated = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-new-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },+ }+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const deleteEffect = watchCallback('DELETED', deleted)+ await cleanupStarted.promise+ const recreateEffect = watchCallback('ADDED', recreated)- afterEach(() => {- vi.useRealTimers()- vi.restoreAllMocks()+ const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]+ expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })+ await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)++ releaseCleanup.resolve(undefined)+ await Promise.all([deleteEffect, recreateEffect])++ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)+ await watcher.stop()})- it('retries failed external egress reconciliation from an ADDED event', async () => {+ it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler+ markMcpServerInventoryAuthoritative(watcher)const reconciler = (watcher as any).reconciler- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))- netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ const noEgressServer = {+ ...serverObject,+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ reconciler.reconcile+ .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))+ .mockResolvedValue(undefined)vi.spyOn(console, 'error').mockImplementation(() => {})vi.spyOn(console, 'warn').mockImplementation(() => {})- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('ADDED', serverObject)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)await vi.advanceTimersByTimeAsync(5000)- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: 'redis-tools' })- )-+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)watcher.stop()})it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)@@ -514,7 +4869,8 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'redis-tools' })+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) })watcher.stop()@@ -522,6 +4878,7 @@ describe('McpServerWatcher external egress retries', () => {it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))@@ -547,14 +4904,16 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'redis-tools' })+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) })watcher.stop()})- it('does not let periodic external egress resync cancel a pending runtime retry', async () => {+ it('continues a pending runtime pipeline during periodic external egress resync', async () => {const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)@@ -563,6 +4922,7 @@ describe('McpServerWatcher external egress retries', () => {vi.spyOn(console, 'error').mockImplementation(() => {})vi.spyOn(console, 'warn').mockImplementation(() => {})vi.spyOn(console, 'log').mockImplementation(() => {})+ vi.spyOn(Math, 'random').mockReturnValue(0)const watchCallback = (watcher as any).getMcpServerWatchCallback()await watchCallback('ADDED', serverObject)@@ -570,18 +4930,152 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- await (watcher as any).runExternalEgressResync()+ await (watcher as any).externalEgressCoordinator.runResync()- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)await vi.advanceTimersByTimeAsync(5000)-expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)watcher.stop()})+ it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress+ .mockRejectedValueOnce(new Error('initial outage'))+ .mockRejectedValueOnce(new Error('outage after 5s'))+ .mockRejectedValueOnce(new Error('outage after 15s'))+ .mockRejectedValueOnce(new Error('outage after 30s'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(15000)+ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)+ expect(reconciler.reconcile).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ watcher.stop()+ })++ it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.cleanupExternalEgress+ .mockRejectedValueOnce(new Error('initial cleanup outage'))+ .mockRejectedValueOnce(new Error('cleanup outage after 5s'))+ .mockRejectedValueOnce(new Error('cleanup outage after 15s'))+ .mockRejectedValueOnce(new Error('cleanup outage after 30s'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(15000)+ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })+ watcher.stop()+ })++ it('bounds concurrent durable retries across McpServers', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const reconciler = (watcher as any).reconciler+ const release = deferred()+ let active = 0+ let maxActive = 0+ reconciler.reconcile.mockImplementation(async () => {+ active += 1+ maxActive = Math.max(maxActive, active)+ await release.promise+ active -= 1+ })+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ for (let index = 0; index < 11; index += 1) {+ const server = {+ name: `server-${index}`,+ namespace: 'mcp-server',+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ ;(watcher as any).servers.set(server.name, server)+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)+ }++ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks()++ expect(maxActive).toBe(10)++ release.resolve(undefined)+ await flushMicrotasks()+ watcher.stop()+ })++ it('replays a retry against the watcher generation current when the timer executes', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const selected = {+ name: 'generation-fenced-server',+ namespace: 'mcp-server',+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).mcpWatchGeneration = 41+ const replay = vi+ .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)+ ;(watcher as any).mcpWatchGeneration = 42+ await vi.advanceTimersByTimeAsync(5000)++ expect(replay).toHaveBeenCalledWith(+ 'MODIFIED',+ expect.objectContaining({ name: selected.name }),+ 42,+ expect.objectContaining({+ isCurrent: expect.any(Function),+ complete: expect.any(Function),+ })+ )+ watcher.stop()+ })+it('reconciles external egress before runtime reconciliation on ADDED events', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler@@ -611,36 +5105,337 @@ describe('McpServerWatcher external egress retries', () => {vi.spyOn(console, 'warn').mockImplementation(() => {})const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('ADDED', serverObject)+ await watchCallback('ADDED', serverObject)++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ watcher.stop()+ })++ it('retries failed external egress cleanup from a DELETED event', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('DELETED', serverObject)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(+ 'redis-tools',+ 'mcp-server',+ undefined,+ expect.any(Function)+ )+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })++ watcher.stop()+ })++ it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {+ response: { statusCode: 503 },+ })+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()++ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)+ expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)++ watcher.stop()+ })++ it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ reconciler.reconcileDelete.mockImplementationOnce(async () => {+ ;(watcher as any).mcpServerCacheSynced = false+ })+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()+ ;(watcher as any).mcpServerCacheSynced = true+ await vi.advanceTimersByTimeAsync(14_999)+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(1)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })++ watcher.stop()+ })++ it('retires queued McpServer work immediately when its watch ends', async () => {+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined+ let watchDone: ((error: Error | null) => void) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {+ watchCallback = callback+ watchDone = done+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ await (watcher as any).startMcpServerWatch('mcp-watch-rv')+ const blocker = (watcher as any).enqueueMcpServerReconciliation(+ { name: 'redis-tools', namespace: 'mcp-server' },+ async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }+ )+ await blockerStarted.promise++ const event = watchCallback!('ADDED', serverObject)+ watchDone!(null)+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, event])++ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {+ const deletedServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-old-uid',+ generation: 1,+ },+ spec: serverObject.spec,+ }+ const recreatedServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-new-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))+ netPol.reconcileExternalEgress+ .mockRejectedValueOnce(new Error('recreated policy unavailable'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ await watchCallback('DELETED', deletedServer)+ await watchCallback('ADDED', recreatedServer)+ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ uid: 'redis-tools-new-uid',+ generation: 1,+ spec: expect.objectContaining({ image: 'redis-tools:recreated' }),+ }),+ { isCurrent: expect.any(Function) }+ )++ watcher.stop()+ })++ it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {+ const server = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 3,+ },+ spec: serverObject.spec,+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))+ netPol.cleanupExternalEgress+ .mockRejectedValueOnce(new Error('cleanup unavailable'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ await watchCallback('ADDED', server)+ await watchCallback('DELETED', server)+ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)++ watcher.stop()+ })++ it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const server = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 1,+ annotations: {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { source: 'redis-tools', target: 'downstream' },+ ]),+ },+ },+ spec: serverObject.spec,+ }+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const reconcile = watchCallback('ADDED', server)+ await egressStarted.promise+ await watcher.stop()+ releaseEgress.resolve(undefined)+ await reconcile++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()+ })++ it('does not run an older runtime revision after newer desired state arrives during egress', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const oldServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:old' },+ }+ const newServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 2,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:new' },+ }+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress+ .mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ .mockResolvedValue(undefined)+ const watchCallback = (watcher as any).getMcpServerWatchCallback()- expect(reconciler.reconcile).not.toHaveBeenCalled()+ const oldReconcile = watchCallback('ADDED', oldServer)+ await egressStarted.promise+ const newReconcile = watchCallback('MODIFIED', newServer)+ releaseEgress.resolve(undefined)+ await Promise.all([oldReconcile, newReconcile])++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ generation: 2,+ spec: expect.objectContaining({ image: 'redis-tools:new' }),+ }),+ { isCurrent: expect.any(Function) }+ )watcher.stop()})- it('retries failed external egress cleanup from a DELETED event', async () => {+ it('does not resume periodic external egress work from jitter after stop', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))- netPol.cleanupExternalEgress.mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('DELETED', serverObject)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)-- await vi.advanceTimersByTimeAsync(5000)+ ;(watcher as any).servers.set('redis-tools', {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ })+ vi.spyOn(Math, 'random').mockReturnValue(0.5)- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)- expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith('redis-tools', 'mcp-server')+ const resync = (watcher as any).externalEgressCoordinator.runResync()+ await flushMicrotasks()+ await watcher.stop()+ await vi.advanceTimersByTimeAsync(2500)+ await resync- watcher.stop()+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()})it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)vi.spyOn(Math, 'random').mockReturnValue(0)@@ -650,7 +5445,7 @@ describe('McpServerWatcher external egress retries', () => {spec: serverObject.spec,})- await (watcher as any).runExternalEgressResync()+ await (watcher as any).externalEgressCoordinator.runResync()expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)@@ -660,55 +5455,92 @@ describe('McpServerWatcher external egress retries', () => {it('does not run parallel external egress reconciles for the same McpServer', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()const server = {name: 'redis-tools',namespace: 'mcp-server',spec: serverObject.spec,};(watcher as any).servers.set('redis-tools', server)- ;(watcher as any).externalEgressInFlight.set(- 'mcp-server/redis-tools',- new Promise(() => undefined)- )+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ })vi.spyOn(Math, 'random').mockReturnValue(0)- await (watcher as any).runExternalEgressResync()+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ await (watcher as any).externalEgressCoordinator.runResync()- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ releaseMutation.resolve(undefined)+ await activewatcher.stop()})it('waits for already in-flight external egress before reconciling the current server', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()const server = {name: 'redis-tools',namespace: 'mcp-server',spec: serverObject.spec,}- let releaseExisting!: () => void- const existing = new Promise<void>(resolve => {- releaseExisting = resolve+ ;(watcher as any).servers.set(server.name, server)+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise})- ;(watcher as any).externalEgressInFlight.set('mcp-server/redis-tools', existing)const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})- const run = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await Promise.resolve()+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await flushMicrotasks()- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()expect(warn).toHaveBeenCalledWith('[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight')- releaseExisting()- await expect(run).resolves.toBeUndefined()- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ releaseMutation.resolve(undefined)+ await Promise.all([active, waiting])+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)warn.mockRestore()watcher.stop()})++ it('does not start another external egress mutation after stop releases an in-flight wait', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ const server = {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ }+ ;(watcher as any).servers.set(server.name, server)+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ })++ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await flushMicrotasks()+ await watcher.stop()+ releaseMutation.resolve(undefined)+ await Promise.all([active, waiting])++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ })})describe('McpServerWatcher.countCommunicationChannels', () => {@@ -779,9 +5611,139 @@ describe('McpServerWatcher wires CC counter into HostReconciler', () => {})})+describe('McpServerWatcher Host mutation dependency wiring', () => {+ it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {+ const watcher = new McpServerWatcher()+ const reflect = (+ watcher.getHostReconciler() as unknown as {+ _reflectHostOutcomeFn:+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)+ | null+ }+ )._reflectHostOutcomeFn+ if (!reflect) {+ throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')+ }+ const entry = {+ name: 'h2-host',+ namespace: 'mcp-host',+ uid: 'uid-1',+ spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },+ } as unknown as HostCRD+ ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)+ const apply = vi.fn()+ reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid+ expect(apply).toHaveBeenCalledTimes(1)+ expect(apply).toHaveBeenCalledWith(entry)+ apply.mockClear()+ reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip+ expect(apply).not.toHaveBeenCalled()+ reflect('h2-host', undefined, apply) // fail-closed on missing uid+ expect(apply).not.toHaveBeenCalled()+ reflect('absent-host', 'uid-1', apply) // absent entry → skip+ expect(apply).not.toHaveBeenCalled()+ })++ it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {+ const watcher = new McpServerWatcher()+ const host = {+ name: 'alpha-host',+ namespace: 'mcp-host',+ spec: { contextRef: 'alpha-context' },+ } as HostCRD+ const context = {+ name: 'alpha-context',+ namespace: 'mcp-server',+ spec: {+ contextId: 'alpha-context',+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],+ },+ }+ const documentsV1 = {+ name: 'documents',+ namespace: 'mcp-host',+ spec: { storageClassName: 'fast-v1' },+ }+ const alphaChannelV1 = {+ name: 'telegram-alpha',+ namespace: 'channels',+ spec: { hostRef: 'alpha-host' },+ }+ ;(watcher as any).contexts.set(context.name, context)+ ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)+ ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)+ ;(watcher as any).ccCacheSynced = true+ ;(watcher as any).ccWatchGeneration = 7+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler+ sfsReconciler.isMountable.mockReturnValue(true)+ const hostReconciler = watcher.getHostReconciler() as any+ const resolveDependencies = hostReconciler._hostMutationDependenciesFn++ const original = resolveDependencies(host)+ expect(original).toEqual([+ { name: context.name, namespace: context.namespace, spec: context.spec },+ true,+ 7,+ { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },+ true,+ {+ name: alphaChannelV1.name,+ namespace: alphaChannelV1.namespace,+ spec: alphaChannelV1.spec,+ },+ ])+ ;(watcher as any).sharedFileSystems.set('unrelated', {+ name: 'unrelated',+ namespace: 'mcp-host',+ spec: {},+ })+ ;(watcher as any).communicationChannels.set('telegram-beta', {+ name: 'telegram-beta',+ namespace: 'channels',+ spec: { hostRef: 'beta-host' },+ })+ const afterUnrelatedChanges = resolveDependencies(host)+ expect(afterUnrelatedChanges).toEqual(original)++ const documentsV2 = {+ ...documentsV1,+ spec: { storageClassName: 'fast-v2' },+ }+ ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)+ const afterSfsRevision = resolveDependencies(host)+ expect(afterSfsRevision[3]).toEqual({+ name: documentsV2.name,+ namespace: documentsV2.namespace,+ spec: documentsV2.spec,+ })+ expect(afterSfsRevision[3]).not.toEqual(original[3])++ sfsReconciler.isMountable.mockReturnValue(false)+ const afterMountabilityChange = resolveDependencies(host)+ expect(afterMountabilityChange[4]).toBe(false)++ const alphaChannelV2 = {+ ...alphaChannelV1,+ spec: {+ ...alphaChannelV1.spec,+ credentialsSecretRef: { name: 'telegram-alpha-v2' },+ },+ }+ ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)+ const afterChannelRevision = resolveDependencies(host)+ expect(afterChannelRevision.at(-1)).toEqual({+ name: alphaChannelV2.name,+ namespace: alphaChannelV2.namespace,+ spec: alphaChannelV2.spec,+ })+ expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))+ })+})+describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('re-reconciles the Host whose name matches the CC hostRef', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>hosts.set('marketing', {name: 'marketing',@@ -800,6 +5762,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('is a no-op when the hostRef does not match any cached Host', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const reconcileSpy = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -811,6 +5774,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>hosts.set('marketing', {name: 'marketing',@@ -828,6 +5792,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher);(watcher as any).hosts.set('marketing', {name: 'marketing',namespace: 'mcp-host',@@ -896,8 +5861,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {})const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')@@ -936,8 +5901,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)const ccWatchSpy = vi@@ -1005,7 +5970,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('partial-channel', {name: 'partial-channel',@@ -1045,7 +6010,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {if (plural === 'hosts') return { items: [host] }return { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('watch-observed-channel', {name: 'watch-observed-channel',@@ -1086,7 +6051,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockImplementationOnce(() => firstFleetPass.promise).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const failClosedPass = (watcher as any).requestHostFleetReconcile(@@ -1162,7 +6127,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {})const fleetFailure = new Error('fleet did not converge')mocks.hostReconcileHosts.mockRejectedValueOnce(fleetFailure).mockResolvedValueOnce(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -1204,7 +6169,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {if (hostListAttempts === 1) throw inventoryFailurereturn { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -1214,14 +6179,12 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {lifecycleGeneration)expect((watcher as any).ccAppliedLifecycleGeneration).not.toBe(lifecycleGeneration)- expect((watcher as any).ccFleetRetryTimer).not.toBeNull()expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()await vi.advanceTimersByTimeAsync(5000)await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())expect(hostListAttempts).toBe(2)expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)- expect((watcher as any).ccFleetRetryTimer).toBeNull()} finally {errorSpy.mockRestore()watcher.stop()@@ -1237,7 +6200,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([], [cleanupFailure]))- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -1249,7 +6212,6 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {)expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)- expect((watcher as any).ccFleetRetryTimer).toBeNull()await vi.advanceTimersByTimeAsync(300000)expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()} finally {@@ -1270,7 +6232,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([hostFailure], [cleanupFailure])).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher();(watcher as any).ccCacheSynced = trueconst errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -1287,7 +6249,6 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {await vi.advanceTimersByTimeAsync(5000)await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)- expect((watcher as any).ccFleetRetryTimer).toBeNull()await (watcher as any).performHostResync()expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2)@@ -1312,7 +6273,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockRejectedValueOnce(new Error('lifecycle pass failed')).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition();(watcher as any).ccCacheSynced = true@@ -1357,7 +6318,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.hostReconcileHosts.mockRejectedValue(new Error('persistent fleet failure'))- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -1403,15 +6364,13 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockRejectedValueOnce(new Error('old generation failed')).mockResolvedValueOnce(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})try {const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()await (watcher as any).requestHostFleetReconcile('fail-closed', staleGeneration)- expect((watcher as any).ccFleetRetryTimer).not.toBeNull()const currentGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()- expect((watcher as any).ccFleetRetryTimer).toBeNull()await (watcher as any).requestHostFleetReconcile('recovered', currentGeneration)expect(mocks.hostReconcileHosts).toHaveBeenCalledTimes(2)@@ -1439,7 +6398,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {}return { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('previous-channel', {name: 'previous-channel',@@ -1480,7 +6439,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -1538,7 +6497,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {.mockImplementationOnce(() => firstHostConvergence.promise).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const firstRecovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>try {@@ -1596,7 +6555,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {}).mockResolvedValue({ abort: vi.fn() })- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()try {await expect((watcher as any).recoverCommunicationChannelCache()).resolves.toBe(false)@@ -1629,7 +6588,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.watch.mockImplementationOnce(() => pendingWatch.promise)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const recovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>try {@@ -1674,7 +6633,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -1736,9 +6695,9 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {maxActivePasses = Math.max(maxActivePasses, activePasses)activePasses -= 1})- const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)+ const watcher = newContextAuthoritativeWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(@@ -1758,8 +6717,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()expect(maxActivePasses).toBe(1)- initialPass.resolve(undefined)+ // Startup now resolves after safe bootstrap; the serialized fleet tail+ // keeps running independently and must still drain the queued lifecycle pass.await start+ initialPass.resolve(undefined)+ await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce()@@ -1786,7 +6748,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher();(watcher as any).ccCacheSynced = trueawait (watcher as any).startCommunicationChannelWatch('504')@@ -1820,7 +6782,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return requests[callbacks.length - 1]})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -1871,7 +6833,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {const abort = vi.fn()const watchRequest = { abort }mocks.watch.mockReset().mockResolvedValue(watchRequest)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()await (watcher as any).startHostWatch('stream-rv')await vi.advanceTimersByTimeAsync(30_001)@@ -1906,9 +6868,11 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => activePass).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher();(watcher as any).ccCacheSynced = true- ;(watcher as any).ccAppliedLifecycleGeneration = (watcher as any).ccLifecycleGeneration+ ;(watcher as any).hostFleetScheduler.markLifecycleApplied(+ (watcher as any).ccLifecycleGeneration+ )const first = (watcher as any).runHostResync() as Promise<void>await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))@@ -1927,9 +6891,53 @@ describe('McpServerWatcher Host periodic resync serialization', () => {expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)})+ it('queues a trailing full Host pass when Context recovery lands during an active pass', async () => {+ vi.clearAllMocks()+ const activePass = deferred()+ mocks.hostFullReconcile+ .mockReset()+ .mockImplementationOnce(() => activePass.promise)+ .mockResolvedValue(undefined)+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return {+ metadata: { resourceVersion: 'recovered-context-rv' },+ items: [+ {+ metadata: { name: 'recovered-context', namespace: 'mcp-server' },+ spec: {+ contextId: 'recovered-context',+ mcpServers: [],+ sharedFileSystems: [{ name: 'recovered-sfs' }],+ },+ },+ ],+ }+ }+ return { items: [] }+ })+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })+ const watcher = newContextAuthoritativeWatcher()+ ;(watcher as any).hostCacheSynced = true++ const first = (watcher as any).requestHostFleetReconcile(+ 'before Context recovery'+ ) as Promise<void>+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))++ await expect((watcher as any).recoverContextInventoryAndWatch()).resolves.toBe(true)+ expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)++ activePass.resolve(undefined)+ await first+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2))++ await watcher.stop()+ })+it('does not invent another fleet transition during a failed periodic cache recovery', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher();(watcher as any).ccCacheSynced = falseconst recover = vi.spyOn(watcher as any, 'recoverCommunicationChannelCache')@@ -1945,6 +6953,33 @@ describe('McpServerWatcher Host periodic resync serialization', () => {watcher.stop()})+ it('coalesces the newest Host input revision into the serialized trailing pass', async () => {+ vi.clearAllMocks()+ const watcher = newContextAuthoritativeWatcher()+ const activePass = deferred()+ const requests: Array<{ inputRevision: number }> = []+ vi.spyOn(watcher as any, 'performHostFleetReconcile').mockImplementation(+ async (...args: unknown[]) => {+ const request = args[0] as { inputRevision: number }+ requests.push({ ...request })+ if (requests.length === 1) await activePass.promise+ }+ )+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()+ const first = (watcher as any).requestHostFleetReconcile('active') as Promise<void>+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()+ const second = (watcher as any).requestHostFleetReconcile('pending-v2') as Promise<void>+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()+ const third = (watcher as any).requestHostFleetReconcile('pending-v3') as Promise<void>++ activePass.resolve(undefined)+ await Promise.all([first, second, third])++ expect(requests).toHaveLength(2)+ expect(requests[1].inputRevision).toBe(3)+ watcher.stop()+ })+it('collapses overlapping fleet requests into one serialized trailing pass', async () => {vi.clearAllMocks()const fleetV1 = [@@ -1988,7 +7023,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {maxActivePasses = Math.max(maxActivePasses, activePasses)activePasses -= 1})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const first = (watcher as any).requestHostFleetReconcile('first',@@ -2041,7 +7076,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {it('uses fresh lifecycle inventory without replacing newer Host watch-cache entries', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const initialHost = {name: 'stateless-primary',namespace: 'mcp-host',@@ -2106,7 +7141,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()const lifecycle = (watcher as any).requestHostFleetReconcile('CommunicationChannel interruption',@@ -2151,7 +7186,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()const lifecycle = (watcher as any).requestHostFleetReconcile('CommunicationChannel recovery',@@ -2196,7 +7231,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const active = (watcher as any).requestHostFleetReconcile('CommunicationChannel interruption',@@ -2243,7 +7278,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const full = (watcher as any).requestHostFleetReconcile('initial Host reconciliation',@@ -2295,7 +7330,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { items: [] }})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')@@ -2380,13 +7415,14 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { items: [] }})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient event failure')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)await hostWatchCallback('MODIFIED', {metadata: { name: 'retry-before-list', namespace: 'mcp-host', generation: 1 },@@ -2395,6 +7431,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {expect(reconcile).toHaveBeenCalledTimes(1)expect((watcher as any).hostWatchRetryTimers.size).toBe(1)+ // The active Host watch is then lost. A full request must recover a fresh+ // snapshot and retire the failed event's pending retry.+ ;(watcher as any).hostCacheSynced = falseawait (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')expect((watcher as any).hostWatchRetryTimers.size).toBe(0)@@ -2447,7 +7486,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {await fullReconcile.promiseorder.push('full:end')})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const hostReconciler = watcher.getHostReconciler()const reconcile = vi.spyOn(hostReconciler, 'reconcile').mockImplementation(async host => {order.push(`watch:${host.name}:${host.generation}`)@@ -2500,7 +7539,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})mocks.listNamespacedCustomObject.mockImplementation(async () => ({ items: [] }))- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const hostReconciler = watcher.getHostReconciler()// The reconcile()/reconcileDelete() spies stand in for the per-Host chain;// the actual serializeByHost non-overlap guarantee is proven in@@ -2551,7 +7590,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const order: string[] = []const timeout = new HostK8sRequestTimeoutError('AppsV1Api.readNamespacedDeployment', 30_000)const reconcile = vi@@ -2562,6 +7601,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)const first = hostWatchCallback('MODIFIED', {metadata: { name: 'host-a', namespace: 'mcp-host', generation: 1 },@@ -2597,7 +7637,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {if (plural === 'hosts') return { items: [] }return { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -2646,7 +7686,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {// wins). Block that full pass so it stays running until shutdown unblocks// the promoted trailing waiter — the semantic this test protects.mocks.hostFullReconcile.mockReset().mockImplementation(() => trailingPass)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const activeRequest = (watcher as any).requestHostFleetReconcile('startup',@@ -2692,7 +7732,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {resolveActivePass = resolve})mocks.hostReconcileHosts.mockReset().mockImplementationOnce(() => activePass)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const activeRequest = (watcher as any).requestHostFleetReconcile('active',@@ -2731,7 +7771,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return inventory})mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const staleRequest = (watcher as any).requestHostFleetReconcile('stale transition',@@ -2785,7 +7825,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {if (plural === 'hosts') return hostInventoryreturn { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const run = (watcher as any).requestHostFleetReconcile('shutdown race') as Promise<void>await vi.waitFor(() => {@@ -2810,7 +7850,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {it('releases the periodic single-flight guard after a rejected pass', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const failure = new Error('transient resync failure')const performSpy = vi.spyOn(watcher as any, 'performHostResync')@@ -2843,12 +7883,189 @@ describe('McpServerWatcher Host watch generation', () => {vi.useRealTimers()})+ it('folds a newer lifecycle generation into an in-flight Host inventory recovery', async () => {+ const hostListStarted = deferred()+ const hostList = deferred<{+ metadata: { resourceVersion: string }+ items: unknown[]+ }>()+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListStarted.resolve(undefined)+ return hostList.promise+ }+ return { items: [] }+ })+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile')+ .mockResolvedValue(undefined)++ const recovery = (watcher as any).recoverHostInventoryAndWatch(+ 'Host watch recovery convergence',+ 1,+ 'watch-recovery'+ ) as Promise<unknown>+ await hostListStarted.promise+ const joined = (watcher as any).recoverHostInventoryAndWatch(+ 'initial Host reconciliation',+ 2,+ 'cold-start'+ ) as Promise<unknown>++ expect(joined).toBe(recovery)+ hostList.resolve({ metadata: { resourceVersion: 'joined-host-rv' }, items: [] })+ await Promise.all([recovery, joined])++ expect(fleetRequest).toHaveBeenCalledOnce()+ expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')+ await watcher.stop()+ })++ it('adopts a pending cold-start intent when a direct Host recovery starts first', async () => {+ vi.useFakeTimers()+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ return { metadata: { resourceVersion: 'adopted-host-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile')+ .mockResolvedValue(undefined)++ ;(watcher as any).scheduleHostCacheRecovery({+ convergenceReason: 'initial Host reconciliation',+ ccLifecycleGeneration: 2,+ cause: 'cold-start',+ })+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()++ await (watcher as any).recoverHostInventoryAndWatch(+ 'Host watch recovery convergence',+ 1,+ 'watch-recovery'+ )++ expect(mocks.listNamespacedCustomObject).toHaveBeenCalledTimes(1)+ expect(fleetRequest).toHaveBeenCalledOnce()+ expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()+ await watcher.stop()+ })++ it('folds a recovery intent scheduled while Host LIST is in flight into the successful pass', async () => {+ vi.useFakeTimers()+ const hostListStarted = deferred()+ const hostList = deferred<{+ metadata: { resourceVersion: string }+ items: unknown[]+ }>()+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListStarted.resolve(undefined)+ return hostList.promise+ }+ return { items: [] }+ })+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile')+ .mockResolvedValue(undefined)++ const recovery = (watcher as any).recoverHostInventoryAndWatch(+ 'Host watch recovery convergence',+ 1,+ 'watch-recovery'+ ) as Promise<unknown>+ await hostListStarted.promise+ ;(watcher as any).scheduleHostCacheRecovery({+ convergenceReason: 'CommunicationChannel event Host convergence',+ ccLifecycleGeneration: 3,+ cause: 'cold-start',+ })+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()+ hostList.resolve({ metadata: { resourceVersion: 'in-flight-intent-rv' }, items: [] })+ await recovery++ expect(fleetRequest).toHaveBeenCalledOnce()+ expect(fleetRequest).toHaveBeenCalledWith(+ 'CommunicationChannel event Host convergence',+ 3,+ 'full'+ )+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()+ await watcher.stop()+ })++ it('preserves an in-flight recovery intent across failure and the scheduled retry', async () => {+ vi.useFakeTimers()+ const firstHostListStarted = deferred()+ const firstHostList = deferred<{+ metadata: { resourceVersion: string }+ items: unknown[]+ }>()+ let hostListCalls = 0+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural !== 'hosts') return { items: [] }+ hostListCalls += 1+ if (hostListCalls === 1) {+ firstHostListStarted.resolve(undefined)+ return firstHostList.promise+ }+ return { metadata: { resourceVersion: 'retry-preserved-rv' }, items: [] }+ })+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile')+ .mockResolvedValue(undefined)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ const recovery = (watcher as any).recoverHostInventoryAndWatch(+ 'Host watch recovery convergence',+ 1,+ 'watch-recovery'+ ) as Promise<unknown>+ await firstHostListStarted.promise+ ;(watcher as any).scheduleHostCacheRecovery({+ convergenceReason: 'CommunicationChannel event Host convergence',+ ccLifecycleGeneration: 3,+ cause: 'cold-start',+ })+ firstHostList.reject(new Error('transient Host LIST failure'))+ await expect(recovery).rejects.toThrow('transient Host LIST failure')++ expect((watcher as any).hostCacheRecoveryIntent).toEqual({+ convergenceReason: 'CommunicationChannel event Host convergence',+ ccLifecycleGeneration: 3,+ cause: 'cold-start',+ })+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()++ await vi.advanceTimersByTimeAsync(5_000)+ await vi.waitFor(() => expect(fleetRequest).toHaveBeenCalledOnce())++ expect(hostListCalls).toBe(2)+ expect(fleetRequest).toHaveBeenCalledWith(+ 'CommunicationChannel event Host convergence',+ 3,+ 'full'+ )+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ errorSpy.mockRestore()+ await watcher.stop()+ })+it('rejects a Host snapshot that cannot anchor a continuing watch', async () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'hosts') return { metadata: {}, items: [] }return { items: [] }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -2884,7 +8101,7 @@ describe('McpServerWatcher Host watch generation', () => {if (watchQueries.length === 1) throw new Error('watch connection failed')return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -2928,7 +8145,7 @@ describe('McpServerWatcher Host watch generation', () => {if (path.endsWith('/hosts')) return lateRequest.promisereturn { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const requested = (watcher as any).requestHostFleetReconcile('Periodic resync',@@ -2937,7 +8154,8 @@ describe('McpServerWatcher Host watch generation', () => {) as Promise<void>await vi.waitFor(() => expect(mocks.watch).toHaveBeenCalledOnce())- const internal = (watcher as any).hostFleetReconcileInFlight.promise as Promise<void>+ const internal = (watcher as any).hostFleetScheduler.hostFleetReconcileInFlight+ .promise as Promise<void>expect((watcher as any).hosts.has('snapshot-host')).toBe(true)watcher.stop()@@ -2970,7 +8188,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()await (watcher as any).startHostWatch('initial-rv');(watcher as any).hostCacheSynced = true@@ -3010,7 +8228,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient event failure'))@@ -3087,7 +8305,7 @@ describe('McpServerWatcher Host watch generation', () => {mocks.hostFullReconcile.mockImplementationOnce(() => firstFullReconcile.promise).mockResolvedValue(undefined)- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const full = (watcher as any).requestHostFleetReconcile('Periodic resync',undefined,@@ -3153,7 +8371,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()await (watcher as any).startHostWatch('test-host-rv')await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -3194,7 +8412,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -3248,7 +8466,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -3270,7 +8488,8 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()+ markHostInventoryAuthoritative(watcher)const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -3301,7 +8520,8 @@ describe('McpServerWatcher Host watch generation', () => {// short-circuits on snapshotNames.has(name) and never dispatches at all.it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()+ markHostInventoryAuthoritative(watcher)const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -3335,7 +8555,8 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()+ markHostInventoryAuthoritative(watcher)const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -3371,7 +8592,7 @@ describe('McpServerWatcher Host watch generation', () => {})it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher();(watcher as any).hosts.set('x', {name: 'x',namespace: 'mcp-host',@@ -3400,7 +8621,8 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()+ markHostInventoryAuthoritative(watcher)const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -3442,7 +8664,7 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {vi.clearAllMocks()- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -3490,13 +8712,14 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)await hostWatchCallback('MODIFIED', {metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },@@ -3532,7 +8755,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const unrelatedHost = {name: 'unrelated-host',namespace: 'mcp-host',@@ -3547,6 +8770,7 @@ describe('McpServerWatcher Host watch generation', () => {await (watcher as any).startHostWatch('test-host-rv')const watchGeneration = (watcher as any).hostWatchGenerationif (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },@@ -3582,12 +8806,13 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('persistent failure'))await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },@@ -3617,13 +8842,14 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = new McpServerWatcher()+ const watcher = newContextAuthoritativeWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('stale event failed')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },@@ -3645,6 +8871,7 @@ describe('McpServerWatcher Host watch generation', () => {describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('caches ADDED events and triggers Host reconcile', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',@@ -3668,6 +8895,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('removes from cache on DELETED and triggers Host reconcile', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-marketing', {name: 'cc-marketing',@@ -3696,6 +8924,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-1', {name: 'cc-1',@@ -3760,6 +8989,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',@@ -3782,6 +9012,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('preserves credentialsSecretRef from the watch event into the cache', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {@@ -3812,6 +9043,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-1', {name: 'cc-1',@@ -3846,6 +9078,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-marketing', {name: 'cc-marketing',@@ -3874,6 +9107,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 7fe7ceaec..1dc96adcf 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -14,9 +14,19 @@ import {} from './administrativeOutcomeReporter'import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'import { config } from './config'+import {+ ExternalEgressConvergenceCoordinator,+ type ExternalEgressRetryHandle,+ type ExternalEgressWatchEventType,+} from './externalEgressConvergenceCoordinator'import { gfsDefaultFactoryConfig } from './gfsConfig'import { GfsReconciler } from './gfsReconciler'import { ControlApiGfsSeedClient } from './gfsSeedClient'+import {+ type HostFleetReconcileMode,+ type HostFleetReconcileRequest,+ HostFleetScheduler,+} from './hostFleetScheduler'import {HostFleetReconcileError,type HostReconcileSource,@@ -30,8 +40,19 @@ import {import { K8sGfsApi } from './k8s/gfsK8sApi'import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'-import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'-import { NetworkPolicyReconciler } from './networkPolicyReconciler'+import {+ confirmAuthoritativeMcpServerAbsence,+ isMcpServerStatusOnlyUpdate,+ sameMcpServerDesiredRevision,+} from './mcpServerSafety'+import {+ hostDeleteCleanupTotal,+ hostFleetRequestsTotal,+ hostWatchRecoverySeconds,+ initialConvergenceLastSuccessTimestampSeconds,+ initialConvergenceRetriesTotal,+} from './metrics'+import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'import { McpServerReconciler } from './reconciler'import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'import {@@ -48,6 +69,7 @@ import {SharedFileSystemCRD,SharedFileSystemSpec,} from './types'+import { getErrorCode } from './utils'// Only initialize K8s client if not in dev modelet customObjectsApi: k8s.CustomObjectsApi | null = null@@ -83,30 +105,13 @@ const PLURAL_HOSTS = 'hosts'const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'-const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000]-const EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY = 10-const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000+// Before readiness was decoupled, a failed initial McpServer or NetworkPolicy+// sweep made provider.start() fail and Kubernetes restarted HCC. These retries+// retain that convergence guarantee now that the sweeps run in the background.+const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]+const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000-/**- * External-egress resync delay (ms), H2 (issue #299). The configured interval is- * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a- * rotating low-TTL host is sampled every rotation. The floor is the hard lower- * bound to avoid a hot-loop. Pure and unit-testable.- */-export function externalEgressResyncDelayMs(- intervalSec: number,- floorSec: number,- minObservedTtlMs = Infinity-): number {- const floorMs = floorSec * 1000- const configuredMs = intervalSec * 1000- const ttlAwareMs = Number.isFinite(minObservedTtlMs)- ? Math.min(configuredMs, minObservedTtlMs / 2)- : configuredMs- return Math.max(ttlAwareMs, floorMs)-}const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000-const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]const HOST_CACHE_RECOVERY_RETRY_MS = 5000const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]// Wake-pending Hosts get immediate per-Host admission after watch recovery@@ -118,57 +123,92 @@ type CommunicationChannelSnapshot = {resourceVersion?: string}+type McpServerSnapshot = {+ servers: McpServerCRD[]+ resourceVersion?: string+}++type ContextSnapshot = {+ contexts: ContextCRD[]+ resourceVersion?: string+}+type HostSnapshot = {hosts: HostCRD[]resourceVersion?: string}-type HostFleetReconcileMode = 'full' | 'lifecycle'+type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'+type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'-type HostFleetReconcileRequest = {- reason: string- mode: HostFleetReconcileMode- ccLifecycleGeneration?: number+type NetworkPolicySafetyCertificate = {+ contextGeneration: number+ serverGeneration: number+ contextRevision: number+ serverRevision: number}-type PendingHostFleetReconcile = HostFleetReconcileRequest & {+type ActiveInitialConvergenceRun = {+ trailingRequested: booleanpromise: Promise<void>- resolve: () => void}-type ActiveHostFleetReconcile = HostFleetReconcileRequest & { promise: Promise<void> }+type HostInventoryRecoveryRequest = {+ convergenceReason: string+ ccLifecycleGeneration?: number+ cause: HostInventoryRecoveryCause+}++type ActiveHostInventoryRecovery = {+ request: HostInventoryRecoveryRequest+ promise: Promise<HostCRD[]>+}-function hostFleetRequestCovers(- active: HostFleetReconcileRequest,- requested: HostFleetReconcileRequest-): boolean {- const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'- if (!modeCovered) return false- if (requested.ccLifecycleGeneration === undefined) return true- return active.ccLifecycleGeneration === requested.ccLifecycleGeneration+function mergeHostInventoryRecoveryRequest(+ target: HostInventoryRecoveryRequest,+ requested: HostInventoryRecoveryRequest+): void {+ // Cold start is the stronger admission mode: its snapshot is consumed only+ // by the bounded full pass, whereas watch recovery also dispatches urgent+ // per-Host work. Preserve that mode when either joined caller requires it.+ if (requested.cause === 'cold-start' || target.cause !== 'cold-start') {+ target.convergenceReason = requested.convergenceReason+ }+ if (target.cause === 'cold-start' || requested.cause === 'cold-start') {+ target.cause = 'cold-start'+ }+ if (requested.ccLifecycleGeneration !== undefined) {+ target.ccLifecycleGeneration =+ target.ccLifecycleGeneration === undefined+ ? requested.ccLifecycleGeneration+ : Math.max(target.ccLifecycleGeneration, requested.ccLifecycleGeneration)+ }}-function mergeHostFleetRequests(- pending: HostFleetReconcileRequest,- requested: HostFleetReconcileRequest-): HostFleetReconcileRequest {- const pendingGeneration = pending.ccLifecycleGeneration- const requestedGeneration = requested.ccLifecycleGeneration- return {- reason: requested.reason,- mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',- ccLifecycleGeneration:- pendingGeneration === undefined- ? requestedGeneration- : requestedGeneration === undefined- ? pendingGeneration- : Math.max(pendingGeneration, requestedGeneration),+/**+ * Serializes a reconciliation domain without turning a rejected operation into+ * a permanent queue failure. Kubernetes Watch invokes async callbacks without+ * awaiting the preceding callback, while a full reconciliation can delete+ * orphaned resources owned by the same domain. A single ordered effect stream+ * therefore makes a later watch event the final writer after an older sweep.+ */+class SerializedReconciliationQueue {+ private tail: Promise<void> = Promise.resolve()++ enqueue(work: () => Promise<void>): Promise<void> {+ const result = this.tail.then(work)+ this.tail = result.then(+ () => undefined,+ () => undefined+ )+ return result}}-function delay(ms: number): Promise<void> {- return new Promise(resolve => setTimeout(resolve, ms))+type KeyedReconciliationQueue = {+ queue: SerializedReconciliationQueue+ references: number}type McpServerWatchObject = {@@ -205,7 +245,7 @@ export interface McpServerProvider {/*** List all McpServer CRDs in the namespace.*/-export async function listAllMcpServers(): Promise<McpServerCRD[]> {+async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {if (!customObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}@@ -221,6 +261,7 @@ export async function listAllMcpServers(): Promise<McpServerCRD[]> {})const list = response as {+ metadata?: { resourceVersion?: string }items: Array<{metadata: {name: string@@ -247,17 +288,21 @@ export async function listAllMcpServers(): Promise<McpServerCRD[]> {}))console.log(`[K8s] Found ${servers.length} McpServer(s)`)- return servers+ return { servers, resourceVersion: list.metadata?.resourceVersion }} catch (error) {console.error('[K8s] Failed to list McpServers:', error)throw error}}+export async function listAllMcpServers(): Promise<McpServerCRD[]> {+ return (await listMcpServerSnapshot()).servers+}+/*** List all Context CRDs in the namespace.*/-export async function listAllContexts(): Promise<ContextCRD[]> {+async function listContextSnapshot(): Promise<ContextSnapshot> {if (!customObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}@@ -273,8 +318,14 @@ export async function listAllContexts(): Promise<ContextCRD[]> {})const list = response as {+ metadata?: { resourceVersion?: string }items: Array<{- metadata: { name: string; namespace?: string }+ metadata: {+ name: string+ namespace?: string+ uid?: string+ generation?: number+ }spec: ContextSpec}>}@@ -282,17 +333,23 @@ export async function listAllContexts(): Promise<ContextCRD[]> {const contexts = list.items.map(item => ({name: item.metadata.name,namespace: item.metadata.namespace || config.namespace,+ uid: item.metadata.uid,+ generation: item.metadata.generation,spec: item.spec,}))console.log(`[K8s] Found ${contexts.length} Context(s)`)- return contexts+ return { contexts, resourceVersion: list.metadata?.resourceVersion }} catch (error) {console.error('[K8s] Failed to list Contexts:', error)throw error}}+export async function listAllContexts(): Promise<ContextCRD[]> {+ return (await listContextSnapshot()).contexts+}+/*** List all Host CRDs in host namespace.*/@@ -399,12 +456,16 @@ export async function getAuthToken(* SharedFileSystems are allowed to live in, per CRD validation).*/export async function listAllSharedFileSystems(): Promise<SharedFileSystemCRD[]> {- if (!customObjectsApi) {+ if (!hostCustomObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}try {console.log(`[K8s] Listing all SharedFileSystems in namespace ${config.hostNamespace}`)- const response = await customObjectsApi.listNamespacedCustomObject({+ // Deadline-bearing client, like listHostSnapshot: the cold-start Host fleet+ // pass waits on this inventory, so an apiserver that never answers would+ // otherwise strand the fleet behind an already-certified readiness — and+ // leave the SharedFileSystem watch, which starts after this await, dead.+ const response = await hostCustomObjectsApi.listNamespacedCustomObject({group: GROUP,version: VERSION,namespace: config.hostNamespace,@@ -446,12 +507,17 @@ export async function listAllGlobalFileSystems(): Promise<GlobalFileSystemCRD[]>plural: PLURAL_GLOBALFILESYSTEMS,})const list = response as {- items: Array<{ metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }>+ items: Array<{+ metadata: { name: string; namespace?: string }+ spec: GlobalFileSystemSpec+ status?: GlobalFileSystemCRD['status']+ }>}return list.items.map(item => ({name: item.metadata.name,namespace: item.metadata.namespace || namespace,spec: item.spec,+ status: item.status,}))} catch (error) {console.error('[K8s] Failed to list GlobalFileSystems:', error)@@ -552,7 +618,7 @@ export async function getContext(contextId: string): Promise<ContextCRD | null>return null}console.error(`[K8s] Failed to read Context CRD:`, error)- return null+ throw error}}@@ -572,7 +638,26 @@ export class McpServerWatcher implements McpServerProvider {private hosts: Map<string, HostCRD> = new Map()private contexts: Map<string, ContextCRD> = new Map()private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()+ private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()+ private mcpWatchGeneration = 0+ private contextWatchGeneration = 0+ private mcpServerDesiredRevision = 0+ private contextDesiredRevision = 0+ // Readiness covers authoritative revocation, not additive fleet completion.+ // Generations bind the completed safety sweep to the exact LIST -> WATCH+ // pair that supplied its absence decisions. A recovered watch invalidates+ // this marker until the new inventory's orphan-allow sweep completes.+ private networkPolicyRevocationContextGeneration = 0+ private networkPolicyRevocationServerGeneration = 0+ private networkPolicyRevocationContextRevision = 0+ private networkPolicyRevocationServerRevision = 0+ private mcpServerCacheSynced = false+ private contextCacheSynced = false+ private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null+ private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null+ private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null+ private contextCacheRecoveryInFlight: Promise<boolean> | null = null// A watch callback can still settle after its stream reports completion.// Host reconciles change the mcp-host pod template, so stale callbacks must// never replay an older Host spec after a replacement watch is active.@@ -585,11 +670,18 @@ export class McpServerWatcher implements McpServerProvider {private readonly hostWatchRetryAttempts = new Map<string, number>()private hostCacheSynced = falseprivate hostCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null+ private sfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null+ private gfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null// Host watch LIST-to-WATCH recovery is a dedicated, deduplicated operation// (§10.2). Concurrent recovery signals reuse the in-flight promise so exactly// one LIST + WATCH is installed. Independent per-Host events no longer share a// process-wide convergence tail — they enter the per-Host serializer directly.- private hostRecoveryInFlight: Promise<HostCRD[]> | null = null+ private hostRecoveryInFlight: ActiveHostInventoryRecovery | null = null+ private hostCacheRecoveryIntent: HostInventoryRecoveryRequest | null = null+ // Only the cold-start Host fleet pass waits for the initial SFS inventory.+ // The inventory itself remains asynchronous with respect to provider.start+ // and NetworkPolicy readiness certification.+ private initialHostFleetSfsInventory: Promise<boolean> | null = null// B2: tracks whether the CC snapshot is paired with a continuing watch.// Set to true only while a complete snapshot is paired with a live watch;// used by HostReconciler to make fail-closed lifecycle decisions.@@ -597,16 +689,16 @@ export class McpServerWatcher implements McpServerProvider {private ccWatchGeneration = 0private ccCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = nullprivate ccCacheRecoveryInFlight: Promise<boolean> | null = null- private ccLifecycleGeneration = 0- private ccAppliedLifecycleGeneration = -1- private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null- private ccFleetRetryAttempt = 0- private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null- private hostFleetReconcilePending: PendingHostFleetReconcile | null = null- private resolveHostFleetShutdown: () => void = () => {}- private readonly hostFleetShutdown = new Promise<void>(resolve => {- this.resolveHostFleetShutdown = resolve+ private readonly hostFleetScheduler = new HostFleetScheduler({+ perform: request => this.performHostFleetReconcile(request),+ recordRequest: result => hostFleetRequestsTotal.inc({ result }),})+ private get ccLifecycleGeneration(): number {+ return this.hostFleetScheduler.currentLifecycleGeneration+ }+ private get ccAppliedLifecycleGeneration(): number {+ return this.hostFleetScheduler.appliedLifecycleGeneration+ }private hostResyncInFlight: Promise<void> | null = nullprivate changeCallback?: () => voidprivate stopped = false@@ -618,15 +710,46 @@ export class McpServerWatcher implements McpServerProvider {private gfsReconciler: GfsReconcilerprivate readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporterprivate readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter- private readonly externalEgressRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()- private readonly externalEgressRetryAttempts = new Map<string, number>()- private readonly externalEgressInFlight = new Map<string, Promise<void>>()+ private readonly externalEgressCoordinator = new ExternalEgressConvergenceCoordinator({+ listServers: () => [...this.servers.values()],+ getCurrentServer: name => this.servers.get(name),+ inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,+ sameDesiredRevision: sameMcpServerDesiredRevision,+ enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),+ mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),+ replay: (type, server, retry) =>+ this.reconcileMcpServerWatchEvent(type, server, this.mcpWatchGeneration, retry),+ externalEgressRefreshMinTtlMs: () => this.netPolReconciler.externalEgressRefreshMinTtlMs,+ })+ private readonly initialConvergenceRetryTimers = new Map<+ InitialConvergenceLane,+ ReturnType<typeof setTimeout>+ >()+ private readonly initialConvergenceRetryAttempts = new Map<InitialConvergenceLane, number>()+ private readonly initialConvergenceRuns = new Map<+ InitialConvergenceLane,+ ActiveInitialConvergenceRun+ >()+ // Every resource effect serializes by resource identity, so one slow fleet+ // member cannot block unrelated live events while same-resource updates+ // still retain last-writer ordering.+ private readonly mcpServerReconciliationQueues = new Map<string, KeyedReconciliationQueue>()+ private readonly contextReconciliationQueues = new Map<string, KeyedReconciliationQueue>()+ private readonly sharedFileSystemReconciliationQueues = new Map<+ string,+ KeyedReconciliationQueue+ >()+ private readonly globalFileSystemReconciliationQueues = new Map<+ string,+ KeyedReconciliationQueue+ >()+ private sharedFileSystemCacheRevision = 0+ private globalFileSystemCacheRevision = 0// Periodic resync timer: K8s watches can drop events on long disconnects;// running fullReconcile every N minutes guarantees mcp-host-runtime-token Secret// rotation eventually catches up even after a missed MODIFIED event.// Disabled when interval <= 0 (tests).private resyncTimer: ReturnType<typeof setInterval> | null = null- private externalEgressResyncTimer: ReturnType<typeof setTimeout> | null = null// Periodic SharedFileSystem resync (#592): the SFS watch fires only on SFS CRD// changes, not on PVC binding / wfc pod readiness, so a SharedFileSystem that// reported Initializing/Degraded needs a periodic re-reconcile to converge to a@@ -644,6 +767,11 @@ export class McpServerWatcher implements McpServerProvider {}this.watch = new k8s.Watch(kc)this.reconciler = new McpServerReconciler(kc)+ this.reconciler.setInventoryAuthority(() => ({+ known: this.mcpServerCacheSynced,+ generation: this.mcpWatchGeneration,+ }))+ this.reconciler.setResolveCurrentServer(name => this.servers.get(name))const infrastructureTelemetryReporter = config.controlApiBaseUrl? createInfrastructureTelemetryReporter(config.governedTracingEnabled, {baseUrl: config.controlApiBaseUrl,@@ -693,6 +821,14 @@ export class McpServerWatcher implements McpServerProvider {// entries a delete/newer event has since removed), and orphan cleanup// compares candidates with the CURRENT inventory rather than a pass snapshot.this.hostReconciler.setResolveCurrentHost(name => this.hosts.get(name))+ // H2: reflect committed lifecycle outcomes onto the CURRENT cache entry —+ // but never onto a same-name recreation. uid === undefined fails closed+ // (production watch objects always carry uid).+ this.hostReconciler.setReflectHostOutcome((name, uid, apply) => {+ const cached = this.hosts.get(name)+ if (!cached || uid === undefined || cached.uid !== uid) return+ apply(cached)+ })// Orphan cleanup requires known watch authority and a stable watch// generation; expose both so cleanup fail-closes while authority is unknown// or the watch has been retired mid-pass.@@ -700,6 +836,40 @@ export class McpServerWatcher implements McpServerProvider {known: this.hostCacheSynced,generation: this.hostWatchGeneration,}))+ this.hostReconciler.setHostMutationAuthority(() => ({+ known: this.hostCacheSynced && this.contextCacheSynced,+ hostGeneration: this.hostWatchGeneration,+ contextGeneration: this.contextWatchGeneration,+ }))+ this.hostReconciler.setResolveHostMutationDependencies(host => {+ const context = this.contexts.get(host.spec.contextRef)+ // Lease desired semantics, not cache-object identity or observed status.+ // Kubernetes watch events replace objects and status-only changes may+ // legitimately advance resourceVersion without changing Host inputs.+ const desiredRevision = <T extends { name: string; namespace: string; spec: unknown }>(+ resource: T | undefined+ ) =>+ resource+ ? { name: resource.name, namespace: resource.namespace, spec: resource.spec }+ : undefined+ const dependencies: unknown[] = [+ desiredRevision(context),+ this.ccCacheSynced,+ this.ccWatchGeneration,+ ]+ for (const ref of context?.spec.sharedFileSystems ?? []) {+ const sfs = this.sharedFileSystems.get(ref.name)+ dependencies.push(+ desiredRevision(sfs),+ sfs ? this.sharedFileSystemReconciler.isMountable(sfs) : false+ )+ }+ const channels = this.findCommunicationChannelsByHostRef(host.name).sort((a, b) =>+ a.name.localeCompare(b.name)+ )+ dependencies.push(...channels.map(channel => desiredRevision(channel)))+ return dependencies+ })// Per-CC credentials Secret migration: HostReconciler computes the// channel-reader credentials-revision annotation by hashing the Secret(s)// referenced by all CCs of a host. Wire both finders so@@ -728,65 +898,487 @@ export class McpServerWatcher implements McpServerProvider {return this.ccCacheSynced}- private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {- this.communicationChannels.clear()- for (const channel of snapshot.channels) {- this.communicationChannels.set(channel.name, channel)+ /**+ * Readiness authority covers the inventories that back HCC discovery,+ * Context policy decisions, and Host API/runtime ownership. Communication+ * Channel loss is intentionally excluded: that subsystem already fails safe+ * by preserving replicas and disabling stateless suspension until recovery.+ * SFS/GFS retain their established eventual-resync contract.+ */+ isReadinessInventoryAuthoritative(): boolean {+ return (+ !this.stopped &&+ this.mcpServerCacheSynced &&+ this.contextCacheSynced &&+ this.hostCacheSynced &&+ // The generation/revision equalities below prove no newer desired state+ // is outstanding; they cannot prove the last pass actually revoked what+ // it set out to revoke. Losing a delete fence bumps no generation and+ // moves no revision, so it is invisible to every other term here while+ // leaving live an allow the pass classified as stale. The reconciler+ // degrades this fence at the point of loss; readiness reads it so the+ // gate re-closes for the same reason it opened.+ this.netPolReconciler.hasCertifiedSafetyInventory() &&+ this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&+ this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&+ this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&+ this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision+ )+ }++ /**+ * Host-managed runtime mutation requires a LIST snapshot paired with its+ * continuing WATCH. This narrower predicate is also used by non-HTTP effect+ * producers while the broader readiness predicate additionally covers+ * McpServer and Context discovery authority.+ */+ isHostInventoryAuthoritative(): boolean {+ return !this.stopped && this.hostCacheSynced+ }++ /**+ * Host runtime state is derived from both the Host and Context inventories:+ * Context owns the SharedFileSystem mount set rendered into each Host pod.+ * Mutation must therefore stop when either LIST -> WATCH pair is unavailable.+ */+ private isHostEffectInventoryAuthoritative(): boolean {+ return this.isHostInventoryAuthoritative() && this.contextCacheSynced+ }++ /**+ * Fail closed at every Host-effect admission boundary while recovery owns+ * inventory authority. Preserve the reason/generation in the retry intent so+ * the successful LIST -> WATCH recovery's full pass applies the deferred+ * effect from current state instead of acting on the stale cache.+ */+ private admitHostDependentEffects(+ convergenceReason: string,+ ccLifecycleGeneration?: number+ ): boolean {+ if (this.isHostEffectInventoryAuthoritative()) return true+ if (!this.stopped && !this.hostCacheSynced) {+ this.scheduleHostCacheRecovery({+ convergenceReason,+ ccLifecycleGeneration,+ cause: 'watch-recovery',+ })}+ return false}- private beginCommunicationChannelLifecycleTransition(): number {- this.ccLifecycleGeneration += 1- this.ccFleetRetryAttempt = 0- if (this.ccFleetRetryTimer) {- clearTimeout(this.ccFleetRetryTimer)- this.ccFleetRetryTimer = null+ private retireMcpServerWatch(expectedGeneration?: number): boolean {+ if (expectedGeneration !== undefined && expectedGeneration !== this.mcpWatchGeneration) {+ return false}- return this.ccLifecycleGeneration+ this.mcpServerCacheSynced = false+ this.mcpWatchGeneration += 1+ this.mcpWatchRequest?.abort()+ this.mcpWatchRequest = null+ return true}- private markCommunicationChannelLifecycleApplied(generation: number): void {- if (generation !== this.ccLifecycleGeneration) return- this.ccAppliedLifecycleGeneration = generation- this.ccFleetRetryAttempt = 0- if (this.ccFleetRetryTimer) {- clearTimeout(this.ccFleetRetryTimer)- this.ccFleetRetryTimer = null+ private retireContextWatch(expectedGeneration?: number): boolean {+ if (expectedGeneration !== undefined && expectedGeneration !== this.contextWatchGeneration) {+ return false}+ this.contextCacheSynced = false+ this.contextWatchGeneration += 1+ this.ctxWatchRequest?.abort()+ this.ctxWatchRequest = null+ return true}- private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {- const generation = request.ccLifecycleGeneration- if (generation === undefined) return+ private async enqueueKeyedReconciliation(+ queues: Map<string, KeyedReconciliationQueue>,+ key: string,+ work: () => Promise<void>+ ): Promise<void> {+ let entry = queues.get(key)+ if (!entry) {+ entry = { queue: new SerializedReconciliationQueue(), references: 0 }+ queues.set(key, entry)+ }+ entry.references+++ try {+ await entry.queue.enqueue(async () => {+ if (this.stopped) return+ await work()+ })+ } finally {+ entry.references--+ if (entry.references === 0 && queues.get(key) === entry) queues.delete(key)+ }+ }++ private enqueueMcpServerReconciliation(+ server: Pick<McpServerCRD, 'name' | 'namespace'>,+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedReconciliation(+ this.mcpServerReconciliationQueues,+ `${server.namespace}/${server.name}`,+ work+ )+ }++ private enqueueContextReconciliation(+ contextId: string,+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedReconciliation(this.contextReconciliationQueues, contextId, work)+ }++ private enqueueContextIdentityReconciliation(+ contextIds: string[],+ work: () => Promise<void>+ ): Promise<void> {+ const keys = [...new Set(contextIds)].sort()+ const enter = (index: number): Promise<void> => {+ const key = keys[index]+ if (!key) return work()+ return this.enqueueContextReconciliation(key, () => enter(index + 1))+ }+ return enter(0)+ }++ private hasMcpServerInventoryAuthority(watchGeneration: number): boolean {+ return this.mcpServerCacheSynced && this.mcpWatchGeneration === watchGeneration+ }++ private hasContextInventoryAuthority(watchGeneration: number): boolean {+ return this.contextCacheSynced && this.contextWatchGeneration === watchGeneration+ }++ private async mcpServerAbsentForDelete(+ name: string,+ namespace: string,+ watchGeneration: number+ ): Promise<boolean> {+ if (!customObjectsApi) return false+ return confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => this.hasMcpServerInventoryAuthority(watchGeneration),+ resolveCurrent: () => this.servers.get(name),+ readCurrent: () =>+ customObjectsApi.getNamespacedCustomObject({+ group: GROUP,+ version: VERSION,+ namespace,+ plural: PLURAL_MCPSERVERS,+ name,+ }),+ })+ }++ private async authorizeMcpServerDeleteOrRetry(+ server: McpServerCRD,+ watchGeneration: number+ ): Promise<boolean> {+ try {+ const authorized = await this.mcpServerAbsentForDelete(+ server.name,+ server.namespace,+ watchGeneration+ )+ if (!authorized && !this.hasMcpServerInventoryAuthority(watchGeneration)) {+ console.warn(+ `[K8s] McpServer inventory authority changed while deleting ${server.name}; ` ++ 'full cleanup deferred until retry'+ )+ this.scheduleExternalEgressRetry('DELETED', server)+ this.changeCallback?.()+ }+ return authorized+ } catch (error) {+ console.error(+ `[K8s] Authoritative absence check failed for deleted McpServer ${server.name}; ` ++ 'cleanup blocked until retry:',+ error+ )+ this.scheduleExternalEgressRetry('DELETED', server)+ this.changeCallback?.()+ return false+ }+ }++ private async contextAbsentForDelete(+ name: string,+ namespace: string,+ watchGeneration: number+ ): Promise<boolean> {+ if (+ !this.contextCacheSynced ||+ this.contextWatchGeneration !== watchGeneration ||+ this.contexts.has(name) ||+ !customObjectsApi+ ) {+ return false+ }+ try {+ await customObjectsApi.getNamespacedCustomObject({+ group: GROUP,+ version: VERSION,+ namespace,+ plural: PLURAL_CONTEXTS,+ name,+ })+ return false+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404) throw error+ }+ return (+ this.contextCacheSynced &&+ this.contextWatchGeneration === watchGeneration &&+ !this.contexts.has(name)+ )+ }++ private async contextIdAbsentForDelete(+ contextId: string,+ watchGeneration: number+ ): Promise<boolean> {+ const inventoryAuthoritative = () =>+ this.contextCacheSynced && this.contextWatchGeneration === watchGeneration+ const resolveCurrent = () =>+ [...this.contexts.values()].find(context => context.spec.contextId === contextId)+ if (!customObjectsApi || !inventoryAuthoritative() || resolveCurrent()) return false++ const response = (await customObjectsApi.listNamespacedCustomObject({+ group: GROUP,+ version: VERSION,+ namespace: config.namespace,+ plural: PLURAL_CONTEXTS,+ })) as { items?: Array<{ spec?: { contextId?: string } }> }+ if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false++ return inventoryAuthoritative() && !resolveCurrent()+ }++ private enqueueSharedFileSystemReconciliation(+ name: string,+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedReconciliation(this.sharedFileSystemReconciliationQueues, name, work)+ }++ private enqueueGlobalFileSystemReconciliation(+ name: string,+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedReconciliation(this.globalFileSystemReconciliationQueues, name, work)+ }++ /**+ * Fleet passes acquire every selected object lane in stable order. This+ * preserves their snapshot-wide ordering without making two unrelated live+ * watch events wait behind one another.+ */+ private enqueueKeyedFleetReconciliation(+ queues: Map<string, KeyedReconciliationQueue>,+ keys: string[],+ work: () => Promise<void>+ ): Promise<void> {+ const orderedKeys = [...new Set(keys)].sort()+ const enter = (index: number): Promise<void> => {+ const key = orderedKeys[index]+ if (!key) return work()+ return this.enqueueKeyedReconciliation(queues, key, () => enter(index + 1))+ }+ return enter(0)+ }++ private enqueueSharedFileSystemFleetReconciliation(+ names: string[],+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedFleetReconciliation(+ this.sharedFileSystemReconciliationQueues,+ names,+ work+ )+ }++ private enqueueGlobalFileSystemFleetReconciliation(+ names: string[],+ work: () => Promise<void>+ ): Promise<void> {+ return this.enqueueKeyedFleetReconciliation(+ this.globalFileSystemReconciliationQueues,+ names,+ work+ )+ }++ private requireInventoryResourceVersion(+ kind: 'McpServer' | 'Context',+ resourceVersion: string | undefined+ ): string {+ if (!resourceVersion) {+ throw new Error(`${kind} snapshot missing resourceVersion`)+ }+ return resourceVersion+ }++ private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {+ this.servers.clear()+ for (const server of snapshot.servers) {+ this.servers.set(server.name, server)+ }+ }++ private installContextSnapshot(snapshot: ContextSnapshot): void {+ this.contexts.clear()+ for (const context of snapshot.contexts) {+ this.contexts.set(context.name, context)+ }+ }++ private async restartMcpServerWatch(snapshot: McpServerSnapshot): Promise<number> {+ const resourceVersion = this.requireInventoryResourceVersion(+ 'McpServer',+ snapshot.resourceVersion+ )+ this.mcpServerCacheSynced = false+ this.installMcpServerSnapshot(snapshot)+ const watchGeneration = await this.startMcpServerWatch(resourceVersion)if (this.stopped ||- generation !== this.ccLifecycleGeneration ||- generation === this.ccAppliedLifecycleGeneration ||- this.ccFleetRetryTimer+ watchGeneration !== this.mcpWatchGeneration ||+ this.mcpWatchRequest === null) {- return+ throw new Error('McpServer snapshot could not be paired with an active watch')}- const delayIndex = Math.min(- this.ccFleetRetryAttempt,- COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1+ this.mcpServerCacheSynced = true+ return watchGeneration+ }++ private async restartContextWatch(snapshot: ContextSnapshot): Promise<number> {+ const resourceVersion = this.requireInventoryResourceVersion(+ 'Context',+ snapshot.resourceVersion)- const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]- this.ccFleetRetryAttempt += 1- this.ccFleetRetryTimer = setTimeout(() => {- this.ccFleetRetryTimer = null- if (- this.stopped ||- generation !== this.ccLifecycleGeneration ||- generation === this.ccAppliedLifecycleGeneration- ) {- return+ this.contextCacheSynced = false+ this.installContextSnapshot(snapshot)+ const watchGeneration = await this.startContextWatch(resourceVersion)+ if (+ this.stopped ||+ watchGeneration !== this.contextWatchGeneration ||+ this.ctxWatchRequest === null+ ) {+ throw new Error('Context snapshot could not be paired with an active watch')+ }+ this.contextCacheSynced = true+ return watchGeneration+ }++ private recoverMcpServerInventoryAndWatch(): Promise<boolean> {+ if (this.stopped) return Promise.resolve(false)+ if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight++ const recovery = (async () => {+ try {+ const snapshot = await listMcpServerSnapshot()+ if (this.stopped) return false+ await this.restartMcpServerWatch(snapshot)+ if (this.stopped) return false+ this.changeCallback?.()+ if (this.contextCacheSynced) {+ void this.runInitialNetworkPolicyConvergence()+ }+ return true+ } catch (error) {+ this.mcpServerCacheSynced = false+ console.error('[K8s] McpServer cache recovery failed:', error)+ return false}- void this.requestHostFleetReconcile(- 'CommunicationChannel lifecycle convergence retry',- generation,- request.mode- )- }, retryDelay)+ })()++ this.mcpServerCacheRecoveryInFlight = recovery+ void recovery.finally(() => {+ if (this.mcpServerCacheRecoveryInFlight === recovery) {+ this.mcpServerCacheRecoveryInFlight = null+ }+ })+ return recovery+ }++ private recoverContextInventoryAndWatch(): Promise<boolean> {+ if (this.stopped) return Promise.resolve(false)+ if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight++ const recovery = (async () => {+ try {+ const snapshot = await listContextSnapshot()+ if (this.stopped) return false+ await this.restartContextWatch(snapshot)+ if (this.stopped) return false+ this.hostFleetScheduler.bumpInputRevision()+ void this.runInitialNetworkPolicyConvergence()+ void this.runInitialSharedFileSystemConvergence()+ if (this.hostCacheSynced) {+ const pendingCcLifecycleGeneration =+ this.ccLifecycleGeneration > this.ccAppliedLifecycleGeneration+ ? this.ccLifecycleGeneration+ : undefined+ void this.requestHostFleetReconcile(+ 'Context cache recovery',+ pendingCcLifecycleGeneration,+ 'full'+ )+ }+ return true+ } catch (error) {+ this.contextCacheSynced = false+ console.error('[K8s] Context cache recovery failed:', error)+ return false+ }+ })()++ this.contextCacheRecoveryInFlight = recovery+ void recovery.finally(() => {+ if (this.contextCacheRecoveryInFlight === recovery) {+ this.contextCacheRecoveryInFlight = null+ }+ })+ return recovery+ }++ private scheduleMcpServerCacheRecovery(): void {+ if (this.stopped || this.mcpServerCacheRecoveryTimer) return+ this.mcpServerCacheRecoveryTimer = setTimeout(() => {+ this.mcpServerCacheRecoveryTimer = null+ void this.recoverMcpServerInventoryAndWatch().then(recovered => {+ if (!recovered) this.scheduleMcpServerCacheRecovery()+ })+ }, INVENTORY_CACHE_RECOVERY_RETRY_MS)+ }++ private scheduleContextCacheRecovery(): void {+ if (this.stopped || this.contextCacheRecoveryTimer) return+ this.contextCacheRecoveryTimer = setTimeout(() => {+ this.contextCacheRecoveryTimer = null+ void this.recoverContextInventoryAndWatch().then(recovered => {+ if (!recovered) this.scheduleContextCacheRecovery()+ })+ }, INVENTORY_CACHE_RECOVERY_RETRY_MS)+ }++ private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {+ this.communicationChannels.clear()+ for (const channel of snapshot.channels) {+ this.communicationChannels.set(channel.name, channel)+ }+ }++ private beginCommunicationChannelLifecycleTransition(): number {+ return this.hostFleetScheduler.beginLifecycleTransition()+ }++ private markCommunicationChannelLifecycleApplied(generation: number): void {+ this.hostFleetScheduler.markLifecycleApplied(generation)+ }++ private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {+ this.hostFleetScheduler.scheduleLifecycleRetry(request)}/**@@ -884,19 +1476,45 @@ export class McpServerWatcher implements McpServerProvider {* Hosts through their per-Host chains (step 7) and requests — without awaiting* — a coalesced background full pass (step 8).*/- private recoverHostInventoryAndWatch(): Promise<HostCRD[]> {+ private recoverHostInventoryAndWatch(+ convergenceReason = 'Host watch recovery convergence',+ ccLifecycleGeneration?: number,+ cause: HostInventoryRecoveryCause = 'watch-recovery'+ ): Promise<HostCRD[]> {if (this.stopped) return Promise.resolve([])- if (this.hostRecoveryInFlight) return this.hostRecoveryInFlight- const recovery = this.performHostInventoryRecovery()- this.hostRecoveryInFlight = recovery- const clear = (): void => {- if (this.hostRecoveryInFlight === recovery) this.hostRecoveryInFlight = null- }+ const requested = { convergenceReason, ccLifecycleGeneration, cause }+ // A scheduled retry and an immediate caller are two entrances to the same+ // monotonic recovery intent. Adopt the pending request before choosing or+ // joining the active operation so a direct recovery cannot erase a stronger+ // cold-start cause or newer lifecycle generation when it cancels the timer.+ const request = this.hostCacheRecoveryIntent+ ? { ...this.hostCacheRecoveryIntent }+ : { ...requested }+ if (this.hostCacheRecoveryIntent) {+ mergeHostInventoryRecoveryRequest(request, requested)+ this.hostCacheRecoveryIntent = null+ if (this.hostCacheRecoveryTimer) {+ clearTimeout(this.hostCacheRecoveryTimer)+ this.hostCacheRecoveryTimer = null+ }+ }+ if (this.hostRecoveryInFlight) {+ mergeHostInventoryRecoveryRequest(this.hostRecoveryInFlight.request, request)+ return this.hostRecoveryInFlight.promise+ }+ const recovery = this.performHostInventoryRecovery(request)+ const active = { request, promise: recovery }+ this.hostRecoveryInFlight = active+ const clear = (): void => {+ if (this.hostRecoveryInFlight === active) this.hostRecoveryInFlight = null+ }void recovery.then(clear, clear)return recovery}- private async performHostInventoryRecovery(): Promise<HostCRD[]> {+ private async performHostInventoryRecovery(+ request: HostInventoryRecoveryRequest+ ): Promise<HostCRD[]> {const startedAt = Date.now()// Capture the pre-recovery inventory so step 7 can distinguish genuinely// new/changed Hosts from ones the fleet pass will converge anyway.@@ -941,24 +1559,58 @@ export class McpServerWatcher implements McpServerProvider {{ phase: 'total', outcome: 'success' },(Date.now() - startedAt) / 1000)+ // Effect producers can discover newer work while LIST/WATCH recovery is+ // already in flight. Fold that scheduled intent into this successful+ // operation before cancelling its timer so no lifecycle generation,+ // convergence reason, or stronger cold-start cause is discarded.+ if (this.hostCacheRecoveryIntent) {+ mergeHostInventoryRecoveryRequest(request, this.hostCacheRecoveryIntent)+ }if (this.hostCacheRecoveryTimer) {clearTimeout(this.hostCacheRecoveryTimer)this.hostCacheRecoveryTimer = null}- this.enqueueRecoveredUrgentHosts(- snapshot.hosts,- previousNames,- previousGenerations,- previousUids- )- // Addendum 4 (#827): a Host present before recovery and absent from the- // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate- // per-Host delete cleanup so a DELETE lost to watch retirement is never- // silently dropped.- this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)+ this.hostCacheRecoveryIntent = null+ if (request.cause === 'watch-recovery') {+ this.enqueueRecoveredUrgentHosts(+ snapshot.hosts,+ previousNames,+ previousGenerations,+ previousUids+ )+ // Addendum 4 (#827): a Host present before recovery and absent from the+ // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate+ // per-Host delete cleanup so a DELETE lost to watch retirement is never+ // silently dropped.+ this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)+ }// Request a coalesced background full pass, but do NOT await it before- // declaring watch recovery complete.- void this.requestHostFleetReconcile('Host watch recovery convergence', undefined, 'full')+ // declaring watch recovery complete. The cold-start pass alone waits for+ // the concurrent SFS LIST: otherwise it can write every Host template+ // without its declared mounts, then immediately roll the fleet again+ // when SFS discovery finishes. NetworkPolicy readiness and start() stay+ // independent from that finite inventory request.+ const requestFleetPass = () => {+ if (this.stopped) return+ void this.requestHostFleetReconcile(+ request.convergenceReason,+ request.ccLifecycleGeneration,+ 'full'+ )+ }+ const initialSfsInventory =+ request.cause === 'cold-start' ? this.initialHostFleetSfsInventory : null+ if (initialSfsInventory) {+ // Waiting for the inventory is deliberate: a Host fleet pass without it+ // writes mount-less templates and immediately rerolls the whole fleet.+ // The wait cannot strand the fleet, because the LIST underneath runs on+ // the deadline-bearing client: an apiserver that never answers aborts+ // the request, `startSharedFileSystemInventoryAndWatch` logs and+ // resolves false, and this settles. One path, one pass.+ void initialSfsInventory.then(() => requestFleetPass())+ } else {+ requestFleetPass()+ }return snapshot.hosts} catch (error) {// Keep authority unknown, record the failure, and use the existing@@ -970,7 +1622,7 @@ export class McpServerWatcher implements McpServerProvider {)if (!this.stopped) {console.error('[K8s] Host watch recovery failed:', error)- this.scheduleHostCacheRecovery()+ this.scheduleHostCacheRecovery(request)}throw error}@@ -1010,6 +1662,7 @@ export class McpServerWatcher implements McpServerProvider {private async dispatchUrgentHostReconcile(name: string): Promise<void> {if (this.stopped) return+ if (!this.admitHostDependentEffects('Recovered urgent Host convergence')) returnconst host = this.hosts.get(name)if (!host) returntry {@@ -1041,6 +1694,7 @@ export class McpServerWatcher implements McpServerProvider {private async dispatchRecoveredHostDelete(name: string): Promise<void> {if (this.stopped) return+ if (!this.admitHostDependentEffects('Recovered Host deletion convergence')) return// The fresh authoritative LIST already confirmed absence and the cache no// longer holds this Host, so cleanup is authorized. Route it through the// per-Host serializer (reconcileDelete → serializeByHost) so an older@@ -1072,13 +1726,30 @@ export class McpServerWatcher implements McpServerProvider {}}- private scheduleHostCacheRecovery(): void {- if (this.stopped || this.hostCacheRecoveryTimer) return+ private scheduleHostCacheRecovery(+ requested: HostInventoryRecoveryRequest = {+ convergenceReason: 'Host watch recovery convergence',+ cause: 'watch-recovery',+ }+ ): void {+ if (this.stopped) return+ if (this.hostCacheRecoveryIntent) {+ mergeHostInventoryRecoveryRequest(this.hostCacheRecoveryIntent, requested)+ } else {+ this.hostCacheRecoveryIntent = { ...requested }+ }+ if (this.hostCacheRecoveryTimer) returnthis.hostCacheRecoveryTimer = setTimeout(() => {this.hostCacheRecoveryTimer = null+ const request = this.hostCacheRecoveryIntent ?? requested+ this.hostCacheRecoveryIntent = null// Watch recovery is an independent operation, no longer coupled to a full// fleet pass. Recovery itself requests the background convergence pass.- void this.recoverHostInventoryAndWatch().catch(() => undefined)+ void this.recoverHostInventoryAndWatch(+ request.convergenceReason,+ request.ccLifecycleGeneration,+ request.cause+ ).catch(() => undefined)}, HOST_CACHE_RECOVERY_RETRY_MS)}@@ -1123,6 +1794,7 @@ export class McpServerWatcher implements McpServerProvider {source: HostReconcileSource = 'urgent'): Promise<void> {if (this.stopped) return+ if (!this.admitHostDependentEffects('Host watch event convergence')) returnif (!this.isCurrentHostWatchEvent(type, host, eventRevision)) {// #827: a DELETE that is no longer current because the cache now holds a// (recreated) same-name Host is a superseded delete — record it and do@@ -1217,6 +1889,7 @@ export class McpServerWatcher implements McpServerProvider {await this.recoverHostInventoryAndWatch()if (this.stopped) return}+ if (!this.isHostEffectInventoryAuthoritative()) return// Reconcile the CURRENT cache. Fleet workers dispatch Host keys and// resolve the freshest cached spec at execution (see collectHostReconcile// Failures); only fullReconcile runs the authority-gated orphan cleanup.@@ -1279,78 +1952,7 @@ export class McpServerWatcher implements McpServerProvider {ccLifecycleGeneration?: number,mode?: HostFleetReconcileMode): Promise<void> {- if (this.stopped) return Promise.resolve()- const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')- if (- ccLifecycleGeneration !== undefined &&- (ccLifecycleGeneration !== this.ccLifecycleGeneration ||- (requestMode === 'lifecycle' &&- ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))- ) {- return Promise.resolve()- }-- const request: HostFleetReconcileRequest = {- reason,- mode: requestMode,- ccLifecycleGeneration,- }- const active = this.hostFleetReconcileInFlight- if (!active) {- hostFleetRequestsTotal.inc({ result: 'started' })- return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))- }- if (hostFleetRequestCovers(active, request)) {- hostFleetRequestsTotal.inc({ result: 'coalesced' })- return this.waitForHostFleetOrShutdown(active.promise)- }-- if (!this.hostFleetReconcilePending) {- let resolve!: () => void- const promise = new Promise<void>(resolvePromise => {- resolve = resolvePromise- })- this.hostFleetReconcilePending = {- ...request,- promise,- resolve,- }- } else {- const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)- this.hostFleetReconcilePending.reason = merged.reason- this.hostFleetReconcilePending.mode = merged.mode- this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration- }- // The request did not start its own pass; it is queued behind the active one.- hostFleetRequestsTotal.inc({ result: 'coalesced' })- return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)- }-- private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {- return Promise.race([coverage, this.hostFleetShutdown])- }-- private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {- const promise = this.performHostFleetReconcile(request)- const active: ActiveHostFleetReconcile = { ...request, promise }- this.hostFleetReconcileInFlight = active-- const settle = () => {- if (this.hostFleetReconcileInFlight !== active) return- this.hostFleetReconcileInFlight = null- const pending = this.hostFleetReconcilePending- this.hostFleetReconcilePending = null- if (!pending) return- if (this.stopped) {- pending.resolve()- return- }- hostFleetRequestsTotal.inc({ result: 'trailing' })- const trailing = this.startHostFleetReconcile(pending)- void trailing.then(pending.resolve, pending.resolve)- }- void promise.then(settle, settle)- return promise+ return this.hostFleetScheduler.request(reason, ccLifecycleGeneration, mode)}private contextsReferencingSfs(sfsName: string): Array<{ namespace: string; name: string }> {@@ -1410,14 +2012,35 @@ export class McpServerWatcher implements McpServerProvider {return out}- private async reconcileSharedFileSystemsReferencedByContext(+ private reconcileSharedFileSystemsReferencedByContext(+ prevContext: ContextCRD | undefined,+ nextContext: ContextCRD | undefined,+ isCurrent: () => boolean = () => true+ ): Promise<void> {+ const names = [+ ...(prevContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),+ ...(nextContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),+ ]+ return this.enqueueSharedFileSystemFleetReconciliation(names, async () => {+ if (!isCurrent()) return+ await this.reconcileSharedFileSystemsReferencedByContextCore(+ prevContext,+ nextContext,+ isCurrent+ )+ })+ }++ private async reconcileSharedFileSystemsReferencedByContextCore(prevContext: ContextCRD | undefined,- nextContext: ContextCRD | undefined+ nextContext: ContextCRD | undefined,+ isCurrent: () => boolean): Promise<void> {const names = new Set<string>()for (const r of prevContext?.spec.sharedFileSystems ?? []) names.add(r.name)for (const r of nextContext?.spec.sharedFileSystems ?? []) names.add(r.name)for (const name of names) {+ if (!isCurrent()) returnconst sfs = this.sharedFileSystems.get(name)if (!sfs) continuetry {@@ -1434,6 +2057,9 @@ export class McpServerWatcher implements McpServerProvider {* Deployment template picks up (or drops) the volume + volumeMount.*/private async reconcileHostsReferencingSfs(sfsName: string): Promise<void> {+ if (!this.admitHostDependentEffects(`SharedFileSystem "${sfsName}" Host convergence`)) {+ return+ }const affectedContexts = new Set<string>()for (const ctx of this.contexts.values()) {if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {@@ -1451,6 +2077,24 @@ export class McpServerWatcher implements McpServerProvider {}}+ /**+ * Reconcile Hosts affected by a Context change only while the Host cache is+ * authoritative. A recovery full pass consumes the current Context cache, so+ * deferral loses no effect and never acts on an unpaired Host LIST.+ */+ private async reconcileHostsReferencingContext(+ contextName: string,+ isCurrent: () => boolean = () => true+ ): Promise<void> {+ if (!isCurrent()) return+ if (!this.admitHostDependentEffects(`Context "${contextName}" Host convergence`)) return+ for (const host of this.hosts.values()) {+ if (!isCurrent()) return+ if (host.spec.contextRef !== contextName) continue+ await this.hostReconciler.reconcile(host)+ }+ }+/*** Re-reconcile the Host whose name matches the given hostRef. Called by the* CommunicationChannel watch handler so per-Host channel-reader Deployment@@ -1463,6 +2107,9 @@ export class McpServerWatcher implements McpServerProvider {* its caller schedules bounded fleet convergence.*/private async reconcileHostsReferencingCC(hostRef: string): Promise<boolean> {+ if (!this.admitHostDependentEffects(`CommunicationChannel Host "${hostRef}" convergence`)) {+ return false+ }const host = this.hosts.get(hostRef)if (!host) return truetry {@@ -1554,9 +2201,28 @@ export class McpServerWatcher implements McpServerProvider {* tracker resolves the heartbeat's hostRef through this).*/getHost(name: string): HostCRD | undefined {+ if (!this.isHostInventoryAuthoritative()) return undefinedreturn this.hosts.get(name)}+ /**+ * Route channel credentials-Secret events through the same Host authority+ * boundary as watch, Context, SFS, and heartbeat effects.+ */+ async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {+ const convergenceReason = `CommunicationChannel Secret "${secretNamespace}/${secretName}" Host convergence`+ if (!this.ccCacheSynced) {+ console.warn(+ `[K8s] Deferring ${convergenceReason}; CommunicationChannel cache is not authoritative`+ )+ return+ }+ if (!this.admitHostDependentEffects(convergenceReason)) {+ return+ }+ await this.hostReconciler.reconcileChannelReaderRevision(secretName, secretNamespace)+ }+/*** Trigger a reconcile for any cached McpServer whose `envSecret.name`* matches the given Secret. Used by SecretInformer to react to Secret@@ -1564,18 +2230,45 @@ export class McpServerWatcher implements McpServerProvider {* event.*/async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {- for (const server of this.servers.values()) {- if (server.namespace !== secretNamespace) continue- if (server.spec.envSecret?.name !== secretName) continue- try {- console.log(- `[K8s] Re-reconciling McpServer "${server.name}" after Secret "${secretName}" change`- )- await this.reconciler.reconcile(server)- } catch (err) {- console.error(`[K8s] Secret-triggered reconcile failed for "${server.name}":`, err)- }- }+ const inventoryGeneration = this.mcpWatchGeneration+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return+ const selected = [...this.servers.values()].filter(+ server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName+ )+ await Promise.all(+ selected.map(server =>+ this.enqueueMcpServerReconciliation(server, async () => {+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return+ const current = this.servers.get(server.name)+ if (+ !current ||+ current.namespace !== secretNamespace ||+ current.spec.envSecret?.name !== secretName+ ) {+ return+ }+ try {+ console.log(+ `[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`+ )+ await this.reconciler.reconcile(current, {+ isCurrent: () => {+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false+ const latest = this.servers.get(current.name)+ return (+ latest !== undefined &&+ latest.namespace === secretNamespace &&+ latest.spec.envSecret?.name === secretName &&+ sameMcpServerDesiredRevision(current, latest)+ )+ },+ })+ } catch (err) {+ console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)+ }+ })+ )+ )}/**@@ -1603,7 +2296,7 @@ export class McpServerWatcher implements McpServerProvider {transport,auth: server.spec.auth,enabled: server.spec.enabled !== false,- status: this.reconciler.getStatus(server.name),+ status: this.reconciler.getStatus(server),}}@@ -1658,61 +2351,55 @@ export class McpServerWatcher implements McpServerProvider {* Start watching for changes and run initial reconciliation.*/async start(): Promise<void> {- let initialServers: McpServerCRD[] = []- let serverInventoryComplete = false+ // L0/L1 isolation and removal of legacy permissive policies are mandatory+ // bootstrap barriers. They are a fixed-size safety operation, independent+ // of fleet size, and must succeed before HCC can watch or reconcile runtimes.+ await this.netPolReconciler.ensureDefaultPolicies()++ // Discovery and policy caches become authoritative only when each complete+ // LIST snapshot is paired with a watch continuing from its opaque collection+ // resourceVersion. Full fleet convergence happens later in the background.+ // A transient LIST/WATCH failure on either inventory must not kill the+ // retry that restart*Watch already armed (it sets *CacheSynced=false before+ // it can throw). Mirror the Host lane below: log, keep the flag false, and+ // schedule in-process recovery. The HTTP server stays live, dynamic+ // readiness stays 503, and recovery promotes readiness only after a fresh+ // LIST is paired with its continuing WATCH.try {- initialServers = await listAllMcpServers()- serverInventoryComplete = true- for (const server of initialServers) {- this.servers.set(server.name, server)- }+ const initialServerSnapshot = await listMcpServerSnapshot()+ await this.restartMcpServerWatch(initialServerSnapshot)} catch (error) {+ this.mcpServerCacheSynced = falseconsole.error(- '[K8s] Skipping initial McpServer full reconciliation because server discovery failed:',+ '[K8s] Initial McpServer inventory is unavailable; HCC remains unready while in-process recovery continues:',error)}-- // Ensure baseline network policies exist even if initial Context discovery fails.- try {- await this.netPolReconciler.ensureDefaultPolicies()- } catch (error) {- console.error('[K8s] Failed to ensure default NetworkPolicies before startup:', error)- }-- let initialExternalEgressFailures = new Set<string>()- if (serverInventoryComplete) {- initialExternalEgressFailures =- await this.reconcileInitialExternalEgressBeforeRuntime(initialServers)- }-- // Full McpServer reconciliation (Deployments + Services). Servers with- // egressBindings are reconciled only after their external egress policies- // have converged, matching the ADDED/MODIFIED watch-event contract.- if (serverInventoryComplete) {- const runtimeServers = initialServers.filter(- server => !initialExternalEgressFailures.has(this.externalEgressRetryKey(server))- )- console.log('[K8s] Running initial full reconciliation...')- await this.reconciler.fullReconcile(runtimeServers)+ if (!this.mcpServerCacheSynced) {+ this.scheduleMcpServerCacheRecovery()}try {- const initialContexts = await listAllContexts()- for (const context of initialContexts) {- this.contexts.set(context.name, context)- }- console.log('[K8s] Running initial NetworkPolicy reconciliation...')- await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {- serverInventoryComplete,- ensureDefaults: false,- })+ const initialContextSnapshot = await listContextSnapshot()+ await this.restartContextWatch(initialContextSnapshot)} catch (error) {+ this.contextCacheSynced = falseconsole.error(- '[K8s] Skipping initial NetworkPolicy reconciliation because context discovery failed:',+ '[K8s] Initial Context inventory is unavailable; HCC remains unready while in-process recovery continues:',error)}+ if (!this.contextCacheSynced) {+ this.scheduleContextCacheRecovery()+ }++ // Host templates resolve Context SharedFileSystem references from this+ // cache. Inventory is therefore a fixed startup boundary, not part of the+ // unbounded SFS reconciliation fleet: letting the first Host pass run+ // without it writes mount-less templates and immediately rolls the entire+ // Host fleet a second time once discovery catches up.+ const initialSharedFileSystemInventory = this.startSharedFileSystemInventoryAndWatch()+ this.initialHostFleetSfsInventory = initialSharedFileSystemInventory// ── CommunicationChannel snapshot + watch — MUST complete before Host fullReconcile ──// A stateless Host may suspend only after this controller has a complete@@ -1761,54 +2448,30 @@ export class McpServerWatcher implements McpServerProvider {}console.log(- `[K8s] Running initial Host reconciliation... (ccCacheSynced=${this.ccCacheSynced})`+ `[K8s] Starting initial Host background convergence... (ccCacheSynced=${this.ccCacheSynced})`)const initialLifecycleGeneration =this.ccLifecycleGeneration === 0? this.beginCommunicationChannelLifecycleTransition(): this.ccLifecycleGeneration- await this.requestHostFleetReconcile(- 'initial Host reconciliation',- initialLifecycleGeneration,- 'full'- )-- // ── SharedFileSystem initial load + reconciliation ──try {- const initialSfses = await listAllSharedFileSystems()- for (const sfs of initialSfses) {- this.sharedFileSystems.set(sfs.name, sfs)- }- console.log('[K8s] Running initial SharedFileSystem reconciliation...')- await this.sharedFileSystemReconciler.fullReconcile(initialSfses)- } catch (error) {- console.error(- '[K8s] Skipping initial SharedFileSystem reconciliation because discovery failed:',- error+ await this.recoverHostInventoryAndWatch(+ 'initial Host reconciliation',+ initialLifecycleGeneration,+ 'cold-start')- }-- // ── GlobalFileSystem (gfs) initial load + reconciliation ──- // gfs is a cluster singleton (DISTINCT from SharedFileSystem). A failed- // discovery (e.g. the CRD not installed yet) is skipped, not fatal — the- // next HCC restart / resync retries. Live updates land via the watch- // (follow-up); the deploy applies the singleton before HCC reconciles it.- try {- const initialGfses = await listAllGlobalFileSystems()- console.log('[K8s] Running initial GlobalFileSystem reconciliation...')- await this.gfsReconciler.fullReconcile(initialGfses)} catch (error) {- console.error(- '[K8s] Skipping initial GlobalFileSystem reconciliation because discovery failed:',+ // Host inventory is an essential readiness authority, but a transient+ // LIST/WATCH failure must not kill the retry that recovery just armed.+ // Continue startup with hostCacheSynced=false: the HTTP server stays live,+ // dynamic readiness remains 503, and recovery promotes readiness only+ // after a fresh LIST is paired with its continuing WATCH.+ console.warn(+ '[K8s] Initial Host inventory is unavailable; HCC remains unready while in-process recovery continues:',error)}- await this.startMcpServerWatch()- await this.startContextWatch()- await this.startSharedFileSystemWatch()- await this.startGlobalFileSystemWatch()-const resyncSec = config.hostResyncIntervalSecif (resyncSec > 0) {this.resyncTimer = setInterval(() => {@@ -1846,35 +2509,375 @@ export class McpServerWatcher implements McpServerProvider {}const externalEgressResyncSec = config.externalEgressResyncIntervalSec- if (externalEgressResyncSec > 0) {- // Self-rescheduling (not a fixed setInterval) so the delay can advance to- // <= observed TTL/2 (H2, issue #299) and rescheduling AFTER each pass- // completes guarantees no overlapping resyncs.- const scheduleNext = (): void => {- if (this.stopped) return- const delayMs = externalEgressResyncDelayMs(- externalEgressResyncSec,- config.externalEgressRefreshFloorSec,- this.netPolReconciler.externalEgressRefreshMinTtlMs- )- this.externalEgressResyncTimer = setTimeout(() => {- void this.runExternalEgressResync().finally(() => scheduleNext())- }, delayMs)- this.externalEgressResyncTimer.unref?.()- }- // Idempotency guard (audit F5): never arm a second self-rescheduling chain- // if one is already live (start() runs once today, but a future restart- // path must not double the cadence).- if (!this.externalEgressResyncTimer) scheduleNext()- console.log(- `[K8s] External egress periodic DNS resync enabled (every ${externalEgressResyncSec}s, advancing to <= TTL/2)`+ // #205 delegates external-egress periodic resync to the convergence+ // coordinator. Its runResyncCore drives reconcileExternalEgress, so the+ // #299 sliding-window accumulation still converges on DNS rotation between+ // McpServer events. The coordinator owns cadence with a self-rescheduling+ // timer that advances to <= observed TTL/2 (H2, issue #299), bounded by the+ // configured floor.+ this.externalEgressCoordinator.startPeriodicResync(+ externalEgressResyncSec,+ config.externalEgressRefreshFloorSec+ )++ // Full convergence is observable and retains each lane's existing retry or+ // periodic-resync contract, but it no longer extends provider.start().+ // NetworkPolicy safety starts first and launches McpServer convergence from+ // its certification callback, before additive policy convergence completes.+ // This prevents the startup egress refresh from racing or undoing the safety+ // pass while keeping readiness independent from both additive fleets.+ void this.runInitialNetworkPolicyConvergence()+ void initialSharedFileSystemInventory.then(inventoryComplete => {+ if (inventoryComplete && !this.stopped) {+ void this.runInitialSharedFileSystemConvergence()+ }+ })+ void this.startGlobalFileSystemBackgroundLane()+ }++ private async startSharedFileSystemInventoryAndWatch(): Promise<boolean> {+ let inventoryComplete = false+ try {+ const initialSfses = await listAllSharedFileSystems()+ inventoryComplete = true+ for (const sfs of initialSfses) this.sharedFileSystems.set(sfs.name, sfs)+ } catch (error) {+ console.error(+ '[K8s] Skipping initial SharedFileSystem background convergence because discovery failed:',+ error)- } else {- console.warn(- '[K8s] External egress periodic DNS resync DISABLED (HCC_EXTERNAL_EGRESS_RESYNC_SEC=0); ' +- 'exact-host FQDN egress will NOT converge on DNS rotation between McpServer events — ' +- 'accumulated IPs can go stale and block workloads (issue #299). Enable it unless you have a specific reason.'+ }+ // The cold-start Host fleet pass gates on this promise, so it must settle on+ // the inventory alone. `k8s.Watch` carries no transport deadline — the+ // client builds `AbortSignal.any([controller, timeout])` and then overwrites+ // it with the bare controller — so awaiting the watch start here would+ // re-strand the fleet behind an already-certified readiness, which is the+ // exact failure bounding the LIST was meant to close.+ void this.startSharedFileSystemWatch().catch(error => {+ console.error('[K8s] SharedFileSystem background watch failed to start:', error)+ this.scheduleSharedFileSystemWatchRestart(5000)+ })+ if (inventoryComplete && !this.stopped) {+ return true+ }+ return false+ }++ private async startGlobalFileSystemBackgroundLane(): Promise<void> {+ let inventoryComplete = false+ try {+ const initialGfses = await listAllGlobalFileSystems()+ inventoryComplete = true+ for (const gfs of initialGfses) this.globalFileSystems.set(gfs.name, gfs)+ } catch (error) {+ console.error(+ '[K8s] Skipping initial GlobalFileSystem background convergence because discovery failed:',+ error+ )+ }+ try {+ await this.startGlobalFileSystemWatch()+ } catch (error) {+ console.error('[K8s] GlobalFileSystem background watch failed to start:', error)+ this.scheduleGlobalFileSystemWatchRestart(5000)+ }+ if (inventoryComplete && !this.stopped) {+ void this.runInitialGlobalFileSystemConvergence()+ }+ }++ private runInitialMcpServerConvergence(): Promise<void> {+ return this.runInitialConvergence('McpServer')+ }++ private currentNetworkPolicySafetyCertificate(): NetworkPolicySafetyCertificate | null {+ if (+ this.stopped ||+ !this.contextCacheSynced ||+ !this.mcpServerCacheSynced ||+ !this.netPolReconciler.hasCertifiedSafetyInventory() ||+ this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||+ this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||+ this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||+ this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision+ ) {+ return null+ }+ return {+ contextGeneration: this.contextWatchGeneration,+ serverGeneration: this.mcpWatchGeneration,+ contextRevision: this.contextDesiredRevision,+ serverRevision: this.mcpServerDesiredRevision,+ }+ }++ private recordNetworkPolicySafetyCertificate(+ certificate: NetworkPolicySafetyCertificate+ ): boolean {+ if (+ this.stopped ||+ !this.contextCacheSynced ||+ !this.mcpServerCacheSynced ||+ this.contextWatchGeneration !== certificate.contextGeneration ||+ this.mcpWatchGeneration !== certificate.serverGeneration ||+ this.contextDesiredRevision !== certificate.contextRevision ||+ this.mcpServerDesiredRevision !== certificate.serverRevision+ ) {+ return false+ }+ this.networkPolicyRevocationContextGeneration = certificate.contextGeneration+ this.networkPolicyRevocationServerGeneration = certificate.serverGeneration+ this.networkPolicyRevocationContextRevision = certificate.contextRevision+ this.networkPolicyRevocationServerRevision = certificate.serverRevision+ return true+ }++ private isNetworkPolicySafetyCertificateCurrent(+ certificate: NetworkPolicySafetyCertificate+ ): boolean {+ const current = this.currentNetworkPolicySafetyCertificate()+ return (+ current !== null &&+ current.contextGeneration === certificate.contextGeneration &&+ current.serverGeneration === certificate.serverGeneration &&+ current.contextRevision === certificate.contextRevision &&+ current.serverRevision === certificate.serverRevision+ )+ }++ private async runInitialMcpServerConvergenceCore(): Promise<void> {+ const safetyCertificate = this.currentNetworkPolicySafetyCertificate()+ if (safetyCertificate === null) return+ const inventoryGeneration = this.mcpWatchGeneration+ const inventoryAuthoritative = () =>+ this.hasMcpServerInventoryAuthority(inventoryGeneration) &&+ this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)+ try {+ const initialServers = [...this.servers.values()]+ const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(+ initialServers,+ inventoryAuthoritative+ )+ if (this.stopped) return+ console.log('[K8s] Running initial McpServer background reconciliation...')+ await this.reconciler.fullReconcile(initialServers, {+ runEffect: async (serverName, work) => {+ const selected = this.servers.get(serverName)+ const laneOwner = selected ?? {+ name: serverName,+ namespace: config.namespace,+ }+ if (!(await initialExternalEgressGates.waitFor(laneOwner))) return+ if (!inventoryAuthoritative()) return+ await this.enqueueMcpServerReconciliation(laneOwner, async () => {+ if (!inventoryAuthoritative()) return+ await work()+ })+ },+ })+ if (!inventoryAuthoritative()) return+ initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)+ this.clearInitialConvergenceRetry('McpServer')+ } catch (error) {+ console.error('[K8s] Initial McpServer background reconciliation failed:', error)+ this.scheduleInitialConvergenceRetry('McpServer')+ }+ }++ private runInitialNetworkPolicyConvergence(): Promise<void> {+ return this.runInitialConvergence('NetworkPolicy')+ }++ private runInitialConvergence(lane: InitialConvergenceLane): Promise<void> {+ const active = this.initialConvergenceRuns.get(lane)+ if (active) {+ active.trailingRequested = true+ return active.promise+ }++ const run: ActiveInitialConvergenceRun = {+ trailingRequested: false,+ promise: Promise.resolve(),+ }+ this.initialConvergenceRuns.set(lane, run)+ run.promise = Promise.resolve()+ .then(async () => {+ do {+ run.trailingRequested = false+ if (lane === 'McpServer') {+ await this.runInitialMcpServerConvergenceCore()+ } else {+ await this.runInitialNetworkPolicyConvergenceCore()+ }+ } while (!this.stopped && run.trailingRequested)+ // Retire the active slot synchronously before this promise settles.+ // A request arriving in the following settlement microtask must start+ // a new run instead of attaching to a pass whose loop already exited.+ if (this.initialConvergenceRuns.get(lane) === run) {+ this.initialConvergenceRuns.delete(lane)+ }+ })+ .finally(() => {+ // Errors bypass the normal retirement above.+ if (this.initialConvergenceRuns.get(lane) === run) {+ this.initialConvergenceRuns.delete(lane)+ }+ })+ return run.promise+ }++ private async runInitialNetworkPolicyConvergenceCore(): Promise<void> {+ // A NetworkPolicy full pass combines authoritative Context/McpServer allow+ // revocation with additive Context policy effects. External egress+ // creation and DNS refresh have one separate owner: the startup/resync+ // coordinator. Running safety from a partial inventory would be neither a+ // full convergence nor safe evidence of success. Recovery of either+ // missing LIST -> WATCH pair schedules a fresh current-cache pass.+ if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return+ let authoritativeRevocationCompleted = false+ try {+ const initialContexts = [...this.contexts.values()]+ const initialServers = [...this.servers.values()]+ const contextInventoryGeneration = this.contextWatchGeneration+ const serverInventoryGeneration = this.mcpWatchGeneration+ const safetyCertificate: NetworkPolicySafetyCertificate = {+ contextGeneration: contextInventoryGeneration,+ serverGeneration: serverInventoryGeneration,+ contextRevision: this.contextDesiredRevision,+ serverRevision: this.mcpServerDesiredRevision,+ }+ const serverInventoryComplete = this.mcpServerCacheSynced+ const contextInventoryAuthoritative = () =>+ this.hasContextInventoryAuthority(contextInventoryGeneration)+ const serverInventoryAuthoritative = () =>+ this.hasMcpServerInventoryAuthority(serverInventoryGeneration)+ console.log('[K8s] Running initial NetworkPolicy background reconciliation...')+ await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {+ serverInventoryComplete,+ ensureDefaults: false,+ contextInventoryAuthoritative,+ serverInventoryAuthoritative,+ runContextEffect: (contextId, work) =>+ this.enqueueContextReconciliation(contextId, async () => {+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return+ await work()+ }),+ runServerEffect: (serverName, work) => {+ const selected = this.servers.get(serverName)+ return this.enqueueMcpServerReconciliation(+ selected ?? { name: serverName, namespace: config.namespace },+ async () => {+ if (!serverInventoryAuthoritative()) return+ await work()+ }+ )+ },+ resolveCurrentContext: name => this.contexts.get(name),+ resolveCurrentContextById: contextId =>+ [...this.contexts.values()].find(context => context.spec.contextId === contextId),+ resolveCurrentServer: name => this.servers.get(name),+ contextDesiredRevision: () => this.contextDesiredRevision,+ serverDesiredRevision: () => this.mcpServerDesiredRevision,+ // B3 piece-2: queue an additive recreation for every server whose+ // external-egress allow the authoritative pass revoked — fired at the+ // deletion, so an abort/throw cannot strand a revoked-but-desired allow.+ // The coordinator maps against the current cache (absent → DELETED, no+ // recreate) and rebuilds from the current spec with fresh DNS.+ onExternalEgressRevoked: server =>+ this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),+ onAuthoritativeRevocationComplete: () => {+ if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return+ authoritativeRevocationCompleted = true+ // Start runtime convergence at the safety boundary, before this full+ // pass enters additive Context work. Startup egress gates may now+ // preserve only policies certified by this exact authoritative+ // inventory generation.+ void this.runInitialMcpServerConvergence()+ },+ })+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return+ initialConvergenceLastSuccessTimestampSeconds.set(+ { lane: 'NetworkPolicy' },+ Date.now() / 1000)+ this.clearInitialConvergenceRetry('NetworkPolicy')+ } catch (error) {+ console.error(+ authoritativeRevocationCompleted+ ? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'+ : '[K8s] Initial NetworkPolicy background reconciliation failed:',+ error+ )+ this.scheduleInitialConvergenceRetry('NetworkPolicy')+ }+ }++ private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {+ const timer = this.initialConvergenceRetryTimers.get(lane)+ if (timer) {+ clearTimeout(timer)+ this.initialConvergenceRetryTimers.delete(lane)+ }+ this.initialConvergenceRetryAttempts.delete(lane)+ }++ private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {+ if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return++ const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1+ const delayMs =+ INITIAL_CONVERGENCE_RETRY_DELAYS_MS[+ Math.min(attempt - 1, INITIAL_CONVERGENCE_RETRY_DELAYS_MS.length - 1)+ ]+ this.initialConvergenceRetryAttempts.set(lane, attempt)+ initialConvergenceRetriesTotal.inc({ lane })+ console.warn(+ `[K8s] Scheduling initial ${lane} background convergence retry ${attempt} in ${delayMs}ms`+ )++ const timer = setTimeout(() => {+ this.initialConvergenceRetryTimers.delete(lane)+ if (this.stopped) return+ if (lane === 'McpServer') {+ void this.runInitialMcpServerConvergence()+ return+ }+ void this.runInitialNetworkPolicyConvergence()+ }, delayMs)+ this.initialConvergenceRetryTimers.set(lane, timer)+ }++ private runInitialSharedFileSystemConvergence(): Promise<void> {+ const names = [...this.sharedFileSystems.keys()]+ return this.enqueueSharedFileSystemFleetReconciliation(names, () =>+ this.runInitialSharedFileSystemConvergenceCore()+ )+ }++ private async runInitialSharedFileSystemConvergenceCore(): Promise<void> {+ try {+ const initialSfses = [...this.sharedFileSystems.values()]+ console.log('[K8s] Running initial SharedFileSystem background reconciliation...')+ await this.sharedFileSystemReconciler.fullReconcile(initialSfses)+ } catch (error) {+ console.error('[K8s] Initial SharedFileSystem background reconciliation failed:', error)+ }+ }++ private runInitialGlobalFileSystemConvergence(): Promise<void> {+ const names = [...this.globalFileSystems.keys()]+ return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>+ this.runInitialGlobalFileSystemConvergenceCore()+ )+ }++ private async runInitialGlobalFileSystemConvergenceCore(): Promise<void> {+ try {+ const initialGfses = [...this.globalFileSystems.values()]+ console.log('[K8s] Running initial GlobalFileSystem background reconciliation...')+ await this.gfsReconciler.fullReconcile(initialGfses)+ } catch (error) {+ console.error('[K8s] Initial GlobalFileSystem background reconciliation failed:', error)}}@@ -1902,42 +2905,58 @@ export class McpServerWatcher implements McpServerProvider {} else if (type === 'DELETED') {this.sharedFileSystems.delete(sfs.name)}+ this.sharedFileSystemCacheRevision += 1- try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.sharedFileSystemReconciler.reconcile(sfs)- } else if (type === 'DELETED') {- await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)- }- } catch (error) {- console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)- }+ await this.enqueueSharedFileSystemReconciliation(sfs.name, async () => {+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.sharedFileSystemReconciler.reconcile(sfs)+ } else if (type === 'DELETED') {+ await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)+ }+ } catch (error) {+ console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)+ }- // Re-reconcile any Host whose Context references this SharedFileSystem.- // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on- // the next mcp-host pod template diff so the pod stops failing to mount.- try {- await this.reconcileHostsReferencingSfs(sfs.name)- } catch (error) {- console.error(- `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,- error- )- }+ // Re-reconcile any Host whose Context references this SharedFileSystem.+ // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on+ // the next mcp-host pod template diff so the pod stops failing to mount.+ try {+ await this.reconcileHostsReferencingSfs(sfs.name)+ } catch (error) {+ console.error(+ `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,+ error+ )+ }+ })}const doneCallback = (err: Error | null) => {if (this.stopped) return+ this.sfsWatchRequest = nullif (err) {console.error('[K8s] SharedFileSystem watch error:', err)}console.log('[K8s] SharedFileSystem watch ended, restarting...')- setTimeout(() => this.startSharedFileSystemWatch(), err ? 5000 : 1000)+ this.scheduleSharedFileSystemWatchRestart(err ? 5000 : 1000)}this.sfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}+ private scheduleSharedFileSystemWatchRestart(delayMs: number): void {+ if (this.stopped || this.sfsWatchRestartTimer) return+ this.sfsWatchRestartTimer = setTimeout(() => {+ this.sfsWatchRestartTimer = null+ if (this.stopped) return+ void this.startSharedFileSystemWatch().catch(error => {+ console.error('[K8s] SharedFileSystem background watch restart failed:', error)+ this.scheduleSharedFileSystemWatchRestart(5000)+ })+ }, delayMs)+ }+/*** Watch the GlobalFileSystem singleton (gfs). ADDED/MODIFIED → reconcile the* drive stack; DELETED → tear it down (respecting retainOnDelete). DISTINCT@@ -1951,37 +2970,63 @@ export class McpServerWatcher implements McpServerProvider {const watchCallback = async (type: string,- apiObj: { metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }+ apiObj: {+ metadata: { name: string; namespace?: string }+ spec: GlobalFileSystemSpec+ status?: GlobalFileSystemCRD['status']+ }) => {const gfs: GlobalFileSystemCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || namespace,spec: apiObj.spec,+ status: apiObj.status,}console.log(`[K8s] GlobalFileSystem watch event: ${type} for ${gfs.name}`)- try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.gfsReconciler.reconcile(gfs)- } else if (type === 'DELETED') {- await this.gfsReconciler.reconcileDelete(gfs)- }- } catch (error) {- console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)+ if (type === 'ADDED' || type === 'MODIFIED') {+ this.globalFileSystems.set(gfs.name, gfs)+ } else if (type === 'DELETED') {+ this.globalFileSystems.delete(gfs.name)}+ this.globalFileSystemCacheRevision += 1+ await this.enqueueGlobalFileSystemReconciliation(gfs.name, async () => {+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.gfsReconciler.reconcile(gfs)+ } else if (type === 'DELETED') {+ await this.gfsReconciler.reconcileDelete(gfs)+ }+ } catch (error) {+ console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)+ }+ })}const doneCallback = (err: Error | null) => {if (this.stopped) return+ this.gfsWatchRequest = nullif (err) {console.error('[K8s] GlobalFileSystem watch error:', err)}console.log('[K8s] GlobalFileSystem watch ended, restarting...')- setTimeout(() => this.startGlobalFileSystemWatch(), err ? 5000 : 1000)+ this.scheduleGlobalFileSystemWatchRestart(err ? 5000 : 1000)}this.gfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}+ private scheduleGlobalFileSystemWatchRestart(delayMs: number): void {+ if (this.stopped || this.gfsWatchRestartTimer) return+ this.gfsWatchRestartTimer = setTimeout(() => {+ this.gfsWatchRestartTimer = null+ if (this.stopped) return+ void this.startGlobalFileSystemWatch().catch(error => {+ console.error('[K8s] GlobalFileSystem background watch restart failed:', error)+ this.scheduleGlobalFileSystemWatchRestart(5000)+ })+ }, delayMs)+ }+/*** Extracted so unit tests can invoke the handler without spinning up a real* K8s watch. Production calls this from `startCommunicationChannelWatch` and@@ -2036,6 +3081,14 @@ export class McpServerWatcher implements McpServerProvider {if (previousHostRef && previousHostRef !== cc.spec.hostRef) {affectedHostRefs.add(previousHostRef)}+ if (!this.isHostInventoryAuthoritative()) {+ const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()+ this.admitHostDependentEffects(+ 'CommunicationChannel event Host convergence',+ lifecycleGeneration+ )+ return+ }let needsFleetRetry = falsefor (const hostRef of affectedHostRefs) {if (!(await this.reconcileHostsReferencingCC(hostRef))) {@@ -2197,8 +3250,41 @@ export class McpServerWatcher implements McpServerProvider {private async runGfsResync(): Promise<void> {if (this.stopped) returntry {+ const cacheRevisionAtListStart = this.globalFileSystemCacheRevisionconst gfses = await listAllGlobalFileSystems()- await this.gfsReconciler.fullReconcile(gfses)+ const names = [+ ...this.globalFileSystems.keys(),+ ...gfses.map(globalFileSystem => globalFileSystem.name),+ ]+ return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>+ this.runGfsResyncCore(gfses, cacheRevisionAtListStart)+ )+ } catch (err) {+ console.error(+ `[K8s] GlobalFileSystem periodic resync failed: ${+ err instanceof Error ? err.message : String(err)+ }`+ )+ }+ }++ private async runGfsResyncCore(+ listedGfses?: GlobalFileSystemCRD[],+ listedAtCacheRevision?: number+ ): Promise<void> {+ if (this.stopped) return+ try {+ const cacheRevisionAtListStart = listedAtCacheRevision ?? this.globalFileSystemCacheRevision+ const gfses = listedGfses ?? (await listAllGlobalFileSystems())+ let reconcileInventory: GlobalFileSystemCRD[]+ if (cacheRevisionAtListStart === this.globalFileSystemCacheRevision) {+ this.globalFileSystems.clear()+ for (const gfs of gfses) this.globalFileSystems.set(gfs.name, gfs)+ reconcileInventory = gfses+ } else {+ reconcileInventory = [...this.globalFileSystems.values()]+ }+ await this.gfsReconciler.fullReconcile(reconcileInventory)} catch (err) {console.error(`[K8s] GlobalFileSystem periodic resync failed: ${@@ -2211,9 +3297,36 @@ export class McpServerWatcher implements McpServerProvider {private async runSfsResync(): Promise<void> {if (this.stopped) returntry {+ const cacheRevisionAtListStart = this.sharedFileSystemCacheRevisionconst sfses = await listAllSharedFileSystems()- this.sharedFileSystems.clear()- for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)+ const names = [+ ...this.sharedFileSystems.keys(),+ ...sfses.map(sharedFileSystem => sharedFileSystem.name),+ ]+ return this.enqueueSharedFileSystemFleetReconciliation(names, () =>+ this.runSfsResyncCore(sfses, cacheRevisionAtListStart)+ )+ } catch (error) {+ console.error('[K8s] Periodic SharedFileSystem resync failed:', error)+ }+ }++ private async runSfsResyncCore(+ listedSfses?: SharedFileSystemCRD[],+ listedAtCacheRevision?: number+ ): Promise<void> {+ if (this.stopped) return+ try {+ const cacheRevisionAtListStart = listedAtCacheRevision ?? this.sharedFileSystemCacheRevision+ const sfses = listedSfses ?? (await listAllSharedFileSystems())+ let reconcileInventory: SharedFileSystemCRD[]+ if (cacheRevisionAtListStart === this.sharedFileSystemCacheRevision) {+ this.sharedFileSystems.clear()+ for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)+ reconcileInventory = sfses+ } else {+ reconcileInventory = [...this.sharedFileSystems.values()]+ }// Capture each SFS's mountability (PVC Bound) BEFORE reconciling so we can// detect a flip. resolveContextMounts() injects the RO mount + podAffinity@@ -2222,11 +3335,11 @@ export class McpServerWatcher implements McpServerProvider {// on phase==='Ready': a transient wfc readiness dip while the PVC stays// Bound must NOT re-roll consumers — see resolveContextMounts.)const wasMountable = new Map<string, boolean>()- for (const sfs of sfses) {+ for (const sfs of reconcileInventory) {wasMountable.set(sfs.name, this.sharedFileSystemReconciler.isMountable(sfs))}- await this.sharedFileSystemReconciler.fullReconcile(sfses)+ await this.sharedFileSystemReconciler.fullReconcile(reconcileInventory)// #592 gap fix: heal the consuming mcp-host Deployments on a mountability// flip — inject the mount + affinity once the PVC binds, or drop it if the@@ -2236,7 +3349,7 @@ export class McpServerWatcher implements McpServerProvider {// an already-Bound SFS picked up on HCC restart still re-injects its mount// within one resync interval instead of waiting for the (much longer) Host// resync.- for (const sfs of sfses) {+ for (const sfs of reconcileInventory) {const nowMountable = this.sharedFileSystemReconciler.isMountable(sfs)if (nowMountable !== (wasMountable.get(sfs.name) ?? false)) {await this.reconcileHostsReferencingSfs(sfs.name)@@ -2247,86 +3360,14 @@ export class McpServerWatcher implements McpServerProvider {}}- private async runExternalEgressResync(): Promise<void> {- if (this.stopped) return- const servers = [...this.servers.values()]- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)- .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))-- if (servers.length === 0) return-- console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)- let index = 0- const workers = Array.from(- { length: Math.min(EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY, servers.length) },- async () => {- while (!this.stopped) {- const server = servers[index++]- if (!server) return- const key = this.externalEgressRetryKey(server)- if (this.externalEgressInFlight.has(key)) {- console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)- continue- }- if (this.externalEgressRetryTimers.has(key)) {- console.log(`[K8s] Skipping external egress resync for ${key}; retry already scheduled`)- continue- }- const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)- if (jitterMs > 0) await delay(jitterMs)- try {- await this.runExternalEgressOnce('MODIFIED', server)- } catch (error) {- console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)- this.scheduleExternalEgressRetry('MODIFIED', server)- }- }- }- )- await Promise.all(workers)- }-- private async reconcileInitialExternalEgressBeforeRuntime(- servers: McpServerCRD[]- ): Promise<Set<string>> {- const failures = new Set<string>()- const serversWithExternalEgress = servers- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)- .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))-- if (serversWithExternalEgress.length === 0) return failures-- console.log(- `[K8s] Reconciling external egress before startup runtime reconciliation for ` +- `${serversWithExternalEgress.length} McpServer(s)`- )-- for (const server of serversWithExternalEgress) {- const key = this.externalEgressRetryKey(server)- try {- await this.runExternalEgressOnce('MODIFIED', server)- } catch (error) {- failures.add(key)- console.error(- `[K8s] Initial external egress reconciliation failed for ${key}; ` +- 'runtime reconciliation will stay blocked until retry succeeds:',- error- )- this.scheduleExternalEgressRetry('MODIFIED', server)- }- }-- return failures- }-/*** Start watching McpServer CRDs.*/- private getMcpServerWatchCallback(): (- type: string,- apiObj: McpServerWatchObject- ) => Promise<void> {+ private getMcpServerWatchCallback(+ watchGeneration = this.mcpWatchGeneration+ ): (type: string, apiObj: McpServerWatchObject) => Promise<void> {return async (type, apiObj) => {+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) returnconst server: McpServerCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.namespace,@@ -2340,227 +3381,324 @@ export class McpServerWatcher implements McpServerProvider {console.log(`[K8s] McpServer watch event: ${type} for ${server.name}`)+ const previous = this.servers.get(server.name)+// Update cache+ let desiredStateChanged = falseif (type === 'ADDED' || type === 'MODIFIED') {+ desiredStateChanged =+ previous === undefined || !sameMcpServerDesiredRevision(previous, server)this.servers.set(server.name, server)} else if (type === 'DELETED') {- this.servers.delete(server.name)+ if (!previous?.uid || !server.uid || previous.uid === server.uid) {+ desiredStateChanged = this.servers.delete(server.name)+ }+ }+ if (desiredStateChanged) {+ this.mcpServerDesiredRevision += 1+ void this.runInitialNetworkPolicyConvergence()}- // External egress is part of the workload's pre-start contract. Reconcile- // it before HCC creates or updates any managed runtime Deployment so a- // stdio MCP with egressBindings cannot start before ExternalEgressReady.- if (type === 'ADDED' || type === 'MODIFIED') {- try {- await this.runExternalEgressOnce(type, server)- } catch (error) {- console.error(- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,- error- )- this.scheduleExternalEgressRetry(type, server)- this.changeCallback?.()- return- }+ // HCC writes McpServer status during reconciliation. Those writes emit+ // MODIFIED events but do not change the desired runtime or policy state.+ // Suppress that self-induced work while still publishing the fresh status+ // through discovery. Any UID/spec/annotation/label change remains live.+ if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {+ this.changeCallback?.()+ return}- // Trigger deployment reconciliation+ await this.enqueueMcpServerReconciliation(server, () =>+ this.reconcileMcpServerWatchEvent(+ type as ExternalEgressWatchEventType,+ server,+ watchGeneration+ )+ )+ }+ }++ private async reconcileMcpServerWatchEvent(+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ watchGeneration: number,+ retry?: ExternalEgressRetryHandle+ ): Promise<void> {+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return+ if (retry && !retry.isCurrent()) return+ if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {+ this.scheduleExternalEgressRetry(type, server)+ return+ }+ const current = this.servers.get(server.name)+ if (type === 'DELETED') {+ // A same-name recreation may already be accepted while this callback+ // waits for its per-server effect. The newer identity owns the key.+ if (current) return+ } else {+ if (!current || current.namespace !== server.namespace) return+ server = current+ }+ const deleteAllowed = () =>+ this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)+ const contextInventoryGeneration = this.contextWatchGeneration+ const contextPolicyInventoryIsCurrent = (): boolean =>+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&+ this.hasContextInventoryAuthority(contextInventoryGeneration)++ // External egress is part of the workload's pre-start contract. Reconcile+ // it before HCC creates or updates any managed runtime Deployment so a+ // stdio MCP with egressBindings cannot start before ExternalEgressReady.+ let retryCompletion: ExternalEgressRetryHandle | undefined+ if (type === 'ADDED' || type === 'MODIFIED') {try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.reconciler.reconcile(server)- } else if (type === 'DELETED') {- await this.reconciler.reconcileDelete(server.name, server.namespace)- }+ retryCompletion = await this.runExternalEgressOnce(type, server, {+ retry,+ isCurrent: () =>+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&+ (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),+ })} catch (error) {- console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)- if (- (type === 'ADDED' || type === 'MODIFIED') &&- (server.spec.egressBindings?.length ?? 0) > 0- ) {- this.scheduleExternalEgressRetry(type, server)- }+ console.error(+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,+ error+ )+ this.scheduleExternalEgressRetry(type, server)+ this.changeCallback?.()+ return}-- // Trigger binding policy reconciliation (L3 ingress/egress)+ } else {+ if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) returntry {- if (type === 'ADDED' || type === 'MODIFIED') {- const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']- if (bindingsJson) {- const bindings: BindingDef[] = JSON.parse(bindingsJson)- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name- const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name- await this.bindingReconciler.reconcileBindings(- recipeName,- bindings,- mcpWorkloadName,- server.name- )- }- // Re-reconcile any cached Context that references this server.- // Fixes race condition where Context MODIFIED arrives before- // the McpServer ADDED event populates the cache.- for (const ctx of this.contexts.values()) {- if (ctx.spec.mcpServers?.includes(server.name)) {- console.log(- `[K8s] Re-reconciling context "${ctx.name}" after McpServer "${server.name}" cached`- )- await this.netPolReconciler.reconcileContext(ctx)- }- }- } else if (type === 'DELETED') {+ retryCompletion = await this.runExternalEgressOnce(type, server, {+ deleteAllowed,+ retry,+ isCurrent: () =>+ !this.stopped &&+ watchGeneration === this.mcpWatchGeneration &&+ (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),+ })+ } catch (error) {+ console.error(+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,+ error+ )+ this.scheduleExternalEgressRetry(type, server)+ this.changeCallback?.()+ return+ }+ }++ if (type === 'ADDED' || type === 'MODIFIED') {+ if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {+ this.scheduleExternalEgressRetry(type, server)+ return+ }+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return+ server = this.servers.get(server.name)!+ } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {+ return+ }++ // Trigger deployment reconciliation.+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ const currentServer = server+ await this.reconciler.reconcile(currentServer, {+ isCurrent: () => this.mcpServerWatchEffectIsCurrent(currentServer, watchGeneration),+ })+ } else if (type === 'DELETED') {+ await this.reconciler.reconcileDelete(server.name, server.namespace)+ }+ } catch (error) {+ console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)+ this.scheduleExternalEgressRetry(type, server)+ this.changeCallback?.()+ return+ }++ if (type === 'ADDED' || type === 'MODIFIED') {+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return+ server = this.servers.get(server.name)!+ } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {+ return+ }++ // Trigger binding policy reconciliation (L3 ingress/egress).+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']+ if (bindingsJson) {+ const bindings: BindingDef[] = JSON.parse(bindingsJson)const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name- await this.bindingReconciler.cleanupBindings(recipeName)- await this.runExternalEgressOnce(type, server)+ const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name+ const bindingServer = server+ await this.bindingReconciler.reconcileBindings(+ recipeName,+ bindings,+ mcpWorkloadName,+ bindingServer.name,+ {+ isCurrent: () => this.mcpServerWatchEffectIsCurrent(bindingServer, watchGeneration),+ }+ )+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return+ server = this.servers.get(server.name)!}- } catch (error) {- console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)- if (type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED') {- this.scheduleExternalEgressRetry(type, server)+ // Re-reconcile cached Contexts after the server cache is populated.+ for (const selectedContext of this.contexts.values()) {+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return+ if (!selectedContext.spec.mcpServers?.includes(server.name)) continue+ const selectedContextId = selectedContext.spec.contextId+ console.log(+ `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`+ )+ await this.enqueueContextReconciliation(selectedContextId, async () => {+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return+ const currentContext = this.contexts.get(selectedContext.name)+ if (+ currentContext?.spec.contextId !== selectedContextId ||+ !currentContext.spec.mcpServers?.includes(server.name)+ ) {+ return+ }+ await this.netPolReconciler.reconcileContext(currentContext, {+ isCurrent: () =>+ contextPolicyInventoryIsCurrent() &&+ this.contexts.get(currentContext.name) === currentContext,+ })+ })}+ } else if (type === 'DELETED') {+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name+ await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })}-- // Notify listeners+ } catch (error) {+ console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)+ this.scheduleExternalEgressRetry(type, server)this.changeCallback?.()+ return}++ retryCompletion?.complete()+ this.changeCallback?.()}- private async startMcpServerWatch(): Promise<void> {+ private async startMcpServerWatch(resourceVersion: string): Promise<number> {+ this.requireInventoryResourceVersion('McpServer', resourceVersion)const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_MCPSERVERS}`console.log(`[K8s] Starting McpServer watch`)+ const watchGeneration = ++this.mcpWatchGeneration+ if (this.mcpWatchRequest) {+ this.mcpWatchRequest.abort()+ this.mcpWatchRequest = null+ }+ let watchEnded = false- const watchCallback = this.getMcpServerWatchCallback()+ const watchCallback = this.getMcpServerWatchCallback(watchGeneration)const doneCallback = (err: Error | null) => {- if (this.stopped) return+ if (watchEnded) return+ watchEnded = true+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return+ if (!this.retireMcpServerWatch(watchGeneration)) returnif (err) {console.error('[K8s] McpServer watch error:', err)}- console.log('[K8s] McpServer watch ended, restarting...')- setTimeout(() => this.startMcpServerWatch(), err ? 5000 : 1000)+ console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')+ this.scheduleMcpServerCacheRecovery()}- this.mcpWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)+ const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {+ request.abort()+ return watchGeneration+ }+ this.mcpWatchRequest = request+ return watchGeneration}- private externalEgressRetryKey(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {- return `${server.namespace}/${server.name}`+ private mcpServerWatchEffectIsCurrent(expected: McpServerCRD, watchGeneration: number): boolean {+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return false+ const current = this.servers.get(expected.name)+ return current !== undefined && sameMcpServerDesiredRevision(expected, current)}- private async runExternalEgressOnce(- type: string,+ private runExternalEgressOnce(+ type: ExternalEgressWatchEventType,server: McpServerCRD,- options: { clearRetryOnSuccess?: boolean } = {}- ): Promise<void> {- const clearRetryOnSuccess = options.clearRetryOnSuccess ?? true- const key = this.externalEgressRetryKey(server)- const existing = this.externalEgressInFlight.get(key)- if (existing) {- console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)- await existing- }-- const run = (async () => {- if (type === 'DELETED') {- await this.netPolReconciler.cleanupExternalEgress(server.name, server.namespace)- } else {- await this.netPolReconciler.reconcileExternalEgress(server)- }- })()- this.externalEgressInFlight.set(key, run)- try {- await run- if (clearRetryOnSuccess) {- this.clearExternalEgressRetry(server)- }- } finally {- if (this.externalEgressInFlight.get(key) === run) {- this.externalEgressInFlight.delete(key)- }- }- }-- private clearExternalEgressRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {- const key = this.externalEgressRetryKey(server)- const timer = this.externalEgressRetryTimers.get(key)- if (timer) {- clearTimeout(timer)- this.externalEgressRetryTimers.delete(key)- }- this.externalEgressRetryAttempts.delete(key)+ options: {+ deleteAllowed?: () => Promise<boolean>+ retry?: ExternalEgressRetryHandle+ isCurrent?: () => boolean+ } = {}+ ): Promise<ExternalEgressRetryHandle | undefined> {+ return this.externalEgressCoordinator.reconcile(type, server, options)}private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {- if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return- if (this.stopped) return-- const key = this.externalEgressRetryKey(server)- if (this.externalEgressRetryTimers.has(key)) return-- const attempt = (this.externalEgressRetryAttempts.get(key) ?? 0) + 1- const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]- if (!delayMs) {- console.error(- `[K8s] External egress retry exhausted for McpServer "${server.name}" in namespace "${server.namespace}"`- )- return- }-- this.externalEgressRetryAttempts.set(key, attempt)- console.warn(- `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length} ` +- `for McpServer "${server.name}" in ${delayMs}ms`- )-- const timer = setTimeout(() => {- this.externalEgressRetryTimers.delete(key)- void this.retryExternalEgress(type, server)- }, delayMs)- this.externalEgressRetryTimers.set(key, timer)+ this.externalEgressCoordinator.scheduleRetry(type, server)}- private async retryExternalEgress(type: string, server: McpServerCRD): Promise<void> {- if (this.stopped) return- try {- if (type === 'DELETED') {- await this.runExternalEgressOnce(type, server)- } else {- const current = this.servers.get(server.name)- if (!current || current.namespace !== server.namespace) {- this.clearExternalEgressRetry(server)- return- }- await this.runExternalEgressOnce(type, current, { clearRetryOnSuccess: false })- try {- await this.reconciler.reconcile(current)- this.clearExternalEgressRetry(current)- } catch (error) {- console.error(- `[K8s] Runtime reconciliation after external egress retry failed for ${server.name}:`,- error- )- this.scheduleExternalEgressRetry(type, current)- }+ private async performExternalEgressMutation(+ type: ExternalEgressWatchEventType,+ server: McpServerCRD,+ options: {+ deleteAllowed?: () => Promise<boolean>+ isCurrent: () => boolean+ }+ ): Promise<void> {+ if (type === 'DELETED') {+ const deleteAllowed = async (): Promise<boolean> => {+ if (!options.isCurrent()) return false+ if (options.deleteAllowed && !(await options.deleteAllowed())) return false+ return options.isCurrent()}- } catch (error) {- console.error(`[K8s] External egress retry failed for ${server.name}:`, error)- this.scheduleExternalEgressRetry(type, server)+ await this.netPolReconciler.cleanupExternalEgress(+ server.name,+ server.namespace,+ undefined,+ deleteAllowed+ )+ return}+ await this.netPolReconciler.reconcileExternalEgress(server, {+ isCurrent: options.isCurrent,+ })}/*** Start watching Context CRDs for NetworkPolicy reconciliation.*/- private async startContextWatch(): Promise<void> {+ private async startContextWatch(resourceVersion: string): Promise<number> {+ this.requireInventoryResourceVersion('Context', resourceVersion)const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_CONTEXTS}`console.log(`[K8s] Starting Context watch`)+ const watchGeneration = ++this.contextWatchGeneration+ if (this.ctxWatchRequest) {+ this.ctxWatchRequest.abort()+ this.ctxWatchRequest = null+ }+ let watchEnded = falseconst watchCallback = async (type: string,- apiObj: { metadata: { name: string; namespace?: string }; spec: ContextSpec }+ apiObj: {+ metadata: {+ name: string+ namespace?: string+ uid?: string+ generation?: number+ }+ spec: ContextSpec+ }) => {+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) returnconst context: ContextCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.namespace,+ uid: apiObj.metadata.uid,+ generation: apiObj.metadata.generation,spec: apiObj.spec,}@@ -2569,58 +3707,168 @@ export class McpServerWatcher implements McpServerProvider {// Capture the previous spec before mutating the cache so we can// re-reconcile any SFS that *was* referenced but is no longer.const previous = this.contexts.get(context.name)+ // A scoped Context delta may replace only a prior, fully authoritative+ // safety certificate. Capturing it before this event changes desired+ // state proves there was no outstanding Context or McpServer revocation+ // work that the scoped reconciliation could accidentally certify.+ const previousSafetyCertificate = this.currentNetworkPolicySafetyCertificate()// Cache the context for cross-resource re-reconciliation+ let desiredStateChanged = false+ let deltaSafetyCertificate: NetworkPolicySafetyCertificate | undefinedif (type === 'ADDED' || type === 'MODIFIED') {+ desiredStateChanged =+ previous === undefined || !sameContextDesiredRevision(previous, context)this.contexts.set(context.name, context)} else if (type === 'DELETED') {- this.contexts.delete(context.name)+ desiredStateChanged = this.contexts.delete(context.name)+ }+ if (desiredStateChanged) {+ this.contextDesiredRevision += 1+ // A same-identity Context MODIFIED event can complete its own stale+ // allow revocation below. Once that exact delta is current, it is safe+ // to restore readiness without waiting for an older pass's additive+ // fleet. Deletes and identity changes remain fail-closed until the+ // authoritative full pass certifies their broader cleanup.+ if (+ type === 'MODIFIED' &&+ previous?.uid === context.uid &&+ previous?.spec.contextId === context.spec.contextId &&+ previousSafetyCertificate !== null+ ) {+ deltaSafetyCertificate = {+ contextGeneration: previousSafetyCertificate.contextGeneration,+ serverGeneration: previousSafetyCertificate.serverGeneration,+ contextRevision: this.contextDesiredRevision,+ serverRevision: previousSafetyCertificate.serverRevision,+ }+ }+ void this.runInitialNetworkPolicyConvergence()}// Re-reconcile every SFS this Context referenced before or after the// change so SharedFileSystem.status.mountedByContexts stays in sync// without waiting for an SFS-level event.const nextForSfs = type === 'DELETED' ? undefined : context- void this.reconcileSharedFileSystemsReferencedByContext(previous, nextForSfs)+ const contextInventoryAuthoritative = () => this.hasContextInventoryAuthority(watchGeneration)+ const serverInventoryGeneration = this.mcpWatchGeneration+ void this.reconcileSharedFileSystemsReferencedByContext(+ previous,+ nextForSfs,+ contextInventoryAuthoritative+ )- try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.netPolReconciler.reconcileContext(context)- } else if (type === 'DELETED') {- await this.netPolReconciler.reconcileDeleteContext(context.spec.contextId)+ const previousContextId = previous?.spec.contextId+ const effectContextIds =+ previousContextId && previousContextId !== context.spec.contextId+ ? [previousContextId, context.spec.contextId]+ : [context.spec.contextId]+ await this.enqueueContextIdentityReconciliation(effectContextIds, async () => {+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return++ try {+ if (previousContextId && previousContextId !== context.spec.contextId) {+ const oldIdDeleteAllowed = () =>+ this.contextIdAbsentForDelete(previousContextId, watchGeneration)+ if (await oldIdDeleteAllowed()) {+ await this.netPolReconciler.reconcileDeleteContext(+ previousContextId,+ oldIdDeleteAllowed+ )+ }+ }++ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return+ const current = this.contexts.get(context.name)+ if (type === 'DELETED') {+ if (current) return+ } else if (current?.spec.contextId !== context.spec.contextId) {+ return+ }+ if (type === 'ADDED' || type === 'MODIFIED') {+ const selectedContext = current!+ // A scoped delta may only replace the safety certificate when its+ // own stale-allow revocation ran to completion — `reconcileContext`+ // aborts mid-pass whenever its authority fence breaks — and when+ // the last authoritative pass certified its namespace-wide+ // inventory, which a label-scoped delta cannot vouch for.+ // `deltaSafetyCertificate` is only ever built for MODIFIED, so this+ // is the one branch where it can be recorded.+ const scopedRevocationCompleted = await this.netPolReconciler.reconcileContext(+ selectedContext,+ {+ isCurrent: () =>+ contextInventoryAuthoritative() &&+ this.hasMcpServerInventoryAuthority(serverInventoryGeneration) &&+ this.contexts.get(selectedContext.name) === selectedContext,+ // Only opt in when there is a certificate to withhold. Without+ // one the result below is discarded, so a reported lost fence+ // would vanish: no throw to reach the retry, and the recovery+ // in the desired-state branch does not run for an event that+ // did not move the revision.+ honorsLostFence: deltaSafetyCertificate !== undefined,+ }+ )+ if (deltaSafetyCertificate && scopedRevocationCompleted) {+ if (this.netPolReconciler.hasCertifiedSafetyInventory()) {+ this.recordNetworkPolicySafetyCertificate(deltaSafetyCertificate)+ } else {+ console.warn(+ `[K8s] Not certifying the scoped delta for context "${context.spec.contextId}": the last authoritative safety pass ended without certifying its namespace-wide inventory`+ )+ }+ }+ } else if (type === 'DELETED') {+ const deleteAllowed = () =>+ this.contextAbsentForDelete(context.name, context.namespace, watchGeneration)+ if (!(await deleteAllowed())) return+ await this.netPolReconciler.reconcileDeleteContext(+ context.spec.contextId,+ deleteAllowed+ )+ }+ } catch (error) {+ console.error(+ `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,+ error+ )+ void this.runInitialNetworkPolicyConvergence()+ return}- } catch (error) {- console.error(- `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,- error- )- }- // Re-reconcile any Host that points at this Context so that changes to- // spec.sharedFileSystems[] propagate into the mcp-host pod template.- try {- for (const host of this.hosts.values()) {- if (host.spec.contextRef !== context.name) continue- await this.hostReconciler.reconcile(host)+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return+ // Re-reconcile any Host that points at this Context so that changes to+ // spec.sharedFileSystems[] propagate into the mcp-host pod template.+ try {+ await this.reconcileHostsReferencingContext(context.name, contextInventoryAuthoritative)+ } catch (error) {+ console.error(+ `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,+ error+ )}- } catch (error) {- console.error(- `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,- error- )- }+ })}const doneCallback = (err: Error | null) => {- if (this.stopped) return+ if (watchEnded) return+ watchEnded = true+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return+ if (!this.retireContextWatch(watchGeneration)) returnif (err) {console.error('[K8s] Context watch error:', err)}- console.log('[K8s] Context watch ended, restarting...')- setTimeout(() => this.startContextWatch(), err ? 5000 : 1000)+ console.log('[K8s] Context watch ended; recovering authoritative inventory...')+ this.scheduleContextCacheRecovery()}- this.ctxWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)+ const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)+ if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {+ request.abort()+ return watchGeneration+ }+ this.ctxWatchRequest = request+ return watchGeneration}/**@@ -2722,25 +3970,39 @@ export class McpServerWatcher implements McpServerProvider {*/async stop(): Promise<void> {this.stopped = true+ this.externalEgressCoordinator.stop()+ this.retireMcpServerWatch()+ this.retireContextWatch()this.ccCacheSynced = falsethis.hostCacheSynced = falsethis.hostWatchGeneration += 1this.ccWatchGeneration += 1- this.resolveHostFleetShutdown()- this.hostFleetReconcilePending?.resolve()- this.hostFleetReconcilePending = null+ this.hostFleetScheduler.stop()if (this.ccCacheRecoveryTimer) {clearTimeout(this.ccCacheRecoveryTimer)this.ccCacheRecoveryTimer = null}- if (this.ccFleetRetryTimer) {- clearTimeout(this.ccFleetRetryTimer)- this.ccFleetRetryTimer = null- }if (this.hostCacheRecoveryTimer) {clearTimeout(this.hostCacheRecoveryTimer)this.hostCacheRecoveryTimer = null}+ if (this.sfsWatchRestartTimer) {+ clearTimeout(this.sfsWatchRestartTimer)+ this.sfsWatchRestartTimer = null+ }+ if (this.gfsWatchRestartTimer) {+ clearTimeout(this.gfsWatchRestartTimer)+ this.gfsWatchRestartTimer = null+ }+ this.hostCacheRecoveryIntent = null+ if (this.mcpServerCacheRecoveryTimer) {+ clearTimeout(this.mcpServerCacheRecoveryTimer)+ this.mcpServerCacheRecoveryTimer = null+ }+ if (this.contextCacheRecoveryTimer) {+ clearTimeout(this.contextCacheRecoveryTimer)+ this.contextCacheRecoveryTimer = null+ }this.clearAllHostWatchRetries()if (this.resyncTimer) {clearInterval(this.resyncTimer)@@ -2754,16 +4016,11 @@ export class McpServerWatcher implements McpServerProvider {clearInterval(this.gfsResyncTimer)this.gfsResyncTimer = null}- if (this.externalEgressResyncTimer) {- clearTimeout(this.externalEgressResyncTimer)- this.externalEgressResyncTimer = null- }- for (const timer of this.externalEgressRetryTimers.values()) {+ for (const timer of this.initialConvergenceRetryTimers.values()) {clearTimeout(timer)}- this.externalEgressRetryTimers.clear()- this.externalEgressRetryAttempts.clear()- this.externalEgressInFlight.clear()+ this.initialConvergenceRetryTimers.clear()+ this.initialConvergenceRetryAttempts.clear()if (this.mcpWatchRequest) {console.log('[K8s] Stopping McpServer watch')this.mcpWatchRequest.abort()diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.tsindex 0e1502aa5..565c537d3 100644--- a/host-context-controller/src/main.ts+++ b/host-context-controller/src/main.ts@@ -18,6 +18,7 @@ import {createMcpServerProvider,getKubeConfig,} from './k8sClient'+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'import { ContextMapperServer } from './server'import { StatelessLifecycleTracker } from './statelessLifecycleTracker'import {@@ -113,6 +114,16 @@ async function main(): Promise<void> {const hasDesktopFn = hostReconciler? (hostRef: string) => hostReconciler.hasDesktop(hostRef): undefined+ // Dev (no watcher) gets an explicit always-authoritative gate: no inventory+ // to certify there, and the server's fail-closed default would otherwise pin+ // /ready at 503 forever (R3-B1). See resolveProviderAuthoritativeFn.+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(watcher)+ // Desktop status gates on Host inventory authority alone (not full readiness).+ // Dev (no watcher) mirrors the provider gate above: there is no Host inventory+ // to certify, so authority is unconditional (permissive) — otherwise the+ // server's fail-closed default would pin /api/v1/desktop/* at 503 forever in+ // dev (R2-M1). See resolveHostAuthoritativeFn.+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)// Stateless heartbeat consumption — mcp-host pods authenticate their// heartbeats toward control-api's /mcp-host facade (control-api is the@@ -145,16 +156,23 @@ async function main(): Promise<void> {`target=${config.controlApiBaseUrl})`)}- server = new ContextMapperServer(provider, config.port, hostReconciler, hasDesktopFn)+ server = new ContextMapperServer(+ provider,+ config.port,+ hostReconciler,+ hasDesktopFn,+ providerAuthoritativeFn,+ hostAuthoritativeFn+ )await server.start()// One-shot legacy sweep: delete the static `clerum-channel-reader`// Deployment if it still exists in the channels namespace. MUST run- // BEFORE provider.start(): provider.start() invokes fullReconcile on- // the initial Host list, which creates per-Host channel-reader Deployments- // that immediately try to long-poll Telegram. If the static is still- // alive at that moment, both pods compete on the same bot's getUpdates- // and Telegram 409s one of them. Sweeping first guarantees zero overlap.+ // BEFORE provider.start(): provider startup establishes the Host inventory+ // watch and schedules its initial fleet convergence, which creates per-Host+ // channel-reader Deployments in the background. If the static is still alive+ // when that work begins, both pods compete on the same bot's getUpdates and+ // Telegram 409s one of them. Sweeping first guarantees zero overlap.// Idempotent; 404 (already gone) is the steady state. See issue #273// for the empirical reproduction.if (!config.devMode && hostReconciler) {@@ -199,12 +217,11 @@ async function main(): Promise<void> {// reconcileChannelReaderRevision is self-contained (does not throw), so// the .catch here is defensive — it catches any unexpected runtime error// in the SecretInformer's own callback path.- if (!config.devMode && hostReconciler) {+ if (!config.devMode && watcher) {const kc = getKubeConfig()if (kc) {- const reconciler = hostReconcilerchannelSecretInformer = new SecretInformer(kc, config.channelsNamespace, evt => {- reconciler.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {+ watcher.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {console.error('[Main] reconcileChannelReaderRevision failed:', err)})})diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.tsnew file mode 100644index 000000000..19f75caf3--- /dev/null+++ b/host-context-controller/src/mcpServerSafety.test.ts@@ -0,0 +1,267 @@+import { describe, expect, it, vi } from 'vitest'+import {+ confirmAuthoritativeMcpServerAbsence,+ isMcpServerStatusOnlyUpdate,+ sameMcpServerDesiredRevision,+} from './mcpServerSafety'+import type { McpServerCRD } from './types'++function makeServer(overrides: Partial<McpServerCRD> = {}): McpServerCRD {+ return {+ name: 'web-search',+ namespace: 'mcp-server',+ uid: 'web-search-uid',+ generation: 7,+ annotations: { 'clerum.io/recipe': 'research' },+ labels: { 'clerum.io/workload': 'web-search' },+ spec: {+ contextRef: 'default',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ ...overrides,+ }+}++describe('sameMcpServerDesiredRevision', () => {+ it('matches equal desired state while ignoring status and one-sided identity metadata', () => {+ const expected = makeServer({+ status: { conditions: [{ type: 'Ready', status: 'False' }] },+ })+ const current = makeServer({+ uid: undefined,+ generation: undefined,+ status: { conditions: [{ type: 'Ready', status: 'True' }] },+ })++ expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)+ expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)+ })++ it('rejects a different resource key', () => {+ const expected = makeServer()++ expect(sameMcpServerDesiredRevision(expected, makeServer({ name: 'doc-generator' }))).toBe(+ false+ )+ expect(sameMcpServerDesiredRevision(expected, makeServer({ namespace: 'other' }))).toBe(false)+ })++ it('rejects differing UID or generation only when both sides provide the field', () => {+ const expected = makeServer()++ expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: 'replacement-uid' }))).toBe(+ false+ )+ expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: 8 }))).toBe(false)+ expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: undefined }))).toBe(true)+ expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: undefined }))).toBe(true)+ })++ it('compares desired object fields by semantic content, not Kubernetes map key order', () => {+ const expected = makeServer({+ annotations: { first: '1', second: '2' },+ labels: { first: '1', second: '2' },+ })++ expect(+ sameMcpServerDesiredRevision(+ expected,+ makeServer({+ spec: {+ ...expected.spec,+ image: 'clerum/web-search:next',+ },+ labels: expected.labels,+ })+ )+ ).toBe(false)+ expect(+ sameMcpServerDesiredRevision(+ expected,+ makeServer({+ annotations: { second: '2', first: '1' },+ labels: expected.labels,+ })+ )+ ).toBe(true)+ expect(+ sameMcpServerDesiredRevision(+ expected,+ makeServer({+ annotations: { 'clerum.io/recipe': 'other' },+ labels: expected.labels,+ })+ )+ ).toBe(false)+ expect(+ sameMcpServerDesiredRevision(+ expected,+ makeServer({+ labels: { second: '2', first: '1' },+ annotations: expected.annotations,+ })+ )+ ).toBe(true)+ })++ it('ignores the controller-owned network-ready handshake annotation', () => {+ const expected = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })+ const current = makeServer({+ annotations: {+ 'clerum.io/pre-deploy': 'true',+ 'clerum.io/network-ready': 'true',+ },+ })++ expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)+ expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)+ })+})++describe('isMcpServerStatusOnlyUpdate', () => {+ it('accepts only an exact identity and generation with unchanged desired state', () => {+ const previous = makeServer({+ status: { conditions: [{ type: 'Ready', status: 'False' }] },+ })+ const current = makeServer({+ status: { conditions: [{ type: 'Ready', status: 'True' }] },+ })++ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)+ })++ it('does not suppress events when UID or generation equality is only one-sided', () => {+ const previous = makeServer()++ expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ uid: undefined }))).toBe(false)+ expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ generation: undefined }))).toBe(false)+ })++ it('does not suppress a desired-state change', () => {+ const previous = makeServer()+ const current = makeServer({+ labels: { 'clerum.io/workload': 'replacement' },+ })++ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(false)+ })++ it('treats the controller-owned network-ready handshake as a status-only update', () => {+ const previous = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })+ const current = makeServer({+ annotations: {+ 'clerum.io/pre-deploy': 'true',+ 'clerum.io/network-ready': 'true',+ },+ })++ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)+ })+})++describe('confirmAuthoritativeMcpServerAbsence', () => {+ it('stops before the API read when inventory is not authoritative', async () => {+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)+ const readCurrent = vi.fn()++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => false,+ resolveCurrent,+ readCurrent,+ })+ ).resolves.toBe(false)+ expect(resolveCurrent).not.toHaveBeenCalled()+ expect(readCurrent).not.toHaveBeenCalled()+ })++ it('stops before the API read when the cache already contains the current server', async () => {+ const readCurrent = vi.fn()++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => true,+ resolveCurrent: () => makeServer(),+ readCurrent,+ })+ ).resolves.toBe(false)+ expect(readCurrent).not.toHaveBeenCalled()+ })++ it('rejects absence when the API read returns a current object', async () => {+ const readCurrent = vi.fn().mockResolvedValue({+ metadata: { name: 'web-search', namespace: 'mcp-server' },+ })++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => true,+ resolveCurrent: () => undefined,+ readCurrent,+ })+ ).resolves.toBe(false)+ expect(readCurrent).toHaveBeenCalledOnce()+ })++ it('propagates a non-404 API read failure', async () => {+ const apiError = Object.assign(new Error('unavailable'), { code: 503 })++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => true,+ resolveCurrent: () => undefined,+ readCurrent: vi.fn().mockRejectedValue(apiError),+ })+ ).rejects.toBe(apiError)+ })++ it('rejects absence when inventory authority is lost after a 404', async () => {+ const inventoryAuthoritative = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative,+ resolveCurrent,+ readCurrent: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ })+ ).resolves.toBe(false)+ expect(resolveCurrent).toHaveBeenCalledOnce()+ })++ it('rejects absence when the server is recreated in cache after a 404', async () => {+ const resolveCurrent = vi+ .fn<() => McpServerCRD | undefined>()+ .mockReturnValueOnce(undefined)+ .mockReturnValueOnce(makeServer())++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => true,+ resolveCurrent,+ readCurrent: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ })+ ).resolves.toBe(false)+ expect(resolveCurrent).toHaveBeenCalledTimes(2)+ })++ it('confirms absence after a 404 when authority and cache absence remain stable', async () => {+ const inventoryAuthoritative = vi.fn(() => true)+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)++ await expect(+ confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative,+ resolveCurrent,+ readCurrent: vi.fn().mockRejectedValue({ response: { statusCode: 404 } }),+ })+ ).resolves.toBe(true)+ expect(inventoryAuthoritative).toHaveBeenCalledTimes(2)+ expect(resolveCurrent).toHaveBeenCalledTimes(2)+ })+})diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.tsnew file mode 100644index 000000000..7f8d8438d--- /dev/null+++ b/host-context-controller/src/mcpServerSafety.ts@@ -0,0 +1,91 @@+import type { McpServerCRD } from './types'+import { getErrorCode } from './utils'++const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'++/**+ * This acknowledgement is written by HCC after it has applied a server's+ * NetworkPolicies so WRC can admit its workload. It is not user intent and it+ * cannot change any NetworkPolicy decision. Treating that write as desired+ * state makes HCC invalidate its own authoritative revocation pass under a+ * large startup fleet, coupling /ready to additive runtime convergence again.+ */+function desiredAnnotations(annotations: Record<string, string> | undefined): Record<string, string> {+ if (!annotations?.[NETWORK_READY_ANNOTATION]) return annotations ?? {}+ const { [NETWORK_READY_ANNOTATION]: _networkReady, ...remaining } = annotations+ return remaining+}++/**+ * Kubernetes represents objects such as labels and annotations as maps, whose+ * wire key order is not meaningful or stable across LIST and WATCH payloads.+ * Preserve array order (it is semantic) while sorting object keys so a+ * status-only event cannot masquerade as a desired-state revision.+ */+function stableJson(value: unknown): string {+ if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`+ if (value && typeof value === 'object') {+ const record = value as Record<string, unknown>+ return `{${Object.keys(record)+ .sort()+ .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)+ .join(',')}}`+ }+ return JSON.stringify(value) ?? 'undefined'+}++export function sameMcpServerDesiredRevision(+ expected: McpServerCRD,+ current: McpServerCRD+): boolean {+ if (expected.name !== current.name || expected.namespace !== current.namespace) return false+ if (expected.uid && current.uid && expected.uid !== current.uid) return false+ if (+ expected.generation !== undefined &&+ current.generation !== undefined &&+ expected.generation !== current.generation+ ) {+ return false+ }++ return (+ stableJson(expected.spec) === stableJson(current.spec) &&+ stableJson(desiredAnnotations(expected.annotations)) ===+ stableJson(desiredAnnotations(current.annotations)) &&+ stableJson(expected.labels ?? {}) === stableJson(current.labels ?? {})+ )+}++export function isMcpServerStatusOnlyUpdate(+ previous: McpServerCRD,+ current: McpServerCRD+): boolean {+ return (+ previous.uid === current.uid &&+ previous.generation === current.generation &&+ sameMcpServerDesiredRevision(previous, current)+ )+}++export interface AuthoritativeMcpServerAbsenceCheck {+ inventoryAuthoritative: () => boolean+ resolveCurrent: () => McpServerCRD | undefined+ readCurrent: () => Promise<unknown>+}++export async function confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative,+ resolveCurrent,+ readCurrent,+}: AuthoritativeMcpServerAbsenceCheck): Promise<boolean> {+ if (!inventoryAuthoritative() || resolveCurrent()) return false++ try {+ await readCurrent()+ return false+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404) throw error+ }++ return inventoryAuthoritative() && !resolveCurrent()+}diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 75b8cc5f0..59465c3b3 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -79,6 +79,30 @@ export const networkPoliciesTotal = gauge({labelNames: ['layer'] as const,})+export const initialConvergenceRetriesTotal = counter({+ name: 'clerum_hcc_initial_convergence_retries_total',+ help: 'Initial background convergence retries scheduled after a failed fleet pass.',+ labelNames: ['lane'] as const,+})++export const initialConvergenceLastSuccessTimestampSeconds = gauge({+ name: 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ help: 'Unix timestamp of the last successful initial background convergence pass.',+ labelNames: ['lane'] as const,+})++export const networkPolicySafetyPassDurationSeconds = histogram({+ name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',+ help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',+ labelNames: ['outcome'] as const,+})++export const networkPolicySafetyPassPoliciesTotal = counter({+ name: 'clerum_hcc_networkpolicy_safety_pass_policies_total',+ help: 'NetworkPolicies listed and revoked by authoritative HCC safety passes.',+ labelNames: ['operation'] as const,+})+export const contextReconciliationsTotal = counter({name: 'clerum_hcc_context_reconciliations_total',help: 'Total Context CRD reconciliations',@@ -201,3 +225,13 @@ export const hostDeleteCleanupTotal = counter({help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',labelNames: ['outcome'] as const,})++// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding+// whose DNS never resolves is retried forever at the capped (maximum) backoff;+// without this signal the still-denied binding converges nowhere yet stays+// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server+// label — so cardinality stays flat regardless of fleet size.+export const externalEgressRetriesAtCap = gauge({+ name: 'clerum_hcc_external_egress_retries_at_cap',+ help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',+})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 8f3d3f3e1..047ccd88e 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -4,9 +4,37 @@ import type { RecordWithTtl } from 'node:dns'import * as dns from 'node:dns/promises'import { readFileSync } from 'node:fs'import { join } from 'node:path'-import { NetworkPolicyReconciler, PUBLIC_EGRESS_EXCEPT_CIDRS } from './networkPolicyReconciler'+import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'+import {+ networkPolicySafetyPassDurationSeconds,+ networkPolicySafetyPassPoliciesTotal,+} from './metrics'+import {+ NetworkPolicyReconciler,+ PUBLIC_EGRESS_EXCEPT_CIDRS,+ sameContextDesiredRevision,+} from './networkPolicyReconciler'import { ContextCRD, McpServerCRD } from './types'+function deferred<T = void>(): {+ promise: Promise<T>+ resolve: (value: T | PromiseLike<T>) => void+} {+ let resolve!: (value: T | PromiseLike<T>) => void+ const promise = new Promise<T>(res => {+ resolve = res+ })+ return { promise, resolve }+}++vi.mock('./mcpServerSafety', async () => {+ const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')+ return {+ ...actual,+ confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),+ }+})+// resolve4({ ttl: true }) returns RecordWithTtl[]; `rec` builds that shape and// `resolve4Mock` sidesteps the string[]/RecordWithTtl[] overload inference.const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))@@ -22,6 +50,7 @@ vi.mock('./config', () => ({port: 8081,runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],k8sApiCidrs: [],+ externalEgressDnsResolveTimeoutMs: 5_000,// #299 sliding-window knobs read by reconcileExternalEgress.externalEgressOverlapSec: 300,externalEgressMaxEntries: 128,@@ -39,10 +68,23 @@ vi.mock('node:dns/promises', () => ({]),}))+vi.mock('./metrics', () => ({+ networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },+ networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },+}))+function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {+ if (+ name === 'allow-desktop-egress-rpc-proxy' ||+ name === 'allow-rpc-proxy-to-managed-mcp-servers'+ ) {+ throw Object.assign(new Error('not found'), { code: 404 })+ }+ return { metadata: { resourceVersion: '1' } }+ }),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -51,7 +93,17 @@ function makeMockNetworkingApi() {function makeMockCustomApi() {return {- getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({ status: {} }),+ getNamespacedCustomObject: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({+ metadata: {+ generation: 7,+ resourceVersion: 'status-rv-default',+ },+ status: {},+ }),patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),}}@@ -70,7 +122,48 @@ function makeReconciler(})const cache = serverCache ?? new Map()const reconciler = new NetworkPolicyReconciler(kc, cache)- ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = mockApi+ const listedPolicies = new Map<string, k8s.V1NetworkPolicy>()+ const networkingApi = {+ ...mockApi,+ listNamespacedNetworkPolicy: async (args: { namespace: string }) => {+ const response = await mockApi.listNamespacedNetworkPolicy(args)+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {+ const name = policy.metadata?.name ?? `unnamed-${index}`+ const materialized = {+ ...policy,+ metadata: {+ ...policy.metadata,+ name,+ namespace: policy.metadata?.namespace ?? args.namespace,+ uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,+ resourceVersion: policy.metadata?.resourceVersion ?? '1',+ },+ }+ listedPolicies.set(`${args.namespace}/${name}`, materialized)+ return materialized+ })+ return { ...response, items }+ },+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {+ const response = await mockApi.readNamespacedNetworkPolicy(args)+ const listed = listedPolicies.get(`${args.namespace}/${args.name}`)+ return {+ ...listed,+ ...response,+ metadata: {+ ...listed?.metadata,+ ...response.metadata,+ name: response.metadata?.name ?? listed?.metadata?.name ?? args.name,+ namespace: response.metadata?.namespace ?? listed?.metadata?.namespace ?? args.namespace,+ uid:+ response.metadata?.uid ?? listed?.metadata?.uid ?? `${args.namespace}:${args.name}:uid`,+ resourceVersion:+ response.metadata?.resourceVersion ?? listed?.metadata?.resourceVersion ?? '1',+ },+ }+ },+ }+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApireturn reconciler}@@ -81,6 +174,7 @@ describe('NetworkPolicyReconciler', () => {let reconciler: NetworkPolicyReconcilerbeforeEach(() => {+ vi.useRealTimers()mockApi = makeMockNetworkingApi()mockCustomApi = makeMockCustomApi()reconciler = makeReconciler(mockApi, undefined, mockCustomApi)@@ -88,6 +182,102 @@ describe('NetworkPolicyReconciler', () => {resolve4Mock.mockResolvedValue(rec('1.2.3.4', '5.6.7.8'))})+ it('compares Context desired revisions canonically and fences identity changes', () => {+ const expected: ContextCRD = {+ name: 'default',+ namespace: 'mcp-server',+ uid: 'context-uid',+ generation: 7,+ spec: {+ contextId: 'default',+ mcpServers: ['alpha'],+ sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace' }],+ },+ }+ const reordered: ContextCRD = {+ namespace: 'mcp-server',+ name: 'default',+ generation: 7,+ uid: 'context-uid',+ spec: {+ sharedFileSystems: [{ mountPath: '/workspace', name: 'workspace' }],+ mcpServers: ['alpha'],+ contextId: 'default',+ },+ }++ expect(sameContextDesiredRevision(expected, reordered)).toBe(true)+ expect(sameContextDesiredRevision(expected, { ...reordered, uid: 'replacement-uid' })).toBe(+ false+ )+ expect(sameContextDesiredRevision(expected, { ...reordered, generation: 8 })).toBe(false)+ })++ it('records the authoritative safety-pass inventory, completed revocations, and duration', async () => {+ mockApi.listNamespacedNetworkPolicy+ .mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-context',+ labels: { 'clerum.io/context': 'missing-context' },+ },+ },+ ],+ })+ .mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-host-egress',+ labels: { 'clerum.io/context': 'missing-context' },+ },+ },+ ],+ })+ .mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-rpc-egress',+ labels: { 'clerum.io/context': 'missing-context' },+ },+ },+ ],+ })+ .mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-external-egress',+ labels: { 'clerum.io/mcpserver': 'missing-server' },+ },+ },+ ],+ })+ const complete = vi.fn()++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ onAuthoritativeRevocationComplete: complete,+ })++ expect(complete).toHaveBeenCalledOnce()+ expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(+ { operation: 'listed' },+ 4+ )+ expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(+ { operation: 'revoked' },+ 4+ )+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(+ { outcome: 'completed' },+ expect.any(Number)+ )+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)+ })+describe('public egress blocklist parity', () => {it('keeps the HCC and WRC IPv4 exclusion lists in sync', () => {const wrcFactory = readFileSync(@@ -150,6 +340,50 @@ describe('NetworkPolicyReconciler', () => {// ─── L1: Infrastructure Policies ───────────────────────────────────describe('L1 — ensureInfrastructurePolicies', () => {+ it.each([+ ['deny-all-mcp-server', 'default-deny'],+ ['allow-dns-egress-mcp-server', 'infrastructure'],+ ['allow-hcc-api-egress-mcp-server', 'infrastructure'],+ ['allow-k8s-api-egress-mcp-server', 'infrastructure'],+ ])(+ 'refuses to adopt a foreign policy that collides with %s',+ async (targetName, policyType) => {+ mockApi.createNamespacedNetworkPolicy.mockImplementation(+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {+ if (body.metadata?.name === targetName) {+ throw Object.assign(new Error('already exists'), { code: 409 })+ }+ return {}+ }+ )+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (+ name === 'allow-desktop-egress-rpc-proxy' ||+ name === 'allow-rpc-proxy-to-managed-mcp-servers'+ ) {+ throw Object.assign(new Error('not found'), { code: 404 })+ }+ return {+ metadata: {+ name,+ namespace,+ uid: 'foreign-baseline-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': policyType,+ },+ },+ }+ })++ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: targetName })+ )+ }+ )+it('creates DNS egress policies in every runtime namespace', async () => {await reconciler.ensureDefaultPolicies()@@ -308,34 +542,204 @@ describe('NetworkPolicyReconciler', () => {})it('removes legacy static policies now replaced by generated scoped policies', async () => {+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `${name}-uid`,+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ }))+await reconciler.ensureDefaultPolicies()expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({namespace: 'rpc-proxy',name: 'allow-desktop-egress-rpc-proxy',+ body: {+ preconditions: {+ uid: 'allow-desktop-egress-rpc-proxy-uid',+ resourceVersion: '7',+ },+ },})expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({namespace: 'mcp-server',name: 'allow-rpc-proxy-to-managed-mcp-servers',+ body: {+ preconditions: {+ uid: 'allow-rpc-proxy-to-managed-mcp-servers-uid',+ resourceVersion: '7',+ },+ },})})- it('fails closed when legacy static policy cleanup fails', async () => {- const cleanupError = Object.assign(new Error('rbac denied'), { code: 403 })- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {- if (name === 'allow-rpc-proxy-to-managed-mcp-servers') {- throw cleanupError+ it.each([+ ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],+ ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],+ ])('refuses to delete a foreign legacy policy %s/%s', async (targetNamespace, targetName) => {+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name !== targetName) {+ throw Object.assign(new Error('not found'), { code: 404 })+ }+ return {+ metadata: {+ name,+ namespace,+ uid: 'foreign-legacy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ },+ },}- return {}})- await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(cleanupError)+ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ namespace: targetNamespace, name: targetName })+ )})++ it.each([+ ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],+ ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],+ ])(+ 'propagates an identity conflict deleting legacy policy %s/%s',+ async (targetNamespace, targetName) => {+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name !== targetName) {+ throw Object.assign(new Error('not found'), { code: 404 })+ }+ return {+ metadata: {+ name,+ namespace,+ uid: 'legacy-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ }+ })+ const conflict = Object.assign(new Error('identity changed'), { code: 409 })+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name === targetName && namespace === targetNamespace) throw conflict+ return {}+ })++ await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(conflict)+ }+ )})// ─── L1: Allow API ─────────────────────────────────────────────────describe('L1 — ensureAllowContextMapperApi', () => {+ it('updates the existing HCC-owned allow-api policy without requiring Context owner labels', async () => {+ mockApi.createNamespacedNetworkPolicy.mockImplementation(+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {+ if (body.metadata?.name === 'allow-host-context-controller-api') {+ throw Object.assign(new Error('already exists'), { code: 409 })+ }+ return {}+ }+ )+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: 'existing-default-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'allow-api',+ },+ },+ }))++ await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'allow-host-context-controller-api',+ namespace: 'mcp-server',+ })+ )+ })++ it('refuses to adopt a foreign policy that collides with the allow-api policy name', async () => {+ mockApi.createNamespacedNetworkPolicy.mockImplementation(+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {+ if (body.metadata?.name === 'allow-host-context-controller-api') {+ throw Object.assign(new Error('already exists'), { code: 409 })+ }+ return {}+ }+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({+ metadata: {+ name: 'allow-host-context-controller-api',+ namespace: 'mcp-server',+ uid: 'foreign-default-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'allow-api',+ },+ },+ })++ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('revalidates allow-api ownership after a replace conflict', async () => {+ mockApi.createNamespacedNetworkPolicy.mockImplementation(+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {+ if (body.metadata?.name === 'allow-host-context-controller-api') {+ throw Object.assign(new Error('already exists'), { code: 409 })+ }+ return {}+ }+ )+ mockApi.readNamespacedNetworkPolicy+ .mockResolvedValueOnce({+ metadata: {+ name: 'allow-host-context-controller-api',+ namespace: 'mcp-server',+ uid: 'owned-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'allow-api',+ },+ },+ })+ .mockResolvedValueOnce({+ metadata: {+ name: 'allow-host-context-controller-api',+ namespace: 'mcp-server',+ uid: 'foreign-uid',+ resourceVersion: '8',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'allow-api',+ },+ },+ })+ mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('conflict'), { code: 409 })+ )++ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })+it('creates allow-api ingress policy using _from (K8s client convention)', async () => {await reconciler.ensureDefaultPolicies()@@ -355,6 +759,82 @@ describe('NetworkPolicyReconciler', () => {// ─── L2: Context-Based Policies ────────────────────────────────────describe('L2 — reconcileContext', () => {+ it('refuses to adopt a foreign policy that collides with a live Context policy name', async () => {+ const server: McpServerCRD = {+ name: 'mongo',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))+ const context: ContextCRD = {+ name: 'dev',+ namespace: 'mcp-server',+ spec: { contextId: 'dev', mcpServers: [server.name] },+ }+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: 'ctx-dev-mongo',+ namespace: 'mcp-server',+ uid: 'foreign-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'dev',+ 'clerum.io/mcpserver': 'mongo',+ },+ },+ })++ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('refuses to replace an HCC Context policy owned by a different Context/server pair', async () => {+ const server: McpServerCRD = {+ name: 'b-c',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'a',+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))+ const context: ContextCRD = {+ name: 'a',+ namespace: 'mcp-server',+ spec: { contextId: 'a', mcpServers: ['b-c'] },+ }+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: 'ctx-a-b-c',+ namespace: 'mcp-server',+ uid: 'other-hcc-owner-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'a-b',+ 'clerum.io/mcpserver': 'c',+ },+ },+ })++ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+it('creates context-allow ingress policy per server in mcp-server namespace', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {@@ -374,7 +854,7 @@ describe('NetworkPolicyReconciler', () => {spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context)+ await rec.reconcileContext(context, { isCurrent: () => true })// First call: ingress in mcp-server namespaceconst ingressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {@@ -416,109 +896,355 @@ describe('NetworkPolicyReconciler', () => {)})- it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {- const cache = new Map<string, McpServerCRD>()- cache.set('mongo', {+ it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {+ const oldServer: McpServerCRD = {name: 'mongo',namespace: 'mcp-server',spec: {contextRef: 'dev',image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ transport: { type: 'streamableHttp', port: 3000 },},- })-+ }+ const cache = new Map([[oldServer.name, oldServer]])const rec = makeReconciler(mockApi, cache)const context: ContextCRD = {name: 'dev',namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['mongo'] },+ spec: { contextId: 'dev', mcpServers: [oldServer.name] },}- await rec.reconcileContext(context)+ await rec.reconcileContext(context, { isCurrent: () => true })+ const livePolicies = new Map(+ mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body+ return [`${policy.metadata?.namespace}/${policy.metadata?.name}`, policy]+ })+ )- // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {- namespace: string- body: {- metadata: { name: string; namespace: string; labels: Record<string, string> }- spec: {- podSelector: { matchLabels: Record<string, string> }- policyTypes: string[]- egress: Array<{- to: Array<{- namespaceSelector: { matchLabels: Record<string, string> }- podSelector: { matchLabels: Record<string, string> }- }>- ports: Array<{ port: number }>- }>- }+ vi.clearAllMocks()+ cache.set(oldServer.name, {+ ...oldServer,+ generation: 2,+ spec: {+ ...oldServer.spec,+ transport: { type: 'streamableHttp', port: 4000 },+ },+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => ({+ items: [...livePolicies.values()].filter(+ policy => policy.metadata?.namespace === namespace+ ),+ })+ )+ mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {+ livePolicies.set(`${namespace}/${body.metadata?.name}`, body)+ return {}+ })+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, name, body }) => {+ livePolicies.set(`${namespace}/${name}`, body)+ return {}}- }- expect(egressCall.namespace).toBe('mcp-host')- expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')- expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')- expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'dev',+ )+ // The delete mock MUST remove from livePolicies, or the port-4000+ // assertion below survives a spurious deletion and the test passes with+ // the bug reinjected (R3-H7).+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {+ livePolicies.delete(`${namespace}/${name}`)+ return {}})- // Egress targets mcp-server namespace + specific server pod- const egressRule = egressCall.body.spec.egress[0]- expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(- 'mcp-server'++ await rec.reconcileContext(context, { isCurrent: () => true })++ // Filter the real calls by name+namespace: the production delete always+ // carries `preconditions`, so `not.toHaveBeenCalledWith({name,namespace})`+ // never matched any call and was vacuously true (R3-H7).+ const deletedCtxDevMongo = mockApi.deleteNamespacedNetworkPolicy.mock.calls.some(+ call =>+ (call[0] as { name?: string; namespace?: string })?.name === 'ctx-dev-mongo' &&+ (call[0] as { name?: string; namespace?: string })?.namespace === 'mcp-server')- expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')- expect(egressRule.ports[0].port).toBe(3000)+ expect(deletedCtxDevMongo).toBe(false)+ const ingress = livePolicies.get('mcp-server/ctx-dev-mongo')+ expect(ingress?.spec?.ingress?.[0]?.ports).toEqual([expect.objectContaining({ port: 4000 })])})- it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {+ it('stops remaining policy mutations after its combined authority lease is retired', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',namespace: 'mcp-server',spec: {- contextRef: 'ctx-name',+ contextRef: 'dev',image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ transport: { type: 'streamableHttp', port: 3000 },},})-+ const firstCreateStarted = deferred()+ const releaseFirstCreate = deferred()+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ firstCreateStarted.resolve(undefined)+ await releaseFirstCreate.promise+ return {}+ })const rec = makeReconciler(mockApi, cache)+ let contextGeneration = 4+ let serverGeneration = 8+ const capturedContextGeneration = contextGeneration+ const capturedServerGeneration = serverGenerationconst context: ContextCRD = {- name: 'ctx-name',+ name: 'dev',namespace: 'mcp-server',- spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },+ spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context)-- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {- body: {- metadata: { labels: Record<string, string> }- spec: { podSelector: { matchLabels: Record<string, string> } }- }- }- expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'ctx-name',+ const reconcile = rec.reconcileContext(context, {+ isCurrent: () =>+ contextGeneration === capturedContextGeneration &&+ serverGeneration === capturedServerGeneration,})+ await firstCreateStarted.promise++ contextGeneration = 6+ serverGeneration = 10+ releaseFirstCreate.resolve(undefined)+ await reconcile++ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()})- it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {+ it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',namespace: 'mcp-server',+ uid: 'mongo-uid',+ generation: 1,spec: {contextRef: 'dev',- image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ image: 'mongo:old',+ transport: { type: 'streamableHttp', port: 3000 },},})-+ const firstCreateStarted = deferred()+ const releaseFirstCreate = deferred()+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ firstCreateStarted.resolve(undefined)+ await releaseFirstCreate.promise+ return {}+ })+ const rec = makeReconciler(mockApi, cache)+ const context: ContextCRD = {+ name: 'dev',+ namespace: 'mcp-server',+ spec: { contextId: 'dev', mcpServers: ['mongo'] },+ }++ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })+ await firstCreateStarted.promise+ cache.set('mongo', {+ name: 'mongo',+ namespace: 'mcp-server',+ uid: 'mongo-uid',+ generation: 2,+ spec: {+ contextRef: 'dev',+ image: 'mongo:new',+ transport: { type: 'streamableHttp', port: 4000 },+ },+ })+ releaseFirstCreate.resolve(undefined)+ await reconcile++ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('continues a Context policy pass across a status-only McpServer replacement', async () => {+ const cache = new Map<string, McpServerCRD>()+ const selected: McpServerCRD = {+ name: 'mongo',+ namespace: 'mcp-server',+ uid: 'mongo-uid',+ generation: 1,+ spec: {+ contextRef: 'dev',+ image: 'mongo:stable',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ status: { conditions: [{ type: 'Ready', status: 'False' }] },+ }+ cache.set(selected.name, selected)+ const firstCreateStarted = deferred()+ const releaseFirstCreate = deferred()+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ firstCreateStarted.resolve(undefined)+ await releaseFirstCreate.promise+ return {}+ })+ const rec = makeReconciler(mockApi, cache)+ const context: ContextCRD = {+ name: 'dev',+ namespace: 'mcp-server',+ spec: { contextId: 'dev', mcpServers: ['mongo'] },+ }++ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })+ await firstCreateStarted.promise+ cache.set(selected.name, {+ ...selected,+ status: { conditions: [{ type: 'Ready', status: 'True' }] },+ })+ releaseFirstCreate.resolve(undefined)+ await reconcile++ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)+ })++ it('does not adopt an McpServer that appeared after Context selection', async () => {+ const cache = new Map<string, McpServerCRD>()+ const inventoryListed = deferred()+ const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ inventoryListed.resolve(undefined)+ return releaseInventory.promise+ })+ const rec = makeReconciler(mockApi, cache)+ const context: ContextCRD = {+ name: 'dev',+ namespace: 'mcp-server',+ spec: { contextId: 'dev', mcpServers: ['late-server'] },+ }++ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })+ await inventoryListed.promise+ cache.set('late-server', {+ name: 'late-server',+ namespace: 'mcp-server',+ uid: 'late-server-uid',+ generation: 1,+ spec: {+ contextRef: 'dev',+ image: 'late-server:new',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ })+ releaseInventory.resolve({ items: [] })+ await reconcile++ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {+ const cache = new Map<string, McpServerCRD>()+ cache.set('mongo', {+ name: 'mongo',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ },+ })++ const rec = makeReconciler(mockApi, cache)+ const context: ContextCRD = {+ name: 'dev',+ namespace: 'mcp-server',+ spec: { contextId: 'dev', mcpServers: ['mongo'] },+ }++ await rec.reconcileContext(context, { isCurrent: () => true })++ // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {+ namespace: string+ body: {+ metadata: { name: string; namespace: string; labels: Record<string, string> }+ spec: {+ podSelector: { matchLabels: Record<string, string> }+ policyTypes: string[]+ egress: Array<{+ to: Array<{+ namespaceSelector: { matchLabels: Record<string, string> }+ podSelector: { matchLabels: Record<string, string> }+ }>+ ports: Array<{ port: number }>+ }>+ }+ }+ }+ expect(egressCall.namespace).toBe('mcp-host')+ expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')+ expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')+ expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'dev',+ })+ // Egress targets mcp-server namespace + specific server pod+ const egressRule = egressCall.body.spec.egress[0]+ expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(+ 'mcp-server'+ )+ expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')+ expect(egressRule.ports[0].port).toBe(3000)+ })++ it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {+ const cache = new Map<string, McpServerCRD>()+ cache.set('mongo', {+ name: 'mongo',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'ctx-name',+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ },+ })++ const rec = makeReconciler(mockApi, cache)+ const context: ContextCRD = {+ name: 'ctx-name',+ namespace: 'mcp-server',+ spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },+ }++ await rec.reconcileContext(context, { isCurrent: () => true })++ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {+ body: {+ metadata: { labels: Record<string, string> }+ spec: { podSelector: { matchLabels: Record<string, string> } }+ }+ }+ expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'ctx-name',+ })+ })++ it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {+ const cache = new Map<string, McpServerCRD>()+ cache.set('mongo', {+ name: 'mongo',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ },+ })+const rec = makeReconciler(mockApi, cache)const context: ContextCRD = {name: 'dev',@@ -526,7 +1252,7 @@ describe('NetworkPolicyReconciler', () => {spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context)+ await rec.reconcileContext(context, { isCurrent: () => true })const rpcProxyEgressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[2][0] as {namespace: string@@ -566,6 +1292,118 @@ describe('NetworkPolicyReconciler', () => {// ─── L3: External Egress ───────────────────────────────────────────describe('L3 — reconcileExternalEgress', () => {+ it('removes a deterministic safety policy created after its desired-state fence is lost', async () => {+ const server: McpServerCRD = {+ name: 'race-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'race:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],+ },+ }+ const createStarted = deferred()+ const releaseCreate = deferred()+ let current = true+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ createStarted.resolve(undefined)+ await releaseCreate.promise+ return {+ metadata: {+ name: 'ext-egress-race-server-104-18-0-0-16-443',+ namespace: server.namespace,+ uid: 'created-race-policy',+ resourceVersion: '7',+ },+ }+ })++ const safety = (reconciler as any).reconcileExternalEgressSafety(+ server,+ [{ metadata: { name: 'ext-egress-race-server-stale' } }],+ () => current+ ) as Promise<boolean>++ await createStarted.promise+ current = false+ releaseCreate.resolve(undefined)++ expect(await safety).toBe(false)+ const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: created.metadata.name,+ namespace: server.namespace,+ body: { preconditions: { uid: 'created-race-policy', resourceVersion: '7' } },+ })+ )+ })++ it('refuses to adopt a foreign policy that collides with an external-egress name', async () => {+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],+ },+ }+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',+ namespace: 'mcp-server',+ uid: 'foreign-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': server.name,+ },+ },+ })++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('refuses to replace an HCC external-egress policy owned by a different server', async () => {+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],+ },+ }+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',+ namespace: 'mcp-server',+ uid: 'other-hcc-owner-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': 'different-server',+ },+ },+ })++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+it('creates egress policy for CIDR binding', async () => {const server: McpServerCRD = {name: 'openai-mcp',@@ -578,7 +1416,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body@@ -601,7 +1439,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body@@ -639,35 +1477,69 @@ describe('NetworkPolicyReconciler', () => {)})- it('fails closed when ExternalEgressReady status cannot be written', async () => {- const statusError = new Error('status update denied')- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)+ it('does not mutate policy or status after its McpServer authority lease is retired', async () => {+ const dnsStarted = deferred()+ const releaseDns = deferred<string[]>()+ vi.mocked(dns.resolve4).mockImplementationOnce(async () => {+ dnsStarted.resolve(undefined)+ return releaseDns.promise+ })+ let inventoryGeneration = 7+ const capturedGeneration = inventoryGenerationconst server: McpServerCRD = {- name: 'openai-mcp',+ name: 'authority-race-mcp',namespace: 'mcp-server',- generation: 7,+ uid: 'authority-race-uid',+ generation: 3,spec: {contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ image: 'authority-race:test',+ transport: { type: 'streamableHttp', port: 3000 },egressBindings: [{ dns: 'api.openai.com', port: 443 }],},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(- 'status update denied'- )+ const reconcile = reconciler.reconcileExternalEgress(server, {+ isCurrent: () => inventoryGeneration === capturedGeneration,+ })+ await dnsStarted.promise- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()+ // Retire and recover to a different generation. The recovered cache is+ // authoritative again, but never for this stale pass.+ inventoryGeneration = 9+ releaseDns.resolve(['1.2.3.4'])+ await reconcile++ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})- it('fails closed when ExternalEgressReady status cannot be read', async () => {- const statusError = new Error('status read denied')- mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)+ it('identity-fences the status patch and preserves unrelated conditions', async () => {+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ metadata: {+ uid: 'openai-mcp-uid',+ generation: 7,+ resourceVersion: 'status-rv-17',+ },+ status: {+ conditions: [+ {+ type: 'RuntimeReady',+ status: 'True',+ reason: 'DeploymentAvailable',+ message: 'Runtime deployment is available',+ lastTransitionTime: '2026-07-29T00:00:00.000Z',+ observedGeneration: 7,+ },+ ],+ },+ })const server: McpServerCRD = {name: 'openai-mcp',namespace: 'mcp-server',+ uid: 'openai-mcp-uid',generation: 7,spec: {contextRef: 'dev',@@ -677,14 +1549,201 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('status read denied')-- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- })+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })- it('creates explicit public-web egress with private and special-use ranges excluded', async () => {- const server: McpServerCRD = {+ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body+ expect(patch.slice(0, 3)).toEqual([+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },+ { op: 'test', path: '/metadata/generation', value: 7 },+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },+ ])+ expect(patch).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: '/status/conditions',+ value: expect.arrayContaining([+ expect.objectContaining({+ type: 'RuntimeReady',+ reason: 'DeploymentAvailable',+ }),+ expect.objectContaining({+ type: 'ExternalEgressReady',+ status: 'True',+ observedGeneration: 7,+ }),+ ]),+ }),+ ])+ )+ })++ it('fences every identity field read from status when synthetic input omits them', async () => {+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ metadata: {+ uid: 'openai-mcp-uid',+ generation: 7,+ resourceVersion: 'status-rv-17',+ },+ status: {},+ })+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })++ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body+ expect(patch.slice(0, 3)).toEqual([+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },+ { op: 'test', path: '/metadata/generation', value: 7 },+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },+ ])+ })++ it.each([+ {+ name: 'UID',+ currentMetadata: {+ uid: 'replacement-uid',+ generation: 7,+ resourceVersion: 'status-rv-18',+ },+ },+ {+ name: 'generation',+ currentMetadata: {+ uid: 'openai-mcp-uid',+ generation: 8,+ resourceVersion: 'status-rv-18',+ },+ },+ ])(+ 'rejects a stale $name before patching external egress status',+ async ({ currentMetadata }) => {+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ metadata: currentMetadata,+ status: {},+ })+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ uid: 'openai-mcp-uid',+ generation: 7,+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/stale McpServer/i)++ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ }+ )++ it('fails closed when a same-generation status writer wins the resourceVersion race', async () => {+ const conflict = Object.assign(new Error('JSON patch test failed'), { code: 409 })+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ metadata: {+ uid: 'openai-mcp-uid',+ generation: 7,+ resourceVersion: 'status-rv-17',+ },+ status: {+ conditions: [+ {+ type: 'RuntimeReady',+ status: 'True',+ reason: 'DeploymentAvailable',+ lastTransitionTime: '2026-07-29T00:00:00.000Z',+ },+ ],+ },+ })+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict)+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ uid: 'openai-mcp-uid',+ generation: 7,+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toBe(conflict)++ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.arrayContaining([+ {+ op: 'test',+ path: '/metadata/resourceVersion',+ value: 'status-rv-17',+ },+ ]),+ })+ )+ })++ it('fails closed when ExternalEgressReady status cannot be written', async () => {+ const statusError = new Error('status update denied')+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ generation: 7,+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ 'status update denied'+ )++ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()+ })++ it('fails closed when ExternalEgressReady status cannot be read', async () => {+ const statusError = new Error('status read denied')+ mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)+ const server: McpServerCRD = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ generation: 7,+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('status read denied')++ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ })++ it('creates explicit public-web egress with private and special-use ranges excluded', async () => {+ const server: McpServerCRD = {name: 'search-mcp',namespace: 'mcp-server',spec: {@@ -695,7 +1754,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(dns.resolve4).not.toHaveBeenCalled()expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)@@ -737,7 +1796,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/public-web external egress bindings/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -755,7 +1814,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/public-web external egress bindings/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -793,7 +1852,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})@@ -839,7 +1898,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})@@ -864,7 +1923,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].bodyconst resolvedPatch = patch.find(@@ -889,7 +1948,7 @@ describe('NetworkPolicyReconciler', () => {}// First reconcile bootstraps and writes the external-egress policy.- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -916,7 +1975,7 @@ describe('NetworkPolicyReconciler', () => {// Second resync: same DNS answers → same IP set → must NOT rewrite the// policy (a timestamp-only refresh is a no-op). This is the churn the audit// caught: before the fix, RESOLVED_AT + renewed expiry forced a write here.- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -932,7 +1991,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -964,7 +2023,7 @@ describe('NetworkPolicyReconciler', () => {// Same DNS → set unchanged (changed=false) but window aging → renewalDue=true// → the policy MUST be re-persisted (deleting `|| egressRenewalDue` breaks this,// reintroducing the M1 overlap-grace loss).- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const wrote =mockApi.createNamespacedNetworkPolicy.mock.calls.length +mockApi.replaceNamespacedNetworkPolicy.mock.calls.length@@ -982,7 +2041,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -1035,7 +2094,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })// The reconcile must rewrite the policy WITHOUT the private IP.const call =mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??@@ -1060,7 +2119,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })// The min observed TTL (ms) must reflect the 15s answer — deleting the// Math.min update leaves it Infinity and the resync degrades to the fixed// interval (the #299 under-sampling failure mode).@@ -1080,7 +2139,7 @@ describe('NetworkPolicyReconciler', () => {}// Round 1: DNS serves A only.resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.3', ttl: 15 }])- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -1093,7 +2152,7 @@ describe('NetworkPolicyReconciler', () => {// Round 2: DNS rotates to B. The written policy must be the UNION {A, B} —// this only holds if previousAnnotations rehydration is actually wired in.resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.4', ttl: 15 }])- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const call =mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??mockApi.createNamespacedNetworkPolicy.mock.calls[0]?.[0]@@ -1115,7 +2174,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -1126,7 +2185,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -1146,7 +2205,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -1163,7 +2222,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -1179,7 +2238,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -1198,7 +2257,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })// The reconciler must rewrite the policy back to the declared CIDR.const wrote =mockApi.createNamespacedNetworkPolicy.mock.calls.length +@@ -1219,7 +2278,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/External egress reconciliation failed/)}@@ -1240,7 +2299,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/resolved disallowed address/)@@ -1275,7 +2334,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/private, internal, metadata/)@@ -1296,7 +2355,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/resolved disallowed address/)@@ -1322,13 +2381,15 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: 'ext-egress-openai-mcp-old-example-com-443',- namespace: 'mcp-server',- })+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-openai-mcp-old-example-com-443',+ namespace: 'mcp-server',+ })+ )})it('fails closed when an orphaned external egress policy cannot be deleted', async () => {@@ -1351,7 +2412,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('delete denied')+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('delete denied')})it('deletes stale policies and fails when DNS resolution for that binding fails', async () => {@@ -1374,17 +2435,21 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/dns timeout/)+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/dns timeout/)expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: 'ext-egress-openai-mcp-api.openai.com-443',- namespace: 'mcp-server',- })- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: 'ext-egress-openai-mcp-old-example-com-443',- namespace: 'mcp-server',- })+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-openai-mcp-api.openai.com-443',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-openai-mcp-old-example-com-443',+ namespace: 'mcp-server',+ })+ )expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(expect.objectContaining({body: expect.arrayContaining([@@ -1415,13 +2480,36 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/must declare dns or cidr/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ it('bounds DNS resolution so a stuck resolver cannot block reconciliation indefinitely', async () => {+ vi.useFakeTimers()+ vi.mocked(dns.resolve4).mockImplementationOnce(() => new Promise(() => undefined))+ const server: McpServerCRD = {+ name: 'stuck-dns-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'stuck-dns:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }++ const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ const rejection = expect(reconcile).rejects.toThrow(/DNS resolution timed out after 5000ms/)+ await vi.advanceTimersByTimeAsync(5_000)+ await rejection++ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+it('skips if no egressBindings', async () => {const server: McpServerCRD = {name: 'basic-mcp',@@ -1433,7 +2521,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server)+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -1454,6 +2542,14 @@ describe('NetworkPolicyReconciler', () => {await expect((reconciler as any).listAllContextPolicies()).rejects.toThrow('api list denied')})+ it('propagates all-context mcp-host egress policy list failures', async () => {+ mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))++ await expect((reconciler as any).listAllContextEgressPolicies()).rejects.toThrow(+ 'api list denied'+ )+ })+it('propagates all external-egress policy list failures', async () => {mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))@@ -1463,81 +2559,2496 @@ describe('NetworkPolicyReconciler', () => {})})- describe('startup fullReconcile', () => {- it('reconciles external egress policies for existing servers on startup', async () => {+ describe('delete-context TOCTOU re-check', () => {+ it('stops mid-inventory when the context is recreated between two deletes', async () => {+ // deleteAllowed is a live GET, so its answer can change while the loop+ // runs. Hoisting the check out of the loop would satisfy every existing+ // test and still delete the recreated context's own policies.+ mockApi.listNamespacedNetworkPolicy.mockResolvedValueOnce({+ items: [+ { metadata: { name: 'ctx-allow-first' } },+ { metadata: { name: 'ctx-allow-second' } },+ ],+ })+ const deleteAllowed = vi+ .fn<() => Promise<boolean>>()+ .mockResolvedValueOnce(true)+ .mockResolvedValue(false)++ await reconciler.reconcileDeleteContext('dev', deleteAllowed)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'ctx-allow-first' })+ )+ expect(deleteAllowed).toHaveBeenCalledTimes(2)+ })++ it('re-checks before every mcp-host egress delete', async () => {+ mockApi.listNamespacedNetworkPolicy+ .mockResolvedValueOnce({ items: [] }) // context-allow+ .mockResolvedValueOnce({+ items: [+ { metadata: { name: 'host-egress-first' } },+ { metadata: { name: 'host-egress-second' } },+ ],+ })+ const deleteAllowed = vi+ .fn<() => Promise<boolean>>()+ .mockResolvedValueOnce(true)+ .mockResolvedValue(false)++ await reconciler.reconcileDeleteContext('dev', deleteAllowed)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'host-egress-first' })+ )+ })++ it('re-checks before every rpc-proxy egress delete', async () => {+ mockApi.listNamespacedNetworkPolicy+ .mockResolvedValueOnce({ items: [] }) // context-allow+ .mockResolvedValueOnce({ items: [] }) // mcp-host egress+ .mockResolvedValueOnce({+ items: [+ { metadata: { name: 'rpc-egress-first' } },+ { metadata: { name: 'rpc-egress-second' } },+ ],+ })+ const deleteAllowed = vi+ .fn<() => Promise<boolean>>()+ .mockResolvedValueOnce(true)+ .mockResolvedValue(false)++ await reconciler.reconcileDeleteContext('dev', deleteAllowed)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'rpc-egress-first' })+ )+ })+ })++ describe('L3 — DNS-derived retention by binding identity (B3)', () => {+ // Live DNS policy built through the reconciler's OWN builder so its identity+ // is exact-by-construction; each test drifts exactly one dimension.+ function makeDnsFixture() {const server: McpServerCRD = {- name: 'airtable-mcp',+ name: 'openai-mcp',namespace: 'mcp-server',spec: {contextRef: 'dev',- image: 'airtable:latest',- transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },- egressBindings: [{ dns: 'api.airtable.com', port: 443 }],+ image: 'openai:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],},}+ const binding = server.spec.egressBindings![0]+ const name = (reconciler as any).externalEgressPolicyName(server.name, binding) as string+ const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [+ '1.2.3.4/32',+ ]) as k8s.V1NetworkPolicy+ live.metadata!.uid = 'live-dns-uid'+ live.metadata!.resourceVersion = '11'+ return { server, binding, name, live }+ }++ async function runSafety(server: McpServerCRD, live: k8s.V1NetworkPolicy): Promise<boolean> {+ return (reconciler as any).reconcileExternalEgressSafety(+ server,+ [live],+ () => true+ ) as Promise<boolean>+ }++ function expectRevoked(name: string) {+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name,+ namespace: 'mcp-server',+ body: { preconditions: { uid: 'live-dns-uid', resourceVersion: '11' } },+ })+ )+ }- await reconciler.fullReconcile([], [server])+ it('(a) retains an unchanged DNS-derived allow — no revocation', async () => {+ const { server, live } = makeDnsFixture()+ await expect(runSafety(server, live)).resolves.toBe(true)+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })- const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(- (call: unknown[]) => {- const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }- return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'- }- )- expect(externalPolicies).toHaveLength(1)- const body = (- externalPolicies[0][0] as {- body: {- metadata: { name: string }- spec: { egress: Array<{ ports: Array<{ port: number }> }> }- }- }- ).body- expect(body.metadata.name).toContain('ext-egress-airtable-mcp')- expect(body.spec.egress[0].ports[0].port).toBe(443)+ it('(b) revokes on spec-level port drift under the same policy name', async () => {+ const { server, name, live } = makeDnsFixture()+ live.spec!.egress![0].ports = [{ port: 8443, protocol: 'TCP' }]+ await expect(runSafety(server, live)).resolves.toBe(true)+ expectRevoked(name)})- it('deletes orphaned external egress policies for servers no longer present', async () => {+ it('(c) revokes on protocol drift TCP→UDP', async () => {+ const { server, name, live } = makeDnsFixture()+ live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]+ await expect(runSafety(server, live)).resolves.toBe(true)+ expectRevoked(name)+ })++ it('(d) revokes a foreign-owned policy even when the spec is identical', async () => {+ const { server, name, live } = makeDnsFixture()+ live.metadata!.labels!['clerum.io/managed-by'] = 'foreign-controller'+ await expect(runSafety(server, live)).resolves.toBe(true)+ expectRevoked(name)+ })++ it('(f) revokes a degenerate zero-cidr "allow" (deny in disguise)', async () => {+ const { server, name, live } = makeDnsFixture()+ live.spec!.egress = []+ await expect(runSafety(server, live)).resolves.toBe(true)+ expectRevoked(name)+ })++ it('(e) queues recreation AT the revocation even when the pass then aborts and throws', async () => {+ const { server, binding, name } = makeDnsFixture()+ // Identity-drifted (protocol UDP) → this DNS allow is revoked (deleted).+ const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [+ '1.2.3.4/32',+ ]) as k8s.V1NetworkPolicy+ live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]+ live.metadata!.uid = 'live-dns-uid'+ live.metadata!.resourceVersion = '11'+mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('context-allow')) return { items: [] }- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-old-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'old-server' },- },- },- ],- }- }- return { items: [] }- }+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: !labelSelector && namespace === 'mcp-server' ? [live] : [],+ }))- await reconciler.fullReconcile([], [], { serverInventoryComplete: true })-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: 'ext-egress-old-server-api-example-com-443',- namespace: 'mcp-server',+ let resolved: McpServerCRD | undefined = server+ const spy = vi.fn()+ // Flip authority false AT the deletion, so the pass aborts right after the+ // revocation hook fires — reproducing the abort/throw window B3 must survive.+ mockApi.deleteNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ resolved = undefined+ return {}})++ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)++ await expect(+ rec.fullReconcile([], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentServer: () => resolved,+ onExternalEgressRevoked: spy,+ })+ ).rejects.toThrow(/inventory changed during authoritative revocation/)++ expect(spy).toHaveBeenCalledTimes(1)+ expect(spy).toHaveBeenCalledWith(+ expect.objectContaining({ name: server.name, namespace: server.namespace })+ )})+ })- it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(+ describe('startup fullReconcile', () => {+ it('discovers and revokes orphaned policies with one missing ownership marker in every safety lane', async () => {+ const partiallyLabelledByNamespace: Record<string, k8s.V1NetworkPolicy[]> = {+ 'mcp-server': [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'orphan-context',+ 'clerum.io/mcpserver': 'old-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ },+ {+ metadata: {+ name: 'ext-egress-orphan-server-old-example-com-443',+ labels: {+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': 'orphan-server',+ 'clerum.io/egress-class': 'exact-host',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ 'mcp-host': [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server-egress',+ labels: {+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'orphan-context',+ 'clerum.io/mcpserver': 'old-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ 'rpc-proxy': [+ {+ metadata: {+ name: 'rpc-egress-orphan-context-old-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'orphan-context',+ 'clerum.io/mcpserver': 'old-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: labelSelector ? [] : (partiallyLabelledByNamespace[namespace ?? ''] ?? []),+ })+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server-egress',+ namespace: 'mcp-host',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'rpc-egress-orphan-context-old-server',+ namespace: 'rpc-proxy',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-orphan-server-old-example-com-443',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it('repairs one-marker live widened policies in every lane before certification', async () => {+ const server: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'live-context',+ image: 'live:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443 }],+ },+ }+ const context: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [server.name] },+ }+ const cache = new Map([[server.name, server]])+ const rec = makeReconciler(mockApi, cache, mockCustomApi)++ await rec.reconcileContext(context, { isCurrent: () => true })+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })+ const generated = mockApi.createNamespacedNetworkPolicy.mock.calls.map(+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body+ )+ const widened = generated.map((policy, index) => ({+ ...policy,+ metadata: {+ ...policy.metadata,+ labels: Object.fromEntries(+ Object.entries(policy.metadata?.labels ?? {}).filter(+ ([label]) =>+ label !== (index % 2 === 0 ? 'clerum.io/managed-by' : 'clerum.io/policy-type')+ )+ ),+ },+ spec: {+ ...policy.spec,+ policyTypes: [...(policy.spec?.policyTypes ?? []), 'Ingress'],+ },+ }))++ vi.clearAllMocks()+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: labelSelector+ ? []+ : widened.filter(policy => policy.metadata?.namespace === namespace),+ })+ )+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ const ordering: string[] = []+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ ordering.push(`replace:${name}`)+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))++ await rec.fullReconcile([context], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentContext: () => context,+ resolveCurrentServer: () => server,+ onAuthoritativeRevocationComplete,+ })++ const certificationIndex = ordering.indexOf('certify')+ expect(certificationIndex).toBeGreaterThanOrEqual(4)+ for (const expectedName of [+ 'ctx-live-context-live-server',+ 'ctx-live-context-live-server-egress',+ 'rpc-egress-live-context-live-server',+ 'ext-egress-live-server-8-8-8-8-32-443',+ ]) {+ expect(ordering.indexOf(`replace:${expectedName}`)).toBeGreaterThanOrEqual(0)+ expect(ordering.indexOf(`replace:${expectedName}`)).toBeLessThan(certificationIndex)+ }+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it.each([+ [+ 'context ingress',+ 'mcp-server',+ {+ metadata: { name: 'ctx-reserved-context-server', labels: {} },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ },+ ],+ [+ 'host egress',+ 'mcp-host',+ {+ metadata: { name: 'ctx-reserved-context-server-egress', labels: {} },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ [+ 'rpc egress',+ 'rpc-proxy',+ {+ metadata: { name: 'rpc-egress-reserved-context-server', labels: {} },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ [+ 'external egress',+ 'mcp-server',+ {+ metadata: { name: 'ext-egress-reserved-server-example-443', labels: {} },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ },+ ],+ ])(+ 'fails closed without takeover for a both-markers-missing reserved %s name',+ async (_lane, namespace, candidate) => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({+ namespace: listedNamespace,+ labelSelector,+ }: {+ namespace?: string+ labelSelector?: string+ }) => ({+ items: !labelSelector && listedNamespace === namespace ? [candidate] : [],+ })+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ }+ )++ it('fails closed for an explicit foreign ownership conflict on an HCC-shaped policy', async () => {+ const candidate: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ext-egress-live-server-api-example-com-443',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': 'live-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],+ })+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('treats a non-reserved external policy missing its manager marker as ambiguous', async () => {+ const candidate: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'custom-egress-policy',+ labels: {+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': 'live-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],+ })+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('lets the broad snapshot shadow a selected-owned object that became unrelated at the same UID', async () => {+ const selected: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'legacy-custom-policy',+ uid: 'stable-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'orphan-context',+ 'clerum.io/mcpserver': 'old-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ const broadRelabelled: k8s.V1NetworkPolicy = {+ ...selected,+ metadata: {+ ...selected.metadata,+ resourceVersion: '8',+ labels: { 'tenant.example/managed-by': 'tenant-controller' },+ },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace !== 'mcp-server') return { items: [] }+ if (labelSelector?.includes('policy-type=context-allow')) return { items: [selected] }+ if (!labelSelector) return { items: [broadRelabelled] }+ return { items: [] }+ }+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it('leaves unrelated namespace policies untouched and still certifies', async () => {+ const unrelated: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'tenant-owned-policy',+ labels: {+ 'app.kubernetes.io/managed-by': 'tenant-operator',+ 'tenant.example/purpose': 'custom-egress',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: !labelSelector && namespace === 'mcp-server' ? [unrelated] : [],+ })+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('fails closed when the broad safety inventory LIST fails', async () => {+ mockApi.listNamespacedNetworkPolicy+ .mockResolvedValueOnce({ items: [] })+ .mockResolvedValueOnce({ items: [] })+ .mockResolvedValueOnce({ items: [] })+ .mockResolvedValueOnce({ items: [] })+ .mockRejectedValueOnce(new Error('broad inventory denied'))+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow('broad inventory denied')++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('withholds certification and mutation when Context authority is lost during the broad LIST', async () => {+ let contextAuthoritative = true+ const candidate: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ctx-orphan-context-old-server',+ labels: {+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'orphan-context',+ 'clerum.io/mcpserver': 'old-server',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (!labelSelector && namespace === 'mcp-server') {+ contextAuthoritative = false+ return { items: [candidate] }+ }+ return { items: [] }+ }+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => contextAuthoritative,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('does not reconcile a queued Context when its current contextId changed', async () => {+ const queuedContext: ContextCRD = {+ name: 'mutable-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'captured-context', mcpServers: [] },+ }+ const currentContext: ContextCRD = {+ ...queuedContext,+ spec: { contextId: 'replacement-context', mcpServers: [] },+ }+ const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)+ const effectKeys: string[] = []++ await reconciler.fullReconcile([queuedContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => false,+ resolveCurrentContext: () => currentContext,+ runContextEffect: async (contextId, work) => {+ effectKeys.push(contextId)+ await work()+ },+ })++ expect(effectKeys).toEqual(['captured-context', 'captured-context'])+ expect(reconcileContext).not.toHaveBeenCalled()+ })++ it('retires a full-pass Context lease when the selected object is replaced', async () => {+ const selected: ContextCRD = {+ name: 'full-pass-context',+ namespace: 'mcp-server',+ spec: { contextId: 'full-pass-context', description: 'old', mcpServers: [] },+ }+ const replacement: ContextCRD = {+ ...selected,+ spec: { ...selected.spec, description: 'new' },+ }+ let current = selected+ let selectedLease: (() => boolean) | undefined+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async (_context, options) => {+ selectedLease = options?.isCurrent+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ return true+ })++ const fullPass = reconciler.fullReconcile([selected], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentContext: () => current,+ })+ await mutationStarted.promise+ expect(selectedLease?.()).toBe(true)++ current = replacement+ expect(selectedLease?.()).toBe(false)+ releaseMutation.resolve(undefined)+ await fullPass+ })++ it('deletes orphaned Context policies in all three namespaces through the canonical contextId effect key', async () => {+ const effectKeys: string[] = []+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server-egress',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'rpc-egress-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ runContextEffect: async (contextId, work) => {+ effectKeys.push(contextId)+ await work()+ },+ })++ expect(effectKeys).toEqual(['orphan-context', 'orphan-context', 'orphan-context'])+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server-egress',+ namespace: 'mcp-host',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'rpc-egress-orphan-context-old-server',+ namespace: 'rpc-proxy',+ })+ )+ })++ it('routes desired and orphan Context policy lanes through canonical contextId effect keys', async () => {+ const desiredContext: ContextCRD = {+ name: 'desired-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'desired-context', mcpServers: [] },+ }+ const effectKeys: string[] = []+ vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'rpc-egress-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([desiredContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ runContextEffect: async (contextId, work) => {+ effectKeys.push(contextId)+ await work()+ },+ })++ expect(effectKeys).toEqual([+ 'orphan-context',+ 'orphan-context',+ 'desired-context',+ 'desired-context',+ ])+ })++ it('revokes every orphan allow lane before a slow additive Context effect', async () => {+ const desiredContext: ContextCRD = {+ name: 'desired-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'desired-context', mcpServers: [] },+ }+ const additiveStarted = deferred()+ const releaseAdditive = deferred()+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {+ additiveStarted.resolve(undefined)+ await releaseAdditive.promise+ return true+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-orphan-context-old-server-egress',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'rpc-egress-orphan-context-old-server',+ labels: { 'clerum.io/context': 'orphan-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-orphan-server-old.example',+ labels: { 'clerum.io/mcpserver': 'orphan-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ const fullPass = reconciler.fullReconcile([desiredContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ })++ await additiveStarted.promise+ try {+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-orphan-context-old-server-egress',+ namespace: 'mcp-host',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'rpc-egress-orphan-context-old-server',+ namespace: 'rpc-proxy',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-orphan-server-old.example',+ namespace: 'mcp-server',+ })+ )+ } finally {+ releaseAdditive.resolve(undefined)+ await fullPass+ }+ })++ it('revokes every stale allow lane for a live Context before signaling authoritative revocation', async () => {+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [] },+ }+ const additiveStarted = deferred()+ const releaseAdditive = deferred()+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {+ additiveStarted.resolve(undefined)+ await releaseAdditive.promise+ return true+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-live-context-removed-server',+ labels: { 'clerum.io/context': 'live-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-live-context-removed-server-egress',+ labels: { 'clerum.io/context': 'live-context' },+ },+ },+ ],+ }+ }+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'rpc-egress-live-context-removed-server',+ labels: { 'clerum.io/context': 'live-context' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ const fullPass = reconciler.fullReconcile([liveContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ await additiveStarted.promise+ try {+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-live-context-removed-server',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ctx-live-context-removed-server-egress',+ namespace: 'mcp-host',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'rpc-egress-live-context-removed-server',+ namespace: 'rpc-proxy',+ })+ )+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ } finally {+ releaseAdditive.resolve(undefined)+ await fullPass+ }+ })++ it('revokes stale external egress for a live McpServer before readiness certification', async () => {+ const liveServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [],+ },+ }+ const ordering: string[] = []+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-live-server-old.example-443',+ labels: { 'clerum.io/mcpserver': 'live-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('delete')+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ ordering.push('certify')+ })++ await reconciler.fullReconcile([], [liveServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-live-server-old.example-443',+ namespace: 'mcp-server',+ })+ )+ expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it('does not let one post-certification Context failure starve later additive owners', async () => {+ const contexts: ContextCRD[] = ['aaa-poisoned', 'bbb-healthy'].map(contextId => ({+ name: `${contextId}-resource`,+ namespace: 'mcp-server',+ spec: { contextId, mcpServers: [] },+ }))+ const ordering: string[] = []+ const additiveFailure = new Error('poisoned additive Context')+ vi.spyOn(reconciler, 'reconcileContext').mockImplementation(async context => {+ ordering.push(`context:${context.spec.contextId}`)+ if (context.spec.contextId === 'aaa-poisoned') throw additiveFailure+ return true+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ ordering.push('certify')+ })+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)++ const error = await reconciler+ .fullReconcile(contexts, [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ .then(+ () => undefined,+ failure => failure+ )++ expect(error).toBeInstanceOf(AggregateError)+ expect((error as AggregateError).errors).toEqual([additiveFailure])+ expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])+ expect(errorLog).toHaveBeenCalledWith(+ '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',+ additiveFailure+ )+ })++ it('replaces same-name Context allow policies in every lane before certification', async () => {+ const oldServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ transport: { type: 'streamableHttp', port: 4000 },+ },+ }+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },+ }+ const serverCache = new Map([[oldServer.name, oldServer]])+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)++ await rec.reconcileContext(liveContext, { isCurrent: () => true })+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body+ )+ expect(oldPolicies).toHaveLength(3)++ vi.clearAllMocks()+ serverCache.set(currentServer.name, currentServer)+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) return { items: [] }+ return {+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),+ }+ }+ )+ vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)+ vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)+ const ordering: string[] = []+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => {+ ordering.push(`replace:${namespace}`)+ return {}+ }+ )+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))++ await rec.fullReconcile([liveContext], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(ordering).toEqual([+ 'replace:mcp-server',+ 'replace:mcp-host',+ 'replace:rpc-proxy',+ 'certify',+ ])+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it.each([+ ['context-ingress', 'mcp-server'],+ ['context-host-egress', 'mcp-host'],+ ['context-rpc-egress', 'rpc-proxy'],+ ])(+ 'refuses a same-name %s safety replacement owned by another server',+ async (_lane, targetNamespace) => {+ const oldServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ transport: { type: 'streamableHttp', port: 4000 },+ },+ }+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },+ }+ const serverCache = new Map([[oldServer.name, oldServer]])+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)++ await rec.reconcileContext(liveContext, { isCurrent: () => true })+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body+ if (policy.metadata?.namespace !== targetNamespace) return policy+ return {+ ...policy,+ metadata: {+ ...policy.metadata,+ labels: {+ ...policy.metadata?.labels,+ 'clerum.io/mcpserver': 'different-server',+ },+ },+ }+ })++ vi.clearAllMocks()+ serverCache.set(currentServer.name, currentServer)+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) return { items: [] }+ return {+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),+ }+ }+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ rec.fullReconcile([liveContext], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/ownership/i)++ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ namespace: targetNamespace })+ )+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ }+ )++ it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {+ const oldServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ transport: { type: 'streamableHttp', port: 4000 },+ },+ }+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },+ }+ const serverCache = new Map([[oldServer.name, oldServer]])+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)++ await rec.reconcileContext(liveContext, { isCurrent: () => true })+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body+ )++ vi.clearAllMocks()+ serverCache.set(currentServer.name, currentServer)+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) return { items: [] }+ return {+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),+ }+ }+ )+ const replacementFailure = new Error('replace unavailable')+ mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(replacementFailure)+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ rec.fullReconcile([liveContext], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toBe(replacementFailure)++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(+ { outcome: 'failed' },+ expect.any(Number)+ )+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)+ })++ it('retains authoritative same-name policies when only API object key order differs', async () => {+ const server: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [server.name] },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)+ const reverseObjectKeys = (value: unknown): unknown => {+ if (Array.isArray(value)) return value.map(reverseObjectKeys)+ if (value === null || typeof value !== 'object') return value+ return Object.fromEntries(+ Object.entries(value as Record<string, unknown>)+ .reverse()+ .map(([key, entry]) => [key, reverseObjectKeys(entry)])+ )+ }++ await rec.reconcileContext(liveContext, { isCurrent: () => true })+ const existingPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body+ return { ...policy, spec: reverseObjectKeys(policy.spec) as k8s.V1NetworkPolicySpec }+ })+ vi.clearAllMocks()+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) return { items: [] }+ return {+ items: existingPolicies.filter(policy => policy.metadata?.namespace === namespace),+ }+ }+ )+ vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)+ vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)++ await rec.fullReconcile([liveContext], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ })++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('revokes same-name DNS egress before certifying and leaves refresh to its coordinator', async () => {+ const oldServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)++ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')+ expect(oldPolicy).toBeDefined()++ vi.clearAllMocks()+ const ordering: string[] = []+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: [oldPolicy!] } : { items: [] }+ )+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('replace')+ return {}+ })+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('delete')+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ ordering.push('certify')+ })++ await rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])+ expect(ordering).not.toContain('replace')+ expect(dns.resolve4).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it('revokes a divergent same-name DNS policy when its safe replacement cannot resolve', async () => {+ const oldServer: McpServerCRD = {+ name: 'live-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'live:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)++ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')+ expect(oldPolicy).toBeDefined()++ vi.clearAllMocks()+ const ordering: string[] = []+ vi.mocked(dns.resolve4).mockImplementation(async () => {+ ordering.push('dns')+ throw new Error('dns unavailable')+ })+ let currentPolicies = [oldPolicy!]+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('delete')+ currentPolicies = []+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ ordering.push('certify')+ })++ await expect(+ rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).resolves.toBeUndefined()++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: oldPolicy!.metadata!.name!,+ namespace: oldServer.namespace,+ })+ )+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(dns.resolve4).not.toHaveBeenCalled()+ expect(ordering).toEqual(['delete', 'certify'])+ })++ it('revokes a renamed DNS binding before certification and performs no DNS lookup on the safety path', async () => {+ const oldServer: McpServerCRD = {+ name: 'renamed-dns-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'dns:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'old.example.com', port: 443 }],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [{ dns: 'new.example.com', port: 443 }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!++ vi.clearAllMocks()+ const ordering: string[] = []+ let currentPolicies = [oldPolicy]+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ ordering.push('delete')+ currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)+ return {}+ })+ vi.mocked(dns.resolve4).mockImplementation(async () => {+ ordering.push('dns')+ return ['1.2.3.4']+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ expect(dns.resolve4).not.toHaveBeenCalled()+ ordering.push('certify')+ })++ await rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(ordering).toEqual(['delete', 'certify'])+ expect(dns.resolve4).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: oldPolicy.metadata!.name!,+ namespace: oldServer.namespace,+ body: {+ preconditions: {+ uid: `${oldServer.namespace}:${oldPolicy.metadata!.name!}:uid`,+ resourceVersion: '1',+ },+ },+ })+ })++ it('replaces deterministic drift and revokes renamed DNS before certification without resolving DNS', async () => {+ const oldServer: McpServerCRD = {+ name: 'mixed-drift-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'mixed:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [+ { cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' },+ { dns: 'old.example.com', port: 443, protocol: 'TCP' },+ ],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [+ { cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' },+ { dns: 'new.example.com', port: 443, protocol: 'TCP' },+ ],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .filter(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')++ vi.clearAllMocks()+ const ordering: string[] = []+ let currentPolicies = oldPolicies+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }+ )+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('replace')+ return {}+ })+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ ordering.push('delete')+ currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)+ return {}+ })+ vi.mocked(dns.resolve4).mockImplementation(async () => {+ ordering.push('dns')+ return ['1.2.3.4']+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ expect(dns.resolve4).not.toHaveBeenCalled()+ ordering.push('certify')+ })++ await rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(ordering).toEqual(['replace', 'delete', 'certify'])+ expect(dns.resolve4).not.toHaveBeenCalled()+ })++ it('aborts certification when an identity-drifted safety delete reports a recreate conflict', async () => {+ const server: McpServerCRD = {+ name: 'recreated-dns-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'dns:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443 }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!+ // B3: identity-drift the live policy (protocol TCP→UDP) so the safety pass+ // still revokes it. A same-intent (identity-stable) DNS allow is now+ // RETAINED (see the retention pins in the B3 describe block); the revoke /+ // abort machinery this test guards fires only for a drifted policy.+ existing.spec!.egress = existing.spec!.egress!.map(rule => ({+ ...rule,+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),+ }))++ vi.clearAllMocks()+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }+ )+ const conflict = Object.assign(new Error('delete precondition conflict'), { code: 409 })+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(conflict)+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ rec.fullReconcile([], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toBe(conflict)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: existing.metadata!.name!,+ namespace: server.namespace,+ body: {+ preconditions: {+ uid: `${server.namespace}:${existing.metadata!.name!}:uid`,+ resourceVersion: '1',+ },+ },+ })+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it.each([+ [+ 'foreign relabel',+ {+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': 'fresh-read-server',+ },+ },+ ],+ ['UID drift', { uid: 'replacement-uid' }],+ ['resourceVersion drift', { resourceVersion: '2' }],+ ])(+ 'aborts a deterministic safety replace after fresh-read %s',+ async (_case, metadataDrift) => {+ const oldServer: McpServerCRD = {+ name: 'fresh-read-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'fresh:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!++ vi.clearAllMocks()+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({+ metadata: {+ ...metadataDrift,+ },+ })+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/ownership/)++ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: existing.metadata!.name!,+ namespace: oldServer.namespace,+ })+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ }+ )++ it('uses the fresh identity to complete a deterministic safety replacement before certification', async () => {+ const oldServer: McpServerCRD = {+ name: 'fresh-success-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'fresh:test',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],+ },+ }+ const currentServer: McpServerCRD = {+ ...oldServer,+ spec: {+ ...oldServer.spec,+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!++ vi.clearAllMocks()+ const ordering: string[] = []+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }+ )+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('read')+ return { metadata: { resourceVersion: '1' } }+ })+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {+ ordering.push('replace')+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))++ await rec.fullReconcile([], [currentServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(ordering.slice(0, 3)).toEqual(['read', 'replace', 'certify'])+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: existing.metadata!.name!,+ namespace: oldServer.namespace,+ body: expect.objectContaining({+ metadata: expect.objectContaining({+ name: existing.metadata!.name!,+ namespace: oldServer.namespace,+ resourceVersion: '1',+ }),+ spec: expect.objectContaining({+ egress: expect.arrayContaining([+ expect.objectContaining({+ ports: [expect.objectContaining({ protocol: 'UDP' })],+ }),+ ]),+ }),+ }),+ })+ })++ it('revokes an identity-drifted DNS policy before readiness and defers its refresh', async () => {+ const server: McpServerCRD = {+ name: 'stable-dns-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'stable:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')+ expect(existing).toBeDefined()+ // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a+ // same-intent DNS allow is now retained. The revoke-before-readiness ++ // defer-refresh machinery this test guards still fires on the drifted one.+ existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({+ ...rule,+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),+ }))++ vi.clearAllMocks()+ let currentPolicies = [existing!]+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {+ currentPolicies = []+ return {}+ })+ const onAuthoritativeRevocationComplete = vi.fn()++ await rec.fullReconcile([], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: existing!.metadata!.name!,+ namespace: server.namespace,+ })+ )+ expect(dns.resolve4).not.toHaveBeenCalled()+ })++ it('does not certify safety when an identity-drifted DNS policy cannot be revoked', async () => {+ const server: McpServerCRD = {+ name: 'undeletable-dns-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'stable:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],+ },+ }+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')+ expect(existing).toBeDefined()+ // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a+ // same-intent DNS allow is now retained. The no-certify-when-revoke-fails+ // machinery this test guards still fires on the drifted one.+ existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({+ ...rule,+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),+ }))++ vi.clearAllMocks()+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) =>+ labelSelector?.includes('external-egress') ? { items: [existing!] } : { items: [] }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('delete denied'))+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ rec.fullReconcile([], [server], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow('delete denied')++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: existing!.metadata!.name!,+ namespace: server.namespace,+ })+ )+ })++ it('does not certify revocation when an earlier Context changes while a later owner is paused', async () => {+ const contexts: ContextCRD[] = ['first', 'later'].map(contextId => ({+ name: `${contextId}-resource`,+ namespace: 'mcp-server',+ spec: { contextId, mcpServers: [] },+ }))+ const currentContexts = new Map(contexts.map(context => [context.name, context]))+ let contextDesiredRevision = 10+ let laterPaused = false+ const laterStarted = deferred()+ const releaseLater = deferred()+ const onAuthoritativeRevocationComplete = vi.fn()++ const fullPass = reconciler.fullReconcile(contexts, [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentContext: name => currentContexts.get(name),+ runContextEffect: async (contextId, work) => {+ if (contextId === 'later' && !laterPaused) {+ laterPaused = true+ laterStarted.resolve(undefined)+ await releaseLater.promise+ }+ await work()+ },+ contextDesiredRevision: () => contextDesiredRevision,+ onAuthoritativeRevocationComplete,+ })++ await laterStarted.promise+ currentContexts.set('first-resource', {+ ...contexts[0],+ spec: { ...contexts[0].spec, mcpServers: ['replacement'] },+ })+ contextDesiredRevision += 1+ releaseLater.resolve(undefined)++ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('does not certify revocation when a new owner is added after the global inventory LISTs', async () => {+ const anchorContext: ContextCRD = {+ name: 'anchor-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'anchor', mcpServers: [] },+ }+ const currentContexts = new Map([[anchorContext.name, anchorContext]])+ let contextDesiredRevision = 21+ const safetyEffectStarted = deferred()+ const releaseSafetyEffect = deferred()+ let safetyEffectPaused = false+ const onAuthoritativeRevocationComplete = vi.fn()++ const fullPass = reconciler.fullReconcile([anchorContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentContext: name => currentContexts.get(name),+ runContextEffect: async (_contextId, work) => {+ if (!safetyEffectPaused) {+ safetyEffectPaused = true+ safetyEffectStarted.resolve(undefined)+ await releaseSafetyEffect.promise+ }+ await work()+ },+ contextDesiredRevision: () => contextDesiredRevision,+ onAuthoritativeRevocationComplete,+ })++ await safetyEffectStarted.promise+ currentContexts.set('late-resource', {+ name: 'late-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'late', mcpServers: [] },+ })+ contextDesiredRevision += 1+ releaseSafetyEffect.resolve(undefined)++ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('does not certify revocation when a new McpServer owner is added after inventory LISTs', async () => {+ const anchorServer: McpServerCRD = {+ name: 'anchor-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'anchor:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const currentServers = new Map([[anchorServer.name, anchorServer]])+ let serverDesiredRevision = 31+ const safetyEffectStarted = deferred()+ const releaseSafetyEffect = deferred()+ let safetyEffectPaused = false+ const onAuthoritativeRevocationComplete = vi.fn()++ const fullPass = reconciler.fullReconcile([], [anchorServer], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ resolveCurrentServer: name => currentServers.get(name),+ runServerEffect: async (_serverName, work) => {+ if (!safetyEffectPaused) {+ safetyEffectPaused = true+ safetyEffectStarted.resolve(undefined)+ await releaseSafetyEffect.promise+ }+ await work()+ },+ serverDesiredRevision: () => serverDesiredRevision,+ onAuthoritativeRevocationComplete,+ })++ await safetyEffectStarted.promise+ currentServers.set('late-server', {+ name: 'late-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'late:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ })+ serverDesiredRevision += 1+ releaseSafetyEffect.resolve(undefined)++ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('does not certify revocation when desired inventory changes while default policies are ensured', async () => {+ const context: ContextCRD = {+ name: 'default-window-context',+ namespace: 'mcp-server',+ spec: { contextId: 'default-window', mcpServers: [] },+ }+ const server: McpServerCRD = {+ name: 'default-window-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default-window',+ image: 'default-window:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ let contextDesiredRevision = 41+ let serverDesiredRevision = 51+ const defaultsStarted = deferred()+ const releaseDefaults = deferred()+ const onAuthoritativeRevocationComplete = vi.fn()+ vi.spyOn(reconciler, 'ensureDefaultPolicies').mockImplementation(async () => {+ defaultsStarted.resolve(undefined)+ await releaseDefaults.promise+ })++ const fullPass = reconciler.fullReconcile([context], [server], {+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ contextDesiredRevision: () => contextDesiredRevision,+ serverDesiredRevision: () => serverDesiredRevision,+ onAuthoritativeRevocationComplete,+ })++ await defaultsStarted.promise+ contextDesiredRevision += 1+ serverDesiredRevision += 1+ releaseDefaults.resolve(undefined)++ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('does not signal authoritative revocation when a live-owner delete fails', async () => {+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-live-context-removed-server',+ labels: { 'clerum.io/context': 'live-context' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(+ new Error('policy delete unavailable')+ )+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([liveContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow('policy delete unavailable')+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('does not delete or signal revocation after live Context authority is lost', async () => {+ const liveContext: ContextCRD = {+ name: 'live-context-resource',+ namespace: 'mcp-server',+ spec: { contextId: 'live-context', mcpServers: [] },+ }+ let authoritative = true+ const inventoryListed = deferred()+ const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ inventoryListed.resolve(undefined)+ return releaseInventory.promise+ })+ const onAuthoritativeRevocationComplete = vi.fn()++ const fullPass = reconciler.fullReconcile([liveContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => authoritative,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ await inventoryListed.promise+ authoritative = false+ releaseInventory.resolve({+ items: [+ {+ metadata: {+ name: 'ctx-live-context-removed-server',+ labels: { 'clerum.io/context': 'live-context' },+ },+ },+ ],+ })++ await fullPass+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('uses selected lane LISTs plus one broad LIST per bounded namespace before additive convergence', async () => {+ const contexts: ContextCRD[] = ['alpha', 'beta', 'gamma'].map(contextId => ({+ name: `${contextId}-resource`,+ namespace: 'mcp-server',+ spec: { contextId, mcpServers: [] },+ }))+ const servers: McpServerCRD[] = ['one', 'two'].map(name => ({+ name,+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: `${name}:latest`,+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }))+ const additiveStarted = deferred()+ const releaseAdditive = deferred()+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {+ additiveStarted.resolve(undefined)+ await releaseAdditive.promise+ return true+ })+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })++ const fullPass = reconciler.fullReconcile(contexts, servers, {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ })++ await additiveStarted.promise+ try {+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'mcp-server',+ labelSelector: expect.stringContaining('policy-type=context-allow'),+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'mcp-host',+ labelSelector: expect.stringContaining('policy-type=context-allow'),+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'rpc-proxy',+ labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'mcp-server',+ labelSelector: expect.stringContaining('policy-type=external-egress'),+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'mcp-server',+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'mcp-host',+ })+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'rpc-proxy',+ })+ } finally {+ releaseAdditive.resolve(undefined)+ await fullPass+ }+ })++ it('leaves startup external egress creation to its dedicated coordinator', async () => {+ const server: McpServerCRD = {+ name: 'airtable-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'airtable:latest',+ transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },+ egressBindings: [{ dns: 'api.airtable.com', port: 443 }],+ },+ }++ await reconciler.fullReconcile([], [server])++ const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(+ (call: unknown[]) => {+ const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }+ return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'+ }+ )+ expect(externalPolicies).toHaveLength(0)+ expect(dns.resolve4).not.toHaveBeenCalled()+ })++ it('deletes orphaned external egress policies for servers no longer present', async () => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('context-allow')) return { items: [] }+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-old-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'old-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], { serverInventoryComplete: true })++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'ext-egress-old-server-api-example-com-443',+ namespace: 'mcp-server',+ })+ )+ })++ it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }+ if (labelSelector?.includes('policy-type=context-allow')) {+ return {+ items: [+ {+ metadata: {+ name:+ namespace === 'mcp-host'+ ? 'malformed-host-egress'+ : 'malformed-context-ingress',+ labels: managedLabels,+ },+ },+ ],+ }+ }+ if (labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'malformed-rpc-proxy-egress',+ labels: managedLabels,+ },+ },+ ],+ }+ }+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'malformed-external-egress',+ labels: managedLabels,+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], {+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ serverInventoryComplete: true,+ })++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'malformed-context-ingress',+ namespace: 'mcp-server',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'malformed-host-egress',+ namespace: 'mcp-host',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'malformed-rpc-proxy-egress',+ namespace: 'rpc-proxy',+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: 'malformed-external-egress',+ namespace: 'mcp-server',+ })+ )+ })++ it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('context-allow')) return { items: [] }+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-old-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'old-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], { serverInventoryComplete: false })++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({+ name: 'ext-egress-old-server-api-example-com-443',+ namespace: 'mcp-server',+ })+ })++ it('finishes desired Context reconciliation but stops Context and rpc-proxy orphan cleanup when authority is lost after inventory listing', async () => {+ const desiredContext: ContextCRD = {+ name: 'current',+ namespace: 'mcp-server',+ spec: { contextId: 'current', mcpServers: [] },+ }+ const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)+ const contextInventoryAuthoritative = vi+ .fn()+ .mockReturnValueOnce(true)+ .mockReturnValueOnce(false)+ .mockReturnValue(true)++ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('policy-type=context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-deleted-context-old-server',+ labels: { 'clerum.io/context': 'deleted-context' },+ },+ },+ ],+ }+ }+ if (labelSelector?.includes('external-egress')) return { items: [] }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([desiredContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative,+ })++ expect(reconcileContext).toHaveBeenCalledWith(desiredContext, {+ isCurrent: expect.any(Function),+ })+ expect(contextInventoryAuthoritative).toHaveBeenCalled()+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({+ name: 'ctx-deleted-context-old-server',+ namespace: 'mcp-server',+ })+ expect(mockApi.listNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: 'rpc-proxy',+ labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),+ })+ )+ })++ it('stops external-egress orphan cleanup when authority is lost after inventory listing', async () => {+ const desiredServer: McpServerCRD = {+ name: 'current-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'current',+ image: 'current:latest',+ transport: { type: 'streamableHttp', url: 'http://current:3000', port: 3000 },+ },+ }+ const serverInventoryAuthoritative = vi+ .fn()+ .mockReturnValueOnce(true)+ .mockReturnValueOnce(false)+ .mockReturnValue(true)++ mockApi.listNamespacedNetworkPolicy.mockImplementation(async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('context-allow')) return { items: [] }- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }if (labelSelector?.includes('external-egress')) {return {items: [{metadata: {- name: 'ext-egress-old-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'old-server' },+ name: 'ext-egress-deleted-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'deleted-server' },},},],@@ -1547,13 +5058,146 @@ describe('NetworkPolicyReconciler', () => {})- await reconciler.fullReconcile([], [], { serverInventoryComplete: false })+ await reconciler.fullReconcile([], [desiredServer], {+ ensureDefaults: false,+ serverInventoryComplete: true,+ serverInventoryAuthoritative,+ })+ expect(serverInventoryAuthoritative).toHaveBeenCalled()expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({- name: 'ext-egress-old-server-api-example-com-443',+ name: 'ext-egress-deleted-server-api-example-com-443',namespace: 'mcp-server',})})++ it('does not delete any Context policy lane when a same-contextId CRD exists before its watch event arrives', async () => {+ mockCustomApi.listNamespacedCustomObject.mockResolvedValue({+ items: [{ metadata: { name: 'recreated-context' }, spec: { contextId: 'recreated' } }],+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-recreated-server',+ labels: { 'clerum.io/context': 'recreated' },+ },+ },+ ],+ }+ }+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {+ return {+ items: [+ {+ metadata: {+ name: 'ctx-recreated-server-egress',+ labels: { 'clerum.io/context': 'recreated' },+ },+ },+ ],+ }+ }+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'rpc-egress-recreated-server',+ labels: { 'clerum.io/context': 'recreated' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ })++ expect(mockCustomApi.listNamespacedCustomObject).toHaveBeenCalled()+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: 'mcp-host',+ labelSelector: expect.stringContaining('policy-type=context-allow'),+ })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('does not delete external-egress policy when a same-name McpServer exists before ADDED arrives', async () => {+ mockCustomApi.getNamespacedCustomObject.mockResolvedValue({+ metadata: { name: 'recreated-server', uid: 'new-uid' },+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-recreated-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'recreated-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ serverInventoryComplete: true,+ serverInventoryAuthoritative: () => true,+ })++ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({ plural: 'mcpservers', name: 'recreated-server' })+ )+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('delegates startup McpServer orphan safety to the canonical absence helper', async () => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-orphan-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'orphan-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )++ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ serverInventoryComplete: true,+ serverInventoryAuthoritative: () => true,+ })++ expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledTimes(1)+ expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledWith({+ inventoryAuthoritative: expect.any(Function),+ resolveCurrent: expect.any(Function),+ readCurrent: expect.any(Function),+ })+ })})describe('L3 — cleanupExternalEgress', () => {@@ -1564,7 +5208,16 @@ describe('NetworkPolicyReconciler', () => {await reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'ext-egress-openai-mcp-api-443',+ namespace: 'mcp-server',+ body: {+ preconditions: {+ uid: 'mcp-server:ext-egress-openai-mcp-api-443:uid',+ resourceVersion: '1',+ },+ },+ })})it('fails closed when stale external egress policy deletion fails', async () => {@@ -1590,5 +5243,467 @@ describe('NetworkPolicyReconciler', () => {reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')).resolves.toBeUndefined()})++ it('does not report a revocation when the policy was already absent', async () => {+ const error = Object.assign(new Error('not found'), { code: 404 })+ const onDeleted = vi.fn()+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [{ metadata: { name: 'ext-egress-openai-mcp-old-api-443' } }],+ })+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(error)++ await reconciler.cleanupExternalEgress(+ 'openai-mcp',+ 'mcp-server',+ undefined,+ undefined,+ onDeleted+ )++ expect(onDeleted).not.toHaveBeenCalled()+ })+ })++ describe('scoped delta certification vs. a stuck safety inventory', () => {+ const ambiguousReservedPolicy: k8s.V1NetworkPolicy = {+ metadata: { name: 'ctx-reserved-context-server', labels: {} },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ const deltaContext: ContextCRD = {+ name: 'default',+ namespace: 'mcp-server',+ spec: { contextId: 'default', mcpServers: [] },+ }++ function stickSafetyInventory(): void {+ // Only the namespace-wide safety inventory sees the foreign policy. The+ // label-scoped LISTs the delta lane uses stay clean, which is exactly why+ // the delta cannot vouch for the namespace.+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items: !labelSelector && namespace === 'mcp-server' ? [ambiguousReservedPolicy] : [],+ })+ )+ }++ it('refuses to certify a scoped delta while the safety pass is stuck on its inventory', async () => {+ stickSafetyInventory()+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()++ // The delta's own revocation still completes — its label-scoped LISTs are+ // clean. What it may not do is vouch for the namespace-wide inventory,+ // which the readiness lane checks separately.+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ })++ it('certifies a scoped delta again once a later safety pass completes', async () => {+ stickSafetyInventory()+ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: vi.fn(),+ })+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)++ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ const onAuthoritativeRevocationComplete = vi.fn()+ await reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })++ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)+ })++ it('does not certify the safety inventory until an authoritative pass has run', async () => {+ // Fail-closed default: before any authoritative safety pass certifies, the+ // namespace-wide inventory is NOT certified, even though the scoped delta's+ // own label-scoped revocation completes (returns true). A delta cannot+ // vouch for the namespace-wide inventory it never enumerated.+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ })+ })++ describe('lost delete fence during stale-allow revocation', () => {+ const staleContext: ContextCRD = {+ name: 'default',+ namespace: 'mcp-server',+ spec: { contextId: 'default', mcpServers: [] },+ }+ const stalePolicy: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ctx-default-alpha',+ namespace: 'mcp-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'default',+ 'clerum.io/mcpserver': 'alpha',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ const lostFence = Object.assign(new Error('the UID in the precondition does not match'), {+ code: 409,+ })++ function listOnlyTheStalePolicy(): void {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => ({+ items: namespace === 'mcp-server' ? [stalePolicy] : [],+ })+ )+ }++ it('refuses to certify a scoped delta instead of throwing the lost fence', async () => {+ listOnlyTheStalePolicy()+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)++ // 409 is what Kubernetes returns when the uid/resourceVersion+ // preconditions no longer match: the object live under that name is not+ // the one this pass classified. Nothing was revoked, so the delta must+ // decline to certify — and it must not blow up the caller, which would+ // only force a spurious full re-convergence.+ // The scoped delta lane reads this boolean and withholds the certificate,+ // so it is the one caller allowed to take the lost fence as an outcome+ // rather than a throw.+ expect(await reconciler.reconcileContext(staleContext, { honorsLostFence: true, isCurrent: () => true })).toBe(false)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'ctx-default-alpha', namespace: 'mcp-server' })+ )+ })++ it('withdraws certification when an additive revocation loses the fence after certifying', async () => {+ // The additive phase re-runs reconcileContext, which carries a second+ // revocation lane over a fresh LIST. Losing the fence there happens after+ // the pass already certified, and the outcome recorder is first-wins — so+ // the duration sample stays 'completed', which is right, but the safety+ // fence must still degrade. It is an assertion about live allows, and it+ // may only ever move toward unsafe.+ const additiveContext: ContextCRD = {+ name: 'additive',+ namespace: 'mcp-server',+ spec: { contextId: 'additive', mcpServers: [] },+ }+ const additiveStalePolicy: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ctx-additive-alpha',+ namespace: 'mcp-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'additive',+ 'clerum.io/mcpserver': 'alpha',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ // Self-synchronising: the stale policy only becomes visible once the pass+ // has already certified, so the safety phase is clean by construction and+ // only the additive re-LIST finds work.+ let certified = false+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => ({+ items: certified && namespace === 'mcp-server' ? [additiveStalePolicy] : [],+ })+ )+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(lostFence)+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ certified = true+ })++ await expect(+ reconciler.fullReconcile([additiveContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow()++ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ })++ it('marks the inventory uncertified the moment the authoritative pass loses the fence', async () => {+ // The doom must be visible mid-pass, not only after the unwind. A lost+ // fence is the one doom cause that does not bump a watch generation, so+ // it is the only one where the whole certification machinery stays green+ // while the pass is already condemned. A scoped delta interleaving in+ // that window would certify readiness over a stale allow.+ //+ // Two contexts, each with its own stale policy: the first loses the fence+ // and the second's delete gives an observation point while the pass is+ // condemned but still unwinding.+ const secondContext: ContextCRD = {+ name: 'second',+ namespace: 'mcp-server',+ spec: { contextId: 'second', mcpServers: [] },+ }+ const secondStalePolicy: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ctx-second-beta',+ namespace: 'mcp-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'context-allow',+ 'clerum.io/context': 'second',+ 'clerum.io/mcpserver': 'beta',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ }+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => ({+ items: namespace === 'mcp-server' ? [stalePolicy, secondStalePolicy] : [],+ })+ )+ let certifiedDuringUnwind: boolean | undefined+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {+ if (+ certifiedDuringUnwind === undefined &&+ mockApi.deleteNamespacedNetworkPolicy.mock.calls.length > 1+ ) {+ certifiedDuringUnwind = reconciler.hasCertifiedSafetyInventory()+ return {}+ }+ throw lostFence+ })++ await expect(+ reconciler.fullReconcile([staleContext, secondContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: vi.fn(),+ })+ ).rejects.toThrow()++ expect(certifiedDuringUnwind).toBe(false)+ })++ it('fails the additive phase when it loses the delete fence', async () => {+ listOnlyTheStalePolicy()+ // Safety phase revokes cleanly and certifies; the additive phase then+ // races another writer and loses the fence. The additive caller discards+ // reconcileContext's boolean, so swallowing the 409 there would record+ // the pass as a success, disarm the convergence retry, and leave a stale+ // allow live with no NetworkPolicy resync to recover it.+ mockApi.deleteNamespacedNetworkPolicy+ .mockResolvedValueOnce({})+ .mockRejectedValueOnce(lostFence)+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([staleContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow()++ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ })++ it('refuses to certify the safety pass instead of throwing the lost fence', async () => {+ listOnlyTheStalePolicy()+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)+ const onAuthoritativeRevocationComplete = vi.fn()++ await expect(+ reconciler.fullReconcile([staleContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/Desired NetworkPolicy inventory changed/)++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ })++ it('still fails loud when the delete fails for any other reason', async () => {+ listOnlyTheStalePolicy()+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)++ await expect(reconciler.reconcileContext(staleContext, { isCurrent: () => true })).rejects.toBe(forbidden)+ })++ it('still certifies when the stale policy was already gone', async () => {+ listOnlyTheStalePolicy()+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('not found'), { code: 404 })+ )++ expect(await reconciler.reconcileContext(staleContext, { isCurrent: () => true })).toBe(true)+ })+ })++ describe('same-name conflict creating a safety external-egress policy', () => {+ const server: McpServerCRD = {+ name: 'renamed-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'renamed:latest',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],+ },+ }+ const desiredName = 'ext-egress-renamed-server-104-18-0-0-16-443'+ const stalePredecessor: k8s.V1NetworkPolicy = {+ metadata: {+ name: 'ext-egress-renamed-server-stale',+ namespace: 'mcp-server',+ uid: 'stale-uid',+ resourceVersion: '3',+ },+ }++ function reconcileExternalEgressSafety(): Promise<boolean> {+ return (+ reconciler as unknown as {+ reconcileExternalEgressSafety: (+ server: McpServerCRD,+ existingPolicies: k8s.V1NetworkPolicy[],+ isCurrent: () => boolean+ ) => Promise<boolean>+ }+ ).reconcileExternalEgressSafety(server, [stalePredecessor], () => true)+ }++ it('converges an HCC-owned same-name policy instead of aborting the pass', async () => {+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: desiredName,+ namespace: server.namespace,+ uid: 'own-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': server.name,+ },+ },+ })++ // This is the only creation route that bypasses applyNetworkPolicy. A+ // same-name object HCC already owns is an idempotent re-run, not a+ // reason to fail the pass and hold readiness down.+ expect(await reconcileExternalEgressSafety()).toBe(true)+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: desiredName, namespace: server.namespace })+ )+ })++ it('still fails closed when the same-name policy is foreign', async () => {+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(+ Object.assign(new Error('already exists'), { code: 409 })+ )+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: desiredName,+ namespace: server.namespace,+ uid: 'foreign-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': server.name,+ },+ },+ })++ await expect(reconcileExternalEgressSafety()).rejects.toThrow(/ownership/i)+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('still fails loud when the create fails for any other reason', async () => {+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)++ await expect(reconcileExternalEgressSafety()).rejects.toBe(forbidden)+ })+ })++ describe('safety-pass duration on the uncertified return path', () => {+ it('samples a failed outcome when the pass returns without certifying', async () => {+ const onAuthoritativeRevocationComplete = vi.fn()++ // Losing inventory authority ends the pass without an exception and+ // without certifying. Operators alert on "the pass never completed", so+ // this path has to produce a `failed` sample — absence is not a signal.+ await expect(+ reconciler.fullReconcile([], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => false,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).resolves.toBeUndefined()++ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(+ { outcome: 'failed' },+ expect.any(Number)+ )+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)+ })++ it('keeps the completed sample when a certified pass fails an additive effect afterwards', async () => {+ const context: ContextCRD = {+ name: 'default',+ namespace: 'mcp-server',+ spec: { contextId: 'default', mcpServers: [] },+ }+ const additiveFailure = new Error('additive list unavailable')+ let certified = false+ const onAuthoritativeRevocationComplete = vi.fn(() => {+ certified = true+ })+ mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {+ if (certified) throw additiveFailure+ return { items: [] }+ })++ await expect(+ reconciler.fullReconcile([context], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete,+ })+ ).rejects.toThrow(/additive Context NetworkPolicy reconciliations failed/)++ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(+ { outcome: 'completed' },+ expect.any(Number)+ )+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)+ })})})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 8778887cd..70eab2a1a 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -27,6 +27,14 @@ import {POLICY_TYPE_LABEL,} from './constants'import { accumulateHostExactHostEgress } from './externalEgressAccumulator'+import {+ confirmAuthoritativeMcpServerAbsence,+ sameMcpServerDesiredRevision,+} from './mcpServerSafety'+import {+ networkPolicySafetyPassDurationSeconds,+ networkPolicySafetyPassPoliciesTotal,+} from './metrics'import {ContextCRD,EgressBinding,@@ -35,21 +43,181 @@ import {McpServerCrdStatus,McpServerResolvedEgressIP,} from './types'-import { applyNetworkPolicy, getErrorCode } from './utils'+import { applyNetworkPolicy, getErrorCode, replaceWithConflictRetry } from './utils'type JsonPatchOperation = {- op: 'add' | 'replace' | 'remove'+ op: 'add' | 'replace' | 'remove' | 'test'path: stringvalue?: unknown}+type NetworkPolicyMutationOptions = {+ // Required (not optional): reconcileContext and reconcileExternalEgress are the+ // two functions that CREATE/RETAIN context-allow (L2) and external-egress ALLOW+ // (L3) policies, so the authority fence must be supplied at compile time. A+ // fail-open `?? (() => true)` default here would silently readmit the exact+ // fail-open class hardened out of bindingPolicyReconciler (required isCurrent).+ isCurrent: () => boolean+ onRevoked?: () => void+ /**+ * Set only by a caller that reads the returned boolean and withholds+ * readiness on false. It downgrades a lost delete fence from a throw to a+ * reported outcome; a caller that discards the boolean must not set it, or a+ * stale allow survives while the pass is recorded as a success.+ */+ honorsLostFence?: boolean+}++export function sameContextDesiredRevision(expected: ContextCRD, current: ContextCRD): boolean {+ return (+ expected.name === current.name &&+ expected.namespace === current.namespace &&+ expected.uid === current.uid &&+ expected.generation === current.generation &&+ JSON.stringify(canonicalizeNetworkPolicyValue(expected.spec)) ===+ JSON.stringify(canonicalizeNetworkPolicyValue(current.spec))+ )+}++function canonicalizeNetworkPolicyValue(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(canonicalizeNetworkPolicyValue)+ if (value === null || typeof value !== 'object') return value+ return Object.fromEntries(+ Object.entries(value as Record<string, unknown>)+ .filter(([, entry]) => entry !== undefined)+ .sort(([left], [right]) => left.localeCompare(right))+ .map(([key, entry]) => [key, canonicalizeNetworkPolicyValue(entry)])+ )+}++function sameNetworkPolicySpec(+ existing: k8s.V1NetworkPolicy,+ desired: k8s.V1NetworkPolicy+): boolean {+ return (+ JSON.stringify(canonicalizeNetworkPolicyValue(existing.spec)) ===+ JSON.stringify(canonicalizeNetworkPolicyValue(desired.spec))+ )+}++function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {+ const labels = policy.metadata?.labels ?? {}+ return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[POLICY_TYPE_LABEL] === policyType+}+const GROUP = 'clerum.io'const VERSION = 'v1alpha1'const PLURAL_MCPSERVERS = 'mcpservers'+const PLURAL_CONTEXTS = 'contexts'const CONTEXT_LABEL = 'clerum.io/context'const EGRESS_CLASS_LABEL = 'clerum.io/egress-class'const RPC_PROXY_EGRESS_POLICY_TYPE = 'rpc-proxy-egress'const RPC_PROXY_APP_LABEL = 'rpc-proxy'++type SafetyInventoryLane =+ | 'context-ingress'+ | 'context-host-egress'+ | 'context-rpc-egress'+ | 'external-egress'++type SafetyInventoryClassification = 'owned' | 'repairable' | 'ambiguous' | 'unrelated'++function policyHasOwnerShape(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {+ const labels = policy.metadata?.labels ?? {}+ const hasServerOwner = Boolean(labels[MCPSERVER_LABEL])+ if (lane === 'external-egress') return hasServerOwner+ return Boolean(labels[CONTEXT_LABEL]) && hasServerOwner+}++function policyHasReservedName(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {+ const name = policy.metadata?.name ?? ''+ switch (lane) {+ case 'context-ingress':+ return name.startsWith('ctx-')+ case 'context-host-egress':+ return name.startsWith('ctx-') && name.endsWith('-egress')+ case 'context-rpc-egress':+ return name.startsWith('rpc-egress-')+ case 'external-egress':+ return name.startsWith('ext-egress-')+ }+}++function policyNameBelongsToDifferentLane(+ policy: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane+): boolean {+ const name = policy.metadata?.name ?? ''+ if (lane === 'context-ingress') return name.startsWith('ext-egress-')+ if (lane === 'external-egress') return name.startsWith('ctx-')+ return false+}++function expectedPolicyType(lane: SafetyInventoryLane): string {+ return lane === 'context-rpc-egress'+ ? RPC_PROXY_EGRESS_POLICY_TYPE+ : lane === 'external-egress'+ ? EXTERNAL_EGRESS_POLICY_TYPE+ : 'context-allow'+}++/**+ * Classify broad namespace inventory without taking ownership of unrelated+ * policies. A single missing HCC marker is recoverable only when owner labels+ * corroborate the lane. Reserved HCC names and conflicting ownership markers+ * are never adopted implicitly.+ */+function classifySafetyInventoryPolicy(+ policy: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane+): SafetyInventoryClassification {+ const labels = policy.metadata?.labels ?? {}+ const managedBy = labels[MANAGED_BY_LABEL]+ const policyType = labels[POLICY_TYPE_LABEL]+ const expectedType = expectedPolicyType(lane)+ const managedMatches = managedBy === MANAGED_BY_VALUE+ const typeMatches = policyType === expectedType+ const managedMissing = managedBy === undefined+ const typeMissing = policyType === undefined+ const reservedName = policyHasReservedName(policy, lane)+ const ownerShape = policyHasOwnerShape(policy, lane)++ if (managedMatches && typeMatches) return 'owned'++ const oneMarkerMissing = (managedMatches && typeMissing) || (managedMissing && typeMatches)+ if (managedMissing && typeMatches && ownerShape) {+ return reservedName ? 'repairable' : 'ambiguous'+ }+ if (managedMatches && typeMissing && ownerShape) {+ if (reservedName) return 'repairable'+ if (policyNameBelongsToDifferentLane(policy, lane)) return 'unrelated'+ return 'ambiguous'+ }++ const knownOtherHccType =+ policyType !== undefined &&+ ['context-allow', RPC_PROXY_EGRESS_POLICY_TYPE, EXTERNAL_EGRESS_POLICY_TYPE].includes(+ policyType+ ) &&+ !typeMatches+ const hasRawConflictingMarker =+ (managedBy !== undefined && !managedMatches) || (policyType !== undefined && !typeMatches)+ const hasConflictingMarker =+ (managedBy !== undefined && !managedMatches) ||+ (policyType !== undefined && !typeMatches && !knownOtherHccType)+ if (+ reservedName &&+ ((managedMissing && typeMissing) || hasRawConflictingMarker || oneMarkerMissing)+ ) {+ return 'ambiguous'+ }+ if (ownerShape && hasConflictingMarker) {+ return 'ambiguous'+ }++ return 'unrelated'+}+export const PUBLIC_EGRESS_EXCEPT_CIDRS = ['0.0.0.0/8','10.0.0.0/8',@@ -198,6 +366,45 @@ function k8sApiEgressPodSelector(namespace: string): k8s.V1LabelSelector {return { matchLabels: { 'clerum.io/k8s-api-egress': 'true' } }}+export interface NetworkPolicyFullReconcileOptions {+ serverInventoryComplete?: boolean+ ensureDefaults?: boolean+ /**+ * Dynamic authority checks are intentionally callbacks: watch authority can+ * be lost after the startup snapshot but before an orphan delete begins.+ * Omitted callbacks preserve the historical authoritative-by-default API.+ */+ contextInventoryAuthoritative?: () => boolean+ serverInventoryAuthoritative?: () => boolean+ runContextEffect?: (contextId: string, work: () => Promise<void>) => Promise<void>+ runServerEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>+ resolveCurrentContext?: (name: string) => ContextCRD | undefined+ resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined+ resolveCurrentServer?: (name: string) => McpServerCRD | undefined+ /**+ * Monotonic desired-state revisions fence ordinary watch events that arrive+ * after an owner has completed its safety turn. Watch generations remain the+ * separate authority fence for LIST -> WATCH recovery.+ */+ contextDesiredRevision?: () => number+ serverDesiredRevision?: () => number+ /**+ * Called only after every authoritative orphan-allow revocation lane has+ * completed. Additive Context/McpServer convergence intentionally happens+ * afterwards and must not delay this safety boundary.+ */+ onAuthoritativeRevocationComplete?: () => void+ /**+ * Fired at the moment each external-egress allow owned by an McpServer is+ * revoked (or replaced) during the authoritative safety pass — before any+ * isCurrent abort or inventory-changed throw can unwind the pass. The owner+ * uses it to queue an additive recreation for the affected server; queuing in+ * a pass epilogue instead would reproduce exactly the abort gap this hook+ * closes.+ */+ onExternalEgressRevoked?: (server: McpServerCRD) => void+}+export class NetworkPolicyReconciler {private networkingApi: k8s.NetworkingV1Apiprivate customApi: k8s.CustomObjectsApi@@ -205,6 +412,22 @@ export class NetworkPolicyReconciler {/** Reference to the MCP server cache so we can look up ports. */private serverCache: Map<string, McpServerCRD>+ /**+ * Set whenever an authoritative safety pass ends without certifying, cleared+ * as soon as one certifies again. Only that pass enumerates the bounded+ * namespaces and classifies every object it finds; the scoped delta lane sees+ * nothing but its own label selectors. A delta therefore cannot vouch for a+ * namespace whose classified inventory is still stuck on an unresolved+ * policy, and must not certify readiness on its behalf.+ *+ * Initialised fail-closed (true): hasCertifiedSafetyInventory() reports false+ * until the first authoritative pass actually certifies, so certification+ * cannot be implied before any pass has run. This does not depend on the+ * readiness gate's generation-equality check for safety — it removes that+ * coupling rather than relying on it.+ */+ private safetyPassLeftUncertified = true+// H2 (issue #299): smallest DNS TTL (ms) observed across external-egress// resolutions. The resync loop advances to <= this/2 so a rotating low-TTL host// is sampled every rotation. Ratchets down and never recovers until restart —@@ -303,7 +526,7 @@ export class NetworkPolicyReconciler {},}- await applyNetworkPolicy(this.networkingApi, name, ns, policy, '[NetPol]')+ await this.applyFixedPolicy(name, ns, policy, 'default-deny')}}@@ -377,7 +600,7 @@ export class NetworkPolicyReconciler {egress,},}- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)}/** Allow egress to host-context-controller API gateway in control-plane namespace. */@@ -414,7 +637,7 @@ export class NetworkPolicyReconciler {],},}- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)}/**@@ -459,7 +682,7 @@ export class NetworkPolicyReconciler {],},}- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)}/**@@ -502,46 +725,26 @@ export class NetworkPolicyReconciler {},}- await this.applyPolicy(name, policy)+ await this.applyFixedPolicy(name, config.namespace, policy, 'allow-api')}// ─── Context-Based Policies ──────────────────────────────────────────- /**- * Reconcile NetworkPolicies for a Context CRD.- * Creates one policy per allowed MCP server.- */- async reconcileContext(context: ContextCRD): Promise<void> {+ private buildContextAllowPolicies(+ context: ContextCRD,+ server: McpServerCRD+ ): {+ ingress: k8s.V1NetworkPolicy+ hostEgress: k8s.V1NetworkPolicy+ rpcProxyEgress: k8s.V1NetworkPolicy+ } {const contextId = context.spec.contextId- const allowedServers = context.spec.mcpServers || []-- console.log(- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`- )+ const serverName = server.name+ const name = this.policyName(contextId, serverName)+ const port = server.spec.transport.port || 3000- // Build desired policy names- const desiredNames = new Set(- allowedServers.map(serverName => this.policyName(contextId, serverName))- )-- // Get existing policies for this context- const existingPolicies = await this.listPoliciesForContext(contextId)-- // Create or update policies for each allowed server- for (const serverName of allowedServers) {- const server = this.serverCache.get(serverName)- if (!server) {- console.warn(- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`- )- continue- }-- const name = this.policyName(contextId, serverName)- const port = server.spec.transport.port || 3000-- // L2 ingress: allow scoped mcp-host/rpc-proxy access to this MCP server.- const ingressPolicy: k8s.V1NetworkPolicy = {+ return {+ ingress: {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {@@ -590,7 +793,6 @@ export class NetworkPolicyReconciler {},},{- // MCP Proxy ingress: allow proxy to reach MCP serversnamespaceSelector: {matchLabels: {'kubernetes.io/metadata.name': config.namespace,@@ -607,18 +809,12 @@ export class NetworkPolicyReconciler {},],},- }-- await this.applyPolicy(name, ingressPolicy)-- // L2 egress counterpart: allow traffic FROM mcp-host pods TO this MCP server- // Without this, L0 egress deny-all blocks agents from reaching MCP servers.- const egressName = `${name}-egress`- const egressPolicy: k8s.V1NetworkPolicy = {+ },+ hostEgress: {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {- name: egressName,+ name: `${name}-egress`,namespace: config.hostNamespace,labels: {[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,@@ -655,78 +851,322 @@ export class NetworkPolicyReconciler {},],},+ },+ rpcProxyEgress: this.buildRpcProxyEgressPolicy(contextId, serverName, port),+ }+ }++ /**+ * Reconcile NetworkPolicies for a Context CRD.+ * Creates one policy per allowed MCP server.+ *+ * Returns whether this Context's stale-allow revocation ran to completion.+ * Callers that certify readiness from a scoped delta MUST honour it: the pass+ * aborts as soon as its authority fence breaks, and an aborted pass leaves+ * stale allows live.+ */+ async reconcileContext(+ context: ContextCRD,+ options: NetworkPolicyMutationOptions+ ): Promise<boolean> {+ const callerIsCurrent = options.isCurrent+ if (!callerIsCurrent()) return false+ const contextId = context.spec.contextId+ const allowedServers = context.spec.mcpServers || []+ const selectedServers = new Map(+ allowedServers.map(serverName => [serverName, this.serverCache.get(serverName)])+ )+ const isCurrent = (): boolean =>+ callerIsCurrent() &&+ allowedServers.every(serverName => {+ const selected = selectedServers.get(serverName)+ const current = this.serverCache.get(serverName)+ if (!selected || !current) return selected === current+ return sameMcpServerDesiredRevision(selected, current)+ })+ if (!isCurrent()) return false++ console.log(+ `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`+ )++ // Create or update policies for each allowed server.+ for (const serverName of allowedServers) {+ if (!isCurrent()) return false+ const server = this.serverCache.get(serverName)+ if (!server) {+ console.warn(+ `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`+ )+ continue}- await applyNetworkPolicy(- this.networkingApi,- egressName,+ const desired = this.buildContextAllowPolicies(context, server)+ const ingressName = desired.ingress.metadata!.name!+ await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)+ if (!isCurrent()) return false++ const hostEgressName = desired.hostEgress.metadata!.name!+ await this.applyOwnedPolicy(+ hostEgressName,config.hostNamespace,- egressPolicy,- '[NetPol]'+ desired.hostEgress,+ 'context-host-egress',+ isCurrent+ )+ if (!isCurrent()) return false++ const rpcProxyEgressName = desired.rpcProxyEgress.metadata!.name!+ await this.applyOwnedPolicy(+ rpcProxyEgressName,+ config.rpcProxyNamespace,+ desired.rpcProxyEgress,+ 'context-rpc-egress',+ isCurrent)+ }++ // Re-LIST every lane after writes so a same-name policy that was just+ // replaced is compared in its new form and cannot delete itself.+ return await this.revokeStaleContextAllows(+ context,+ isCurrent,+ undefined,+ undefined,+ options.honorsLostFence === true+ )+ }- // L2 rpc-proxy egress: allow rpc-proxy pods to reach this MCP server- await this.ensureRpcProxyEgress(contextId, serverName, port)+ /**+ * Whether the last authoritative safety pass certified its namespace-wide+ * inventory. Only that pass enumerates the bounded namespaces and classifies+ * every object it finds; a scoped delta sees nothing but its own label+ * selectors, so it cannot vouch for a namespace whose classified inventory is+ * still stuck. The readiness lane consults this before letting a delta+ * replace the safety certificate.+ */+ hasCertifiedSafetyInventory(): boolean {+ return !this.safetyPassLeftUncertified+ }++ private async revokeStaleContextAllows(+ context: ContextCRD,+ isCurrent: () => boolean,+ listedPolicies?: {+ context?: k8s.V1NetworkPolicy[]+ hostEgress?: k8s.V1NetworkPolicy[]+ rpcProxyEgress?: k8s.V1NetworkPolicy[]+ },+ onRevoked?: () => void,+ callerHonorsLostFence = false+ ): Promise<boolean> {+ if (!isCurrent()) return false+ const safetySnapshotProvided = listedPolicies !== undefined+ // A delete whose uid/resourceVersion preconditions were rejected revoked+ // nothing. Reporting it instead of throwing is only safe for a caller that+ // reads the returned boolean: the authoritative snapshot lane aborts its+ // pass on it, and the scoped delta lane declines to certify. Every other+ // caller discards the boolean, so for them a swallowed 409 would leave a+ // stale allow live while the pass is recorded as a success — they keep the+ // loud default and let the failure reach the retry policy.+ const lostFenceOutcome =+ safetySnapshotProvided || callerHonorsLostFence ? ('report' as const) : ('throw' as const)+ let lostDeleteFence = false+ /**+ * Revoke one orphaned policy in any lane. Returns false only when the+ * caller must abort the pass outright, which is what the authoritative+ * safety-snapshot lane does with a lost delete fence; the scoped lane+ * records it and keeps going so the rest of its work still lands.+ */+ const revokeOrphanedPolicy = async (+ namespace: string,+ existing: k8s.V1NetworkPolicy+ ): Promise<boolean> => {+ const deleted = await this.deleteSafetyPolicySnapshot(+ namespace,+ existing,+ isCurrent,+ undefined,+ lostFenceOutcome+ )+ if (deleted) return true+ if (safetySnapshotProvided) {+ // The authoritative pass is condemned from here: it will unwind and+ // throw. Losing the fence is the only doom cause that does not bump a+ // watch generation, so without this the certification machinery stays+ // green for the whole unwind and a concurrent scoped delta can certify+ // readiness over the allow this pass failed to revoke. Record it the+ // moment it is known rather than when the pass finally reports.+ this.safetyPassLeftUncertified = true+ return false+ }+ lostDeleteFence = true+ return true+ }+ const contextId = context.spec.contextId+ const allowedServers = context.spec.mcpServers || []+ const desiredIngress = new Map<string, k8s.V1NetworkPolicy>()+ const desiredHostEgress = new Map<string, k8s.V1NetworkPolicy>()+ const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()+ for (const serverName of allowedServers) {+ const server = this.serverCache.get(serverName)+ if (!server) continue+ const desired = this.buildContextAllowPolicies(context, server)+ desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)+ desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)+ desiredRpcProxyEgress.set(desired.rpcProxyEgress.metadata!.name!, desired.rpcProxyEgress)}- // Delete policies for servers no longer in the context+ const existingPolicies =+ listedPolicies?.context ?? (await this.listPoliciesForContext(contextId))+ if (!isCurrent()) return falsefor (const existing of existingPolicies) {+ if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- if (!desiredNames.has(existingName)) {+ const desired = desiredIngress.get(existingName)+ if (!desired) {console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)- await this.deletePolicy(existingName)+ if (!(await revokeOrphanedPolicy(config.namespace, existing))) return false+ onRevoked?.()+ } else if (+ !sameNetworkPolicySpec(existing, desired) ||+ !hasExpectedPolicyOwnership(existing, 'context-allow')+ ) {+ console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)+ if (safetySnapshotProvided) {+ if (+ !(await this.replaceSafetyPolicySnapshot(+ config.namespace,+ existing,+ desired,+ 'context-ingress',+ isCurrent+ ))+ ) {+ return false+ }+ } else {+ await this.replaceSafetyPolicySnapshot(+ config.namespace,+ existing,+ desired,+ 'context-ingress',+ isCurrent+ )+ }+ onRevoked?.()}}- // Delete orphaned L2 egress counterparts in mcp-host namespace- const existingEgressPolicies = await this.listEgressPoliciesForContext(contextId)- const desiredEgressNames = new Set(- allowedServers.map(serverName => `${this.policyName(contextId, serverName)}-egress`)- )+ const existingEgressPolicies =+ listedPolicies?.hostEgress ?? (await this.listEgressPoliciesForContext(contextId))+ if (!isCurrent()) return falsefor (const existing of existingEgressPolicies) {+ if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- if (!desiredEgressNames.has(existingName)) {+ const desired = desiredHostEgress.get(existingName)+ if (!desired) {console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)- await this.deleteEgressPolicy(existingName)+ if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return false+ onRevoked?.()+ } else if (+ !sameNetworkPolicySpec(existing, desired) ||+ !hasExpectedPolicyOwnership(existing, 'context-allow')+ ) {+ console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)+ if (safetySnapshotProvided) {+ if (+ !(await this.replaceSafetyPolicySnapshot(+ config.hostNamespace,+ existing,+ desired,+ 'context-host-egress',+ isCurrent+ ))+ ) {+ return false+ }+ } else {+ await this.replaceSafetyPolicySnapshot(+ config.hostNamespace,+ existing,+ desired,+ 'context-host-egress',+ isCurrent+ )+ }+ onRevoked?.()}}- // Delete orphaned rpc-proxy egress counterparts. Keeping these after a- // server is removed from a Context would preserve an old in-cluster route.- const existingRpcProxyPolicies = await this.listRpcProxyEgressPoliciesForContext(contextId)- const desiredRpcProxyNames = new Set(- allowedServers.map(serverName => `rpc-egress-${contextId}-${serverName}`)- )+ const existingRpcProxyPolicies =+ listedPolicies?.rpcProxyEgress ?? (await this.listRpcProxyEgressPoliciesForContext(contextId))+ if (!isCurrent()) return falsefor (const existing of existingRpcProxyPolicies) {+ if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- if (!desiredRpcProxyNames.has(existingName)) {+ const desired = desiredRpcProxyEgress.get(existingName)+ if (!desired) {console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)- await this.deletePolicyInNamespace(config.rpcProxyNamespace, existingName)+ if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return false+ onRevoked?.()+ } else if (+ !sameNetworkPolicySpec(existing, desired) ||+ !hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)+ ) {+ console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)+ if (safetySnapshotProvided) {+ if (+ !(await this.replaceSafetyPolicySnapshot(+ config.rpcProxyNamespace,+ existing,+ desired,+ 'context-rpc-egress',+ isCurrent+ ))+ ) {+ return false+ }+ } else {+ await this.replaceSafetyPolicySnapshot(+ config.rpcProxyNamespace,+ existing,+ desired,+ 'context-rpc-egress',+ isCurrent+ )+ }+ onRevoked?.()}}++ return isCurrent() && !lostDeleteFence}/*** Delete all NetworkPolicies for a deleted Context.*/- async reconcileDeleteContext(contextId: string): Promise<void> {+ async reconcileDeleteContext(+ contextId: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)const policies = await this.listPoliciesForContext(contextId)for (const policy of policies) {- const name = policy.metadata?.name || ''- await this.deletePolicy(name)+ if (deleteAllowed && !(await deleteAllowed())) return+ await this.deleteSafetyPolicySnapshot(config.namespace, policy)}// Also delete L2 egress counterparts in mcp-host namespaceconst egressPolicies = await this.listEgressPoliciesForContext(contextId)for (const policy of egressPolicies) {- const name = policy.metadata?.name || ''- await this.deleteEgressPolicy(name)+ if (deleteAllowed && !(await deleteAllowed())) return+ await this.deleteSafetyPolicySnapshot(config.hostNamespace, policy)}// Also delete L2 rpc-proxy egress policies- await this.deleteRpcProxyPoliciesForContext(contextId)+ await this.deleteRpcProxyPoliciesForContext(contextId, deleteAllowed)}/**@@ -735,80 +1175,742 @@ export class NetworkPolicyReconciler {async fullReconcile(contexts: ContextCRD[],servers: McpServerCRD[] = [],- options: { serverInventoryComplete?: boolean; ensureDefaults?: boolean } = {}+ options: NetworkPolicyFullReconcileOptions = {}+ ): Promise<void> {+ const onAuthoritativeRevocationComplete = options.onAuthoritativeRevocationComplete+ if (!onAuthoritativeRevocationComplete) {+ await this.fullReconcileInternal(contexts, servers, options)+ return+ }++ const safetyPassStartedAt = performance.now()+ let safetyPassOutcomeRecorded = false+ /**+ * Settles both observable results of this pass: the duration sample and the+ * cross-lane fence the scoped delta reads. First outcome wins for both — a+ * pass that certified and only then failed an additive effect still revoked+ * every stale allow its inventory implied.+ */+ const recordSafetyPassOutcome = (outcome: 'completed' | 'failed'): void => {+ // The fence is an assertion about live allows, so it only ever moves+ // toward unsafe. The additive phase re-runs reconcileContext, which+ // carries a second revocation lane over a fresh LIST; losing the fence+ // there happens after this pass certified, and first-outcome-wins would+ // otherwise leave the gate green over an allow that was not revoked.+ if (outcome === 'failed') this.safetyPassLeftUncertified = true+ // The duration sample describes the pass's primary outcome, so it stays+ // first-wins: a certified pass that later fails an additive effect did+ // complete its authoritative revocation.+ if (safetyPassOutcomeRecorded) return+ safetyPassOutcomeRecorded = true+ this.safetyPassLeftUncertified = outcome === 'failed'+ networkPolicySafetyPassDurationSeconds.observe(+ { outcome },+ (performance.now() - safetyPassStartedAt) / 1_000+ )+ }++ try {+ await this.fullReconcileInternal(contexts, servers, {+ ...options,+ onAuthoritativeRevocationComplete: () => {+ onAuthoritativeRevocationComplete()+ recordSafetyPassOutcome('completed')+ },+ })+ // The pass returned without certifying: no exception, no certificate.+ // Operators alert on this exact condition, so it must produce a sample+ // rather than the absence of one. Stated as a condition rather than+ // leaning on first-outcome-wins to swallow it, because the fence below is+ // deliberately not first-wins.+ if (!safetyPassOutcomeRecorded) recordSafetyPassOutcome('failed')+ } catch (error) {+ recordSafetyPassOutcome('failed')+ throw error+ }+ }++ private async fullReconcileInternal(+ contexts: ContextCRD[],+ servers: McpServerCRD[] = [],+ options: NetworkPolicyFullReconcileOptions = {}): Promise<void> {console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)+ // This is intentionally scoped to callers that use the authoritative+ // revocation callback. Ordinary additive reconciles must not be presented+ // as readiness safety passes.+ const measureSafetyPass = options.onAuthoritativeRevocationComplete !== undefined+ let safetyPassListedPolicies = 0+ let safetyPassRevokedPolicies = 0+ const recordSafetyPassRevocation = (): void => {+ if (measureSafetyPass) safetyPassRevokedPolicies += 1+ }++ // The revision fence starts at invocation, before any asynchronous+ // default-policy setup can interleave a watch update with this snapshot.+ const selectedContextDesiredRevision = options.contextDesiredRevision?.()+ const selectedServerDesiredRevision = options.serverDesiredRevision?.()+ const runContextEffect =+ options.runContextEffect ?? ((_contextId: string, work: () => Promise<void>) => work())+ const runServerEffect =+ options.runServerEffect ?? ((_serverName: string, work: () => Promise<void>) => work())+ const contextInventoryIsCurrent = (): boolean =>+ (options.contextInventoryAuthoritative?.() ?? true) &&+ (options.serverInventoryAuthoritative?.() ?? true)+if (options.ensureDefaults !== false) {await this.ensureDefaultPolicies()}- // Reconcile each context- for (const context of contexts) {- await this.reconcileContext(context)+ // Authority is monotonic within this pass. Once a watch loses authority,+ // this inventory snapshot cannot safely become authoritative again even if+ // the callback flips back before the next delete.+ let contextAuthorityLost = false+ let contextAuthorityWarningLogged = false+ const contextCleanupAuthoritative = (): boolean => {+ if (+ !contextAuthorityLost &&+ options.contextInventoryAuthoritative &&+ !options.contextInventoryAuthoritative()+ ) {+ contextAuthorityLost = true+ }+ if (contextAuthorityLost && !contextAuthorityWarningLogged) {+ contextAuthorityWarningLogged = true+ console.warn(+ '[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost'+ )+ }+ return !contextAuthorityLost}- // Reconcile external egress for all existing servers on startup so a- // controller restart converges pre-existing McpServer CRDs too.- for (const server of servers) {- await this.reconcileExternalEgress(server)+ let serverAuthorityLost = false+ let serverAuthorityWarningLogged = false+ const serverCleanupAuthoritative = (): boolean => {+ if (+ !serverAuthorityLost &&+ options.serverInventoryAuthoritative &&+ !options.serverInventoryAuthoritative()+ ) {+ serverAuthorityLost = true+ }+ if (serverAuthorityLost && !serverAuthorityWarningLogged) {+ serverAuthorityWarningLogged = true+ console.warn(+ '[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost'+ )+ }+ return !serverAuthorityLost}- // Delete policies for contexts that no longer existconst desiredContextIds = new Set(contexts.map(c => c.spec.contextId))- const allContextPolicies = await this.listAllContextPolicies()-- for (const policy of allContextPolicies) {- const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]- if (contextId && !desiredContextIds.has(contextId)) {- const name = policy.metadata?.name || ''- console.log(- `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`- )- await this.deletePolicy(name)+ const cleanupOrphanedContextPolicies = async (+ policies: k8s.V1NetworkPolicy[],+ deletePolicy: (policy: k8s.V1NetworkPolicy) => Promise<boolean>,+ describePolicy: (name: string, contextId: string) => string+ ): Promise<void> => {+ if (!contextCleanupAuthoritative()) return+ for (const policy of policies) {+ const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]+ if (!contextId) {+ if (!contextCleanupAuthoritative()) return+ const name = policy.metadata?.name+ if (!name) {+ throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')+ }+ console.warn(+ `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`+ )+ if (await deletePolicy(policy)) recordSafetyPassRevocation()+ continue+ }+ if (!desiredContextIds.has(contextId)) {+ const name = policy.metadata?.name || ''+ await runContextEffect(contextId, async () => {+ if (+ !(await this.contextOrphanDeleteAllowed(+ contextId,+ contextCleanupAuthoritative,+ options.resolveCurrentContextById+ ))+ ) {+ return+ }+ console.log(describePolicy(name, contextId))+ if (await deletePolicy(policy)) recordSafetyPassRevocation()+ })+ }}}- // Also clean up orphaned rpc-proxy egress policies- const allRpcProxyPolicies = await this.listAllRpcProxyEgressPolicies()- for (const policy of allRpcProxyPolicies) {- const ctxId = policy.metadata?.labels?.[CONTEXT_LABEL]- if (ctxId && !desiredContextIds.has(ctxId)) {- const pName = policy.metadata?.name || ''- console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${pName}"`)- await this.deletePolicyInNamespace(config.rpcProxyNamespace, pName)- }+ let allContextPolicies = contextCleanupAuthoritative()+ ? await this.listAllContextPolicies()+ : []+ let allContextEgressPolicies = contextCleanupAuthoritative()+ ? await this.listAllContextEgressPolicies()+ : []+ let allRpcProxyEgressPolicies = contextCleanupAuthoritative()+ ? await this.listAllRpcProxyEgressPolicies()+ : []+ let allExternalPolicies: k8s.V1NetworkPolicy[] = []+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {+ allExternalPolicies = await this.listAllExternalEgressPolicies()+ }++ // Label selectors alone cannot prove safety: removing either ownership+ // marker would hide an active allow from the selected LIST. Audit the+ // bounded namespaces, then merge only owned or conservatively repairable+ // candidates into their lane. Ambiguous reserved names fail the pass.+ const needsMcpServerBroadInventory =+ contextCleanupAuthoritative() ||+ (options.serverInventoryComplete !== false && serverCleanupAuthoritative())+ const broadMcpServerPolicies = needsMcpServerBroadInventory+ ? await this.listNamespacePoliciesForSafety(config.namespace)+ : []+ const broadHostPolicies = contextCleanupAuthoritative()+ ? await this.listNamespacePoliciesForSafety(config.hostNamespace)+ : []+ const broadRpcProxyPolicies = contextCleanupAuthoritative()+ ? await this.listNamespacePoliciesForSafety(config.rpcProxyNamespace)+ : []++ if (contextCleanupAuthoritative()) {+ allContextPolicies = this.mergeSafetyLaneInventory(+ allContextPolicies,+ broadMcpServerPolicies,+ 'context-ingress'+ )+ allContextEgressPolicies = this.mergeSafetyLaneInventory(+ allContextEgressPolicies,+ broadHostPolicies,+ 'context-host-egress'+ )+ allRpcProxyEgressPolicies = this.mergeSafetyLaneInventory(+ allRpcProxyEgressPolicies,+ broadRpcProxyPolicies,+ 'context-rpc-egress'+ )+ }+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {+ allExternalPolicies = this.mergeSafetyLaneInventory(+ allExternalPolicies,+ broadMcpServerPolicies,+ 'external-egress'+ )+ }+ safetyPassListedPolicies +=+ allContextPolicies.length ++ allContextEgressPolicies.length ++ allRpcProxyEgressPolicies.length ++ allExternalPolicies.length++ // Delete orphaned Context policies across every L2 lane. Each lane uses+ // the canonical contextId effect key and rechecks authority plus live CRD+ // presence immediately before its delete.+ await cleanupOrphanedContextPolicies(+ allContextPolicies,+ policy =>+ this.deleteSafetyPolicySnapshot(config.namespace, policy, contextCleanupAuthoritative),+ (name, contextId) =>+ `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`+ )+ await cleanupOrphanedContextPolicies(+ allContextEgressPolicies,+ policy =>+ this.deleteSafetyPolicySnapshot(config.hostNamespace, policy, contextCleanupAuthoritative),+ name => `[NetPol] Deleting orphaned L2 egress policy "${name}"`+ )+ await cleanupOrphanedContextPolicies(+ allRpcProxyEgressPolicies,+ policy =>+ this.deleteSafetyPolicySnapshot(+ config.rpcProxyNamespace,+ policy,+ contextCleanupAuthoritative+ ),+ name => `[NetPol] Deleting orphaned rpc-proxy egress policy "${name}"`+ )++ let liveDesiredRevisionChanged = false+ for (const context of contexts) {+ const contextId = context.spec.contextId+ await runContextEffect(contextId, async () => {+ const current = options.resolveCurrentContext+ ? options.resolveCurrentContext(context.name)+ : context+ if (!current || current.spec.contextId !== contextId) {+ liveDesiredRevisionChanged = true+ return+ }+ const contextEffectIsCurrent = (): boolean => {+ if (!contextCleanupAuthoritative() || !serverCleanupAuthoritative()) return false+ if (!options.resolveCurrentContext) return true+ const latest = options.resolveCurrentContext(current.name)+ return latest !== undefined && sameContextDesiredRevision(current, latest)+ }+ const forContext = (policy: k8s.V1NetworkPolicy): boolean =>+ policy.metadata?.labels?.[CONTEXT_LABEL] === contextId+ const completed = await this.revokeStaleContextAllows(+ current,+ contextEffectIsCurrent,+ {+ context: allContextPolicies.filter(forContext),+ hostEgress: allContextEgressPolicies.filter(forContext),+ rpcProxyEgress: allRpcProxyEgressPolicies.filter(forContext),+ },+ recordSafetyPassRevocation+ )+ if (!completed && contextCleanupAuthoritative() && serverCleanupAuthoritative()) {+ liveDesiredRevisionChanged = true+ }+ })}// Clean up external egress policies for servers that no longer exist.- if (options.serverInventoryComplete !== false) {+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {const desiredServerNames = new Set(servers.map(s => s.name))- const allExternalPolicies = await this.listAllExternalEgressPolicies()for (const policy of allExternalPolicies) {const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]- if (serverName && !desiredServerNames.has(serverName)) {+ if (!serverName) {+ if (!serverCleanupAuthoritative()) break+ const name = policy.metadata?.name+ if (!name) {+ throw new Error(+ 'HCC-managed external egress NetworkPolicy is missing its name and owner label'+ )+ }+ console.warn(+ `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`+ )+ if (+ await this.deleteSafetyPolicySnapshot(+ config.namespace,+ policy,+ serverCleanupAuthoritative+ )+ ) {+ recordSafetyPassRevocation()+ }+ continue+ }+ if (!desiredServerNames.has(serverName)) {const name = policy.metadata?.name || ''- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)- await this.deletePolicyInNamespace(config.namespace, name)+ await runServerEffect(serverName, async () => {+ if (+ !(await confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: serverCleanupAuthoritative,+ resolveCurrent: () => options.resolveCurrentServer?.(serverName),+ readCurrent: () =>+ this.customApi.getNamespacedCustomObject({+ group: GROUP,+ version: VERSION,+ namespace: config.namespace,+ plural: PLURAL_MCPSERVERS,+ name: serverName,+ }),+ }))+ ) {+ return+ }+ console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)+ if (+ await this.deleteSafetyPolicySnapshot(+ config.namespace,+ policy,+ serverCleanupAuthoritative+ )+ ) {+ recordSafetyPassRevocation()+ }+ })}}- } else {+ } else if (options.serverInventoryComplete === false) {console.warn('[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete')}+ if (options.serverInventoryComplete !== false) {+ for (const server of servers) {+ await runServerEffect(server.name, async () => {+ const current = options.resolveCurrentServer+ ? options.resolveCurrentServer(server.name)+ : server+ if (!current) {+ liveDesiredRevisionChanged = true+ return+ }+ const serverEffectIsCurrent = (): boolean => {+ if (!serverCleanupAuthoritative()) return false+ if (!options.resolveCurrentServer) return true+ const latest = options.resolveCurrentServer(current.name)+ return latest !== undefined && sameMcpServerDesiredRevision(current, latest)+ }+ const completed = await this.reconcileExternalEgressSafety(+ current,+ allExternalPolicies.filter(+ policy => policy.metadata?.labels?.[MCPSERVER_LABEL] === server.name+ ),+ serverEffectIsCurrent,+ () => {+ recordSafetyPassRevocation()+ options.onExternalEgressRevoked?.(current)+ }+ )+ if (!completed && serverCleanupAuthoritative()) {+ liveDesiredRevisionChanged = true+ }+ })+ }+ }++ const inventoryDesiredRevisionChanged =+ (selectedContextDesiredRevision !== undefined &&+ options.contextDesiredRevision?.() !== selectedContextDesiredRevision) ||+ (selectedServerDesiredRevision !== undefined &&+ options.serverDesiredRevision?.() !== selectedServerDesiredRevision)+ if (+ (liveDesiredRevisionChanged || inventoryDesiredRevisionChanged) &&+ contextCleanupAuthoritative() &&+ serverCleanupAuthoritative()+ ) {+ throw new Error('Desired NetworkPolicy inventory changed during authoritative revocation')+ }++ if (+ options.serverInventoryComplete !== false &&+ contextCleanupAuthoritative() &&+ serverCleanupAuthoritative()+ ) {+ if (measureSafetyPass) {+ networkPolicySafetyPassPoliciesTotal.inc({ operation: 'listed' }, safetyPassListedPolicies)+ networkPolicySafetyPassPoliciesTotal.inc(+ { operation: 'revoked' },+ safetyPassRevokedPolicies+ )+ }+ options.onAuthoritativeRevocationComplete?.()+ }++ // Additive Context convergence happens only after every stale allow policy+ // implied by the authoritative inventory has been revoked. A slow or+ // failing positive effect therefore cannot extend a stale-permission+ // window after readiness becomes available. External egress creation and+ // DNS refresh intentionally do not run here:+ // ExternalEgressConvergenceCoordinator is their sole startup, retry, and+ // periodic-resync owner.+ const contextAdditiveFailures: unknown[] = []+ for (const context of contexts) {+ const contextId = context.spec.contextId+ try {+ await runContextEffect(contextId, async () => {+ const current = options.resolveCurrentContext+ ? options.resolveCurrentContext(context.name)+ : context+ if (!current || current.spec.contextId !== contextId) return+ const contextEffectIsCurrent = (): boolean => {+ if (!contextInventoryIsCurrent()) return false+ if (!options.resolveCurrentContext) return true+ const latest = options.resolveCurrentContext(current.name)+ return latest !== undefined && sameContextDesiredRevision(current, latest)+ }+ await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })+ })+ } catch (error) {+ contextAdditiveFailures.push(error)+ console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)+ }+ }++ if (contextAdditiveFailures.length > 0) {+ throw new AggregateError(+ contextAdditiveFailures,+ 'One or more additive Context NetworkPolicy reconciliations failed'+ )+ }+console.log('[NetPol] Full reconciliation complete')}+ private async contextOrphanDeleteAllowed(+ contextId: string,+ inventoryAuthoritative: () => boolean,+ resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined+ ): Promise<boolean> {+ if (!inventoryAuthoritative() || resolveCurrentContextById?.(contextId)) return false++ const response = (await this.customApi.listNamespacedCustomObject({+ group: GROUP,+ version: VERSION,+ namespace: config.namespace,+ plural: PLURAL_CONTEXTS,+ })) as { items?: Array<{ spec?: { contextId?: string } }> }+ if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false++ return inventoryAuthoritative() && !resolveCurrentContextById?.(contextId)+ }+// ─── L3 External Egress ────────────────────────────────────────────+ private buildPublicWebEgressPolicy(server: McpServerCRD, name: string): k8s.V1NetworkPolicy {+ return {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name,+ namespace: server.namespace,+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,+ [MCPSERVER_LABEL]: server.name,+ [EGRESS_CLASS_LABEL]: 'public-web',+ },+ },+ spec: {+ podSelector: {+ matchLabels: { [MCPSERVER_LABEL]: server.name },+ },+ policyTypes: ['Egress'],+ egress: [+ {+ to: [+ {+ ipBlock: {+ cidr: '0.0.0.0/0',+ except: PUBLIC_EGRESS_EXCEPT_CIDRS,+ },+ },+ ],+ ports: [+ { port: 443, protocol: 'TCP' },+ { port: 80, protocol: 'TCP' },+ ],+ },+ ],+ },+ }+ }++ private buildExactHostEgressPolicy(+ server: McpServerCRD,+ name: string,+ binding: EgressBinding,+ cidrs: string[]+ ): k8s.V1NetworkPolicy {+ return {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name,+ namespace: server.namespace,+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,+ [MCPSERVER_LABEL]: server.name,+ [EGRESS_CLASS_LABEL]: 'exact-host',+ },+ },+ spec: {+ podSelector: {+ matchLabels: { [MCPSERVER_LABEL]: server.name },+ },+ policyTypes: ['Egress'],+ egress: cidrs.map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: binding.port!, protocol: binding.protocol ?? 'TCP' }],+ })),+ },+ }+ }++ private async resolveExternalEgressDns(+ hostname: string+ ): Promise<{ address: string; ttl: number }[]> {+ // Resolve with TTL (issue #299 H2 ratchet needs the smallest observed TTL)+ // while keeping the #205 bounded-timeout race so a silent resolver cannot+ // block safety convergence indefinitely.+ let timeout: ReturnType<typeof setTimeout> | undefined+ try {+ return await Promise.race([+ dns.resolve4(hostname, { ttl: true }),+ new Promise<never>((_, reject) => {+ timeout = setTimeout(+ () =>+ reject(+ new Error(+ `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`+ )+ ),+ config.externalEgressDnsResolveTimeoutMs+ )+ }),+ ])+ } finally {+ if (timeout !== undefined) clearTimeout(timeout)+ }+ }++ private async reconcileExternalEgressSafety(+ server: McpServerCRD,+ existingPolicies: k8s.V1NetworkPolicy[],+ isCurrent: () => boolean,+ onRevoked?: () => void+ ): Promise<boolean> {+ if (!isCurrent()) return false+ const desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()+ for (const binding of server.spec.egressBindings ?? []) {+ const name = this.externalEgressPolicyName(server.name, binding)+ if (!name) continue+ const egressClass = binding.egressClass ?? 'exact-host'+ if (+ egressClass === 'public-web' &&+ !binding.dns &&+ !binding.cidr &&+ binding.port === undefined &&+ binding.protocol === undefined+ ) {+ desired.set(name, { binding, policy: this.buildPublicWebEgressPolicy(server, name) })+ } else if (+ egressClass === 'exact-host' &&+ binding.cidr &&+ isAllowedExternalEgressCidr(binding.cidr) &&+ binding.port !== undefined &&+ Number.isInteger(binding.port) &&+ binding.port >= 1 &&+ binding.port <= 65535+ ) {+ desired.set(name, {+ binding,+ policy: this.buildExactHostEgressPolicy(server, name, binding, [binding.cidr]),+ })+ } else if (+ egressClass === 'exact-host' &&+ binding.dns &&+ isPublicDnsHostname(binding.dns) &&+ binding.port !== undefined &&+ Number.isInteger(binding.port) &&+ binding.port >= 1 &&+ binding.port <= 65535+ ) {+ desired.set(name, { binding })+ }+ }++ const stale: k8s.V1NetworkPolicy[] = []+ const unprovableDnsPolicies: k8s.V1NetworkPolicy[] = []+ const deterministicReplacements: Array<{+ snapshot: k8s.V1NetworkPolicy+ desired: k8s.V1NetworkPolicy+ }> = []+ for (const existing of existingPolicies) {+ const name = existing.metadata?.name+ const selected = name ? desired.get(name) : undefined+ if (!selected) {+ stale.push(existing)+ continue+ }+ desired.delete(name!)+ if (!selected.policy) {+ // DNS binding: the /32s cannot be proven without a fresh lookup, but+ // the binding IDENTITY (protocol, spec-level ports, selector,+ // ownership) can. Retain the live allow when identity is unchanged;+ // revoke only on an identity change (a port move in the CRD changes the+ // policy NAME and is handled by the stale lane above). The comparison is+ // "modulo cidr": the desired policy is rebuilt WITH the live cidrs, so+ // the cidrs compare against themselves and everything else must match+ // exactly. A policy with no ipBlock peers is a deny disguised as an+ // allow and is never retained.+ const existingCidrs = (existing.spec?.egress ?? [])+ .flatMap(rule => rule.to ?? [])+ .map(peer => peer.ipBlock?.cidr)+ .filter((cidr): cidr is string => typeof cidr === 'string')+ const identityUnchanged =+ existingCidrs.length > 0 &&+ hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE) &&+ sameNetworkPolicySpec(+ existing,+ this.buildExactHostEgressPolicy(server, name!, selected.binding, existingCidrs)+ )+ if (!identityUnchanged) unprovableDnsPolicies.push(existing)+ continue+ }+ const retainable =+ sameNetworkPolicySpec(existing, selected.policy) &&+ hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE)+ if (!retainable) {+ deterministicReplacements.push({ snapshot: existing, desired: selected.policy })+ }+ }++ // Preserve traffic only with deterministic CIDR/public-web policies. A+ // renamed deterministic binding is created before its stale predecessor is+ // removed; a same-name conflict is converged only under the ownership+ // guard, which still refuses to adopt a foreign live object.+ if (stale.length > 0) {+ for (const selected of desired.values()) {+ if (!selected.policy) continue+ if (+ !(await this.createSafetyPolicy(+ server.namespace,+ selected.policy,+ 'external-egress',+ isCurrent+ ))+ ) {+ return false+ }+ }+ }+ for (const replacement of deterministicReplacements) {+ if (+ !(await this.replaceSafetyPolicySnapshot(+ server.namespace,+ replacement.snapshot,+ replacement.desired,+ 'external-egress',+ isCurrent+ ))+ ) {+ return false+ }+ onRevoked?.()+ }++ // DNS-derived /32s are unprovable without a fresh lookup. Revoke them+ // immediately; replacement belongs to the additive lane and must not put+ // external DNS latency back on the global readiness path.+ await this.cleanupExternalEgress(+ server.name,+ server.namespace,+ unprovableDnsPolicies,+ async () => isCurrent(),+ onRevoked+ )+ if (!isCurrent()) return false++ await this.cleanupExternalEgress(+ server.name,+ server.namespace,+ stale,+ async () => isCurrent(),+ onRevoked+ )+ return isCurrent()+ }+/*** Reconcile external egress policies for an McpServer with egressBindings.* For DNS bindings, resolves hostnames to IPs and creates ipBlock rules.* For CIDR bindings, creates ipBlock rules directly.*/- async reconcileExternalEgress(server: McpServerCRD): Promise<void> {+ async reconcileExternalEgress(+ server: McpServerCRD,+ options: NetworkPolicyMutationOptions+ ): Promise<void> {+ const isCurrent = options.isCurrent+ if (!isCurrent()) returnlet existingPolicies: k8s.V1NetworkPolicy[]try {existingPolicies = await this.listExternalEgressPoliciesForServer(@@ -816,36 +1918,48 @@ export class NetworkPolicyReconciler {server.namespace)} catch (error) {+ if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'False','InventoryListFailed',- `Failed to list existing external egress policies: ${this.errorMessage(error)}`+ `Failed to list existing external egress policies: ${this.errorMessage(error)}`,+ isCurrent)throw error}+ if (!isCurrent()) returnconst bindings = server.spec.egressBindingsif (!bindings || bindings.length === 0) {try {- await this.cleanupExternalEgress(server.name, server.namespace, existingPolicies)+ await this.cleanupExternalEgress(+ server.name,+ server.namespace,+ existingPolicies,+ async () => isCurrent()+ )} catch (error) {+ if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'False','CleanupFailed',- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,+ isCurrent)throw error}+ if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'True','NoEgressBindings',- 'No external egress bindings declared'+ 'No external egress bindings declared',+ isCurrent)return}@@ -857,9 +1971,11 @@ export class NetworkPolicyReconciler {const desiredPolicyNames = new Set<string>()const resolvedEgressIPs: McpServerResolvedEgressIP[] = []const failures: string[] = []- const resolvedAt = new Date().toISOString()+ const resolvedAtMs = Date.now()+ const resolvedAt = new Date(resolvedAtMs).toISOString()for (const binding of bindings) {+ if (!isCurrent()) returnconst egressClass = binding.egressClass ?? 'exact-host'if (egressClass !== 'exact-host' && egressClass !== 'public-web') {failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)@@ -905,47 +2021,13 @@ export class NetworkPolicyReconciler {desiredPolicyNames.add(name)- const policy: k8s.V1NetworkPolicy = {- apiVersion: 'networking.k8s.io/v1',- kind: 'NetworkPolicy',- metadata: {- name,- namespace: server.namespace,- labels: {- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,- [MCPSERVER_LABEL]: server.name,- [EGRESS_CLASS_LABEL]: egressClass,- },- },- spec: {- podSelector: {- matchLabels: { [MCPSERVER_LABEL]: server.name },- },- policyTypes: ['Egress'],- egress: [- {- to: [- {- ipBlock: {- cidr: '0.0.0.0/0',- except: PUBLIC_EGRESS_EXCEPT_CIDRS,- },- },- ],- ports: [- { port: 443, protocol: 'TCP' },- { port: 80, protocol: 'TCP' },- ],- },- ],- },- }-- // public-web is a static policy — write only when new/changed/drifted so- // the TTL-accelerated resync does not rewrite it every tick (audit F2).+ // public-web is a static policy. Preserve #205's ownership-fenced write+ // (applyOwnedPolicy asserts ownership + honors the isCurrent generation+ // fence) while keeping dev's F2 no-churn gate so the TTL-accelerated+ // resync does not rewrite it every tick.+ const policy = this.buildPublicWebEgressPolicy(server, name)if (this.externalEgressWriteNeeded(existingPolicy, policy)) {- await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')+ await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)}continue}@@ -993,7 +2075,8 @@ export class NetworkPolicyReconciler {maxEntries: config.externalEgressMaxEntries,}try {- const records = await dns.resolve4(binding.dns, { ttl: true })+ const records = await this.resolveExternalEgressDns(binding.dns)+ if (!isCurrent()) returnconst uniqueIps = [...new Set(records.map(r => r.address))].sort()if (uniqueIps.length === 0) {failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)@@ -1094,43 +2177,25 @@ export class NetworkPolicyReconciler {desiredPolicyNames.add(name)- const policy: k8s.V1NetworkPolicy = {- apiVersion: 'networking.k8s.io/v1',- kind: 'NetworkPolicy',- metadata: {- name,- namespace: server.namespace,- labels: {- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,- [MCPSERVER_LABEL]: server.name,- [EGRESS_CLASS_LABEL]: egressClass,- },- // Persist the sliding-window state so the next reconcile rehydrates the- // accumulated set instead of collapsing to a single snapshot (#299).- ...(stateAnnotations ? { annotations: stateAnnotations } : {}),- },- spec: {- podSelector: {- matchLabels: { [MCPSERVER_LABEL]: server.name },- },- policyTypes: ['Egress'],- egress: cidrs.map(cidr => ({- to: [{ ipBlock: { cidr } }],- ports: [{ port: binding.port, protocol }],- })),- },+ // Build the exact-host policy from #205's helper, then fold in the #299+ // sliding-window state annotations so the next reconcile rehydrates the+ // accumulated set instead of collapsing to a single snapshot.+ const policy = this.buildExactHostEgressPolicy(server, name, binding, cidrs)+ if (stateAnnotations) {+ policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }}// Write only when the enforced rules changed / the policy is new / drifted// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.+ // Route the write through applyOwnedPolicy so #205's ownership assertion+ // and isCurrent generation fence still guard the mutation.if (this.externalEgressWriteNeeded(existingPolicy, policy) ||egressRenewalDue ||egressStateChanged) {- await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')+ await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)} else {console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)}@@ -1143,18 +2208,23 @@ export class NetworkPolicyReconciler {existingPolicies.filter(policy => {const name = policy.metadata?.namereturn Boolean(name) && !desiredPolicyNames.has(name!)- })+ }),+ async () => isCurrent(),+ options.onRevoked)} catch (error) {+ if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,resolvedEgressIPs,'False','CleanupFailed',- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,+ isCurrent)throw error}+ if (!isCurrent()) returnif (failures.length > 0) {const message = failures.join('; ')@@ -1163,7 +2233,8 @@ export class NetworkPolicyReconciler {resolvedEgressIPs,'False','ExternalEgressRejected',- message+ message,+ isCurrent)throw new Error(`External egress reconciliation failed for "${server.name}": ${message}`)}@@ -1175,7 +2246,8 @@ export class NetworkPolicyReconciler {'Reconciled',bindings.some(binding => binding.egressClass === 'public-web')? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'- : 'External egress policies reconciled'+ : 'External egress policies reconciled',+ isCurrent)}@@ -1185,7 +2257,9 @@ export class NetworkPolicyReconciler {async cleanupExternalEgress(serverName: string,namespace: string,- policies?: k8s.V1NetworkPolicy[]+ policies?: k8s.V1NetworkPolicy[],+ deleteAllowed?: () => Promise<boolean>,+ onDeleted?: () => void): Promise<void> {const policiesToDelete =policies ?? (await this.listExternalEgressPoliciesForServer(serverName, namespace))@@ -1194,13 +2268,19 @@ export class NetworkPolicyReconciler {const name = policy.metadata?.nameif (!name) continuetry {- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })- console.log(`[NetPol] Deleted external egress policy "${name}"`)+ if (deleteAllowed && !(await deleteAllowed())) return+ await this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {+ console.log(`[NetPol] Deleted external egress policy "${name}"`)+ onDeleted?.()+ })} catch (error: unknown) {if (getErrorCode(error) !== 404) {console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)throw error}+ // A concurrent actor may already have removed the policy. The safety+ // invariant still holds, but this pass did not revoke it and must not+ // inflate the authoritative safety-pass `revoked` counter.}}}@@ -1282,10 +2362,19 @@ export class NetworkPolicyReconciler {resolvedEgressIPs: McpServerResolvedEgressIP[],status: 'True' | 'False' | 'Unknown',reason: string,- message: string+ message: string,+ isCurrent: () => boolean = () => true): Promise<void> {+ if (!isCurrent()) returnlet currentStatus: McpServerCrdStatus = {}let hasStatusObject = false+ let currentMetadata:+ | {+ uid?: string+ generation?: number+ resourceVersion?: string+ }+ | undefinedtry {const current = (await this.customApi.getNamespacedCustomObjectStatus({@@ -1294,7 +2383,15 @@ export class NetworkPolicyReconciler {namespace: server.namespace,plural: PLURAL_MCPSERVERS,name: server.name,- })) as { status?: McpServerCrdStatus }+ })) as {+ metadata?: {+ uid?: string+ generation?: number+ resourceVersion?: string+ }+ status?: McpServerCrdStatus+ }+ currentMetadata = current.metadatahasStatusObject = typeof current.status === 'object' && current.status !== nullcurrentStatus = current.status ?? {}} catch (error) {@@ -1310,6 +2407,20 @@ export class NetworkPolicyReconciler {)throw error}+ if (!isCurrent()) return++ if (+ (server.uid !== undefined && currentMetadata?.uid !== server.uid) ||+ (server.generation !== undefined && currentMetadata?.generation !== server.generation)+ ) {+ throw new Error(+ `Refusing to write external egress status for stale McpServer "${server.name}": ` ++ `expected uid=${String(server.uid)} generation=${String(server.generation)}, ` ++ `read uid=${String(currentMetadata?.uid)} generation=${String(+ currentMetadata?.generation+ )}`+ )+ }const nextConditions = this.mergeStatusCondition(currentStatus.conditions ?? [], {type: 'ExternalEgressReady',@@ -1334,14 +2445,36 @@ export class NetworkPolicyReconciler {return}- const statusPatch: JsonPatchOperation[] = hasStatusObject+ const identityFence: JsonPatchOperation[] = []+ const expectedUid = server.uid ?? currentMetadata?.uid+ if (expectedUid !== undefined) {+ identityFence.push({ op: 'test', path: '/metadata/uid', value: expectedUid })+ }+ const expectedGeneration = server.generation ?? currentMetadata?.generation+ if (expectedGeneration !== undefined) {+ identityFence.push({+ op: 'test',+ path: '/metadata/generation',+ value: expectedGeneration,+ })+ }+ if (currentMetadata?.resourceVersion !== undefined) {+ identityFence.push({+ op: 'test',+ path: '/metadata/resourceVersion',+ value: currentMetadata.resourceVersion,+ })+ }+ const statusMutation: JsonPatchOperation[] = hasStatusObject? [{ op: 'add', path: '/status/resolvedEgressIPs', value: stableResolvedEgressIPs },{ op: 'add', path: '/status/conditions', value: nextConditions },]: [{ op: 'add', path: '/status', value: nextStatus }]+ const statusPatch = [...identityFence, ...statusMutation]try {+ if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObjectStatus({group: GROUP,version: VERSION,@@ -1394,6 +2527,61 @@ export class NetworkPolicyReconciler {}}+ /** Broad, namespace-bounded inventory used only by the safety pass. */+ private async listNamespacePoliciesForSafety(namespace: string): Promise<k8s.V1NetworkPolicy[]> {+ try {+ const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })+ return response.items || []+ } catch (error) {+ console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)+ throw error+ }+ }++ private mergeSafetyLaneInventory(+ selected: k8s.V1NetworkPolicy[],+ broad: k8s.V1NetworkPolicy[],+ lane: SafetyInventoryLane+ ): k8s.V1NetworkPolicy[] {+ const result = new Map<string, k8s.V1NetworkPolicy>()+ const keyFor = (policy: k8s.V1NetworkPolicy, index: number): string =>+ policy.metadata?.uid ?? policy.metadata?.name ?? `unnamed-${index}`++ selected.forEach((policy, index) => result.set(keyFor(policy, index), policy))+ broad.forEach((policy, index) => {+ const classification = classifySafetyInventoryPolicy(policy, lane)+ if (classification === 'ambiguous') {+ const name = policy.metadata?.name ?? '<unnamed>'+ throw new Error(+ `Ambiguous NetworkPolicy ownership for reserved HCC ${lane} policy "${name}"`+ )+ }+ if (classification === 'owned' || classification === 'repairable') {+ result.set(keyFor(policy, selected.length + index), policy)+ } else {+ // The broad LIST is later and authoritative for this object identity.+ // Do not retain a selector snapshot after the same object was relabelled+ // out of HCC ownership.+ result.delete(keyFor(policy, selected.length + index))+ }+ })+ return [...result.values()]+ }++ /** List all L2 context-allow egress policies in the mcp-host namespace. */+ private async listAllContextEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {+ try {+ const response = await this.networkingApi.listNamespacedNetworkPolicy({+ namespace: config.hostNamespace,+ labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=context-allow`,+ })+ return response.items || []+ } catch (error) {+ console.error('[NetPol] Failed to list all Context egress policies:', error)+ throw error+ }+ }+/** List all external-egress policies in the MCP server namespace. */private async listAllExternalEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {try {@@ -1421,26 +2609,234 @@ export class NetworkPolicyReconciler {}/** Create or update a NetworkPolicy. */- private async applyPolicy(name: string, policy: k8s.V1NetworkPolicy): Promise<void> {- await applyNetworkPolicy(this.networkingApi, name, config.namespace, policy, '[NetPol]')+ private async applyFixedPolicy(+ name: string,+ namespace: string,+ policy: k8s.V1NetworkPolicy,+ policyType: string+ ): Promise<void> {+ await applyNetworkPolicy(+ this.networkingApi,+ name,+ namespace,+ policy,+ '[NetPol]',+ undefined,+ existing => {+ if (!hasExpectedPolicyOwnership(existing, policyType)) {+ throw new Error(+ `NetworkPolicy "${name}" has conflicting ownership for the ${policyType} lane`+ )+ }+ }+ )}- /** Delete a NetworkPolicy in mcp-server namespace. */- private async deletePolicy(name: string): Promise<void> {+ /** Create or update a Context ingress NetworkPolicy. */+ private async applyContextIngressPolicy(+ name: string,+ policy: k8s.V1NetworkPolicy,+ isCurrent?: () => boolean+ ): Promise<void> {+ await this.applyOwnedPolicy(name, config.namespace, policy, 'context-ingress', isCurrent)+ }++ private async applyOwnedPolicy(+ name: string,+ namespace: string,+ policy: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane,+ isCurrent?: () => boolean+ ): Promise<void> {+ await applyNetworkPolicy(+ this.networkingApi,+ name,+ namespace,+ policy,+ '[NetPol]',+ isCurrent,+ existing => this.assertPolicyOwnership(name, existing, policy, lane)+ )+ }++ private assertPolicyOwnership(+ name: string,+ existing: k8s.V1NetworkPolicy,+ desired: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane+ ): void {+ const classification = classifySafetyInventoryPolicy(existing, lane)+ const existingLabels = existing.metadata?.labels ?? {}+ const desiredLabels = desired.metadata?.labels ?? {}+ const ownerLabels =+ lane === 'external-egress' ? [MCPSERVER_LABEL] : [CONTEXT_LABEL, MCPSERVER_LABEL]+ if (+ (classification !== 'owned' && classification !== 'repairable') ||+ ownerLabels.some(label => existingLabels[label] !== desiredLabels[label])+ ) {+ throw new Error(`NetworkPolicy "${name}" has conflicting ownership for the ${lane} lane`)+ }+ }++ private safetyPolicyIdentity(policy: k8s.V1NetworkPolicy): {+ name: string+ uid: string+ resourceVersion: string+ } {+ const name = policy.metadata?.name+ const uid = policy.metadata?.uid+ const resourceVersion = policy.metadata?.resourceVersion+ if (!name || !uid || !resourceVersion) {+ throw new Error(+ `NetworkPolicy safety mutation requires name, uid, and resourceVersion for "${name ?? '<unnamed>'}"`+ )+ }+ return { name, uid, resourceVersion }+ }++ /**+ * Delete exactly the policy version classified by the authoritative safety+ * inventory. Kubernetes rejects the preconditions if the name was recreated+ * or changed after LIST, so HCC cannot delete a new or foreign object.+ */+ private async deleteSafetyPolicySnapshot(+ namespace: string,+ policy: k8s.V1NetworkPolicy,+ isCurrent?: () => boolean,+ onDeleted?: () => void,+ lostFenceOutcome: 'throw' | 'report' = 'throw'+ ): Promise<boolean> {+ if (isCurrent && !isCurrent()) return false+ const { name, uid, resourceVersion } = this.safetyPolicyIdentity(policy)try {await this.networkingApi.deleteNamespacedNetworkPolicy({name,- namespace: config.namespace,+ namespace,+ body: { preconditions: { uid, resourceVersion } },})- console.log(`[NetPol] Deleted policy "${name}"`)+ console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)+ onDeleted?.()+ return true} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[NetPol] Policy "${name}" already gone`)- } else {- console.error(`[NetPol] Failed to delete policy "${name}":`, error)- throw error+ console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)+ return true+ }+ if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {+ // Kubernetes rejected the uid/resourceVersion preconditions, so the+ // object live under this name is not the one the authoritative+ // inventory classified. Nothing was revoked, and nothing may be+ // adopted. Callers that opt in act on this outcome by declining to+ // certify, which is the expected racy result rather than a failure.+ // Callers that cannot act on it keep the loud default.+ console.warn(+ `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`+ )+ return false+ }+ console.error(+ `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,+ error+ )+ throw error+ }+ }++ /**+ * Replace only the exact live object classified by the safety inventory.+ * The fresh read closes ownership drift before mutation; resourceVersion on+ * the replace body closes the remaining read-to-write race.+ */+ private async replaceSafetyPolicySnapshot(+ namespace: string,+ snapshot: k8s.V1NetworkPolicy,+ desired: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane,+ isCurrent: () => boolean+ ): Promise<boolean> {+ if (!isCurrent()) return false+ const expected = this.safetyPolicyIdentity(snapshot)+ const live = await this.networkingApi.readNamespacedNetworkPolicy({+ name: expected.name,+ namespace,+ })+ if (!isCurrent()) return false+ const actual = this.safetyPolicyIdentity(live)+ if (actual.uid !== expected.uid || actual.resourceVersion !== expected.resourceVersion) {+ throw new Error(+ `NetworkPolicy "${expected.name}" changed identity or ownership after the authoritative safety inventory`+ )+ }+ this.assertPolicyOwnership(expected.name, live, desired, lane)+ await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: expected.name,+ namespace,+ body: {+ ...desired,+ metadata: {+ ...desired.metadata,+ name: expected.name,+ namespace,+ resourceVersion: actual.resourceVersion,+ },+ },+ })+ return isCurrent()+ }++ private async createSafetyPolicy(+ namespace: string,+ desired: k8s.V1NetworkPolicy,+ lane: SafetyInventoryLane,+ isCurrent: () => boolean+ ): Promise<boolean> {+ if (!isCurrent()) return false+ let created: k8s.V1NetworkPolicy+ try {+ created = await this.networkingApi.createNamespacedNetworkPolicy({+ namespace,+ body: desired,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) !== 409) throw error+ // This is the only creation route that does not go through+ // applyNetworkPolicy, so a same-name object used to abort the whole+ // safety pass and hold readiness down. Converge it exactly like every+ // other lane does: adopt it only under the ownership guard, which still+ // refuses a foreign or ambiguous object.+ const name = desired.metadata?.name+ if (!name) {+ throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)}+ console.log(+ `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`+ )+ // Deliberately not delegating to applyOwnedPolicy here: that helper owns+ // the create-then-conflict flow, so calling it from inside our own+ // create's catch would issue a second create. Delegating from the top is+ // not an option either — the desired-state fence below needs the object+ // this request created, which applyNetworkPolicy does not return. The+ // wiring below is kept identical to it on purpose.+ await replaceWithConflictRetry<k8s.V1NetworkPolicy>({+ description: `policy "${name}" in ${namespace}`,+ logPrefix: '[NetPol]',+ body: desired,+ read: () => this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),+ replace: body =>+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ mutationAllowed: isCurrent,+ validateExisting: existing => this.assertPolicyOwnership(name, existing, desired, lane),+ })+ return isCurrent()}+ if (isCurrent()) return true++ // The mutation crossed its desired-state fence. Remove exactly the object+ // this request created, even though the old reconciliation is no longer+ // authoritative. The UID/resourceVersion preconditions ensure a+ // replacement or foreign policy with the same name is never deleted.+ await this.deleteSafetyPolicySnapshot(namespace, created)+ return false}/** List L2 egress counterpart policies in mcp-host namespace for a given context. */@@ -1457,37 +2853,15 @@ export class NetworkPolicyReconciler {}}- /** Delete a NetworkPolicy in mcp-host namespace (L2 egress counterpart). */- private async deleteEgressPolicy(name: string): Promise<void> {- try {- await this.networkingApi.deleteNamespacedNetworkPolicy({- name,- namespace: config.hostNamespace,- })- console.log(`[NetPol] Deleted L2 egress policy "${name}"`)- } catch (error: unknown) {- if (getErrorCode(error) === 404) {- console.log(`[NetPol] L2 egress policy "${name}" already gone`)- } else {- console.error(`[NetPol] Failed to delete L2 egress policy "${name}":`, error)- throw error- }- }- }-// ─── RPC-Proxy Egress (L2) ──────────────────────────────────────────- /**- * Create or update an egress policy in the rpc-proxy namespace allowing- * rpc-proxy pods to reach a specific MCP server on a given port.- */- private async ensureRpcProxyEgress(+ private buildRpcProxyEgressPolicy(contextId: string,serverName: string,port: number- ): Promise<void> {+ ): k8s.V1NetworkPolicy {const name = `rpc-egress-${contextId}-${serverName}`- const policy: k8s.V1NetworkPolicy = {+ return {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {@@ -1518,16 +2892,18 @@ export class NetworkPolicyReconciler {],},}- await applyNetworkPolicy(this.networkingApi, name, config.rpcProxyNamespace, policy, '[NetPol]')}/** Delete all rpc-proxy egress policies for a given context. */- private async deleteRpcProxyPoliciesForContext(contextId: string): Promise<void> {+ private async deleteRpcProxyPoliciesForContext(+ contextId: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {try {const policies = await this.listRpcProxyEgressPoliciesForContext(contextId)for (const policy of policies) {- const name = policy.metadata?.name || ''- await this.deletePolicyInNamespace(config.rpcProxyNamespace, name)+ if (deleteAllowed && !(await deleteAllowed())) return+ await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)}} catch (error) {console.error(@@ -1571,34 +2947,19 @@ export class NetworkPolicyReconciler {}}- /** Delete a NetworkPolicy in an arbitrary namespace. */- private async deletePolicyInNamespace(namespace: string, name: string): Promise<void> {- try {- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })- console.log(`[NetPol] Deleted rpc-proxy egress policy "${name}"`)- } catch (error: unknown) {- if (getErrorCode(error) === 404) {- console.log(`[NetPol] rpc-proxy egress policy "${name}" already gone`)- } else {- console.error(`[NetPol] Failed to delete policy "${name}" in ${namespace}:`, error)- throw error- }- }- }-/** Delete a formerly static NetworkPolicy that is now generated dynamically. */private async deleteLegacyStaticPolicy(namespace: string, name: string): Promise<void> {+ let existing: k8s.V1NetworkPolicytry {- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)+ existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })} catch (error: unknown) {- if (getErrorCode(error) !== 404) {- console.error(- `[NetPol] Failed to delete legacy static policy "${name}" in "${namespace}":`,- error- )- throw error- }+ if (getErrorCode(error) === 404) return+ throw error+ }+ if (existing.metadata?.labels?.[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE) {+ throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)}+ await this.deleteSafetyPolicySnapshot(namespace, existing)+ console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)}}diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.tsnew file mode 100644index 000000000..a9b522ab0--- /dev/null+++ b/host-context-controller/src/readinessGate.test.ts@@ -0,0 +1,53 @@+import { describe, expect, it } from 'vitest'+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'++describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {+ it('dev provider (no watcher) is unconditionally authoritative', () => {+ // Regression for R3-B1: main.ts left this undefined for DevMcpServerProvider,+ // so the ContextMapperServer fail-closed default (() => false) pinned /ready+ // and every /api/v1/* endpoint at 503 forever in dev mode. There is no+ // NetworkPolicy inventory to certify in dev, so authority must be granted.+ expect(resolveProviderAuthoritativeFn(null)()).toBe(true)+ })++ it('production watcher delegates to the live inventory certification', () => {+ let asked = false+ const watcher = {+ isReadinessInventoryAuthoritative: () => {+ asked = true+ return false+ },+ }+ // Delegates (returns the watcher's verdict) rather than a constant.+ expect(resolveProviderAuthoritativeFn(watcher)()).toBe(false)+ expect(asked).toBe(true)+ watcher.isReadinessInventoryAuthoritative = () => true+ expect(resolveProviderAuthoritativeFn(watcher)()).toBe(true)+ })+})++describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {+ it('dev provider (no watcher) is unconditionally authoritative', () => {+ // Regression for R2-M1: main.ts left this undefined for DevMcpServerProvider,+ // so the ContextMapperServer fail-closed default (() => false) pinned every+ // /api/v1/desktop/* response at 503 after setReady(true) — where the base+ // answered 200 {status:'inactive'}. There is no Host inventory to certify in+ // dev, so authority must mirror the provider gate and be granted.+ expect(resolveHostAuthoritativeFn(null)()).toBe(true)+ })++ it('production watcher delegates to the live Host inventory certification', () => {+ let asked = false+ const watcher = {+ isHostInventoryAuthoritative: () => {+ asked = true+ return false+ },+ }+ // Delegates (returns the watcher's verdict) rather than a constant.+ expect(resolveHostAuthoritativeFn(watcher)()).toBe(false)+ expect(asked).toBe(true)+ watcher.isHostInventoryAuthoritative = () => true+ expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)+ })+})diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.tsnew file mode 100644index 000000000..88ff77ee6--- /dev/null+++ b/host-context-controller/src/readinessGate.ts@@ -0,0 +1,41 @@+/**+ * Resolve the readiness-authority gate for the server from the provider.+ *+ * Production (McpServerWatcher present): readiness follows the live inventory+ * certification — /ready is 200 only once stale allows are revoked.+ *+ * Dev (DevMcpServerProvider — no watcher): there is NO NetworkPolicy inventory+ * to certify, so authority is unconditional. The ContextMapperServer default+ * for a missing gate is fail-closed (`() => false`), which is correct for a+ * caller that forgets to wire one — but in dev it would pin /ready (and every+ * data endpoint) at 503 forever. Dev must therefore pass an explicit+ * always-authoritative gate; this function is that single, named decision so+ * the affirmative literal never gets copy-pasted into a production path.+ */+export function resolveProviderAuthoritativeFn(+ watcher: { isReadinessInventoryAuthoritative(): boolean } | null+): () => boolean {+ return watcher ? () => watcher.isReadinessInventoryAuthoritative() : () => true+}++/**+ * Resolve the Host-inventory-authority gate the desktop route uses.+ *+ * Production (McpServerWatcher present): desktop status follows the live Host+ * inventory certification — a degraded Host lane 503s desktop rather than+ * answering a stale 200 'inactive'.+ *+ * Dev (DevMcpServerProvider — no watcher): there is NO Host inventory to+ * certify, exactly as there is no NetworkPolicy inventory for the provider+ * gate. The ContextMapperServer default for a missing gate is fail-closed+ * (`() => false`), which is correct for a caller that forgets to wire one — but+ * in dev it would pin every /api/v1/desktop/* response at 503 forever (R2-M1).+ * Dev must therefore mirror the provider gate and pass an explicit+ * always-authoritative gate; this function is that single, named decision so+ * the affirmative literal never gets copy-pasted into a production path.+ */+export function resolveHostAuthoritativeFn(+ watcher: { isHostInventoryAuthoritative(): boolean } | null+): () => boolean {+ return watcher ? () => watcher.isHostInventoryAuthoritative() : () => true+}diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.tsnew file mode 100644index 000000000..4bd14f5e6--- /dev/null+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts@@ -0,0 +1,513 @@+/**+ * Crosses the whole readiness chain with a real NetworkPolicyReconciler.+ *+ * Every other test of this behaviour mocks one side. `k8sClient.test.ts` mocks+ * `./networkPolicyReconciler`, so it can prove the readiness predicate *reads*+ * the certification getter but not that the reconciler ever *degrades* it.+ * `networkPolicyReconciler.test.ts` drives the real fence but never reaches an+ * HTTP response. Each half is verified by mutation; the seam between them was+ * not, and the seam is where the guarantee actually lives.+ *+ * Here `@kubernetes/client-node` is the only mock. The reconciler, the watcher+ * predicate, and the HTTP server are the production classes, so the assertion+ * is end-to-end: a delete that loses its uid/resourceVersion precondition+ * during the additive phase — the 409 a second writer produces, and every+ * rollout opens a two-replica window for one — must turn /ready from 200 into+ * 503, and a later clean pass must turn it back.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as http from 'http'+import { McpServerWatcher } from './k8sClient'+import { ContextMapperServer } from './server'++const mocks = vi.hoisted(() => ({+ listNamespacedNetworkPolicy: vi.fn(),+ deleteNamespacedNetworkPolicy: vi.fn(),+ readNamespacedNetworkPolicy: vi.fn(),+ createNamespacedNetworkPolicy: vi.fn(),+ replaceNamespacedNetworkPolicy: vi.fn(),+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),+ getNamespacedCustomObject: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ getNamespacedCustomObjectStatus: vi+ .fn()+ .mockResolvedValue({ metadata: { generation: 1, resourceVersion: 'rv-1' }, status: {} }),+ patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),+ watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),+}))++vi.mock('./config', () => ({+ config: {+ devMode: false,+ namespace: 'mcp-server',+ hostNamespace: 'mcp-host',+ rpcProxyNamespace: 'rpc-proxy',+ channelsNamespace: 'channels',+ port: 0,+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],+ hostK8sRequestTimeoutMs: 30_000,+ hostResyncIntervalSec: 0,+ externalEgressResyncIntervalSec: 0,+ controlApiBaseUrl: 'http://control-api.test:8090',+ governedTracingEnabled: false,+ },+}))++vi.mock('@kubernetes/client-node', () => {+ class CustomObjectsApi {}+ class CoreV1Api {}+ class NetworkingV1Api {}+ class PolicyV1Api {}+ class AppsV1Api {}+ class BatchV1Api {}+ class RbacAuthorizationV1Api {}+ class Watch {+ constructor(_kc: unknown) {}+ watch = mocks.watch+ }+ class KubeConfig {+ loadFromDefault(): void {}+ loadFromOptions(_o: unknown): void {}+ makeApiClient(api: unknown): unknown {+ if (api === NetworkingV1Api) {+ return {+ listNamespacedNetworkPolicy: mocks.listNamespacedNetworkPolicy,+ deleteNamespacedNetworkPolicy: mocks.deleteNamespacedNetworkPolicy,+ readNamespacedNetworkPolicy: mocks.readNamespacedNetworkPolicy,+ createNamespacedNetworkPolicy: mocks.createNamespacedNetworkPolicy,+ replaceNamespacedNetworkPolicy: mocks.replaceNamespacedNetworkPolicy,+ }+ }+ if (api === CustomObjectsApi) {+ return {+ listNamespacedCustomObject: mocks.listNamespacedCustomObject,+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,+ getNamespacedCustomObjectStatus: mocks.getNamespacedCustomObjectStatus,+ patchNamespacedCustomObjectStatus: mocks.patchNamespacedCustomObjectStatus,+ }+ }+ return new Proxy({}, { get: () => vi.fn().mockResolvedValue({ items: [] }) })+ }+ }+ return {+ CustomObjectsApi,+ CoreV1Api,+ NetworkingV1Api,+ PolicyV1Api,+ AppsV1Api,+ BatchV1Api,+ RbacAuthorizationV1Api,+ Watch,+ KubeConfig,+ }+})++const MANAGED = 'clerum.io/managed-by'+const POLICY_TYPE = 'clerum.io/policy-type'+const CONTEXT_LABEL = 'clerum.io/context'+const SERVER_LABEL = 'clerum.io/mcpserver'++/** A stale context-allow: HCC-owned, for a server the Context no longer lists. */+const stalePolicy = {+ metadata: {+ name: 'ctx-alpha-ghost',+ namespace: 'mcp-server',+ uid: 'ghost-uid',+ resourceVersion: '7',+ labels: {+ [MANAGED]: 'host-context-controller',+ [POLICY_TYPE]: 'context-allow',+ [CONTEXT_LABEL]: 'alpha',+ [SERVER_LABEL]: 'ghost',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+}++const alphaContext = {+ name: 'alpha',+ namespace: 'mcp-server',+ spec: { contextId: 'alpha', mcpServers: [] },+}++async function readyStatus(server: InstanceType<typeof ContextMapperServer>): Promise<number> {+ const listener = (server as unknown as { server: http.Server | null }).server+ const address = listener?.address()+ if (!address || typeof address === 'string') throw new Error('readiness listener never bound')+ return new Promise((resolve, reject) => {+ http+ .get({ host: '127.0.0.1', port: address.port, path: '/ready' }, res => {+ res.resume()+ res.on('end', () => resolve(res.statusCode ?? 0))+ })+ .on('error', reject)+ })+}++/** Aligns every revocation counter so only the fence can gate readiness. */+function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>): void {+ const w = watcher as unknown as Record<string, unknown>+ w.mcpServerCacheSynced = true+ w.contextCacheSynced = true+ w.hostCacheSynced = true+ w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration+ w.networkPolicyRevocationServerGeneration = w.mcpWatchGeneration+ w.networkPolicyRevocationContextRevision = w.contextDesiredRevision+ w.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision+}++describe('readiness withdrawal on a real lost delete fence', () => {+ let watcher: InstanceType<typeof McpServerWatcher>+ let server: InstanceType<typeof ContextMapperServer>++ beforeEach(async () => {+ vi.clearAllMocks()+ mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })+ mocks.getNamespacedCustomObject.mockRejectedValue(+ Object.assign(new Error('not found'), { code: 404 })+ )+ mocks.createNamespacedNetworkPolicy.mockResolvedValue({})+ mocks.replaceNamespacedNetworkPolicy.mockResolvedValue({})+ mocks.readNamespacedNetworkPolicy.mockResolvedValue(stalePolicy)+ watcher = new McpServerWatcher()+ server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()+ server.setReady(true)+ })++ afterEach(async () => {+ await server.stop()+ await watcher.stop()+ })++ it('turns /ready from 200 to 503 when the additive phase loses a delete fence, and back on a clean pass', async () => {+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler+ alignRevocationCounters(watcher)++ // Baseline: nothing stale anywhere, so the pass certifies and the gate opens.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(200)++ // The stale allow only becomes visible after the authoritative phase has+ // certified, so the safety phase is clean by construction and the additive+ // re-LIST is what finds it. Its delete then loses the uid/resourceVersion+ // precondition — a 409, exactly what a concurrent writer produces.+ let certified = false+ mocks.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace?: string }) => ({+ items: certified && namespace === 'mcp-server' ? [stalePolicy] : [],+ })+ )+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('the object has been modified'), { code: 409 })+ )++ await expect(+ reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {+ certified = true+ },+ })+ ).rejects.toThrow()++ // Nothing moved: no watch generation bumped, no desired revision changed.+ // Only the fence knows this pass left an allow it classified as stale.+ alignRevocationCounters(watcher)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ expect(await readyStatus(server)).toBe(503)++ // The retry lands: the allow is gone and the pass certifies again.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)+ expect(await readyStatus(server)).toBe(200)+ })++ it('withholds readiness when the authoritative pass loses a delete fence under a provided safety snapshot', async () => {+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler+ alignRevocationCounters(watcher)++ // Baseline: a clean authoritative pass certifies and the gate opens.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(200)++ // Unlike the additive-phase test above, the stale allow is visible to the+ // authoritative LISTs from the very start of the pass, so fullReconcile+ // classifies it into the safety snapshot it provides to the revocation+ // lane. Its identity-bound delete then loses the uid/resourceVersion+ // precondition (409). The external-egress selector stays empty so the+ // fence is lost in the provided-snapshot lane and nowhere else.+ mocks.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({+ items:+ namespace === 'mcp-server' && !(labelSelector ?? '').includes('external-egress')+ ? [stalePolicy]+ : [],+ })+ )+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('the object has been modified'), { code: 409 })+ )++ // The guarantee under test is that the fence degrades at the point of+ // loss, while the condemned pass is still unwinding — not only when the+ // pass finally reports. Sample it through the authority callback the pass+ // keeps polling: every sample before the lost delete must be certified,+ // and every later sample must already be withdrawn.+ let certifications = 0+ const fenceSeenDuringPass: boolean[] = []+ await expect(+ reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => {+ fenceSeenDuringPass.push(reconciler.hasCertifiedSafetyInventory())+ return true+ },+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {+ certifications += 1+ },+ })+ ).rejects.toThrow()++ expect(certifications).toBe(0)+ expect(fenceSeenDuringPass.length).toBeGreaterThan(1)+ expect(fenceSeenDuringPass[0]).toBe(true)+ expect(fenceSeenDuringPass[fenceSeenDuringPass.length - 1]).toBe(false)++ alignRevocationCounters(watcher)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ expect(await readyStatus(server)).toBe(503)++ // A clean authoritative pass re-certifies and reopens the gate.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([alphaContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)+ expect(await readyStatus(server)).toBe(200)+ })++ it('short-circuits a scoped delta at the certificate source while the last authoritative pass left the inventory uncertified', async () => {+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler+ const w = watcher as unknown as Record<string, any>++ // Register the real Context watch so the production watch handler — the+ // scoped-delta certification lane — can be driven through its callback.+ await w.startContextWatch('ctx-rv-1')+ const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>+ String(path).endsWith('/namespaces/mcp-server/contexts')+ )+ if (!contextWatchCall) throw new Error('Context watch was never registered')+ const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>++ // Seed the prior Context so the MODIFIED event below is a same-identity+ // delta: same uid, same contextId, new generation.+ const priorContext = {+ name: 'alpha',+ namespace: 'mcp-server',+ uid: 'ctx-alpha-uid',+ generation: 1,+ spec: { contextId: 'alpha', mcpServers: [] },+ }+ w.contexts.set('alpha', priorContext)+ alignRevocationCounters(watcher)++ // Baseline clean authoritative pass: certified, gate open.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([priorContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(200)++ // The authoritative pass loses its delete fence under the provided+ // snapshot, exactly as in the test above: the broad safety LISTs see the+ // stale allow and its delete keeps 409ing. Every context-scoped LIST+ // stays empty so the scoped delta fired below completes its own+ // revocation cleanly — and because the 409 never clears, the convergence+ // pass the watch event kicks off in the background can never certify+ // either.+ mocks.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ const selector = labelSelector ?? ''+ const scoped =+ selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')+ return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }+ }+ )+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('the object has been modified'), { code: 409 })+ )+ await expect(+ reconciler.fullReconcile([priorContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ ).rejects.toThrow()+ alignRevocationCounters(watcher)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ expect(await readyStatus(server)).toBe(503)++ // Fire a same-identity MODIFIED delta. Its scoped revocation completes+ // (its label-scoped LISTs are clean) and it carries a delta certificate,+ // so the only thing standing between it and+ // recordNetworkPolicySafetyCertificate is the uncertified fence left by+ // the authoritative pass.+ const warnSpy = vi.spyOn(console, 'warn')+ try {+ await fireContextEvent('MODIFIED', {+ metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },+ spec: { contextId: 'alpha', mcpServers: [] },+ })++ // M1: the fence was lost before this delta, so+ // currentNetworkPolicySafetyCertificate() returns null at the capture and no+ // delta certificate is ever minted — the delta short-circuits at the source+ // and never reaches the per-delta certification branch. No warn is emitted;+ // nothing is recorded; readiness stays withdrawn via the independent+ // readiness fence even after every counter is realigned.+ expect(+ warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))+ ).toBe(false)+ expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(503)+ } finally {+ warnSpy.mockRestore()+ }+ })++ it('declines to certify a scoped delta whose certificate outlived a fence lost mid-flight, and warns', async () => {+ // Coverage preservation for the per-delta fence check (k8sClient.ts :3801):+ // the delta certificate is minted while the fence is still certified, then a+ // CONCURRENT authoritative pass loses its delete fence mid-flight. M1's+ // source short-circuit cannot catch this — only the per-delta check can. This+ // test is green with AND without M1; it guards the TOCTOU window, not M1.+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler+ const w = watcher as unknown as Record<string, any>++ await w.startContextWatch('ctx-rv-2')+ const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>+ String(path).endsWith('/namespaces/mcp-server/contexts')+ )+ if (!contextWatchCall) throw new Error('Context watch was never registered')+ const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>++ const priorContext = {+ name: 'alpha',+ namespace: 'mcp-server',+ uid: 'ctx-alpha-uid',+ generation: 1,+ spec: { contextId: 'alpha', mcpServers: [] },+ }+ w.contexts.set('alpha', priorContext)+ alignRevocationCounters(watcher)++ // Baseline clean authoritative pass: certified, gate open.+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ await reconciler.fullReconcile([priorContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(200)++ // The delta certificate is captured now, while the fence is still certified.+ // The FIRST scoped LIST the delta issues runs a concurrent authoritative pass+ // that loses its own delete fence — degrading the fence AFTER the delta minted+ // its certificate but BEFORE the delta reaches its certification branch.+ let fenceBrokenMidFlight = false+ mocks.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {+ const selector = labelSelector ?? ''+ const scoped =+ selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')+ if (scoped && !fenceBrokenMidFlight) {+ fenceBrokenMidFlight = true+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(+ Object.assign(new Error('the object has been modified'), { code: 409 })+ )+ await reconciler+ .fullReconcile([priorContext], [], {+ ensureDefaults: false,+ contextInventoryAuthoritative: () => true,+ serverInventoryAuthoritative: () => true,+ onAuthoritativeRevocationComplete: () => {},+ })+ .then(+ () => {+ throw new Error('interleaved authoritative pass should have lost its fence')+ },+ () => {}+ )+ }+ return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }+ }+ )+ const warnSpy = vi.spyOn(console, 'warn')+ try {+ await fireContextEvent('MODIFIED', {+ metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },+ spec: { contextId: 'alpha', mcpServers: [] },+ })++ // The interleave actually ran (no vacuous pass) and the per-delta check+ // withheld certification: it warned, recorded nothing, readiness stays 503.+ expect(fenceBrokenMidFlight).toBe(true)+ expect(+ warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))+ ).toBe(true)+ expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)+ alignRevocationCounters(watcher)+ expect(await readyStatus(server)).toBe(503)+ } finally {+ warnSpy.mockRestore()+ }+ })+})diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 92e74d6e8..26a7f5ab0 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -1,4 +1,4 @@-import { beforeEach, describe, expect, it, vi } from 'vitest'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'import {asAppsApi,@@ -14,6 +14,17 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './constimport { McpServerReconciler } from './reconciler'import { McpServerCRD } from './types'+function deferred<T = void>(): {+ promise: Promise<T>+ resolve: (value: T | PromiseLike<T>) => void+} {+ let resolve!: (value: T | PromiseLike<T>) => void+ const promise = new Promise<T>(res => {+ resolve = res+ })+ return { promise, resolve }+}+/*** Tests for the managed:false guard in the reconciler (Risk 1.7).*@@ -30,6 +41,7 @@ vi.mock('./config', () => ({controlPlaneNamespace: 'control-plane',hostNamespace: 'mcp-host',rpcProxyNamespace: 'rpc-proxy',+ egressProxyImage: 'clerum/nginx-egress-proxy:test',devMcpServers: [],devContexts: [],devAuthTokens: new Map(),@@ -64,6 +76,7 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -155,6 +168,55 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()})+ it('completes a remote proxy reconcile without rewriting the desired image', async () => {+ const server = makeServer({+ name: 'remote-api',+ image: 'vendor/original-image:1',+ remote: { baseUrl: 'https://api.example.com' },+ })+ const originalSpec = structuredClone(server.spec)+ let desiredRevisionIsCurrent = true+ customApi.patchNamespacedCustomObject.mockImplementation(+ ({ body }: { body?: { spec?: unknown } }) => {+ if (body?.spec !== undefined) {+ desiredRevisionIsCurrent = false+ }+ return Promise.resolve({})+ }+ )++ await reconciler.reconcile(server, {+ isCurrent: () => desiredRevisionIsCurrent,+ })++ expect(server.spec).toEqual(originalSpec)+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(+ expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),+ expect.anything()+ )+ expect(coreApi.createNamespacedConfigMap).toHaveBeenCalledOnce()+ expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({+ spec: expect.objectContaining({+ template: expect.objectContaining({+ spec: expect.objectContaining({+ containers: [+ expect.objectContaining({+ name: 'egress-proxy',+ image: 'clerum/nginx-egress-proxy:test',+ }),+ ],+ }),+ }),+ }),+ }),+ })+ )+ })+it('should log WRC-owned ownership when managed: false', async () => {const consoleSpy = vi.spyOn(console, 'log')const server = makeServer({ name: 'workflow-recipes', managed: false })@@ -223,6 +285,7 @@ describe('PR-B B1 — validateSecret result shape', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -407,7 +470,12 @@ describe('PR-B B1 — writeStatusCondition', () => {const getPatchedConditions = () => {const patch = customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0]- const op = patch.body[0]+ const op = patch.body.find(+ (candidate: { op?: string; path?: string }) =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ if (!op) throw new Error('status condition add operation was not emitted')const conditions = op.path === '/status' ? op.value.conditions : op.valuereturn { patch, op, conditions }}@@ -415,6 +483,7 @@ describe('PR-B B1 — writeStatusCondition', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -509,6 +578,216 @@ describe('PR-B B1 — writeStatusCondition', () => {expect(types).toContain('Ready')})+ it('records the reconciled McpServer generation on status conditions', async () => {+ const server = {+ ...makeServer({ name: 'pg' }),+ generation: 17,+ }++ await reconciler.writeStatusCondition(server, {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ })++ const { conditions } = getPatchedConditions()+ expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(+ expect.objectContaining({+ status: 'True',+ observedGeneration: 17,+ })+ )+ })++ it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {+ const networkReady = {+ type: 'NetworkReady',+ status: 'True',+ reason: 'NetworkPoliciesApplied',+ message: 'network ready',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ }+ const externalEgressReady = {+ type: 'ExternalEgressReady',+ status: 'True',+ reason: 'EgressPolicyApplied',+ message: 'external egress ready',+ lastTransitionTime: '2020-01-02T00:00:00.000Z',+ }+ customApi.getNamespacedCustomObjectStatus+ .mockResolvedValueOnce({+ metadata: { resourceVersion: '100' },+ status: { conditions: [networkReady] },+ })+ .mockResolvedValueOnce({+ metadata: { resourceVersion: '101' },+ status: { conditions: [networkReady, externalEgressReady] },+ })+ customApi.patchNamespacedCustomObjectStatus+ .mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))+ .mockResolvedValueOnce({})++ const server: McpServerCRD = {+ ...makeServer({ name: 'pg' }),+ uid: 'uid-pg',+ generation: 17,+ }+ await reconciler.writeStatusCondition(server, {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ })++ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body).toContainEqual({+ op: 'test',+ path: '/metadata/resourceVersion',+ value: '100',+ })+ const secondPatch = customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body+ expect(secondPatch).toContainEqual({+ op: 'test',+ path: '/metadata/resourceVersion',+ value: '101',+ })+ const conditionPatch = secondPatch.find(+ (candidate: { op?: string; path?: string }) =>+ candidate.op === 'add' && candidate.path === '/status/conditions'+ )+ expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([+ 'NetworkReady',+ 'ExternalEgressReady',+ 'Ready',+ ])+ })++ it('bounds resourceVersion conflict retries', async () => {+ const currentStatus = {+ metadata: { resourceVersion: '100' },+ status: { conditions: [] },+ }+ customApi.getNamespacedCustomObjectStatus+ .mockResolvedValueOnce(currentStatus)+ .mockResolvedValueOnce(currentStatus)+ .mockResolvedValueOnce(currentStatus)+ const conflict = Object.assign(new Error('resourceVersion conflict'), { code: 422 })+ customApi.patchNamespacedCustomObjectStatus+ .mockRejectedValueOnce(conflict)+ .mockRejectedValueOnce(conflict)+ .mockRejectedValueOnce(conflict)+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)++ await reconciler.writeStatusCondition(+ {+ ...makeServer({ name: 'pg' }),+ uid: 'uid-pg',+ generation: 17,+ },+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ }+ )++ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)+ expect(warn).toHaveBeenCalledWith(+ expect.stringContaining('Failed to write status condition Ready=True'),+ conflict+ )+ warn.mockRestore()+ })++ it('guards the status patch with the captured UID, generation, and resourceVersion', async () => {+ const staleServer: McpServerCRD = {+ ...makeServer({ name: 'recreated' }),+ uid: 'uid-before-recreation',+ generation: 1,+ }+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ metadata: { resourceVersion: '100' },+ status: { conditions: [] },+ })+ const writeFailure = Object.assign(new Error('status API unavailable'), {+ code: 500,+ })+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async request => {+ expect(request.body.slice(0, 3)).toEqual([+ {+ op: 'test',+ path: '/metadata/uid',+ value: 'uid-before-recreation',+ },+ {+ op: 'test',+ path: '/metadata/generation',+ value: 1,+ },+ {+ op: 'test',+ path: '/metadata/resourceVersion',+ value: '100',+ },+ ])+ throw writeFailure+ })+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)++ await reconciler.writeStatusCondition(staleServer, {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'stale reconcile result',+ })++ expect(warn).toHaveBeenCalledWith(+ expect.stringContaining('Failed to write status condition Ready=True'),+ writeFailure+ )+ warn.mockRestore()+ })++ it('refreshes observedGeneration even when the condition payload is otherwise unchanged', async () => {+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ status: {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ observedGeneration: 16,+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ },+ ],+ },+ })+ const server = {+ ...makeServer({ name: 'pg' }),+ generation: 17,+ }++ await reconciler.writeStatusCondition(server, {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ })++ const { conditions } = getPatchedConditions()+ expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(+ expect.objectContaining({+ observedGeneration: 17,+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ })+ )+ })+it('creates the status object when the CRD has no status yet', async () => {customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({})@@ -653,6 +932,170 @@ describe('PR-B B1 — writeStatusCondition', () => {})})+describe('restart-safe discovery status', () => {+ const appsApi = createMockAppsApi()+ const coreApi = createMockCoreApi()+ const customApi = createMockCustomApi()+ let reconciler: McpServerReconciler++ beforeEach(() => {+ vi.clearAllMocks()+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(customApi),+ })+ })++ it('uses a persisted Ready condition only when it observes the current generation', () => {+ const server: McpServerCRD = {+ ...makeServer({ name: 'restart-safe' }),+ generation: 7,+ status: {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'Deployment created',+ observedGeneration: 7,+ },+ ],+ },+ }++ expect(reconciler.getStatus(server)).toEqual({+ deployed: true,+ ready: true,+ message: 'Deployment created',+ authoritative: true,+ })+ })++ it.each([+ ['stale', 6],+ ['unversioned', undefined],+ ])('fails closed for a %s persisted Ready condition', (_label, observedGeneration) => {+ const server: McpServerCRD = {+ ...makeServer({ name: 'restart-unsafe' }),+ generation: 7,+ status: {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'Deployment created',+ observedGeneration,+ },+ ],+ },+ }++ expect(reconciler.getStatus(server)).toEqual({+ deployed: false,+ ready: false,+ message: 'Not reconciled yet',+ authoritative: false,+ })+ })++ it('fails closed when persisted Ready conditions conflict for the current generation', () => {+ const server: McpServerCRD = {+ ...makeServer({ name: 'restart-conflict' }),+ generation: 7,+ status: {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ observedGeneration: 7,+ },+ {+ type: 'Ready',+ status: 'False',+ reason: 'SecretValidationFailed',+ observedGeneration: 7,+ },+ ],+ },+ }++ expect(reconciler.getStatus(server)).toEqual({+ deployed: false,+ ready: false,+ message: 'Not reconciled yet',+ authoritative: false,+ })+ })++ it('prefers fresh in-memory status over persisted status from the same generation', async () => {+ const server: McpServerCRD = {+ ...makeServer({ name: 'live-status' }),+ generation: 3,+ status: {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ observedGeneration: 3,+ },+ ],+ },+ }+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({+ status: { readyReplicas: 0 },+ })++ await reconciler.reconcile(server)++ expect(reconciler.getStatus(server)).toEqual(+ expect.objectContaining({+ deployed: true,+ ready: false,+ authoritative: true,+ })+ )+ })++ it.each([+ ['generation changes', 'stable-uid', 4],+ ['the same name is recreated', 'replacement-uid', 3],+ ])('does not reuse in-memory status when %s', async (_case, uid, generation) => {+ const original: McpServerCRD = {+ ...makeServer({ name: 'identity-fenced' }),+ uid: 'original-uid',+ generation: 3,+ }+ await reconciler.reconcile(original)+ // #223 (credential rotation) makes readiness immediately False until the poll+ // upgrades it, so the original caches deployed/authoritative here — not ready.+ // This test's subject is the identity fencing asserted below, not readiness+ // timing; it only needs a real (non-default) cached status to prove it is not+ // reused across an identity change.+ expect(reconciler.getStatus(original)).toEqual(+ expect.objectContaining({ deployed: true, ready: false, authoritative: true })+ )++ const current: McpServerCRD = {+ ...original,+ uid,+ generation,+ status: undefined,+ }++ expect(reconciler.getStatus(current)).toEqual({+ deployed: false,+ ready: false,+ message: 'Not reconciled yet',+ authoritative: false,+ })+ })+})+describe('ownership-safe fail-closed cleanup', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()@@ -662,6 +1105,7 @@ describe('ownership-safe fail-closed cleanup', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -764,6 +1208,480 @@ describe('ownership-safe fail-closed cleanup', () => {})})+describe('full reconciliation inventory authority', () => {+ const appsApi = createMockAppsApi()+ const coreApi = createMockCoreApi()+ const customApi = createMockCustomApi()+ let reconciler: McpServerReconciler++ beforeEach(() => {+ vi.clearAllMocks()+ customApi.getNamespacedCustomObject.mockRejectedValue(+ Object.assign(new Error('not found'), { code: 404 })+ )+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(customApi),+ })+ })++ it('rejects when Deployment inventory LIST fails so startup retry remains possible', async () => {+ const inventoryError = new Error('deployment inventory unavailable')+ const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ appsApi.listNamespacedDeployment.mockRejectedValueOnce(inventoryError)++ await expect(reconciler.fullReconcile([])).rejects.toBe(inventoryError)++ expect(error).toHaveBeenCalledWith(+ '[Reconciler] Failed to list managed deployments:',+ inventoryError+ )+ error.mockRestore()+ })++ it('defaults to fail-closed orphan cleanup until an authority source is wired', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ })++ await reconciler.fullReconcile([])++ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))+ warn.mockRestore()+ })++ it('finishes desired reconciliation but skips orphan cleanup if inventory authority is lost mid-pass', async () => {+ let authority = { known: true, generation: 7 }+ reconciler.setInventoryAuthority(() => authority)+ const desired = makeServer({ name: 'desired-server' })+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {+ authority = { known: false, generation: 8 }+ })+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ })++ await reconciler.fullReconcile([desired])++ expect(reconcile).toHaveBeenCalledOnce()+ expect(reconcile).toHaveBeenCalledWith(desired, {+ isCurrent: expect.any(Function),+ })+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))+ warn.mockRestore()+ })++ it('skips orphan cleanup if the authoritative inventory generation changes mid-pass', async () => {+ let authority = { known: true, generation: 11 }+ reconciler.setInventoryAuthority(() => authority)+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {+ authority = { known: true, generation: 12 }+ })+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ })++ await reconciler.fullReconcile([makeServer({ name: 'desired-server' })])++ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))+ warn.mockRestore()+ })++ it('deletes an orphan when inventory authority remains known and stable', async () => {+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ })++ await reconciler.fullReconcile([])++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({+ name: 'orphan-server',+ namespace: 'mcp-server',+ })+ expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({+ name: 'orphan-server',+ namespace: 'mcp-server',+ })+ })++ it('skips a startup runtime revision superseded before its keyed effect is admitted', async () => {+ const snapshot: McpServerCRD = {+ ...makeServer({ name: 'updated-server' }),+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/updated-server:old',+ transport: { type: 'streamableHttp', port: 3000 },+ },+ }+ const current: McpServerCRD = {+ ...makeServer({ name: 'updated-server' }),+ generation: 2,+ spec: {+ contextRef: 'default',+ image: 'clerum/updated-server:new',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: [{ dns: 'new.example', port: 443 }],+ },+ }+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 40 }))+ reconciler.setResolveCurrentServer(() => current)+ const reconcile = vi.spyOn(reconciler, 'reconcile')++ await reconciler.fullReconcile([snapshot])++ expect(reconcile).not.toHaveBeenCalled()+ })++ it('does not mutate runtime or status after startup authority retires during Secret validation', async () => {+ let authority = { known: true, generation: 51 }+ const server = {+ ...makeServer({+ name: 'secret-authority-race',+ envSecret: {+ name: 'runtime-secret',+ keys: [{ secretKey: 'token', envVar: 'TOKEN' }],+ },+ }),+ uid: 'secret-authority-race-uid',+ generation: 4,+ }+ reconciler.setInventoryAuthority(() => authority)+ reconciler.setResolveCurrentServer(name => (name === server.name ? server : undefined))+ const secretReadStarted = deferred()+ const releaseSecretRead = deferred<{ data: Record<string, string> }>()+ coreApi.readNamespacedSecret.mockImplementationOnce(async () => {+ secretReadStarted.resolve(undefined)+ return releaseSecretRead.promise+ })++ const convergence = reconciler.fullReconcile([server])+ await secretReadStarted.promise++ authority = { known: true, generation: 53 }+ releaseSecretRead.resolve({ data: { token: 'encoded-token' } })+ await convergence++ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ })++ it('retires a same-watch runtime revision superseded during Secret validation', async () => {+ let current: McpServerCRD+ const original: McpServerCRD = {+ ...makeServer({+ name: 'same-watch-runtime',+ image: 'clerum/same-watch-runtime:v1',+ envSecret: {+ name: 'runtime-secret',+ keys: [{ secretKey: 'token', envVar: 'TOKEN' }],+ },+ }),+ uid: 'same-watch-runtime-uid',+ generation: 1,+ }+ const replacement: McpServerCRD = {+ ...original,+ generation: 2,+ spec: {+ ...original.spec,+ image: 'clerum/same-watch-runtime:v2',+ },+ }+ current = original+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 61 }))+ reconciler.setResolveCurrentServer(name => (name === original.name ? current : undefined))++ const originalSecretReadStarted = deferred()+ const releaseOriginalSecretRead = deferred<{ data: Record<string, string> }>()+ coreApi.readNamespacedSecret+ .mockImplementationOnce(async () => {+ originalSecretReadStarted.resolve(undefined)+ return releaseOriginalSecretRead.promise+ })+ .mockResolvedValueOnce({ data: { token: 'encoded-token' } })++ const originalReconcile = reconciler.reconcile(original)+ await originalSecretReadStarted.promise+ current = replacement+ const replacementReconcile = reconciler.reconcile(replacement)+ releaseOriginalSecretRead.resolve({ data: { token: 'encoded-token' } })++ await Promise.all([originalReconcile, replacementReconcile])++ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({+ namespace: replacement.namespace,+ body: expect.objectContaining({+ spec: expect.objectContaining({+ template: expect.objectContaining({+ spec: expect.objectContaining({+ containers: expect.arrayContaining([+ expect.objectContaining({+ image: 'clerum/same-watch-runtime:v2',+ }),+ ]),+ }),+ }),+ }),+ }),+ })+ })++ it('admits independent desired effects while another server effect is blocked', async () => {+ const blocked = makeServer({ name: 'blocked-server' })+ const independent = makeServer({ name: 'independent-server' })+ let markBlockedStarted!: () => void+ let releaseBlocked!: () => void+ const blockedStarted = new Promise<void>(resolve => {+ markBlockedStarted = resolve+ })+ const blockedRelease = new Promise<void>(resolve => {+ releaseBlocked = resolve+ })+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)+ const fullReconcile = reconciler.fullReconcile([blocked, independent], {+ runEffect: async (serverName, work) => {+ if (serverName === blocked.name) {+ markBlockedStarted()+ await blockedRelease+ }+ await work()+ },+ })++ await blockedStarted+ await Promise.resolve()++ expect(reconcile).toHaveBeenCalledWith(independent, {+ isCurrent: expect.any(Function),+ })+ expect(reconcile.mock.calls.some(([server]) => server === blocked)).toBe(false)++ releaseBlocked()+ await fullReconcile++ expect(reconcile).toHaveBeenCalledWith(blocked, {+ isCurrent: expect.any(Function),+ })+ })++ it('bounds desired-effect admission without serializing independent siblings', async () => {+ const desired = Array.from({ length: 25 }, (_, index) =>+ makeServer({ name: `bounded-server-${index}` })+ )+ let active = 0+ let maxActive = 0+ let startedBeforeRelease = 0+ let releaseAll!: () => void+ const release = new Promise<void>(resolve => {+ releaseAll = resolve+ })+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)+ const options = {+ maxConcurrency: 3,+ runEffect: async (_serverName: string, work: () => Promise<void>) => {+ active += 1+ startedBeforeRelease += 1+ maxActive = Math.max(maxActive, active)+ await release+ try {+ await work()+ } finally {+ active -= 1+ }+ },+ }++ const fullReconcile = reconciler.fullReconcile(desired, options)+ await vi.waitFor(() => expect(active).toBeGreaterThanOrEqual(3))+ await Promise.resolve()+ const observedMaxActive = maxActive+ const observedStartedBeforeRelease = startedBeforeRelease++ releaseAll()+ await fullReconcile++ expect(observedMaxActive).toBe(3)+ expect(observedStartedBeforeRelease).toBe(3)+ expect(reconcile).toHaveBeenCalledTimes(desired.length)+ })++ it('waits for every admitted desired effect before reporting a sibling failure', async () => {+ const failed = makeServer({ name: 'failed-server' })+ const slow = makeServer({ name: 'slow-server' })+ let markSlowStarted!: () => void+ let releaseSlow!: () => void+ const slowStarted = new Promise<void>(resolve => {+ markSlowStarted = resolve+ })+ const slowRelease = new Promise<void>(resolve => {+ releaseSlow = resolve+ })+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)++ const fullReconcile = reconciler.fullReconcile([failed, slow], {+ runEffect: async (serverName, work) => {+ if (serverName === failed.name) {+ throw new Error('failed server gate')+ }+ markSlowStarted()+ await slowRelease+ await work()+ },+ })+ let settled = false+ const observed = fullReconcile.then(+ () => undefined,+ error => error+ )+ void observed.then(() => {+ settled = true+ })++ await slowStarted+ await Promise.resolve()+ await Promise.resolve()++ expect(settled).toBe(false)++ releaseSlow()+ const error = await observed++ expect(error).toBeInstanceOf(AggregateError)+ expect(error).toMatchObject({+ message: 'Failed to reconcile desired McpServers',+ })+ expect(reconcile).toHaveBeenCalledWith(slow, {+ isCurrent: expect.any(Function),+ })+ })++ it('aggregates every desired-effect failure after all bounded workers settle', async () => {+ const failures = [+ new Error('first desired failure'),+ new Error('second desired failure'),+ new Error('third desired failure'),+ ]+ const desired = failures.map((_, index) => makeServer({ name: `failed-${index}` }))+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)++ const error = await reconciler+ .fullReconcile(desired, {+ maxConcurrency: 2,+ runEffect: async serverName => {+ throw failures[Number(serverName.replace('failed-', ''))]+ },+ })+ .then(+ () => undefined,+ reason => reason+ )++ expect(error).toBeInstanceOf(AggregateError)+ expect((error as AggregateError).errors).toEqual(failures)+ expect(reconcile).not.toHaveBeenCalled()+ })++ it('does not delete an orphan candidate that was added to the same-generation cache mid-pass', async () => {+ const currentServers = new Map<string, McpServerCRD>()+ ;(reconciler as any).setResolveCurrentServer((name: string) => currentServers.get(name))+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))+ appsApi.listNamespacedDeployment.mockImplementationOnce(async () => {+ currentServers.set('orphan-server', makeServer({ name: 'orphan-server' }))+ return {+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ }+ })++ await reconciler.fullReconcile([])++ expect(customApi.getNamespacedCustomObject).not.toHaveBeenCalled()+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ })++ it('does not delete an orphan candidate when a fresh API GET sees a same-name CRD before ADDED arrives', async () => {+ ;(reconciler as any).setResolveCurrentServer(() => undefined)+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 32 }))+ customApi.getNamespacedCustomObject.mockResolvedValueOnce({+ metadata: { name: 'orphan-server', namespace: 'mcp-server', uid: 'recreated-uid' },+ })+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ {+ metadata: {+ name: 'orphan-server',+ namespace: 'mcp-server',+ },+ },+ ],+ })++ await reconciler.fullReconcile([])++ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: 'mcp-server',+ plural: 'mcpservers',+ name: 'orphan-server',+ })+ )+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ })+})+describe('updateStatusConditions', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()@@ -773,6 +1691,7 @@ describe('updateStatusConditions', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -835,3 +1754,53 @@ describe('updateStatusConditions', () => {expect(types).toContain('DeploymentReady')})})++describe('unconfigured inventory authority fails closed (G1)', () => {+ const appsApi = createMockAppsApi()+ const coreApi = createMockCoreApi()+ const customApi = createMockCustomApi()+ let errorSpy: ReturnType<typeof vi.spyOn>++ // Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO+ // setInventoryAuthority/setResolveCurrentServer wiring: the fence must deny.+ function makeUnwiredReconciler(): McpServerReconciler {+ return new McpServerReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(customApi),+ })+ }++ beforeEach(() => {+ vi.clearAllMocks()+ errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ })++ afterEach(() => {+ errorSpy.mockRestore()+ })++ it('T-G1a: reconcile() denies (no Deployment) and warns once when authority is unwired', async () => {+ const reconciler = makeUnwiredReconciler()+ await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('fail closed'))+ })++ it('T-G1b: fullReconcile() denies when authority is unwired', async () => {+ const reconciler = makeUnwiredReconciler()+ await reconciler.fullReconcile([makeServer({ name: 'g1-mcp', managed: true })])+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ })++ it('T-G1c: the explicit opt-in restores assume-current', async () => {+ const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(customApi),+ assumeInventoryAuthorityWhenUnconfigured: true,+ })+ await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()+ })+})diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex caba6be91..4056c621c 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -16,6 +16,10 @@ import { classifyPluginImage } from '@clerum/image-policy'import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'import { config } from './config'import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'+import {+ confirmAuthoritativeMcpServerAbsence,+ sameMcpServerDesiredRevision,+} from './mcpServerSafety'import { mcpserverMissingSecret } from './metrics'import { McpServerCRD, McpServerStatus } from './types'import {@@ -62,6 +66,7 @@ export interface McpServerCondition {reason: stringmessage: stringlastTransitionTime: string+ observedGeneration?: number}type McpServerReconcilerDeps = {@@ -69,6 +74,28 @@ type McpServerReconcilerDeps = {coreApi?: k8s.CoreV1ApicustomApi?: k8s.CustomObjectsApinetworkingApi?: k8s.NetworkingV1Api+ /**+ * TEST-ONLY escape hatch. When the inventory-authority fence is not wired+ * (setInventoryAuthority / setResolveCurrentServer never called), mutation+ * fences fail CLOSED by default. Unit tests that exercise the reconciler+ * without a watcher opt in to the legacy assume-current behavior here.+ * Production construction (k8sClient) must never set this.+ */+ assumeInventoryAuthorityWhenUnconfigured?: boolean+}++export type McpServerInventoryAuthoritySnapshot = {+ known: boolean+ generation: number+}++export type McpServerFullReconcileOptions = {+ runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>+ maxConcurrency?: number+}++export type McpServerMutationOptions = {+ isCurrent?: () => boolean}// G2: Pre-deploy handshake constants@@ -88,6 +115,13 @@ const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/// Bound the value length so a pathological CRD cannot blow up the ConfigMap.const MAX_AUTH_HEADER_VALUE_LENGTH = 2048const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'+// Keep fleet admission aligned with the external-egress convergence lane:+// enough parallelism for independent servers without allocating one pending+// effect promise per fleet member.+const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10+// Status is a read/merge/write transaction shared with other controllers.+// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.+const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3function throwCleanupFailures(failures: unknown[], message: string): void {if (failures.length > 0) {@@ -95,6 +129,37 @@ function throwCleanupFailures(failures: unknown[], message: string): void {}}+async function mapSettledWithConcurrency<T, TResult>(+ items: readonly T[],+ maxConcurrency: number,+ work: (item: T, index: number) => Promise<TResult>+): Promise<Array<PromiseSettledResult<TResult>>> {+ if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {+ throw new RangeError('maxConcurrency must be a positive integer')+ }++ const results = new Array<PromiseSettledResult<TResult>>(items.length)+ let nextIndex = 0+ const worker = async (): Promise<void> => {+ while (nextIndex < items.length) {+ const index = nextIndex+ nextIndex += 1+ try {+ results[index] = {+ status: 'fulfilled',+ value: await work(items[index], index),+ }+ } catch (reason) {+ results[index] = { status: 'rejected', reason }+ }+ }+ }++ const workerCount = Math.min(maxConcurrency, items.length)+ await Promise.all(Array.from({ length: workerCount }, () => worker()))+ return results+}+/*** State of the readiness poll window for one McpServer (issue #223).*@@ -146,6 +211,35 @@ interface ReadinessPollState {* WaitingForReplicas by the very reconcile the terminal write triggers.*/detail: string+ /**+ * Mutation fence and CRD object of the LATEST reconcile that owns this+ * window. pollReadiness() REBINDS both on every call that finds the window+ * live: a superseding reconcile (e.g. the fullReconcile after a routine+ * McpServer watch recovery bumps the inventory-authority generation) becomes+ * the window's owner, so the next tick evaluates the CURRENT fence instead of+ * the arming reconcile's stale closure. Without the rebind the stale fence+ * self-terminates the window with no re-arm and — status-only MODIFIED events+ * being suppressed, with no periodic McpServer resync — DeploymentReady=False+ * sticks forever for a healthy rollout (the #223 rotation-timeout symptom,+ * reintroduced under a routine reconnect).+ *+ * Two ORTHOGONAL supersession guards protect this window; do not assume one+ * covers the other (that assumption is exactly how the F1 race was born):+ * - Identity retirement: beginStatusTracking()/clearStatus() retire the+ * window on any CRD uid or CRD-generation change (spec edit, recreate),+ * so a surviving window always has the caller's status identity — a+ * rebind can never adopt a recreate or a superseded desired revision.+ * - In-tick generation reset + recheckRequested: a credential rotation+ * bumps neither CRD uid nor CRD generation (it changes the Deployment's+ * metadata.generation only), so identity retirement does NOT fire; the+ * rotation is instead guarded by the tick's live Deployment read (a new+ * generation resets attempts/exhausted) and by recheckRequested deferring+ * a mid-flight tick's terminal verdict past the rotation's write.+ * A rebind never touches the budget fields (generation/attempts/exhausted/+ * detail) above.+ */+ isCurrent: () => boolean+ server: McpServerCRD}export class McpServerReconciler {@@ -154,8 +248,26 @@ export class McpServerReconciler {private readonly customApi: k8s.CustomObjectsApiprivate readonly networkingApi?: k8s.NetworkingV1Api+ /**+ * Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.+ * The fail-closed default prevents orphan cleanup until the watcher wires+ * the real authority source.+ */+ private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({+ known: false,+ generation: 0,+ })+ private inventoryAuthorityConfigured = false+ private resolveCurrentServer?: (name: string) => McpServerCRD | undefined+ /** See McpServerReconcilerDeps: test-only opt-in to assume-current when the+ * inventory-authority fence is unwired. Production leaves it false. */+ private readonly assumeInventoryAuthorityWhenUnconfigured: boolean+ private warnedUnconfiguredAuthority = false+/** Tracks the deployment status of each managed McpServer. */private readonly statusMap: Map<string, McpServerStatus> = new Map()+ private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =+ new Map()/** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */private readonly managedSnapshot: Map<string, boolean> = new Map()@@ -190,20 +302,120 @@ export class McpServerReconciler {this.networkingApi =deps?.networkingApi ??(typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)+ this.assumeInventoryAuthorityWhenUnconfigured =+ deps?.assumeInventoryAuthorityWhenUnconfigured ?? false+ }++ /**+ * Late-bound authority source for destructive full-reconcile cleanup.+ * A full pass may finish reconciling desired servers, but it may delete an+ * orphan only while the same authoritative inventory generation remains+ * active from pass start through delete admission.+ */+ setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {+ this.inventoryAuthority = fn+ this.inventoryAuthorityConfigured = true+ }++ /**+ * Resolve current cache membership at effect admission. Full-reconcile+ * snapshots select candidates only; they must never overwrite or delete a+ * same-name CRD accepted by the continuing watch.+ */+ setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {+ this.resolveCurrentServer = fn+ }++ /**+ * Verdict for a mutation fence evaluated while the inventory authority is NOT+ * wired. Fail closed (deny) unless a test explicitly opted in; deny loudly+ * once, because an unconfigured fence in a running controller is a+ * construction bug, never a runtime condition.+ */+ private currentWhenUnconfigured(): boolean {+ if (this.assumeInventoryAuthorityWhenUnconfigured) return true+ if (!this.warnedUnconfiguredAuthority) {+ this.warnedUnconfiguredAuthority = true+ console.error(+ '[Reconciler] inventory authority not configured — mutation fences fail closed. ' ++ 'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'+ )+ }+ return false}// ─── Status ────────────────────────────────────────────────────────/*** Get deployment status for an McpServer.- * Returns unknown status if the server hasn't been reconciled yet.+ * Returns current in-memory status when this process has reconciled the+ * server. After a controller restart, a generation-matched persisted Ready+ * condition is the only safe fallback: a legacy condition without+ * observedGeneration, or a condition for an older spec generation, cannot+ * prove that the current desired runtime is ready.*/- getStatus(name: string): McpServerStatus {+ getStatus(serverOrName: string | McpServerCRD): McpServerStatus {+ const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name+ const current = this.statusMap.get(name)+ if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {+ return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }+ }++ if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {+ const readyConditions =+ serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []+ const persistedReady =+ readyConditions.length === 1 &&+ readyConditions[0].observedGeneration === serverOrName.generation+ ? readyConditions[0]+ : undefined+ if (persistedReady) {+ const ready = persistedReady.status === 'True'+ return {+ deployed: ready,+ ready,+ message: persistedReady.message,+ authoritative: true,+ }+ }+ }++ return {+ deployed: false,+ ready: false,+ message: 'Not reconciled yet',+ authoritative: false,+ }+ }++ private statusIdentityMatches(server: McpServerCRD): boolean {+ const identity = this.statusIdentities.get(server.name)return (- this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' }+ identity !== undefined &&+ identity.uid === server.uid &&+ identity.generation === server.generation)}+ private beginStatusTracking(server: McpServerCRD): void {+ if (!this.statusIdentityMatches(server)) {+ this.statusMap.delete(server.name)+ // A new CRD identity (uid/generation) retires any poll window armed for+ // the previous desired revision: its verdicts would describe a rollout+ // that no longer represents the desired state. The reconcile for the+ // new identity arms its own window after its Deployment write.+ const poll = this.readinessPolls.get(server.name)+ if (poll) {+ if (poll.timer !== undefined) clearTimeout(poll.timer)+ this.readinessPolls.delete(server.name)+ }+ }+ this.statusIdentities.set(server.name, {+ uid: server.uid,+ generation: server.generation,+ })+ }+/*** Update the status of an McpServer after reconciliation.*/@@ -216,6 +428,7 @@ export class McpServerReconciler {*/private clearStatus(name: string): void {this.statusMap.delete(name)+ this.statusIdentities.delete(name)this.managedSnapshot.delete(name)const poll = this.readinessPolls.get(name)if (poll) {@@ -307,14 +520,26 @@ export class McpServerReconciler {* window with a superseded generation's numbers and the new rollout would* never be observed (see ReadinessPollState.recheckRequested).*/- private pollReadiness(server: McpServerCRD, intervalMs = 5000, maxAttempts = 24): void {+ private pollReadiness(+ server: McpServerCRD,+ intervalMs = 5000,+ maxAttempts = 24,+ isCurrent: () => boolean = () => true+ ): void {const { name, namespace } = serverconst previous = this.readinessPolls.get(name)if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {- // A window is already running for this server — leave it alone- // (re-arming would reset the attempt budget: the E2 status-update-loop).- // But if its tick is executing RIGHT NOW, the tick's Deployment read may+ // A window is already running for this server — leave its TIMER and+ // BUDGET alone (re-arming would reset the attempt budget: the E2+ // status-update-loop). But this caller is now the window's owner: rebind+ // the mutation fence (and the same-identity server object) so the next+ // tick evaluates the CURRENT fence instead of the arming reconcile's+ // stale closure, which would otherwise self-terminate the window with no+ // re-arm after a routine watch-generation bump.+ previous.isCurrent = isCurrent+ previous.server = server+ // If its tick is executing RIGHT NOW, the tick's Deployment read may// predate the Deployment write that triggered THIS call (a new// generation), so a silent no-op could let the tick end the window on// stale data. Record that a re-check is owed; the tick honors it before@@ -337,6 +562,8 @@ export class McpServerReconciler {attempts: 0,exhausted: previous?.exhausted ?? false,detail: previous?.detail ?? '',+ isCurrent,+ server,}this.readinessPolls.set(name, state)@@ -346,7 +573,21 @@ export class McpServerReconciler {// can tell "tick running" apart from "window ended" during our awaits.state.timer = undefinedstate.ticking = true+ // Mutation fencing: a superseded reconcile (desired-revision or+ // inventory-authority change) must not keep observing — or writing+ // status — on behalf of a stale desired state. End the window quietly;+ // the current reconcile arms its own window after its Deployment write.+ if (!state.isCurrent()) {+ state.ticking = false+ if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)+ return+ }const rollout = await this.readDeploymentRollout(name, namespace)+ if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {+ state.ticking = false+ if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)+ return+ }// The McpServer may have been deleted (clearStatus) while we were// reading. A cancelled window must not keep ticking or write status@@ -395,7 +636,20 @@ export class McpServerReconciler {// always false, so this is the ONLY place a converged rollout is ever// recorded — without it every successful credential rotation would sit// at DeploymentReady=False forever and the UI would report a timeout.- await this.updateStatusConditions(server, true)+ await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)+ // Readiness is decoupled from fleet convergence: the poll also+ // upgrades the persisted, generation-matched Ready condition so a+ // controller restart can trust the recorded readiness (see getStatus).+ await this.writeStatusCondition(+ state.server,+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'Deployment created',+ },+ state.isCurrent+ )return}@@ -415,7 +669,7 @@ export class McpServerReconciler {// Give up loudly, on the CRD: a stuck rollout must be visible to the// operator with the numbers that prove it, never left as a stale// condition or a silent Unknown.- await this.updateStatusConditions(server, false, rollout.detail)+ await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)return}@@ -679,8 +933,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure ConfigMap exists and is up to date (create or replace).*/- private async ensureConfigMap(server: McpServerCRD): Promise<void> {+ private async ensureConfigMap(+ server: McpServerCRD,+ isCurrent: () => boolean = () => true+ ): Promise<void> {const cm = this.buildNginxConfigMap(server)+ if (!isCurrent()) returntry {await this.coreApi.createNamespacedConfigMap({namespace: server.namespace,@@ -693,6 +951,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvname: `${server.name}-nginx-conf`,namespace: server.namespace,})+ if (!isCurrent()) returnconst next = preserveObjectAnnotations({...cm,@@ -703,6 +962,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},existing)+ if (!isCurrent()) returnawait this.coreApi.replaceNamespacedConfigMap({name: `${server.name}-nginx-conf`,namespace: server.namespace,@@ -810,7 +1070,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {- // ── Platform hardening: sanitize CRD before building PodSpec ──+ // ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──+ // The caller retains this object as the LIST -> WATCH desired-state cache.+ // Mutating it here makes a later status-only watch event look like a+ // user-driven spec revision, which invalidates the authoritative policy+ // revocation marker during startup. Sanitization remains mandatory for the+ // generated PodSpec, but must never rewrite the observed CRD inventory.+ server = structuredClone(server)this.sanitizeCrdSpec(server)const transportPort = server.spec.transport.port || 3000@@ -1206,9 +1472,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure a Deployment exists and is up to date.*/- private async ensureDeployment(server: McpServerCRD, credentialsRevision = ''): Promise<void> {+ private async ensureDeployment(+ server: McpServerCRD,+ credentialsRevision = '',+ isCurrent: () => boolean = () => true+ ): Promise<void> {const deployment = this.buildDeployment(server, credentialsRevision)+ if (!isCurrent()) returntry {await this.appsApi.createNamespacedDeployment({namespace: server.namespace,@@ -1243,6 +1514,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvnamespace: server.namespace,body,}),+ mutationAllowed: isCurrent,})}}@@ -1250,9 +1522,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure a Service exists and is up to date.*/- private async ensureService(server: McpServerCRD): Promise<void> {+ private async ensureService(+ server: McpServerCRD,+ isCurrent: () => boolean = () => true+ ): Promise<void> {const service = this.buildService(server)+ if (!isCurrent()) returntry {await this.coreApi.createNamespacedService({namespace: server.namespace,@@ -1276,6 +1552,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvnamespace: server.namespace,body,}),+ mutationAllowed: isCurrent,})} else {throw error@@ -1288,12 +1565,16 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* Name matches are not enough: WRC-owned `managed:false` runtimes can share* names with McpServer CRDs, so every delete verifies HCC ownership labels.*/- private async deleteResources(name: string, namespace: string): Promise<void> {+ private async deleteResources(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {const failures: unknown[] = []for (const cleanup of [- () => this.deleteDeploymentIfHccOwned(name, namespace),- () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name),- () => this.deleteServiceIfHccOwned(name, namespace),+ () => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),]) {try {await cleanup()@@ -1315,7 +1596,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName}- private async deleteDeploymentIfHccOwned(name: string, namespace: string): Promise<void> {+ private async deleteDeploymentIfHccOwned(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {let deployment: k8s.V1Deploymenttry {deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })@@ -1333,6 +1618,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)return}+ if (deleteAllowed && !(await deleteAllowed())) returntry {await this.appsApi.deleteNamespacedDeployment({ name, namespace })@@ -1350,7 +1636,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvprivate async deleteConfigMapIfHccOwned(configMapName: string,namespace: string,- serverName: string+ serverName: string,+ deleteAllowed?: () => Promise<boolean>): Promise<void> {let configMap: k8s.V1ConfigMaptry {@@ -1369,6 +1656,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)return}+ if (deleteAllowed && !(await deleteAllowed())) returntry {await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })@@ -1383,7 +1671,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async deleteServiceIfHccOwned(name: string, namespace: string): Promise<void> {+ private async deleteServiceIfHccOwned(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {let service: k8s.V1Servicetry {service = await this.coreApi.readNamespacedService({ name, namespace })@@ -1401,6 +1693,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)return}+ if (deleteAllowed && !(await deleteAllowed())) returntry {await this.coreApi.deleteNamespacedService({ name, namespace })@@ -1415,7 +1708,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async deleteNetworkPoliciesForServer(name: string, namespace: string): Promise<void> {+ private async deleteNetworkPoliciesForServer(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {if (!this.networkingApi) returnconst failures: unknown[] = []const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]@@ -1429,6 +1726,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconst policyName = policy.metadata.nameif (!policyName) continuetry {+ if (deleteAllowed && !(await deleteAllowed())) returnawait this.networkingApi.deleteNamespacedNetworkPolicy({name: policyName,namespace: policyNamespace,@@ -1460,15 +1758,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvthrowCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)}- private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {+ private async deleteRuntimeResources(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {const failures: unknown[] = []try {- await this.deleteResources(name, namespace)+ await this.deleteResources(name, namespace, deleteAllowed)} catch (error) {failures.push(error)}try {- await this.deleteNetworkPoliciesForServer(name, namespace)+ await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)} catch (error) {failures.push(error)}@@ -1481,13 +1783,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.* Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.*/- private async setNetworkReadyAnnotation(server: McpServerCRD): Promise<void> {+ private async setNetworkReadyAnnotation(+ server: McpServerCRD,+ isCurrent: () => boolean = () => true+ ): Promise<void> {// Use merge-patch (plain object body) instead of JSON Patch (array body).// JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations// to already exist — it fails with 422 when the CRD was created without annotations.// Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)// knows the handshake was not completed — WRC will time out and fail rather than proceed// silently without NetworkPolicies in place.+ if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObject({group: 'clerum.io',@@ -1509,74 +1815,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}- /**- * Keep remote egress proxy CRDs aligned with the platform-owned runtime image.- *- * Remote servers are rendered as nginx egress proxies and never run- * `server.spec.image`, but stale CR specs can mislead scanners or future- * tooling. Patch only the declarative image field; Deployment rendering still- * uses `config.egressProxyImage` as the authority.- *- * This stays internally non-throwing because image canonicalization is- * best-effort cleanup; a failed CRD patch must not block ConfigMap,- * Service, or Deployment reconciliation for the actual proxy runtime.- */- private async canonicalizeRemoteEgressProxyImage(server: McpServerCRD): Promise<void> {- if (!this.isRemote(server)) {- return- }- if (server.spec.image === config.egressProxyImage) {- await this.writeStatusCondition(server, {- type: 'ImageCanonicalized',- status: 'True',- reason: 'RemoteEgressProxyImageMatches',- message: 'Remote McpServer spec.image matches the platform egress proxy image',- })- return- }-- try {- await this.customApi.patchNamespacedCustomObject(- {- group: 'clerum.io',- version: 'v1alpha1',- namespace: server.namespace,- plural: 'mcpservers',- name: server.name,- body: {- spec: {- image: config.egressProxyImage,- },- },- },- {- middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],- }- )- server.spec.image = config.egressProxyImage- console.log(- `[Reconciler] Canonicalized remote McpServer "${server.name}" image to ${config.egressProxyImage}`- )- await this.writeStatusCondition(server, {- type: 'ImageCanonicalized',- status: 'True',- reason: 'RemoteEgressProxyImageMatches',- message: 'Remote McpServer spec.image matches the platform egress proxy image',- })- } catch (err) {- console.warn(- `[Reconciler] Failed to canonicalize remote McpServer "${server.name}" image to ${config.egressProxyImage}:`,- err- )- await this.writeStatusCondition(server, {- type: 'ImageCanonicalized',- status: 'False',- reason: 'RemoteEgressProxyImagePatchFailed',- message: 'Failed to patch remote McpServer spec.image to the platform egress proxy image',- })- }- }-/*** G11: Update status conditions on McpServer CRD.* Sets NetworkReady and DeploymentReady conditions so WRC can watch@@ -1590,29 +1828,38 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* message so a stuck rollout reports WHAT is stuck (issue #223). Callers* that only know the verdict omit it.*/- rolloutDetail?: string+ rolloutDetail?: string,+ isCurrent: () => boolean = () => true): Promise<void> {try {- await this.writeStatusCondition(server, {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'NetworkPolicies and Service created',- })- await this.writeStatusCondition(server, {- type: 'DeploymentReady',- status: deploymentReady ? 'True' : 'False',- reason: deploymentReady- ? 'ReplicasAvailable'- : rolloutDetail- ? 'RolloutIncomplete'- : 'WaitingForReplicas',- message: rolloutDetail- ? `Rollout did not converge — ${rolloutDetail}`- : deploymentReady- ? 'Deployment has ready replicas'- : 'Waiting for pods to become ready',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'NetworkReady',+ status: 'True',+ reason: 'NetworkPoliciesApplied',+ message: 'NetworkPolicies and Service created',+ },+ isCurrent+ )+ await this.writeStatusCondition(+ server,+ {+ type: 'DeploymentReady',+ status: deploymentReady ? 'True' : 'False',+ reason: deploymentReady+ ? 'ReplicasAvailable'+ : rolloutDetail+ ? 'RolloutIncomplete'+ : 'WaitingForReplicas',+ message: rolloutDetail+ ? `Rollout did not converge — ${rolloutDetail}`+ : deploymentReady+ ? 'Deployment has ready replicas'+ : 'Waiting for pods to become ready',+ },+ isCurrent+ )console.log(`[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`)@@ -1637,12 +1884,29 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv** Failure modes:* - 404 (CRD deleted mid-reconcile) → log + swallow.+ * - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.* - Other errors → log + swallow (status writes are best-effort).*/async writeStatusCondition(server: McpServerCRD,- condition: Omit<McpServerCondition, 'lastTransitionTime'>+ condition: Omit<McpServerCondition, 'lastTransitionTime'>,+ isCurrent: () => boolean = () => true): Promise<void> {+ if (!isCurrent()) return+ const now = new Date().toISOString()++ // Update the missing-secret gauge for SecretResolved conditions.+ // Do this before any no-op short-circuit so the gauge stays current+ // across reconciles even when we skip the API patch.+ if (condition.type === 'SecretResolved' && server.spec.envSecret) {+ const labels = {+ namespace: server.namespace,+ name: server.name,+ secret_name: server.spec.envSecret.name,+ }+ mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)+ }+// Optimistic-concurrency loop (M2). The patch REPLACES the whole// /status/conditions array with a value built from a snapshot read just// before. Without a version guard, a concurrent writer — the readiness poll@@ -1652,12 +1916,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// op on metadata.resourceVersion makes the API server reject the patch (422)// if anything changed since our read; we then re-read and retry, so no// writer's condition is lost.- const MAX_ATTEMPTS = 4- for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {- const now = new Date().toISOString()-- // Fetch current status (and its resourceVersion) to preserve existing- // conditions and to fence the write below.+ for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {+ if (!isCurrent()) return+ // Re-read on every optimistic-conflict retry. Merging against the+ // latest resourceVersion preserves conditions written by peer+ // controllers between our read and patch.let existingConditions: McpServerCondition[] = []let hasStatusObject = falselet resourceVersion: string | undefined@@ -1672,9 +1935,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvmetadata?: { resourceVersion?: string }status?: { conditions?: McpServerCondition[] }}- resourceVersion = current.metadata?.resourceVersionhasStatusObject = typeof current.status === 'object' && current.status !== nullexistingConditions = current.status?.conditions ?? []+ resourceVersion = current.metadata?.resourceVersion} catch (error) {if (getErrorCode(error) === 404) {console.warn(@@ -1688,26 +1951,16 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)return}+ if (!isCurrent()) returnconst prior = existingConditions.find(c => c.type === condition.type)+ const observedGeneration = condition.observedGeneration ?? server.generationconst unchanged =prior !== undefined &&prior.status === condition.status &&prior.reason === condition.reason &&- prior.message === condition.message-- // Update the missing-secret gauge for SecretResolved conditions. Do this- // BEFORE the no-op short-circuit so the gauge stays current across- // reconciles even when we skip the API patch. Idempotent, so a retry- // re-running it is harmless.- if (condition.type === 'SecretResolved' && server.spec.envSecret) {- const labels = {- namespace: server.namespace,- name: server.name,- secret_name: server.spec.envSecret.name,- }- mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)- }+ prior.message === condition.message &&+ prior.observedGeneration === observedGeneration// Skip the patch when the condition is identical to what's already on// the CRD. Without this, every reconcile bumps the McpServer's@@ -1727,26 +1980,49 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreason: condition.reason,message: condition.message,lastTransitionTime,+ ...(observedGeneration !== undefined && { observedGeneration }),}const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]- // The `test` op fences the write to the exact revision we read. If a- // concurrent writer bumped resourceVersion in between, the API server- // fails the patch (422) and we retry with a fresh read.+ const writePreconditions: Array<{+ op: 'test'+ path: string+ value: string | number+ }> = []+ if (server.uid !== undefined) {+ writePreconditions.push({+ op: 'test',+ path: '/metadata/uid',+ value: server.uid,+ })+ }+ if (server.generation !== undefined) {+ writePreconditions.push({+ op: 'test',+ path: '/metadata/generation',+ value: server.generation,+ })+ }+ if (resourceVersion !== undefined) {+ writePreconditions.push({+ op: 'test',+ path: '/metadata/resourceVersion',+ value: resourceVersion,+ })+ }const statusPatch = [- ...(resourceVersion- ? [{ op: 'test', path: '/metadata/resourceVersion', value: resourceVersion }]- : []),- hasStatusObject- ? { op: 'add', path: '/status/conditions', value: nextConditions }- : { op: 'add', path: '/status', value: { conditions: nextConditions } },+ ...writePreconditions,+ ...(hasStatusObject+ ? [{ op: 'add', path: '/status/conditions', value: nextConditions }]+ : [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),]try {// The generated client prefers JSON Patch for this endpoint. Send an// actual patch document so status writes keep working across client// upgrades instead of relying on merge-patch object bodies.+ if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObjectStatus({group: 'clerum.io',version: 'v1alpha1',@@ -1757,22 +2033,25 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv})return} catch (error) {- if (getErrorCode(error) === 404) {+ const errorCode = getErrorCode(error)+ if (errorCode === 404) {console.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`)return}- // 409 (Conflict) or 422 (the resourceVersion `test` op failed) mean a- // concurrent writer won the race. Re-read and retry rather than lose- // our condition. Any other error, or the last attempt, falls through to- // the best-effort log-and-swallow below.- const code = getErrorCode(error)- if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) {+ // 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer+ // won the race. Re-read and retry rather than lose our condition. Any+ // other error, or the last attempt, falls through to the best-effort+ // log-and-swallow below.+ if (+ (errorCode === 409 || errorCode === 422) &&+ attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS+ ) {continue}console.warn(- `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${MAX_ATTEMPTS}):`,+ `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,error)return@@ -1790,10 +2069,24 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* A prior failure does not block the next call; the chain absorbs it so* the next reconcile still runs.*/- async reconcile(server: McpServerCRD): Promise<void> {+ async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {+ const capturedAuthority = this.inventoryAuthority()+ const callerIsCurrent = options.isCurrent ?? (() => true)+ const isCurrent = () => {+ if (!callerIsCurrent()) return false+ if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()+ if (!this.authorityMatches(capturedAuthority)) return false+ const current = this.resolveCurrentServer?.(server.name)+ return current !== undefined && sameMcpServerDesiredRevision(server, current)+ }const key = server.nameconst prev = this.inFlight.get(key) ?? Promise.resolve()- const next = prev.catch(() => undefined).then(() => this.performReconcile(server))+ const next = prev+ .catch(() => undefined)+ .then(async () => {+ if (!isCurrent()) return+ await this.performReconcile(server, isCurrent)+ })this.inFlight.set(key, next)try {await next@@ -1804,8 +2097,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async performReconcile(server: McpServerCRD): Promise<void> {+ private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {+ if (!isCurrent()) return+ this.beginStatusTracking(server)+// G7: Enforce managed field immutability — once set, cannot change+ if (!isCurrent()) returnconst currentManaged = server.spec.managed ?? trueconst previousManaged = this.managedSnapshot.get(server.name)if (previousManaged !== undefined && previousManaged !== currentManaged) {@@ -1813,6 +2110,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +`This is not allowed — delete and recreate the McpServer to change ownership.`)+ if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -1820,6 +2118,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv})return}+ if (!isCurrent()) returnthis.managedSnapshot.set(server.name, currentManaged)// If disabled, respect ownership before any cleanup. managed:false means@@ -1829,57 +2128,72 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.log(`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`)- await this.deleteRuntimeResources(server.name, server.namespace)+ await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())} else {console.log(`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`)}+ if (!isCurrent()) returnthis.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })- await this.writeStatusCondition(server, {- type: 'Ready',- status: 'False',- reason: 'Disabled',- message: 'McpServer is disabled',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',+ status: 'False',+ reason: 'Disabled',+ message: 'McpServer is disabled',+ },+ isCurrent+ )return}if (!currentManaged) {- await this.reconcileWrcOwnedServer(server)+ await this.reconcileWrcOwnedServer(server, isCurrent)return}// Validate secret before creating/updatingconst secretResult = await this.validateSecret(server)+ if (!isCurrent()) returnif (!secretResult.ok) {console.error(`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`)if (this.shouldFailClosedForSecretFailure(secretResult)) {- await this.deleteRuntimeResources(server.name, server.namespace)+ await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())} else {console.warn(`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`)}+ if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,message: secretResult.message,})- await this.writeStatusCondition(server, {- type: 'SecretResolved',- status: 'False',- reason: secretResult.reason,- message: secretResult.message,- })- await this.writeStatusCondition(server, {- type: 'Ready',- status: 'False',- reason: 'SecretValidationFailed',- message: secretResult.message,- })+ await this.writeStatusCondition(+ server,+ {+ type: 'SecretResolved',+ status: 'False',+ reason: secretResult.reason,+ message: secretResult.message,+ },+ isCurrent+ )+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',+ status: 'False',+ reason: 'SecretValidationFailed',+ message: secretResult.message,+ },+ isCurrent+ )return}@@ -1887,8 +2201,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvtry {// Remote egress proxy: create nginx ConfigMap before Deploymentif (this.isRemote(server)) {- await this.canonicalizeRemoteEgressProxyImage(server)- await this.ensureConfigMap(server)+ // The CRD image remains desired-state input owned by its author. Remote+ // workloads use the platform-owned proxy image selected in+ // buildDeployment(); rewriting spec.image here would bump generation+ // and retire this reconcile before it can create the runtime resources.+ await this.ensureConfigMap(server, isCurrent)} else {// Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a// would-be denial and still builds; enforce mode blocks the workload.@@ -1900,12 +2217,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconst message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`if (config.enforcePluginImageAllowlist) {console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)- await this.writeStatusCondition(server, {- type: 'Ready',- status: 'False',- reason: 'ImageNotAllowed',- message,- })+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',+ status: 'False',+ reason: 'ImageNotAllowed',+ message,+ },+ isCurrent+ )+ if (!isCurrent()) returnthis.setStatus(server.name, { deployed: false, ready: false, message })return}@@ -1914,9 +2236,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}}- await this.ensureService(server)- await this.ensureDeployment(server, secretResult.revision)+ if (!isCurrent()) return+ await this.ensureService(server, isCurrent)+ if (!isCurrent()) return+ await this.ensureDeployment(server, secretResult.revision, isCurrent)} catch (err) {+ if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -1933,8 +2258,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvserver.annotations?.[NETWORK_READY_ANNOTATION] !== 'true') {try {- await this.setNetworkReadyAnnotation(server)+ await this.setNetworkReadyAnnotation(server, isCurrent)} catch (err) {+ if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -1946,6 +2272,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// Check readiness after reconciliationconst rollout = await this.readDeploymentRollout(server.name, server.namespace)+ if (!isCurrent()) returnconst ready = rollout.readythis.setStatus(server.name, {deployed: true,@@ -1970,28 +2297,36 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvpollState.generation === rollout.generation? pollState.detail: undefined- await this.updateStatusConditions(server, ready, exhaustedDetail)+ await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)// PR-B B1: record SecretResolved=True + Ready=True after a successful// reconcile. This is additive to updateStatusConditions (NetworkReady /// DeploymentReady); the merge logic in writeStatusCondition preserves// the existing conditions by type.if (server.spec.envSecret) {- await this.writeStatusCondition(server, {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'Secret resolved and validated',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'SecretResolved',+ status: 'True',+ reason: 'SecretFound',+ message: 'Secret resolved and validated',+ },+ isCurrent+ )}- await this.writeStatusCondition(server, {- type: 'Ready',- status: ready ? 'True' : 'Unknown',- reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',- message: ready- ? 'Deployment created'- : 'Deployment created, waiting for pods to become ready',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',+ status: ready ? 'True' : 'Unknown',+ reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',+ message: ready+ ? 'Deployment created'+ : 'Deployment created, waiting for pods to become ready',+ },+ isCurrent+ )// If not ready yet, poll until it becomes ready. With generation-aware// readiness this is the normal path right after any Deployment write —@@ -2008,8 +2343,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// is a deployment whose replica tuple perpetually OSCILLATES, and that is// bounded to one reconcile per poll window (~120s) — not a tight loop, and (per// pollReadiness's atomic check-then-set) not a timer leak.- if (!ready) {- this.pollReadiness(server)+ if (!ready && isCurrent()) {+ this.pollReadiness(server, 5000, 24, isCurrent)}}@@ -2023,13 +2358,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}- private async reconcileWrcOwnedServer(server: McpServerCRD): Promise<void> {+ private async reconcileWrcOwnedServer(+ server: McpServerCRD,+ isCurrent: () => boolean+ ): Promise<void> {+ if (!isCurrent()) returnconsole.log(`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +'skipping HCC runtime creation/deletion and updating discovery status.')const secretResult = await this.validateSecret(server)+ if (!isCurrent()) returnif (!secretResult.ok) {this.setStatus(server.name, {deployed: true,@@ -2037,41 +2377,58 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvmessage: secretResult.message,})if (server.spec.envSecret) {- await this.writeStatusCondition(server, {- type: 'SecretResolved',+ await this.writeStatusCondition(+ server,+ {+ type: 'SecretResolved',+ status: 'False',+ reason: secretResult.reason,+ message: secretResult.message,+ },+ isCurrent+ )+ }+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',status: 'False',- reason: secretResult.reason,+ reason: 'SecretValidationFailed',message: secretResult.message,- })- }- await this.writeStatusCondition(server, {- type: 'Ready',- status: 'False',- reason: 'SecretValidationFailed',- message: secretResult.message,- })+ },+ isCurrent+ )return}+ if (!isCurrent()) returnthis.setStatus(server.name, {deployed: true,ready: true,message: 'WRC-owned runtime registered',})if (server.spec.envSecret) {- await this.writeStatusCondition(server, {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'Secret resolved and validated',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'SecretResolved',+ status: 'True',+ reason: 'SecretFound',+ message: 'Secret resolved and validated',+ },+ isCurrent+ )}- await this.writeStatusCondition(server, {- type: 'Ready',- status: 'True',- reason: WRC_OWNED_RUNTIME_READY_REASON,- message: 'WRC-owned runtime registered for HCC discovery',- })+ await this.writeStatusCondition(+ server,+ {+ type: 'Ready',+ status: 'True',+ reason: WRC_OWNED_RUNTIME_READY_REASON,+ message: 'WRC-owned runtime registered for HCC discovery',+ },+ isCurrent+ )}/**@@ -2079,18 +2436,55 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv*/async reconcileDelete(name: string, namespace: string): Promise<void> {console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)- await this.deleteRuntimeResources(name, namespace)+ const capturedAuthority = this.inventoryAuthority()+ await this.deleteRuntimeResources(name, namespace, () =>+ this.orphanDeleteAllowed(name, namespace, capturedAuthority)+ )this.clearStatus(name)}+ private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {+ const current = this.inventoryAuthority()+ return captured.known && current.known && captured.generation === current.generation+ }++ /**+ * A cache miss is not proof of absence: the API watch may have accepted an+ * ADDED event that has not reached this process yet. Confirm a 404 from the+ * API and then recheck both cache membership and LIST→WATCH authority+ * immediately before every destructive delete.+ */+ private async orphanDeleteAllowed(+ name: string,+ namespace: string,+ capturedAuthority: McpServerInventoryAuthoritySnapshot+ ): Promise<boolean> {+ return confirmAuthoritativeMcpServerAbsence({+ inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),+ resolveCurrent: () => this.resolveCurrentServer?.(name),+ readCurrent: () =>+ this.customApi.getNamespacedCustomObject({+ group: 'clerum.io',+ version: 'v1alpha1',+ namespace,+ plural: 'mcpservers',+ name,+ }),+ })+ }+/*** Full reconciliation pass — sync desired state with actual state.* Called on startup to handle CRDs that were created while the skill-mapper was down.*/- async fullReconcile(desiredServers: McpServerCRD[]): Promise<void> {+ async fullReconcile(+ desiredServers: McpServerCRD[],+ options: McpServerFullReconcileOptions = {}+ ): Promise<void> {console.log(`[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`)+ const capturedAuthority = this.inventoryAuthority()// Get all deployments managed by skill-mapperconst existingDeployments = await this.listManagedDeployments()@@ -2100,18 +2494,62 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv.map(s => s.name))- // Create or update desired servers- for (const server of desiredServers) {- await this.reconcile(server)- }+ const runEffect =+ options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())+ const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY++ // Create or update desired servers independently. runEffect owns each+ // per-server lane, so a slow fleet member cannot become a process-wide+ // startup barrier. The finite worker pool admits no more than the configured+ // number of effects at once, while still waiting for every desired result+ // before reporting aggregated failures or moving to orphan cleanup.+ const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>+ runEffect(server.name, async () => {+ const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server+ // Cache mutation happens synchronously before its keyed watch effect is+ // queued. If this startup item was superseded, skip it and let the+ // newer queued event perform external-egress-before-runtime using its+ // own spec; reconciling the newer runtime here could otherwise race+ // ahead of that safety policy.+ if (!current || !sameMcpServerDesiredRevision(server, current)) return+ const isCurrent = () => {+ if (!this.inventoryAuthorityConfigured) {+ if (!this.currentWhenUnconfigured()) return false+ } else if (!this.authorityMatches(capturedAuthority)) {+ return false+ }+ const latest = this.resolveCurrentServer?.(current.name)+ return latest+ ? sameMcpServerDesiredRevision(current, latest)+ : this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()+ }+ await this.reconcile(current, { isCurrent })+ })+ )+ throwCleanupFailures(+ desiredResults+ .filter((result): result is PromiseRejectedResult => result.status === 'rejected')+ .map(result => result.reason),+ 'Failed to reconcile desired McpServers'+ )// Delete orphaned deployments (managed by us but no longer desired)for (const deployment of existingDeployments) {const name = deployment.metadata?.name || ''if (!desiredNames.has(name)) {const namespace = deployment.metadata?.namespace || config.namespace- console.log(`[Reconciler] Removing orphaned resources for "${name}"`)- await this.deleteRuntimeResources(name, namespace)+ await runEffect(name, async () => {+ if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {+ console.warn(+ `[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`+ )+ return+ }+ console.log(`[Reconciler] Removing orphaned resources for "${name}"`)+ await this.deleteRuntimeResources(name, namespace, () =>+ this.orphanDeleteAllowed(name, namespace, capturedAuthority)+ )+ })}}@@ -2130,7 +2568,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn response.items || []} catch (error) {console.error('[Reconciler] Failed to list managed deployments:', error)- return []+ throw error}}}diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.tsindex 633a48992..0ab7d03aa 100644--- a/host-context-controller/src/server.test.ts+++ b/host-context-controller/src/server.test.ts@@ -1,5 +1,6 @@-import { afterEach, describe, expect, it } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'import { McpServerProvider } from './k8sClient'+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'import { ContextMapperServer } from './server'class FakeProvider implements McpServerProvider {@@ -75,14 +76,30 @@ describe('ContextMapperServer readiness', () => {afterEach(async () => {await server?.stop()server = null+ vi.restoreAllMocks()})- it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {+ it('withholds readiness when no inventory-authority gate is wired', async () => {+ // The gate decides whether the provider's inventory is authoritative. A+ // caller that forgets to wire it must not get a permanently Ready server:+ // this endpoint is the contract that no stale allow is live, so the default+ // has to fail closed. There is no type error to catch the omission.server = new ContextMapperServer(new FakeProvider(), 0)+ server.setReady(true)++ const response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ })++ it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)let response = await invoke(server, '/health')expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({ status: 'ok', ready: false })+ expect(JSON.parse(response.body)).toEqual({+ status: 'ok',+ ready: false,+ })response = await invoke(server, '/ready')expect(response.statusCode).toBe(503)@@ -108,4 +125,188 @@ describe('ContextMapperServer readiness', () => {contextRef: '*',})})++ it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {+ let providerAuthoritative = true+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,+ () => providerAuthoritative+ )++ let response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })++ server.setReady(true)++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })++ providerAuthoritative = false++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })++ response = await invoke(server, '/health')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({+ status: 'ok',+ ready: false,+ })++ response = await invoke(server, '/api/v1/mcpservers')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Context mapper provider inventory is not authoritative',+ })++ providerAuthoritative = true++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })++ response = await invoke(server, '/api/v1/mcpservers')+ expect(response.statusCode).toBe(200)+ })++ it('fails readiness closed when the provider authority check throws', async () => {+ const authorityError = new Error('authority unavailable')+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {+ throw authorityError+ })+ server.setReady(true)++ const response = await invoke(server, '/ready')++ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })+ expect(errorLog).toHaveBeenCalledWith(+ '[Server] Provider authority readiness check failed:',+ authorityError+ )+ })++ it('withdraws readiness as soon as shutdown begins', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)+ server.setReady(true)++ expect((await invoke(server, '/ready')).statusCode).toBe(200)++ await server.stop()++ const response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })+ })+})++describe('ContextMapperServer desktop status authority', () => {+ let server: ContextMapperServer | null = null++ // A Host reconciler that reports a running desktop, so the only thing that can+ // withhold a 200 is the Host-authority gate under test.+ const runningReconciler = {+ getStatus: () => ({ deployed: true, ready: true, message: '' }),+ hasDesktop: () => true,+ } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]+ const hasDesktopFn = () => true++ afterEach(async () => {+ await server?.stop()+ server = null+ vi.restoreAllMocks()+ })++ it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {+ // The heart of H1: desktop needs only Host authority. A degraded+ // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,+ // while /api/v1/mcpservers still 503s on the same server.+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => false, // provider inventory NOT authoritative+ () => true // Host inventory authoritative+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(200)+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })++ const mcpservers = await invoke(server, '/api/v1/mcpservers')+ expect(mcpservers.statusCode).toBe(503)+ })++ it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => true, // provider authoritative — must not rescue desktop+ () => false // Host inventory NOT authoritative+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(503)+ expect(JSON.parse(desktop.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ })++ it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {+ // Wiring regression for R2-M1: main.ts resolves both authority gates from the+ // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left+ // the Host gate undefined, so the server's fail-closed default pinned every+ // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200+ // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly+ // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ providerAuthoritativeFn,+ hostAuthoritativeFn+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(200)+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })+ })++ it('fails desktop closed when no Host-authority gate is wired (default)', async () => {+ // Omitting the 6th arg must default to fail-closed, exactly like the+ // provider gate — no type error catches the omission.+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => true+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(503)+ expect(JSON.parse(desktop.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ })})diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.tsindex 7be800a8e..03f576dcf 100644--- a/host-context-controller/src/server.ts+++ b/host-context-controller/src/server.ts@@ -8,24 +8,53 @@ import { McpServerProvider } from './k8sClient'import { registry } from './metrics'import { ErrorResponse, McpServersResponse } from './types'+type ReadinessState =+ | { ready: true; status: 'ready' }+ | { ready: false; status: 'starting' | 'degraded'; message: string }+export class ContextMapperServer {private server: http.Server | null = nullprivate provider: McpServerProviderprivate port: numberprivate hostReconciler: HostReconciler | nullprivate hasDesktopFn: ((hostRef: string) => boolean) | null+ private providerAuthoritativeFn: () => boolean+ private hostAuthoritativeFn: () => booleanprivate ready = falseconstructor(provider: McpServerProvider,port: number = config.port,hostReconciler?: HostReconciler,- hasDesktopFn?: (hostRef: string) => boolean+ hasDesktopFn?: (hostRef: string) => boolean,+ // Fails closed. /ready is the assertion that no stale allow is live, and a+ // caller that omits this gate gets no type error, so the default must+ // withhold readiness rather than grant it.+ providerAuthoritativeFn: () => boolean = () => false,+ // Desktop status needs ONLY Host inventory authority, not the full+ // readiness inventory. Fails closed like providerAuthoritativeFn: a caller+ // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.+ hostAuthoritativeFn: () => boolean = () => false) {this.provider = providerthis.port = portthis.hostReconciler = hostReconciler ?? nullthis.hasDesktopFn = hasDesktopFn ?? null+ this.providerAuthoritativeFn = providerAuthoritativeFn+ this.hostAuthoritativeFn = hostAuthoritativeFn+ }++ /**+ * Host inventory authority, fail-closed on error — the desktop route's only+ * authority gate. Same shape as the provider check in getReadinessState().+ */+ private safeHostAuthoritative(): boolean {+ try {+ return this.hostAuthoritativeFn()+ } catch (err) {+ console.error('[Server] Host authority check failed:', err)+ return false+ }}/**@@ -35,6 +64,30 @@ export class ContextMapperServer {this.ready = ready}+ private getReadinessState(): ReadinessState {+ if (!this.ready) {+ return {+ ready: false,+ status: 'starting',+ message: 'Context mapper is still starting',+ }+ }++ try {+ if (this.providerAuthoritativeFn()) {+ return { ready: true, status: 'ready' }+ }+ } catch (err) {+ console.error('[Server] Provider authority readiness check failed:', err)+ }++ return {+ ready: false,+ status: 'degraded',+ message: 'Context mapper provider inventory is not authoritative',+ }+ }+private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {// CORS headersres.setHeader('Access-Control-Allow-Origin', '*')@@ -57,15 +110,20 @@ export class ContextMapperServer {// Health checkif (req.method === 'GET' && url.pathname === '/health') {- this.sendJson(res, 200, { status: 'ok', ready: this.ready })+ const readiness = this.getReadinessState()+ this.sendJson(res, 200, {+ status: 'ok',+ ready: readiness.ready,+ })return}// Readiness checkif (req.method === 'GET' && url.pathname === '/ready') {- this.sendJson(res, this.ready ? 200 : 503, {- status: this.ready ? 'ready' : 'starting',- ready: this.ready,+ const readiness = this.getReadinessState()+ this.sendJson(res, readiness.ready ? 200 : 503, {+ status: readiness.status,+ ready: readiness.ready,})return}@@ -78,10 +136,30 @@ export class ContextMapperServer {return}- if (!this.ready) {+ // Desktop status: Host inventory is the ONLY authority this route needs, so+ // it is gated here — ABOVE the blanket readiness gate — on Host authority+ // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a+ // degraded Host lane must 503 it (never answer 200 'inactive' from a stale+ // cache). See the E3 design note's endpoint→authority matrix.+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {+ if (!this.ready || !this.safeHostAuthoritative()) {+ this.sendJson(res, 503, {+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ return+ }+ if (!this.checkDesktopAuth(req, res)) return+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')+ await this.handleDesktopStatus(res, hostRef)+ return+ }++ const readiness = this.getReadinessState()+ if (!readiness.ready) {this.sendJson(res, 503, {error: 'Service Unavailable',- message: 'Context mapper is still starting',+ message: readiness.message,})return}@@ -110,14 +188,6 @@ export class ContextMapperServer {return}- // Desktop status (requires service token)- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {- if (!this.checkDesktopAuth(req, res)) return- const hostRef = url.pathname.replace('/api/v1/desktop/', '')- await this.handleDesktopStatus(res, hostRef)- return- }-// 404this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })}@@ -198,7 +268,8 @@ export class ContextMapperServer {endpoints: {'GET /': 'This information page','GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",- 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes',+ 'GET /ready':+ 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative','GET /api/v1/mcpservers': 'List all McpServer resources','GET /api/v1/mcpservers/context/:contextRef':'List McpServers filtered by contextRef (only enabled servers)',@@ -346,9 +417,12 @@ export class ContextMapperServer {* Stop the server.*/stop(): Promise<void> {+ this.ready = false+ const activeServer = this.server+ this.server = nullreturn new Promise(resolve => {- if (this.server) {- this.server.close(() => {+ if (activeServer) {+ activeServer.close(() => {console.log('[Server] Context Mapper stopped')resolve()})diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.tsindex 81542832a..f731391ca 100644--- a/host-context-controller/src/statelessLifecycleExecutor.ts+++ b/host-context-controller/src/statelessLifecycleExecutor.ts@@ -42,6 +42,21 @@ const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'*/const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'+/**+ * H2: reflect a committed lifecycle outcome onto the CURRENT canonical cache+ * entry for `name` — but only if that entry still has the admitted `uid`.+ * The receiver (McpServerWatcher) owns the lookup and the uid guard; the+ * executor never touches the cache directly. A same-name recreation with a+ * different uid MUST be skipped silently (the mutation already committed+ * server-side under the old uid; corrupting the new object would be worse+ * than staleness, which the watch/resync repairs).+ */+export type ReflectHostOutcomeFn = (+ name: string,+ uid: string | undefined,+ apply: (target: HostCRD) => void+) => void+/** Constructor-injected seams from HostReconciler (late-bound where needed). */export interface StatelessLifecycleExecutorDeps {appsApi: k8s.AppsV1Api@@ -71,7 +86,16 @@ export interface StatelessLifecycleExecutorDeps {* serialization slot, so re-entering the per-host chain would* self-deadlock.*/- reconcileCore: (host: HostCRD) => Promise<void>+ reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>+ /**+ * Capture Host inventory authority at heartbeat dispatch time and return an+ * admission closure that revalidates that authority and resolves the current+ * same-identity Host inside this executor's per-Host serializer. The two-step+ * shape is intentional: capturing only after the queue drains cannot detect+ * an authority generation change that happened while the work was waiting.+ */+ prepareHostMutationAdmission?: (action: string, host: HostCRD) => () => HostCRD+ reflectHostOutcome?: ReflectHostOutcomeFnonLifecycleStatusCommitted?: (host: HostCRD, lifecycle: HostLifecycleStatus) => void}@@ -82,7 +106,9 @@ export class StatelessLifecycleExecutor {private readonly now: () => Dateprivate readonly countCommunicationChannels: (hostName: string) => numberprivate readonly isCommunicationChannelCacheSynced: () => boolean- private readonly reconcileCore: (host: HostCRD) => Promise<void>+ private readonly reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>+ private readonly prepareHostMutationAdmission: (action: string, host: HostCRD) => () => HostCRD+ private readonly reflectHostOutcome: ReflectHostOutcomeFnprivate readonly onLifecycleStatusCommitted: (host: HostCRD,lifecycle: HostLifecycleStatus@@ -132,6 +158,9 @@ export class StatelessLifecycleExecutor {this.countCommunicationChannels = deps.countCommunicationChannelsthis.isCommunicationChannelCacheSynced = deps.isCommunicationChannelCacheSyncedthis.reconcileCore = deps.reconcileCore+ this.prepareHostMutationAdmission =+ deps.prepareHostMutationAdmission ?? ((_action, host) => () => host)+ this.reflectHostOutcome = deps.reflectHostOutcome ?? (() => {})this.onLifecycleStatusCommitted = deps.onLifecycleStatusCommitted ?? (() => {})StatelessLifecycleExecutor.warnSingleReplicaInvariantOnce()}@@ -671,15 +700,19 @@ export class StatelessLifecycleExecutor {metadata: {name: stringnamespace?: string+ uid?: string+ generation?: numberresourceVersion?: stringannotations?: Record<string, string>}spec: HostCRD['spec']status?: HostCRD['status']}- return {+ const fresh: HostCRD = {name: obj.metadata.name,namespace: obj.metadata.namespace ?? host.namespace,+ uid: obj.metadata.uid,+ generation: obj.metadata.generation,// Preserve resourceVersion so heartbeat-path status writers can pass it// back as an optimistic-concurrency precondition (D3). Dropping it here// would force every write to be an unconditioned last-write-wins patch.@@ -688,6 +721,8 @@ export class StatelessLifecycleExecutor {spec: obj.spec,status: obj.status,}+ this.requireSameHostIdentity(host, fresh)+ return fresh}/**@@ -716,17 +751,24 @@ export class StatelessLifecycleExecutor {reason: string,entryWakeHandledGeneration: number): Promise<SuspendFromHeartbeatOutcome> {- // L2: serialize on the host chain so a tracker-driven suspend never- // interleaves with a watch-driven wake reconcile for the same host.- return this.serializeByHost(host.name, () =>- this.suspendHostFromHeartbeatCore(host, reason, entryWakeHandledGeneration)+ return this.serializeHeartbeatMutation(+ 'heartbeat suspension',+ host,+ (admittedHost, revalidate) =>+ this.suspendHostFromHeartbeatCore(+ admittedHost,+ reason,+ entryWakeHandledGeneration,+ revalidate+ ))}private async suspendHostFromHeartbeatCore(host: HostCRD,reason: string,- entryWakeHandledGeneration: number+ entryWakeHandledGeneration: number,+ revalidate?: () => HostCRD): Promise<SuspendFromHeartbeatOutcome> {// FRESH GET (see readFreshHost): guard against the CURRENT server-side// lifecycle, not the caller's watch-cache snapshot. The guard is re-run@@ -741,6 +783,17 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {+ // UID proves object identity, not decision freshness. A same-identity+ // Host may have advanced from stateless to stateful while the watch+ // cache trails the API. Re-evaluate the complete effective stateless+ // policy from every fresh read/retry before honoring old heartbeat+ // evidence.+ if (+ !this.sameHostSpecRevision(host, fresh) ||+ !this.effectiveLifecycleFromCache(fresh).stateless+ ) {+ return { skip: true }+ }const freshLifecycle = fresh.status?.lifecycleif (freshLifecycle?.state !== 'draining') {// Every legitimate suspend follows a durable `draining` write (the@@ -802,12 +855,36 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh+ initialFresh,+ revalidate)if ('skipped' in result || writtenStatus === undefined) {return skipOutcome}- host.status = writtenStatus+ // Revalidate authority/UID immediately before follow-on effects, then read+ // once more from the API. The successful PATCH attempt's fresh object is+ // the exact spec on which the status write committed, but a wake/spec+ // update can land immediately AFTER that commit and BEFORE runtime+ // reconciliation. A linearizable follow-on GET observes both our durable+ // suspended write and any such newer same-UID update; the watch cache can+ // legitimately trail either one.+ const currentAtFollowOn = revalidate?.()+ if (currentAtFollowOn) this.requireSameHostIdentity(host, currentAtFollowOn)+ this.reflectOutcome(host, target => {+ target.status = writtenStatus+ })+ const reconcileHost = await this.readFreshHost(result.fresh)+ if (!this.sameHostSpecRevision(host, reconcileHost)) {+ const error = new Error(+ `Host spec generation changed before heartbeat suspension follow-on for "${host.name}"`+ )+ error.name = 'HostMutationSpecRevisionChangedError'+ throw error+ }+ const currentAfterFollowOnRead = revalidate?.()+ if (currentAfterFollowOnRead) {+ this.requireSameHostIdentity(host, currentAfterFollowOnRead)+ }this.lastWrittenLifecycleStatus.delete(host.name)this.logSuspendedApplied(host.name)console.log(@@ -816,18 +893,32 @@ export class StatelessLifecycleExecutor {// L2: call reconcileCore (not the serialized reconcile) — this method// already runs inside this host's serialization slot, so re-entering the// per-host chain here would self-deadlock.- await this.reconcileCore(host)+ await this.reconcileCore(reconcileHost, revalidate)+ // Preserve the historical caller-visible reflection contract even when+ // follow-on work correctly used a fresher API object. The tracker may hold+ // the originally admitted instance until the watch delivers the next+ // MODIFIED event; reflect the final authoritative outcome immediately so+ // it cannot make a second decision from the pre-follow-on status.+ this.reflectOutcome(host, target => {+ target.uid = reconcileHost.uid+ target.generation = reconcileHost.generation+ target.resourceVersion = reconcileHost.resourceVersion+ target.annotations = reconcileHost.annotations+ target.spec = reconcileHost.spec+ target.status = reconcileHost.status+ })// The wake fast-path / drained-pre-scale guard inside reconcile() may// have flipped the host back to active (a wake raced the suspension) —// in that case no scale-down happened, so no phase/metric is emitted.const wakePending =- this.wakeRequestedGeneration(host) > (host.status?.lifecycle?.wakeHandledGeneration ?? 0)- if (host.status?.lifecycle?.state !== 'suspended' || wakePending) {+ this.wakeRequestedGeneration(reconcileHost) >+ (reconcileHost.status?.lifecycle?.wakeHandledGeneration ?? 0)+ if (reconcileHost.status?.lifecycle?.state !== 'suspended' || wakePending) {// The durable suspended write DID commit (a racing wake revived the// host afterwards), so the drained evidence was honored: 'suspended'.return 'suspended'}- this.recordSuspendedApplied(host.name)+ this.recordSuspendedApplied(reconcileHost.name)return 'suspended'}@@ -838,12 +929,19 @@ export class StatelessLifecycleExecutor {* Throws on API failure — the tracker logs it loudly.*/async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {- // L2: serialize on the host chain (see serializeByHost) so this status- // write never interleaves with a concurrent reconcile for the same host.- return this.serializeByHost(host.name, () => this.publishSuspendBlockedReasonCore(host, reason))+ return this.serializeHeartbeatMutation(+ 'heartbeat suspend-blocked reason publication',+ host,+ (admittedHost, revalidate) =>+ this.publishSuspendBlockedReasonCore(admittedHost, reason, revalidate)+ )}- private async publishSuspendBlockedReasonCore(host: HostCRD, reason: string): Promise<void> {+ private async publishSuspendBlockedReasonCore(+ host: HostCRD,+ reason: string,+ revalidate?: () => HostCRD+ ): Promise<void> {// FRESH GET (see readFreshHost): this is a suspend-BLOCKED reason// ANNOTATOR, not a state writer — it must NEVER change state, only stamp// the reason on top of the CURRENT server-side state. AP-1 (the 9th@@ -866,6 +964,12 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {+ if (+ !this.sameHostSpecRevision(host, fresh) ||+ !this.effectiveLifecycleFromCache(fresh).stateless+ ) {+ return { skip: true }+ }const freshLifecycle = fresh.status?.lifecycleconst freshState = freshLifecycle?.state ?? 'active'if (freshState !== 'active') {@@ -889,12 +993,16 @@ export class StatelessLifecycleExecutor {writtenLifecycle = lifecyclereturn [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]},- initialFresh+ initialFresh,+ revalidate)if ('skipped' in result || writtenLifecycle === undefined) {return}- host.status = { ...(host.status ?? {}), lifecycle: writtenLifecycle }+ const reflectedStatus = { ...(host.status ?? {}), lifecycle: writtenLifecycle }+ this.reflectOutcome(host, target => {+ target.status = reflectedStatus+ })this.lastWrittenLifecycleStatus.delete(host.name)}@@ -926,17 +1034,18 @@ export class StatelessLifecycleExecutor {host: HostCRD,entryWakeHandledGeneration: number): Promise<void> {- // L2: serialize on the host chain (see serializeByHost) so the drain- // status write never interleaves with a concurrent reconcile for the- // same host.- return this.serializeByHost(host.name, () =>- this.markHostDrainingFromHeartbeatCore(host, entryWakeHandledGeneration)+ return this.serializeHeartbeatMutation(+ 'heartbeat draining transition',+ host,+ (admittedHost, revalidate) =>+ this.markHostDrainingFromHeartbeatCore(admittedHost, entryWakeHandledGeneration, revalidate))}private async markHostDrainingFromHeartbeatCore(host: HostCRD,- entryWakeHandledGeneration: number+ entryWakeHandledGeneration: number,+ revalidate?: () => HostCRD): Promise<void> {// FRESH GET (see readFreshHost): a stale `active` snapshot over an// actually-`suspended` CR would write `draining` (a running state) and@@ -948,6 +1057,12 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {+ if (+ !this.sameHostSpecRevision(host, fresh) ||+ !this.effectiveLifecycleFromCache(fresh).stateless+ ) {+ return { skip: true }+ }const freshLifecycle = fresh.status?.lifecycleif ((freshLifecycle?.state ?? 'active') !== 'active') {return { skip: true }@@ -983,12 +1098,15 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh+ initialFresh,+ revalidate)if ('skipped' in result || writtenStatus === undefined) {return}- host.status = writtenStatus+ this.reflectOutcome(host, target => {+ target.status = writtenStatus+ })this.lastWrittenLifecycleStatus.delete(host.name)console.log(`[HostReconciler] Marked stateless Host "${host.name}" draining — control-api now answers {drain:true} from Host.status`@@ -1010,13 +1128,17 @@ export class StatelessLifecycleExecutor {* heartbeat retries.*/async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {- // L2: serialize on the host chain (see serializeByHost) so the- // cancel-drain status write never interleaves with a concurrent- // reconcile for the same host.- return this.serializeByHost(host.name, () => this.markHostActiveFromHeartbeatCore(host))+ return this.serializeHeartbeatMutation(+ 'heartbeat cancel-drain transition',+ host,+ (admittedHost, revalidate) => this.markHostActiveFromHeartbeatCore(admittedHost, revalidate)+ )}- private async markHostActiveFromHeartbeatCore(host: HostCRD): Promise<void> {+ private async markHostActiveFromHeartbeatCore(+ host: HostCRD,+ revalidate?: () => HostCRD+ ): Promise<void> {// FRESH GET (see readFreshHost): a stale `draining` snapshot over an// actually-`suspended` CR must never resurrect it (and would regress// wakeHandledGeneration). Proceed only when the CURRENT server-side@@ -1029,6 +1151,12 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {+ if (+ !this.sameHostSpecRevision(host, fresh) ||+ !this.effectiveLifecycleFromCache(fresh).stateless+ ) {+ return { skip: true }+ }const freshLifecycle = fresh.status?.lifecycleif (freshLifecycle?.state !== 'draining') {skippedNotDraining = freshLifecycle?.state ?? 'active'@@ -1041,7 +1169,8 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh+ initialFresh,+ revalidate)if ('skipped' in result || writtenStatus === undefined) {console.log(@@ -1049,7 +1178,9 @@ export class StatelessLifecycleExecutor {)return}- host.status = writtenStatus+ this.reflectOutcome(host, target => {+ target.status = writtenStatus+ })this.lastWrittenLifecycleStatus.delete(host.name)console.log(`[StatelessLifecycle] host=${host.name} cancel-drain: activity evidence while draining — reverting status to active`@@ -1165,8 +1296,10 @@ export class StatelessLifecycleExecutor {// Reflect the fresh truth on the in-memory object so the heavy// reconcile body derives replicas from the current durable state// instead of the stale event payload.- host.annotations = initialFresh.annotations- host.status = initialFresh.status+ this.reflectOutcome(host, target => {+ target.annotations = initialFresh.annotations+ target.status = initialFresh.status+ })return false}@@ -1225,8 +1358,10 @@ export class StatelessLifecycleExecutor {initialFresh)if ('skipped' in result || writtenLifecycle === undefined) {- host.annotations = latestFresh.annotations- host.status = latestFresh.status+ this.reflectOutcome(host, target => {+ target.annotations = latestFresh.annotations+ target.status = latestFresh.status+ })return false}} catch (err) {@@ -1245,8 +1380,11 @@ export class StatelessLifecycleExecutor {// (assessLifecycle reads host.status) derives replicas from the woken// state instead of re-suspending from the stale event payload. The base// is the FRESH status the write committed over, never the cached spread.- host.annotations = latestFresh.annotations- host.status = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }+ const reflectedStatus = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }+ this.reflectOutcome(host, target => {+ target.annotations = latestFresh.annotations+ target.status = reflectedStatus+ })console.log(`[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=status_flipped ts=${this.now().getTime()}`)@@ -1417,19 +1555,31 @@ export class StatelessLifecycleExecutor {private async patchStatusWithPrecondition(host: HostCRD,build: (fresh: HostCRD) => Array<{ op: 'add'; path: string; value: unknown }> | { skip: true },- initialFresh?: HostCRD+ initialFresh?: HostCRD,+ revalidate?: () => HostCRD): Promise<{ fresh: HostCRD } | { skipped: true }> {const maxAttempts = 5let fresh = initialFreshfor (let attempt = 1; attempt <= maxAttempts; attempt += 1) {+ const currentAtAttempt = revalidate?.()+ if (currentAtAttempt) this.requireSameHostIdentity(host, currentAtAttempt)if (fresh === undefined) {fresh = await this.readFreshHost(host)}+ if (host.uid !== undefined && fresh.resourceVersion === undefined) {+ const error = new Error(+ `Host status mutation for "${host.name}" has no fresh resourceVersion; refusing an unconditioned write`+ )+ error.name = 'HostMutationPreconditionUnavailableError'+ throw error+ }const built = build(fresh)if ('skip' in built) {return { skipped: true }}try {+ const currentBeforePatch = revalidate?.()+ if (currentBeforePatch) this.requireSameHostIdentity(host, currentBeforePatch)await this.customApi.patchNamespacedCustomObjectStatus({group: GROUP,version: VERSION,@@ -1499,6 +1649,79 @@ export class StatelessLifecycleExecutor {return result}+ /**+ * Capture admission before enqueueing, then resolve the current Host only+ * after this mutation reaches the front of its own serializer. Keeping the+ * check here avoids wrapping a lifecycle call in HostReconciler's serializer,+ * which would recursively acquire the same Host chain and deadlock.+ */+ private serializeHeartbeatMutation<T>(+ action: string,+ host: HostCRD,+ work: (admittedHost: HostCRD, revalidate: () => HostCRD) => Promise<T>+ ): Promise<T> {+ const revalidate = this.prepareHostMutationAdmission(action, host)+ return this.serializeByHost(host.name, () => work(revalidate(), revalidate))+ }++ /**+ * A resourceVersion conflict can be caused by ordinary writes or by a+ * delete/recreate. Every fresh read, including every bounded 409 retry, must+ * remain on the Kubernetes UID that was admitted. Name equality is not an+ * identity fence. Production watch snapshots always carry UID; if only one+ * side carries it, fail closed rather than guessing.+ */+ /**+ * H2: apply a committed outcome to BOTH visible surfaces:+ * 1. the local working object (`host`) — with a resolver wired this is the+ * admitted CLONE (serializeHeartbeatMutation), so this keeps same-pass+ * logic (assessLifecycle, drained-pre-scale guard, follow-on reconcile)+ * reading the outcome exactly as before; standalone callers (no resolver)+ * get the legacy caller-visible reflection unchanged;+ * 2. the CURRENT cache entry, via the uid-guarded reflector (no-op unless+ * McpServerWatcher wired it).+ * `host.uid` is the admitted identity (admission proved requested.uid ===+ * current.uid; readFreshHost re-proves it), so passing it after mutate() ran+ * is equivalent to passing it before.+ *+ * `mutate` closures MUST only ASSIGN precomputed values — never derive from+ * `target` — so both surfaces receive the identical outcome.+ */+ private reflectOutcome(host: HostCRD, mutate: (target: HostCRD) => void): void {+ mutate(host)+ this.reflectHostOutcome(host.name, host.uid, mutate)+ }++ private requireSameHostIdentity(expected: HostCRD, current: HostCRD): void {+ if (expected.uid === undefined && current.uid === undefined) return+ if (expected.uid === undefined || current.uid === undefined || expected.uid !== current.uid) {+ const error = new Error(+ `Host identity changed while mutating "${expected.name}"; refusing to cross a same-name recreation`+ )+ error.name = 'HostMutationIdentityChangedError'+ throw error+ }+ }++ /**+ * Heartbeat evidence belongs to one Host spec epoch. UID equality allows+ * status-only updates and retries, but a generation change can replace the+ * runtime inputs (context, Secret, image, lifecycle policy, etc.) while+ * remaining stateless. Never apply an older runtime's heartbeat decision to+ * that new spec. UID-less standalone fixtures retain their legacy behavior;+ * production objects (which carry UID) fail closed on a missing generation.+ */+ private sameHostSpecRevision(expected: HostCRD, current: HostCRD): boolean {+ if (expected.generation === undefined && current.generation === undefined) {+ return expected.uid === undefined && current.uid === undefined+ }+ return (+ expected.generation !== undefined &&+ current.generation !== undefined &&+ expected.generation === current.generation+ )+ }+/*** Drained-pre-scale guard (Stage 4.3): a pending wake must abort the* suspension IMMEDIATELY before replicas:0 derives from the assessment.@@ -1579,8 +1802,10 @@ export class StatelessLifecycleExecutor {// A wake is pending in the fresh read — run the wake transition// instead of the scale-down. Reflect the fresh wake generation on the// in-memory object so downstream writes preserve it.- host.annotations = fresh.annotations- host.status = fresh.status+ this.reflectOutcome(host, target => {+ target.annotations = fresh.annotations+ target.status = fresh.status+ })return {effective: { stateless: true, state: 'active' },lifecycle: { state: 'active', wakeHandledGeneration: freshWakeRequested },@@ -1606,8 +1831,10 @@ export class StatelessLifecycleExecutor {// Reflect fresh onto the in-memory host (mirroring the wake fast-path's// epoch guard) so every downstream write derives from the current// server-side truth instead of the stale event payload.- host.annotations = fresh.annotations- host.status = fresh.status+ this.reflectOutcome(host, target => {+ target.annotations = fresh.annotations+ target.status = fresh.status+ })const freshReason = fresh.status?.lifecycle?.reasonconst preservedReason = isHeartbeatManagedLifecycleReason(freshReason) ? freshReason : undefinedreturn {diff --git a/host-context-controller/src/types.ts b/host-context-controller/src/types.tsindex a7488dc7d..552ed3f0f 100644--- a/host-context-controller/src/types.ts+++ b/host-context-controller/src/types.ts@@ -350,6 +350,8 @@ export interface ContextStatus {export interface ContextCRD {name: stringnamespace: string+ uid?: string+ generation?: numberspec: ContextSpecstatus?: ContextStatus}@@ -509,6 +511,12 @@ export interface McpServerStatus {ready: boolean/** Human-readable status message. */message?: string+ /**+ * Whether HCC can tie this status to the current McpServer identity and+ * generation. False means discovery is authoritative but runtime status has+ * not yet been re-established after controller startup.+ */+ authoritative?: boolean}/**diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.tsindex 22d63b8b2..a6992eeb8 100644--- a/host-context-controller/src/utils.ts+++ b/host-context-controller/src/utils.ts@@ -36,6 +36,10 @@ export async function replaceWithConflictRetry<* when a meaningful change is still required.*/isUpToDate?: (body: T, existing: T) => boolean+ /** Reject an object whose identity or ownership is unsafe to replace. */+ validateExisting?: (existing: T) => void+ /** Rechecked immediately before every Kubernetes write attempt. */+ mutationAllowed?: () => booleanmaxAttempts?: number}): Promise<void> {const {@@ -47,10 +51,13 @@ export async function replaceWithConflictRetry<replace,mergeExisting,isUpToDate,+ validateExisting,+ mutationAllowed,maxAttempts = 3,} = optsfor (let attempt = 1; attempt <= maxAttempts; attempt++) {const existing = await read()+ validateExisting?.(existing)const desired = resolveBody ? await resolveBody() : bodyconst base: T = {...desired,@@ -58,6 +65,7 @@ export async function replaceWithConflictRetry<}const next = mergeExisting ? mergeExisting(base, existing) : baseif (isUpToDate?.(next, existing)) return+ if (mutationAllowed && !mutationAllowed()) returntry {await replace(next)const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''@@ -171,8 +179,11 @@ export async function applyNetworkPolicy(name: string,namespace: string,policy: k8s.V1NetworkPolicy,- logPrefix = '[NetPol]'+ logPrefix = '[NetPol]',+ mutationAllowed?: () => boolean,+ validateExisting?: (existing: k8s.V1NetworkPolicy) => void): Promise<void> {+ if (mutationAllowed && !mutationAllowed()) returntry {await api.createNamespacedNetworkPolicy({ namespace, body: policy })console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)@@ -188,6 +199,8 @@ export async function applyNetworkPolicy(body: policy,read: () => api.readNamespacedNetworkPolicy({ name, namespace }),replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ mutationAllowed,+ validateExisting,})}diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.tsindex 4e6593024..17ac53f0f 100644--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts@@ -136,6 +136,20 @@ function createReconciler() {return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }}+function deferred<T = void>(): {+ promise: Promise<T>+ resolve: (value: T) => void+ reject: (reason?: unknown) => void+} {+ let resolve!: (value: T) => void+ let reject!: (reason?: unknown) => void+ const promise = new Promise<T>((res, rej) => {+ resolve = res+ reject = rej+ })+ return { promise, resolve, reject }+}+/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {const calls = [@@ -207,7 +221,755 @@ function freshHostRead(lifecycle?: {}}+describe('HostReconciler heartbeat mutation admission', () => {+ const heartbeatMutations = [+ {+ name: 'suspend',+ core: 'suspendHostFromHeartbeatCore',+ coreArgs: ['idle', 0] as const,+ freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,+ invoke: (reconciler: HostReconciler, host: HostCRD) =>+ reconciler.suspendHostFromHeartbeat(host, 'idle', 0),+ },+ {+ name: 'suspend-blocked reason',+ core: 'publishSuspendBlockedReasonCore',+ coreArgs: ['suspend-blocked: activeTask'] as const,+ freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,+ invoke: (reconciler: HostReconciler, host: HostCRD) =>+ reconciler.publishSuspendBlockedReason(host, 'suspend-blocked: activeTask'),+ },+ {+ name: 'draining',+ core: 'markHostDrainingFromHeartbeatCore',+ coreArgs: [0] as const,+ freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,+ invoke: (reconciler: HostReconciler, host: HostCRD) =>+ reconciler.markHostDrainingFromHeartbeat(host, 0),+ },+ {+ name: 'cancel-drain',+ core: 'markHostActiveFromHeartbeatCore',+ coreArgs: [] as const,+ freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,+ invoke: (reconciler: HostReconciler, host: HostCRD) =>+ reconciler.markHostActiveFromHeartbeat(host),+ },+ ] as const++ it.each(heartbeatMutations)(+ 'rejects queued $name work when Host authority changes before serializer admission',+ async ({ core, invoke }) => {+ const { reconciler } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 1 }+ const authority = { current: { known: true, generation: 7 } }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ const lifecycle = (reconciler as any).lifecycle+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = lifecycle.serializeByHost(host.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const pending = invoke(reconciler, host)+ authority.current = { known: false, generation: 8 }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(pending).rejects.toThrow(/Host inventory authority/)+ expect(coreSpy).not.toHaveBeenCalled()+ }+ )++ it.each(heartbeatMutations)(+ 'rejects queued $name work when the Host UID changes before serializer admission',+ async ({ core, invoke }) => {+ const { reconciler } = createReconciler()+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-old-uid', generation: 4 }+ let current = requested+ const authority = { current: { known: true, generation: 9 } }+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ const lifecycle = (reconciler as any).lifecycle+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = lifecycle.serializeByHost(requested.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const pending = invoke(reconciler, requested)+ current = { ...requested, uid: 'heartbeat-host-new-uid', generation: 1 }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(pending).rejects.toThrow(/Host identity/)+ expect(coreSpy).not.toHaveBeenCalled()+ }+ )++ it.each(heartbeatMutations)(+ 'admits the latest same-UID same-spec Host snapshot for $name inside the heartbeat serializer',+ async ({ core, coreArgs, invoke }) => {+ const { reconciler } = createReconciler()+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ const current = {+ ...requested,+ resourceVersion: 'cache-rv-latest',+ status: { lifecycle: { state: 'active' as const, wakeHandledGeneration: 2 } },+ }+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))+ const lifecycle = (reconciler as any).lifecycle+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)++ await invoke(reconciler, requested)++ expect(coreSpy).toHaveBeenCalledOnce()+ expect(coreSpy).toHaveBeenCalledWith(current, ...coreArgs, expect.any(Function))+ }+ )++ it.each(heartbeatMutations)(+ 'rejects a $name 409 retry that resolves to a recreated same-name Host',+ async ({ freshLifecycle, invoke }) => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),+ uid: 'heartbeat-host-old-uid',+ generation: 4,+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 15 }))+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead(freshLifecycle),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-old',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead(freshLifecycle),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: 'heartbeat-host-new-uid',+ generation: 1,+ resourceVersion: 'rv-new',+ },+ })+ customApi.patchNamespacedCustomObjectStatus+ .mockRejectedValueOnce({ code: 409 })+ .mockResolvedValueOnce(undefined)++ await expect(invoke(reconciler, host)).rejects.toThrow(/Host identity/)+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ }+ )++ it.each(heartbeatMutations)(+ 'rejects queued $name work when the same Host advances to a new spec generation',+ async ({ core, invoke }) => {+ const { reconciler } = createReconciler()+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ let current = requested+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))+ const lifecycle = (reconciler as any).lifecycle+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const occupied = lifecycle.serializeByHost(requested.name, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }) as Promise<void>+ await blockerStarted.promise++ const pending = invoke(reconciler, requested)+ current = {+ ...requested,+ generation: 5,+ spec: { ...requested.spec, contextRef: 'new-runtime-context' },+ }+ releaseBlocker.resolve(undefined)+ await occupied++ await expect(pending).rejects.toThrow(/Host spec generation/)+ expect(coreSpy).not.toHaveBeenCalled()+ }+ )++ it.each(heartbeatMutations)(+ 'does not apply stale $name evidence to a newer same-UID stateless spec',+ async ({ freshLifecycle, invoke }) => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),+ uid: 'heartbeat-host-uid',+ generation: 4,+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead(freshLifecycle),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-new-spec',+ },+ spec: { ...host.spec, contextRef: 'new-runtime-context' },+ })+ const reconcileCore = vi+ .spyOn(reconciler as any, 'reconcileCore')+ .mockResolvedValue(undefined)++ await invoke(reconciler, host)++ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(reconcileCore).not.toHaveBeenCalled()+ }+ )++ it.each(heartbeatMutations)(+ 'does not apply stale $name evidence after the same-UID Host becomes stateful',+ async ({ invoke }) => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 16 }))+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-stateful',+ },+ spec: {+ ...host.spec,+ lifecycle: { stateless: false },+ },+ })+ const reconcileCore = vi+ .spyOn(reconciler as any, 'reconcileCore')+ .mockResolvedValue(undefined)++ await invoke(reconciler, host)++ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(reconcileCore).not.toHaveBeenCalled()+ }+ )++ it('does not retry stale heartbeat evidence after a 409 reveals a same-UID stateful spec', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-stateless',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-stateful',+ },+ spec: {+ ...host.spec,+ lifecycle: { stateless: false },+ },+ })+ customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })++ await reconciler.markHostDrainingFromHeartbeat(host, 0)++ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ })++ it('does not retry stale heartbeat evidence after a 409 reveals a newer stateless spec', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-old-spec',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-new-spec',+ },+ spec: { ...host.spec, contextRef: 'new-runtime-context' },+ })+ customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })++ await reconciler.markHostDrainingFromHeartbeat(host, 0)++ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ })++ it('revalidates authority after the fresh read and before the first status patch', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ const authority = { current: { known: true, generation: 18 } }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ customApi.getNamespacedCustomObject.mockImplementation(async () => {+ authority.current = { known: false, generation: 19 }+ return {+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ resourceVersion: 'rv-authority-lost',+ },+ }+ })++ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(+ /Host inventory authority/+ )+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ })++ it('revalidates authority before a 409 retry performs another fresh read', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ const authority = { current: { known: true, generation: 21 } }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => authority.current)+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-before-conflict',+ },+ })+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {+ authority.current = { known: false, generation: 22 }+ throw { code: 409 }+ })++ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(+ /Host inventory authority/+ )+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ })++ it('does not reconcile runtime effects after suspend when the Host UID changes', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-old-uid',+ generation: 4,+ }+ let current = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 24 }))+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-suspend',+ },+ })+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {+ current = { ...host, uid: 'heartbeat-host-new-uid', generation: 1 }+ })++ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(+ /Host identity/+ )+ expect(reconcileCore).not.toHaveBeenCalled()+ })++ it('reconciles suspend follow-on effects from the successful fresh same-UID Host spec', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ spec: { contextRef: 'latest-context' },+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-uid',+ generation: 5,+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 26 }))+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-suspend-current',+ },+ spec: { ...host.spec, contextRef: 'latest-context' },+ })+ .mockResolvedValue({+ ...freshHostRead({+ state: 'suspended',+ wakeHandledGeneration: 0,+ reason: 'idle',+ }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-suspend-committed',+ },+ spec: { ...host.spec, contextRef: 'latest-context' },+ })++ await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)++ expect(reconcileCore).toHaveBeenCalledOnce()+ const reconciledHost = reconcileCore.mock.calls[0][0] as HostCRD+ expect(reconciledHost).toMatchObject({+ uid: host.uid,+ generation: 5,+ spec: expect.objectContaining({ contextRef: 'latest-context' }),+ })+ expect(reconciledHost.status?.lifecycle).toEqual({+ state: 'suspended',+ wakeHandledGeneration: 0,+ reason: 'idle',+ })+ })++ it('rejects a known-identity status write without a fresh resourceVersion', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 28 }))+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: { name: host.name, namespace: host.namespace, uid: host.uid },+ })++ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(+ /no fresh resourceVersion/+ )+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ })++ it('retries a 409 on the same UID and uses the latest Host for suspend follow-on effects', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-uid',+ generation: 4,+ }+ let current = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 30 }))+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-1',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-2',+ },+ spec: host.spec,+ })+ .mockResolvedValueOnce({+ ...freshHostRead({+ state: 'suspended',+ wakeHandledGeneration: 0,+ reason: 'idle',+ }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-3',+ },+ spec: host.spec,+ })+ customApi.patchNamespacedCustomObjectStatus+ .mockImplementationOnce(async () => {+ current = {+ ...host,+ resourceVersion: 'cache-rv-after-conflict',+ }+ throw { code: 409 }+ })+ .mockResolvedValueOnce(undefined)++ await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)++ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(3)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ expect(reconcileCore.mock.calls[0][0]).toMatchObject({+ uid: host.uid,+ generation: 4,+ spec: expect.objectContaining({ contextRef: 'context-a' }),+ })+ })++ it('does not apply or record suspension when only the follow-on GET observes a newer spec', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-uid',+ generation: 4,+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 33 }))+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-suspend-write',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead({+ state: 'suspended',+ wakeHandledGeneration: 0,+ reason: 'idle',+ }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 5,+ resourceVersion: 'rv-new-spec',+ },+ spec: {+ ...host.spec,+ contextRef: 'replacement-context',+ lifecycle: { stateless: false },+ },+ })+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)+ const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')++ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(+ /Host spec generation/+ )++ expect(reconcileCore).not.toHaveBeenCalled()+ expect(recordSuspended).not.toHaveBeenCalled()+ })++ it('rejects stale suspend follow-on work when the Host spec changes during Secret validation', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-uid',+ generation: 4,+ }+ let current = host+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 34 }))+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-suspend-write',+ },+ })+ .mockResolvedValueOnce({+ ...freshHostRead({+ state: 'suspended',+ wakeHandledGeneration: 0,+ reason: 'idle',+ }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-suspend-follow-on',+ },+ })+ const secretReadStarted = deferred()+ const releaseSecretRead = deferred()+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {+ secretReadStarted.resolve(undefined)+ await releaseSecretRead.promise+ return { ok: true }+ })+ const staleMutation = vi+ .spyOn(reconciler as any, 'ensureHostServiceAccount')+ .mockRejectedValue(new Error('stale follow-on mutation was admitted'))++ const pending = reconciler.suspendHostFromHeartbeat(host, 'idle', 0)+ await secretReadStarted.promise+ current = {+ ...host,+ generation: 5,+ spec: {+ ...host.spec,+ contextRef: 'replacement-context',+ lifecycle: { stateless: false },+ },+ }+ releaseSecretRead.resolve(undefined)++ await expect(pending).rejects.toThrow(/Host spec generation/)+ expect(staleMutation).not.toHaveBeenCalled()+ })++ it('does not record a suspended scale-down when follow-on reconcile handles a wake', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },+ }),+ uid: 'heartbeat-host-uid',+ generation: 4,+ }+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 32 }))+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: host.uid,+ generation: 4,+ resourceVersion: 'rv-wake-follow-on',+ },+ })+ vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async (...args: unknown[]) => {+ const reconcileHost = args[0] as HostCRD+ reconcileHost.annotations = { 'clerum.io/wake-requested': '1' }+ reconcileHost.status = {+ lifecycle: { state: 'active', wakeHandledGeneration: 1 },+ }+ })+ const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')++ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).resolves.toBe('suspended')++ expect(recordSuspended).not.toHaveBeenCalled()+ })+})+describe('HostReconciler.suspendHostFromHeartbeat', () => {+ it('does not mutate the admitted watch-cache snapshot while applying a suspension', async () => {+ const { reconciler, customApi } = createReconciler()+ const host = makeStatelessHost({+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },+ })+ host.uid = 'stateless-host-uid'+ host.generation = 1+ host.resourceVersion = 'cache-rv'+ const snapshot = structuredClone(host)+ Object.freeze(host)+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))+ reconciler.setHostMutationAuthority(() => ({+ known: true,+ hostGeneration: 1,+ contextGeneration: 1,+ }))+ const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })+ Object.assign(draining.metadata, {+ uid: host.uid,+ generation: host.generation,+ resourceVersion: 'draining-rv',+ })+ const suspended = freshHostRead({+ state: 'suspended',+ wakeHandledGeneration: 2,+ reason: 'idle',+ })+ Object.assign(suspended.metadata, {+ uid: host.uid,+ generation: host.generation,+ resourceVersion: 'suspended-rv',+ })+ customApi.getNamespacedCustomObject.mockResolvedValueOnce(draining).mockResolvedValue(suspended)++ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')+ expect(host).toEqual(snapshot)+ })+it("writes state='suspended' + reason 'idle' durably, then reconciles to replicas=0", async () => {const { reconciler, appsApi, customApi } = createReconciler()const host = makeStatelessHost({@@ -1313,3 +2075,89 @@ describe('HostReconciler reconcile — stateless replicas derive from FRESH stat}})})++describe('H2: guarded cache reflection', () => {+ it('reflects a committed lifecycle outcome onto the current cache entry without a watch event', async () => {+ const { reconciler, customApi } = createReconciler()+ const entry = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },+ }),+ uid: 'uid-1',+ generation: 1,+ resourceVersion: 'rv-1',+ }+ const hosts = new Map<string, HostCRD>([[entry.name, entry]])+ reconciler.setResolveCurrentHost(name => hosts.get(name))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))+ // Exactly the k8sClient.ts wiring shape (uid-guarded apply onto the cache).+ reconciler.setReflectHostOutcome((name, uid, apply) => {+ const cached = hosts.get(name)+ if (!cached || uid === undefined || cached.uid !== uid) return+ apply(cached)+ })+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: entry.name,+ namespace: entry.namespace,+ uid: 'uid-1',+ generation: 1,+ resourceVersion: 'fresh-rv',+ },+ })++ await reconciler.markHostDrainingFromHeartbeat(entry, 0)++ // The status write committed to the API…+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')+ // …and — the H2 fix — the committed outcome is visible on the CANONICAL cache+ // entry without waiting for a watch MODIFIED. With the resolver wired, the+ // heartbeat core mutates the admitted clone (Shape A), so this can ONLY be+ // true if the guarded reflector routed the outcome onto the cache entry.+ expect(hosts.get(entry.name)?.status?.lifecycle?.state).toBe('draining')+ })++ it('never reflects a committed outcome onto a same-name recreation with a different uid', async () => {+ const { reconciler, customApi } = createReconciler()+ const entry = {+ ...makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },+ }),+ uid: 'uid-1',+ generation: 1,+ resourceVersion: 'rv-1',+ }+ const hosts = new Map<string, HostCRD>([[entry.name, entry]])+ const replacement = { ...makeStatelessHost({ name: entry.name }), uid: 'uid-2' }+ reconciler.setResolveCurrentHost(name => hosts.get(name))+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))+ reconciler.setReflectHostOutcome((name, uid, apply) => {+ const cached = hosts.get(name)+ if (!cached || uid === undefined || cached.uid !== uid) return+ apply(cached)+ })+ customApi.getNamespacedCustomObject.mockResolvedValue({+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),+ metadata: {+ name: entry.name,+ namespace: entry.namespace,+ uid: 'uid-1',+ generation: 1,+ resourceVersion: 'fresh-rv',+ },+ })+ // A watch DELETE+ADD recreates the entry (new uid) mid-commit.+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async () => {+ hosts.set(entry.name, replacement)+ return {}+ })++ await reconciler.markHostDrainingFromHeartbeat(entry, 0)++ // The write committed…+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')+ // …but the recreation (uid-2) was NOT corrupted by the stale reflection.+ expect(replacement.status?.lifecycle).toBeUndefined()+ })+})diff --git a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.tsindex a06465eeb..f8696e57e 100644--- a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts@@ -34,9 +34,12 @@ describe('ensureK8sApiEgress', () => {mockApply.mockClear()// ensureDefaultPolicies() also prunes legacy static policies via the// networking API directly (not applyNetworkPolicy), so the mocked client- // must expose deleteNamespacedNetworkPolicy.+ // must expose the read-before-delete legacy cleanup path.const mockKc = {makeApiClient: () => ({+ readNamespacedNetworkPolicy: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),}),} as unknown as k8s.KubeConfigdiff --git a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.tsindex 6a89edc00..04e7b3b33 100644--- a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts@@ -26,7 +26,9 @@ const mockApply = vi.mocked(applyNetworkPolicy)function createMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),+ readNamespacedNetworkPolicy: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),diff --git a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.tsindex ae59d2dbd..9d0db5860 100644--- a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts@@ -32,7 +32,9 @@ const mockApply = vi.mocked(applyNetworkPolicy)function createMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),+ readNamespacedNetworkPolicy: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),diff --git a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.tsindex a93e20eed..6acb165b3 100644--- a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts@@ -83,7 +83,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('generates egress policy in rpc-proxy namespace when context has servers', async () => {const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx)+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)@@ -91,7 +91,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('egress policy targets correct server pod by mcpserver label', async () => {const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx)+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)@@ -103,7 +103,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('egress policy uses correct port from McpServer spec', async () => {const ctx = makeContext('ctx-beta', ['airtable-mcp'])- await reconciler.reconcileContext(ctx)+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })const rpcProxyCalls = mockApply.mock.calls.filter(call => call[1] === 'rpc-egress-ctx-beta-airtable-mcp'@@ -122,6 +122,8 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {metadata: {name: 'rpc-egress-ctx-gamma-mongodb-mcp',namespace: 'rpc-proxy',+ uid: 'rpc-delete-uid',+ resourceVersion: '7',labels: {'clerum.io/managed-by': 'host-context-controller','clerum.io/policy-type': 'rpc-proxy-egress',@@ -144,7 +146,22 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('deletes orphaned egress policies when server removed from context', async () => {// First reconcile with both serversconst ctx1 = makeContext('ctx-delta', ['mongodb-mcp', 'airtable-mcp'])- await reconciler.reconcileContext(ctx1)+ await reconciler.reconcileContext(ctx1, { isCurrent: () => true })+ const existingRpcProxyPolicies = mockApply.mock.calls+ .filter(call => call[2] === 'rpc-proxy')+ .map(call => {+ const policy = call[3] as k8s.V1NetworkPolicy+ const name = policy.metadata?.name+ return {+ ...policy,+ metadata: {+ ...policy.metadata,+ uid: `rpc-proxy:${name}:uid`,+ resourceVersion: '1',+ },+ }+ })+ expect(existingRpcProxyPolicies).toHaveLength(2)vi.clearAllMocks()mockNetApi.listNamespacedNetworkPolicy.mockImplementation(@@ -154,10 +171,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {String(labelSelector).includes('clerum.io/context=ctx-delta')) {return {- items: [- { metadata: { name: 'rpc-egress-ctx-delta-mongodb-mcp' } },- { metadata: { name: 'rpc-egress-ctx-delta-airtable-mcp' } },- ],+ items: existingRpcProxyPolicies,}}return { items: [] }@@ -166,7 +180,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {// Now reconcile with only one server — orphaned rpc-proxy policy should be cleanedconst ctx2 = makeContext('ctx-delta', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx2)+ await reconciler.reconcileContext(ctx2, { isCurrent: () => true })// applyNetworkPolicy should be called for mongodb-mcp in rpc-proxy but not airtable-mcpconst rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')@@ -176,6 +190,12 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {expect(mockNetApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({name: 'rpc-egress-ctx-delta-airtable-mcp',namespace: 'rpc-proxy',+ body: {+ preconditions: {+ uid: 'rpc-proxy:rpc-egress-ctx-delta-airtable-mcp:uid',+ resourceVersion: '1',+ },+ },})expect(mockNetApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({name: 'rpc-egress-ctx-delta-mongodb-mcp',@@ -185,7 +205,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('skips egress generation when server not found in cache', async () => {const ctx = makeContext('ctx-epsilon', ['nonexistent-server'])- await reconciler.reconcileContext(ctx)+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBe(0)diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 15f46834e..8519e148d 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.240",+ "version": "0.1.239","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.240",+ "version": "0.1.239","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 0fd114fe5..2dcb39dec 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.240",+ "version": "0.1.239","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/main.mcpInitialization.test.ts b/mcp-host/src/__tests__/main.mcpInitialization.test.tsindex c90bad869..02e9e9c75 100644--- a/mcp-host/src/__tests__/main.mcpInitialization.test.ts+++ b/mcp-host/src/__tests__/main.mcpInitialization.test.ts@@ -22,7 +22,12 @@ function readyServer(overrides: Partial<McpServerInfo> = {}): McpServerInfo {transport: { type: 'streamableHttp', url: 'http://secured-server.test/mcp' },auth: { type: 'bearer', secretRef: 'secured-server-auth' },enabled: true,- status: { deployed: true, ready: true },+ // Mirror the real producer contract: HCC's reconciler.getStatus() (surfaced+ // through k8sClient.toServerInfo) always stamps `authoritative` explicitly —+ // a ready, identity-matched snapshot carries authoritative: true. The legacy+ // shape that omitted it is never emitted for a live server, so the default+ // fixture must carry it too.+ status: { deployed: true, ready: true, authoritative: true },...overrides,}}diff --git a/nginx-egress-proxy/README.md b/nginx-egress-proxy/README.mdindex b4c205974..372f8e5aa 100644--- a/nginx-egress-proxy/README.md+++ b/nginx-egress-proxy/README.md@@ -53,10 +53,10 @@ reference is platform-controlled:| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | host-context-controller | `clerum/nginx-egress-proxy:0.1.0` || `CONTROL_API_REMOTE_MCP_EGRESS_PROXY_IMAGE` | control-api (registry install stamps `spec.image` for remote entries) | `clerum/nginx-egress-proxy:0.1.0` |-HCC additionally canonicalizes `spec.image` of remote McpServers back to the-platform image (`canonicalizeRemoteEgressProxyImage`) as best-effort cleanup;-the enforcement is that Deployment rendering always stamps the configured-platform image, so a CRD's `spec.image` is never what runs.+HCC does not rewrite `spec.image` during reconciliation: that field remains+desired state owned by the CRD author. Deployment rendering always stamps the+configured platform image for remote egress proxies, so a remote CRD's+`spec.image` is never what runs.## Portsdiff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex fde59308e..38afc5704 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.64",+ "version": "0.1.65","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.64",+ "version": "0.1.65","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex 2c7a4f3ed..a17ac4862 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.64",+ "version": "0.1.65","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/config.hccBaseUrl.test.ts b/rpc-proxy/src/config.hccBaseUrl.test.tsindex b1cedb606..bc3543a8d 100644--- a/rpc-proxy/src/config.hccBaseUrl.test.ts+++ b/rpc-proxy/src/config.hccBaseUrl.test.ts@@ -27,16 +27,33 @@ describe('rpc-proxy HCC base URL default', () => {)const configmapMatch = configmap.match(/RPC_PROXY_HCC_BASE_URL:\s*"?([^"\n]+)"?/)- it('parses both sources', () => {+ // Derive the real endpoint from the api-gateway manifest itself, so renaming+ // the Service or moving its port breaks this test unless the dev default and+ // configmap are updated in lockstep — no hand-copied literal to drift.+ const gatewayManifest = readFileSync(+ path.join(repoRoot, 'deploy/base/control-plane/host-context-controller-api-gateway.yaml'),+ 'utf8'+ )+ const serviceDoc = gatewayManifest.split(/^---$/m).find(doc => /\bkind:\s*Service\b/.test(doc))+ const serviceName = serviceDoc?.match(/\bmetadata:[\s\S]*?\bname:\s*([\w.-]+)/)?.[1]+ const serviceNamespace = serviceDoc?.match(/\bnamespace:\s*([\w.-]+)/)?.[1]+ const servicePort = serviceDoc?.match(/\bports:[\s\S]*?\bport:\s*(\d+)/)?.[1]++ it('parses all three sources', () => {expect(defaultMatch?.[1]).toBeTruthy()expect(configmapMatch?.[1]).toBeTruthy()+ expect(serviceName).toBeTruthy()+ expect(serviceNamespace).toBeTruthy()+ expect(servicePort).toBeTruthy()})it('dev default matches the deployed configmap endpoint', () => {expect(defaultMatch?.[1]).toBe(configmapMatch?.[1]?.trim())})- it('targets the api-gateway in control-plane, not a nonexistent Service', () => {- expect(defaultMatch?.[1]).toContain('host-context-controller-api-gateway.control-plane')+ it('targets the api-gateway Service exactly as the gateway manifest declares it', () => {+ const url = new URL(defaultMatch![1])+ expect(url.hostname).toBe(`${serviceName}.${serviceNamespace}.svc.cluster.local`)+ expect(url.port).toBe(servicePort)})})diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.shindex bc77ee0f1..ecb97cbdc 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh@@ -1,28 +1,82 @@#!/usr/bin/env bash-# Safety helpers for the HCC CommunicationChannel watch-recovery fault fixture.+# Safety helpers shared by disruptive HCC fault-injection fixtures.++profile_env_value() {+ local key=$1 file=$2+ awk -v key="$key" '+ index($0, key "=") == 1 {+ print substr($0, length(key) + 2)+ exit+ }+ ' "$file"+}require_branch_owned_hcc_gate() {- local repo_root expected_worktree_id expected_head actual_worktree_id actual_head worktree_dirty+ local marker_namespace="${1:-control-plane}"+ local actual_worktree_id actual_head actual_cluster_fingerprint actual_gate+ local worktree_dirty expected_branch profile_env+ local pre_gate_state_root cluster_fingerprint_file expected_cluster_fingerprintis_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||die "fault injection requires a generated branch-scoped context, got '${E2E_KUBECONTEXT}'"- repo_root="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"- worktree_dirty="$(git -C "$repo_root" status --porcelain --untracked-files=normal)"+ HCC_BRANCH_GATE_REPO_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"+ worktree_dirty="$(+ git -C "$HCC_BRANCH_GATE_REPO_ROOT" status --porcelain --untracked-files=normal+ )"[ -z "$worktree_dirty" ] ||die "worktree has uncommitted changes; commit and re-sync before runtime proof"- expected_worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"- expected_head="$(git -C "$repo_root" rev-parse HEAD)"- actual_worktree_id="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \- -o jsonpath='{.data.worktreeId}' 2>/dev/null || true)"- actual_head="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \- -o jsonpath='{.data.gitHead}' 2>/dev/null || true)"+ HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID="$(+ printf '%s' "$HCC_BRANCH_GATE_REPO_ROOT" | shasum | awk '{print $1}'+ )"+ HCC_BRANCH_GATE_EXPECTED_HEAD="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" rev-parse HEAD)"+ expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"+ [ -n "$expected_branch" ] ||+ die "branch-owned runtime proof requires a named branch, not detached HEAD"+ [ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ] ||+ die "MINIKUBE_PROFILE must explicitly select context ${E2E_KUBECONTEXT}, got ${MINIKUBE_PROFILE:-missing}"++ profile_env="${E2E_BRANCH_PROFILE_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/profile.env}"+ [ -r "$profile_env" ] ||+ die "branch-profile helper evidence is missing at ${profile_env}; run a state-writing branch-profile helper action from this worktree"+ [ "$(profile_env_value PROFILE "$profile_env")" = "$E2E_KUBECONTEXT" ] ||+ die "branch-profile helper output does not select context ${E2E_KUBECONTEXT}"+ [ "$(profile_env_value REPO_DIR "$profile_env")" = "$HCC_BRANCH_GATE_REPO_ROOT" ] ||+ die "branch-profile helper output belongs to a different worktree"+ [ "$(profile_env_value BRANCH "$profile_env")" = "$expected_branch" ] ||+ die "branch-profile helper output belongs to a different branch"+ [ "$(profile_env_value DIRTY "$profile_env")" = false ] ||+ die "branch-profile helper recorded a dirty worktree; refresh it after committing"++ pre_gate_state_root="${E2E_PRE_GATE_STATE_ROOT:-${TMPDIR:-/tmp}/clerum-pre-gate-sync}"+ cluster_fingerprint_file="${pre_gate_state_root}/${HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID}/cluster.sha"+ [ -r "$cluster_fingerprint_file" ] ||+ die "pre-gate fingerprint evidence is missing at ${cluster_fingerprint_file}"+ expected_cluster_fingerprint="$(sed -n '1p' "$cluster_fingerprint_file")"+ [[ "$expected_cluster_fingerprint" =~ ^[0-9a-f]{40}$ ]] ||+ die "pre-gate fingerprint evidence is malformed"+ [ -n "${E2E_EXPECTED_PRE_GATE_GATE:-}" ] ||+ die "E2E_EXPECTED_PRE_GATE_GATE must name the gate recorded by the branch-owned pre-gate sync"++ HCC_BRANCH_GATE_SYNC_MARKER="$(+ kctl get configmap clerum-pre-gate-sync-state -n "$marker_namespace" -o json 2>/dev/null+ )" || die "cluster has no readable pre-gate sync marker"+ actual_worktree_id="$(jq -r '.data.worktreeId // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"+ actual_head="$(jq -r '.data.gitHead // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"+ actual_cluster_fingerprint="$(+ jq -r '.data.clusterFingerprint // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER"+ )"+ actual_gate="$(jq -r '.data.gate // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"- [ "$actual_worktree_id" = "$expected_worktree_id" ] ||+ [ "$actual_worktree_id" = "$HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID" ] ||die "cluster ownership marker does not match this worktree; run minikube-pre-gate-sync first"- [ "$actual_head" = "$expected_head" ] ||- die "cluster HEAD marker ${actual_head:-missing} does not match ${expected_head}"+ [ "$actual_head" = "$HCC_BRANCH_GATE_EXPECTED_HEAD" ] ||+ die "cluster HEAD marker ${actual_head:-missing} does not match ${HCC_BRANCH_GATE_EXPECTED_HEAD}"+ [ "$actual_cluster_fingerprint" = "$expected_cluster_fingerprint" ] ||+ die "cluster fingerprint marker does not match this worktree's last pre-gate sync"+ [ "$actual_gate" = "$E2E_EXPECTED_PRE_GATE_GATE" ] ||+ die "cluster gate marker ${actual_gate:-missing} does not match expected ${E2E_EXPECTED_PRE_GATE_GATE}"}create_hcc_api_proxy() {@@ -284,10 +338,10 @@ delete_hcc_proxy_fixture() {}restore_hcc_after_fault_injection() {- local host_override port_override host_aliases desired ready+ local host_override port_override api_cidrs_override host_aliases desired readykctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- >/dev/null || return 1+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=merge \-p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null || return 1@@ -296,10 +350,12 @@ restore_hcc_after_fault_injection() {-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"+ api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"host_aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')"desired="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"ready="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.status.readyReplicas}')"- [ -z "$host_override$port_override$host_aliases" ] && [ "$desired" = "$ready" ]+ [ -z "$host_override$port_override$api_cidrs_override$host_aliases" ] && [ "$desired" = "$ready" ]}print_hcc_repair_instructions() {@@ -308,7 +364,7 @@ HCC restoration failed. Proxy resources were retained for repair.Context: ${E2E_KUBECONTEXT}Proxy: ${HCC_NS}/${PROXY_NAME}Repair commands:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT-+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180sAfter HCC is healthy, delete ${PROXY_NAME}, ${PROXY_EGRESS_NP}, ${HCC_PROXY_NP}, and ${PROBE_EGRESS_NP} in ${HCC_NS}.diff --git a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh b/scripts/e2e/_lib/hcc-watch-recovery-logs.shindex b227248a6..f0857c3d5 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh@@ -2,6 +2,18 @@# Log capture and causal recovery-cycle assertions for the HCC watch gate.+hcc_log_snapshot_contains() {+ local snapshot=$1 marker=$2+ grep -Fq -- "$marker" <<<"$snapshot"+}++hcc_initial_pass_snapshot_is_active() {+ local snapshot=$1 start_marker=$2 complete_marker=$3 fail_marker=$4+ hcc_log_snapshot_contains "$snapshot" "$start_marker" &&+ ! hcc_log_snapshot_contains "$snapshot" "$complete_marker" &&+ ! hcc_log_snapshot_contains "$snapshot" "$fail_marker"+}+hcc_recovery_logs() {[ -n "$START_TIME" ] || return 0grep -E 'CommunicationChannel|Listing all Hosts|Reconciling [0-9]+ Host|Completed Host reconciliation|Periodic resync' \@@ -59,7 +71,8 @@ recovery_cycle_used_fresh_host_inventory() {hcc_recovery_logs | awk \-v target="$cycle" -v host_namespace="$HOST_NS" -v expected="$expected_count" 'BEGIN {- inventory_line = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"+ channel_inventory = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"+ host_inventory = "Reconciling " expected " Host(s) after Host watch recovery convergence"succeeded = 0}/CommunicationChannel watch ended;/ {@@ -68,7 +81,7 @@ recovery_cycle_used_fresh_host_inventory() {if (inside) {listed = 0recovered = 0- inventory_matched = 0+ completion_mode = ""}next}@@ -78,11 +91,20 @@ recovery_cycle_used_fresh_host_inventory() {inside && /Recovered [0-9]+ CommunicationChannel\(s\) into cache/ {recovered = 1}- inside && index($0, inventory_line) > 0 {- inventory_matched = 1+ inside && listed && recovered && index($0, channel_inventory) > 0 {+ completion_mode = "channel"}- inside && /Completed Host reconciliation after CommunicationChannel recovery$/ {- succeeded = listed && recovered && inventory_matched+ inside && listed && recovered && index($0, host_inventory) > 0 {+ completion_mode = "host"+ }+ inside && completion_mode == "channel" &&+ /Completed Host reconciliation after CommunicationChannel recovery$/ {+ succeeded = 1+ exit+ }+ inside && completion_mode == "host" &&+ /Completed Host reconciliation after Host watch recovery convergence$/ {+ succeeded = 1exit}END {@@ -90,3 +112,15 @@ recovery_cycle_used_fresh_host_inventory() {}'}++wait_recovery_cycle_fresh_host_inventory() {+ local cycle=$1 expected_count=$2 timeout=$3 elapsed=0+ while [ "$elapsed" -lt "$timeout" ]; do+ require_hcc_recovery_log_stream || return 1+ recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_count" && return 0+ sleep 2+ elapsed=$((elapsed + 2))+ done+ hcc_recovery_logs >&2+ return 1+}diff --git a/scripts/e2e/e2e-crd-field-injection.sh b/scripts/e2e/e2e-crd-field-injection.shindex af8a25829..d4fe358f8 100755--- a/scripts/e2e/e2e-crd-field-injection.sh+++ b/scripts/e2e/e2e-crd-field-injection.sh@@ -55,6 +55,7 @@ require_safe_kube_context || exit 1cleanup_e2e_resourcesensure_e2e_resources_absent ||fail "Stale e2e-sec resources remain after cleanup; refusing stale-positive E2E run"+# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.[ "$e2e_fail" -gt 0 ] && exit 1# ═══ Test 1: imagePullPolicy override prevention ═════════════════════@@ -78,8 +79,11 @@ spec:url: http://e2e-sec-pullpolicy.mcp-server.svc:3000/mcpEOF-log "Waiting for HCC to create Deployment..."-sleep 15+if wait_for_deployment "$MCP_NS" e2e-sec-pullpolicy "$TIMEOUT_POD"; then+ ok "imagePullPolicy fixture rendered a ready Deployment"+else+ fail "imagePullPolicy fixture did not render a ready Deployment"+fiPULL_POLICY=$(kctl get deployment e2e-sec-pullpolicy -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.containers[0].imagePullPolicy}' 2>/dev/null || echo "NOT_FOUND")@@ -237,7 +241,11 @@ spec:value: postgres://safeEOF-sleep 15+if wait_for_deployment "$MCP_NS" e2e-sec-envvars "$TIMEOUT_POD"; then+ ok "Dangerous-env fixture rendered a ready Deployment"+else+ fail "Dangerous-env fixture did not render a ready Deployment"+fiENV_JSON=$(kctl get deployment e2e-sec-envvars -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.containers[0].env}' 2>/dev/null || echo "[]")@@ -276,7 +284,11 @@ spec:- DAC_OVERRIDEEOF-sleep 15+if wait_for_deployment "$MCP_NS" e2e-sec-uid70 "$TIMEOUT_POD"; then+ ok "Non-root UID fixture rendered a ready Deployment"+else+ fail "Non-root UID fixture did not render a ready Deployment"+fiUID_70=$(kctl get deployment e2e-sec-uid70 -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.securityContext.runAsUser}' 2>/dev/null || echo "NOT_FOUND")@@ -292,5 +304,6 @@ ensure_e2e_resources_absent && ok "All e2e-sec resources cleaned up" || fail "e2# ═══ Summary ═════════════════════════════════════════════════════════header "Summary"+# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.echo -e " ${GREEN}PASS${NC}: ${e2e_pass} ${RED}FAIL${NC}: ${e2e_fail} Total: ${e2e_total}"[ "$e2e_fail" -gt 0 ] && exit 1 || exit 0diff --git a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.shindex d1a38ed5b..480b9abd9 100755--- a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh+++ b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh@@ -35,13 +35,15 @@ command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }exit 1}kctl get nodes -o json | jq -e \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] != null)' >/dev/null || {+ --arg context "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null || {echo "Refusing HCC fault injection: target is not a minikube cluster." >&2exit 1}HCC_NS="${HCC_NS:-control-plane}"HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"+MCP_NS="${MCP_NS:-mcp-server}"SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm-stateless}"HOST_NS="${MCP_HOST_NS:-mcp-host}"CHANNEL_NS="${CHANNELS_NS:-channels}"@@ -61,7 +63,7 @@ HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-stream.XXXXXX")"NON_FIXTURE_BASELINE="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-baseline.XXXXXX")"NON_FIXTURE_AFTER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-after.XXXXXX")"CLEANUP_KINDS="deployments,services,secrets,serviceaccounts,roles,rolebindings,networkpolicies,persistentvolumeclaims"-START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY=""+START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY="" ORIGINAL_REPLICAS=""HCC_LOG_STREAM_PID=""# Mirrors COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS in k8sClient.ts.CC_RECOVERY_RETRY_SECONDS=5@@ -75,6 +77,35 @@ HCC_GATE_FINALIZATION_FAILURE=""die() { fail "$*"; exit 1; }now_rfc3339() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }+# This gate deliberately changes only the HCC Kubernetes API path. If its+# rollout fails before the normal recovery log stream starts, preserve the+# bounded pod state and both container log buffers that distinguish a proxy,+# policy, or process-startup failure. Cleanup still restores HCC afterwards.+print_hcc_proxy_rollout_diagnostics() {+ local pod++ echo "HCC proxy rollout diagnostics (before restoration):" >&2+ kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o json |+ jq -r '+ .items[] |+ .metadata.name as $pod |+ .status.phase as $phase |+ .status.containerStatuses[]? |+ "pod=\($pod) phase=\($phase // "unknown") ready=\(.ready) restarts=\(.restartCount) waiting=\(.state.waiting.reason // "none") terminated=\(.state.terminated.reason // "none") exitCode=\(.state.terminated.exitCode // "none") lastTerminated=\(.lastState.terminated.reason // "none") lastExitCode=\(.lastState.terminated.exitCode // "none")"+ ' >&2 || true++ while IFS= read -r pod; do+ [ -n "$pod" ] || continue+ echo "HCC proxy rollout previous logs for pod/${pod}:" >&2+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \+ --previous --tail=120 >&2 || true+ echo "HCC proxy rollout current logs for pod/${pod}:" >&2+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \+ --tail=120 >&2 || true+ done < <(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)+}+cleanup() {local status=$? cleanup_failed=0 restore_ok=1 host remainingset +e@@ -126,6 +157,8 @@ cleanup() {elsecleanup_failed=1print_hcc_repair_instructions+ echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}" >&2+ echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s" >&2fifinalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1rm -f "$HCC_LOG_BUFFER" "$NON_FIXTURE_BASELINE" "$NON_FIXTURE_AFTER"@@ -143,19 +176,35 @@ header "HCC CommunicationChannel watch recovery"require_branch_owned_hcc_gateacquire_hcc_watch_gate_lock || die "another HCC watch-recovery gate owns this profile"kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" >/dev/null+ORIGINAL_REPLICAS="$(+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}'+)"+[ "$ORIGINAL_REPLICAS" = 1 ] ||+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"kctl get host "$SOURCE_HOST" -n "$HOST_NS" >/dev/nullhost_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"-[ -z "$host_override$port_override" ] || die "HCC already has an explicit Kubernetes service override"+api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"+[ -z "$host_override$port_override$api_cidrs_override" ] ||+ die "HCC already has an explicit Kubernetes service or API CIDR override"[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||die "HCC already has hostAliases; refusing a non-restorable fault injection"HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"+K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ -c host-context-controller -- printenv KUBERNETES_SERVICE_HOST)" ||+ die "could not read the Kubernetes-injected API service IP"+kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \+ node -e 'const { isIP } = require("node:net"); process.exit(isIP(process.argv[1]) === 4 ? 0 : 1)' \+ "$K8S_API_SERVICE_HOST" >/dev/null ||+ die "Kubernetes-injected API service host is not an IPv4 address"+K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"log "Creating an isolated TCP proxy for the Kubernetes API"create_hcc_api_proxy@@ -164,19 +213,27 @@ verify_hcc_proxy_network_policyPROXY_IP="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"[ -n "$PROXY_IP" ] || die "proxy Service has no ClusterIP"HCC_PATCHED=1-hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" '{spec:{template:{spec:{+hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" --arg api_cidr "$K8S_API_CIDR" '{spec:{template:{spec:{hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],containers:[{name:"host-context-controller",env:[{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},- {name:"KUBERNETES_SERVICE_PORT",value:"443"}+ {name:"KUBERNETES_SERVICE_PORT",value:"443"},+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr}]}]}}}}')"kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$hcc_proxy_patch" >/dev/null-kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||+if ! kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null; then+ print_hcc_proxy_rollout_diagnosticsdie "HCC did not become ready through the API proxy"+fiinitial_hcc_identity="$(wait_for_hcc_identity 30)" || die "could not capture HCC pod identity"read -r HCC_UID HCC_RESTARTS <<<"$initial_hcc_identity"[ -n "$HCC_UID" ] && [ "$HCC_UID" != invalid ] || die "could not capture HCC pod identity"+proxy_policy_cidrs="$(kctl get networkpolicy "allow-k8s-api-egress-${MCP_NS}" -n "$MCP_NS" -o json |+ jq -r '[.spec.egress[0].to[]?.ipBlock.cidr] | join(",")')" ||+ die "could not read the proxy-run API egress policy"+[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ] ||+ die "proxy-run API egress policy did not retain the injected Kubernetes API CIDR"ok "HCC is healthy through the proxy (pod=${HCC_UID}, restarts=${HCC_RESTARTS})"log "Creating one stateful control and two stateless Host fixtures"@@ -214,14 +271,13 @@ ok "isolated Host fleet reached its baseline"run_recovery_cycle() {local cycle=$1 cycle_logs recovery_logs outage_started outage_seconds max_retry_failures list_pattern- local host_list_pattern ended_before started_before recovered_before completed_before failed_before+ local host_list_pattern ended_before started_before recovered_before failed_beforelocal list_before host_list_before expected_host_countlocal failed_after retry_failurescycle_logs="$(hcc_recovery_logs)"ended_before="$(log_count_from "$cycle_logs" 'CommunicationChannel watch ended;')"started_before="$(log_count_from "$cycle_logs" 'Starting CommunicationChannel watch')"recovered_before="$(log_count_from "$cycle_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"- completed_before="$(log_count_from "$cycle_logs" 'Completed Host reconciliation after CommunicationChannel recovery$')"failed_before="$(log_count_from "$cycle_logs" 'cache recovery failed;')"list_pattern="Listing all CommunicationChannels in namespace ${CHANNEL_NS}"host_list_pattern="Listing all Hosts in namespace ${HOST_NS}"@@ -265,10 +321,7 @@ run_recovery_cycle() {max_retry_failures=$(((outage_seconds + CC_RECOVERY_RETRY_SECONDS - 1) / CC_RECOVERY_RETRY_SECONDS + 2))- wait_log_count 'Completed Host reconciliation after CommunicationChannel recovery$' \- "$((completed_before + 1))" 180 ||- die "Host fleet did not converge after snapshot recovery in cycle ${cycle}"- recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_host_count" ||+ wait_recovery_cycle_fresh_host_inventory "$cycle" "$expected_host_count" 180 ||die "cycle ${cycle} did not reconcile the fresh ${expected_host_count}-Host inventory"recovery_logs="$(hcc_recovery_logs)"failed_after="$(log_count_from "$recovery_logs" 'cache recovery failed;')"@@ -316,14 +369,18 @@ ended="$(log_count_from "$final_logs" 'CommunicationChannel watch ended;')"started="$(log_count_from "$final_logs" 'Starting CommunicationChannel watch')"recovered="$(log_count_from "$final_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"failed="$(log_count_from "$final_logs" 'cache recovery failed;')"-fleet_passes="$(log_count_from "$final_logs" 'Completed Host reconciliation after CommunicationChannel')"+fleet_passes="$(+ log_count_from "$final_logs" \+ 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'+)"[ "$ended" = 2 ] && [ "$started" = 2 ] && [ "$recovered" = 2 ] && \[ "$failed" = "$TOTAL_RETRY_FAILURES" ] && [ "$failed" -le "$TOTAL_RETRY_FAILURE_LIMIT" ] ||die "unexpected watch lifecycle counts: ended=${ended}, started=${started}, recovered=${recovered}, failed=${failed}"[ "$fleet_passes" -ge 2 ] && [ "$fleet_passes" -le 4 ] ||die "unexpected successful fleet reconcile count ${fleet_passes}; expected 2..4 passes"-# Each recovered snapshot requires one successful pass; at most one already-# queued fail-closed pass may also finish per cycle. The deterministic+# Each recovered snapshot requires one causally ordered successful pass; a+# Host-watch recovery pass may cover the same fresh CC+Host inventories. At+# most one already queued covering pass may also finish per cycle. The deterministic# max-concurrency=1 proof stays in k8sClient unit tests where overlapping passes# can be barrier-controlled.diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.shnew file mode 100755index 000000000..cccb17538--- /dev/null+++ b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh@@ -0,0 +1,1790 @@+#!/usr/bin/env bash+# Proves that HCC readiness is independent from the exact McpServer ->+# external-egress -> runtime -> Context NetworkPolicy initial convergence path.+#+# The gate runs only on an exact-HEAD, branch-owned Minikube profile. It routes+# HCC DNS through an isolated in-cluster proxy that forwards every query except+# one fixture hostname. The gate first lets HCC create a real external-egress+# policy. A same-intent restart then proves the safety pass RETAINS that+# identity-stable DNS-derived allow without attempting DNS and certifies global+# readiness with the allow still present — no deny window — while the affected+# runtime stays blocked. DNS release lets the additive lane refresh the current+# TCP policy and runtime. A second phase changes that binding's protocol while+# HCC is stopped and proves the divergent contract: an identity-changing+# TCP-to-UDP replacement IS revoked before readiness. A real Context watch update+# still converges its three+# NetworkPolicies. The fleet is wider than the bounded worker width, covering+# the original large-fleet symptom.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+# shellcheck source=scripts/e2e/e2e-lib.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/e2e-lib.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"++[ -n "$E2E_KUBECONTEXT" ] || {+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2+ exit 1+}+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {+ echo "Refusing HCC MCP readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2+ exit 1+}+require_safe_kube_context+[ "${E2E_HCC_MCP_READINESS_FAULT_INJECTION:-0}" = 1 ] || {+ echo "Set E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC DNS fault injection." >&2+ exit 1+}+command -v jq >/dev/null 2>&1 || {+ echo "jq is required" >&2+ exit 1+}++HCC_NS="${HCC_NS:-control-plane}"+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"+MCP_NS="${MCP_SERVER_NS:-mcp-server}"+HOST_NS="${MCP_HOST_NS:-mcp-host}"+RPC_PROXY_NS="${RPC_PROXY_NS:-rpc-proxy}"+RUN_ID="$(date +%s)-$$"+SUITE_NAME="hcc-mcp-readiness"+RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"+MCP_NAME="$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}")"+CONTEXT_NAME="$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}")"+CONTEXT_ID="$CONTEXT_NAME"+MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"+TARGET_DNS="$(truncate_rfc1123 "hcc-readiness-${RUN_ID}").example.com"+DNS_BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-dns-${RUN_ID}")"+DNS_BLOCKER_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-policy")"+HCC_DNS_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-from-hcc")"+CONTEXT_POLICY="ctx-${CONTEXT_ID}-${MCP_NAME}"+CONTEXT_EGRESS_POLICY="${CONTEXT_POLICY}-egress"+RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_ID}-${MCP_NAME}"+MANAGED_BY_LABEL="clerum.io/managed-by"+MANAGED_BY_VALUE="host-context-controller"+MCP_SERVER_LABEL="clerum.io/mcpserver"+POLICY_TYPE_LABEL="clerum.io/policy-type"+CONTEXT_LABEL="clerum.io/context"+EXTERNAL_EGRESS_CIDR="93.184.216.34/32"+EXTERNAL_EGRESS_PORT=443+INITIAL_EXTERNAL_EGRESS_PROTOCOL=TCP+EXTERNAL_EGRESS_PROTOCOL=UDP+ORIGINAL_REPLICAS=""+ORIGINAL_DNS_POLICY=""+ORIGINAL_DNS_CONFIG=""+DNS_BLOCKER_IP=""+DNS_RELEASE_COUNT_AT_HOLD=0+DNS_HOLD_COUNT_AT_ARM=0+DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=0+UPSTREAM_DNS_IP=""+NEW_HCC_POD=""+HCC_UID=""+HCC_RESTARTS=""+HCC_PORT=""+HCC_MUTATED=0+DNS_BLOCKER_CREATED=0+MCP_CREATED=0+CONTEXT_CREATED=0+PEER_FLEET_CREATED=0+MCP_FLEET_NAMES=("$MCP_NAME")+CONTEXT_FLEET_NAMES=("$CONTEXT_NAME")+# Read and mutated by the sourced compare-and-swap lock helper.+# shellcheck disable=SC2034+HCC_GATE_LOCK_ACQUIRED=0+# shellcheck disable=SC2034+HCC_GATE_LOCK_NAME=""+# shellcheck disable=SC2034+HCC_GATE_LOCK_UID=""+# shellcheck disable=SC2034+HCC_GATE_FINALIZATION_FAILURE=""+HCC_READY_PROBE_DIAGNOSTIC=""+BASELINE_POLICY_SNAPSHOT=""+SAME_INTENT_EGRESS_SNAPSHOT=""++# Clean-window negative markers. Consumed ONLY while the DNS fault injection is+# NOT applied (pre-injection pre-checks, post-restore cleanup): under the+# injection the additive NetworkPolicy lane legitimately fails and schedules+# retries, so these negatives false-fail inside the injected window (removed+# there by 1abcfb1/15997353/e7064a7f). Inventoried in+# scripts/tests/test-hcc-watch-recovery-gate.sh marker_bindings.+NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'+NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'+NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'+NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '+clean_window_checks_run=0++[[ "$MCP_FLEET_SIZE" =~ ^[0-9]+$ ]] && [ "$MCP_FLEET_SIZE" -gt 10 ] &&+ [ "$MCP_FLEET_SIZE" -le 24 ] || {+ echo "E2E_HCC_MCP_FLEET_SIZE must be an integer from 11 through 24; got ${MCP_FLEET_SIZE}." >&2+ exit 1+}+for ((index = 2; index <= MCP_FLEET_SIZE; index++)); do+ MCP_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}-${index}")")+ CONTEXT_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}-${index}")")+done++die() {+ if [ -n "$NEW_HCC_POD" ]; then+ echo "Recent HCC logs from ${NEW_HCC_POD}:" >&2+ kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller \+ --tail=300 >&2 || true+ fi+ if [ "$DNS_BLOCKER_CREATED" = 1 ]; then+ echo "Recent DNS blocker logs:" >&2+ kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" --tail=120 >&2 || true+ fi+ fail "$*"+ exit 1+}++wait_until() {+ local timeout=$1 description=$2+ shift 2+ local deadline now+ deadline=$(( $(date +%s) + timeout ))+ while :; do+ "$@" && return 0+ now="$(date +%s)"+ [ "$now" -lt "$deadline" ] || break+ sleep 1+ done+ echo "Timed out after ${timeout}s waiting for ${description}" >&2+ return 1+}++wait_until_fast() {+ local timeout=$1 description=$2+ shift 2+ local deadline now rc+ deadline=$(( $(date +%s) + timeout ))+ while :; do+ "$@" && return 0+ rc=$?+ # rc=2 is a fatal, non-retryable ordering violation (e.g. 200 observed with+ # a stale policy still present): stop polling immediately so a later clean+ # poll cannot mask it. rc=1 is benign "not ready yet".+ if [ "$rc" -eq 2 ]; then+ echo "FATAL ordering violation while waiting for ${description}: ${HCC_READY_PROBE_DIAGNOSTIC:-unset}" >&2+ return 2+ fi+ now="$(date +%s)"+ [ "$now" -lt "$deadline" ] || break+ sleep 0.5+ done+ echo "Timed out after ${timeout}s waiting for ${description}" >&2+ return 1+}++running_hcc_pod() {+ local rows+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ --field-selector=status.phase=Running \+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \+ 2>/dev/null)" || return 1+ awk -F '\t' '$1 != "" && $4 == "" { print $1; exit }' <<<"$rows"+}++capture_hcc() {+ NEW_HCC_POD="$(running_hcc_pod)"+ [ -n "$NEW_HCC_POD" ]+}++hcc_pods_absent() {+ local pods+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||+ return 1+ [ -z "$pods" ]+}++hcc_log_contains() {+ local marker=$1 logs+ [ -n "$NEW_HCC_POD" ] || return 1+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \+ -c host-context-controller 2>/dev/null)" || return 1+ grep -Fq "$marker" <<<"$logs"+}++dns_log_contains() {+ local marker=$1 logs+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||+ return 1+ grep -Fq "$marker" <<<"$logs"+}++dns_release_count() {+ local marker="released DNS A ${TARGET_DNS}" logs+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||+ return 1+ grep -Fc "$marker" <<<"$logs" || true+}++dns_hold_count() {+ local marker="holding DNS A ${TARGET_DNS}" logs+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||+ return 1+ grep -Fc "$marker" <<<"$logs" || true+}++dns_hold_observed_since_arm() {+ [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_ARM" ]+}++dns_has_not_released_since_hold() {+ [ "$(dns_release_count)" = "$DNS_RELEASE_COUNT_AT_HOLD" ]+}++dns_released_since_hold() {+ [ "$(dns_release_count)" -gt "$DNS_RELEASE_COUNT_AT_HOLD" ]+}++external_egress_retry_progress_is_observed() {+ local logs marker+ [ -n "$NEW_HCC_POD" ] || return 1+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \+ -c host-context-controller 2>/dev/null)" || return 1+ marker="Scheduling external egress retry "+ grep -Fq \+ "Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};" <<<"$logs" &&+ awk \+ -v marker="$marker" \+ -v server="for McpServer \"${MCP_NAME}\"" '+ index($0, marker) && index($0, server) { found = 1 }+ END { exit found ? 0 : 1 }+ ' <<<"$logs"+}++dns_retry_query_observed_since_schedule() {+ [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_RETRY_SCHEDULE" ]+}++hcc_identity_is_stable() {+ local row uid restarts+ row="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}' \+ 2>/dev/null)" || return 1+ read -r uid restarts <<<"$row"+ [ "$uid" = "$HCC_UID" ] && [ "$restarts" = "$HCC_RESTARTS" ]+}++hcc_kubernetes_readiness_is_exact() {+ [ -n "$NEW_HCC_POD" ] && [ -n "$HCC_UID" ] || return 1+ kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" -o json 2>/dev/null |+ jq -e --arg uid "$HCC_UID" '+ .metadata.uid == $uid and+ any(.status.conditions[]?; .type == "Ready" and .status == "True") and+ any(.status.containerStatuses[]?;+ .name == "host-context-controller" and .ready == true+ )+ ' >/dev/null || return 1+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json 2>/dev/null |+ jq -e '+ .status.observedGeneration == .metadata.generation and+ (.spec.replicas // 0) == 1 and+ (.status.updatedReplicas // 0) == 1 and+ (.status.readyReplicas // 0) == 1 and+ (.status.availableReplicas // 0) == 1 and+ (.status.unavailableReplicas // 0) == 0+ ' >/dev/null+}++resource_absent() {+ local kind=$1 name=$2 namespace=$3 found+ found="$(+ kctl get "$kind" "$name" -n "$namespace" --ignore-not-found -o name 2>/dev/null+ )" || return 1+ [ -z "$found" ]+}++fixture_mcp_runtime_absent() {+ local external_egress_policies+ fixture_runtime_absent || return 1+ external_egress_policies="$(+ kctl get networkpolicy -n "$MCP_NS" \+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \+ -o name 2>/dev/null+ )" || return 1+ [ -z "$external_egress_policies" ]+}++fixture_runtime_absent() {+ resource_absent deployment "$MCP_NAME" "$MCP_NS" &&+ resource_absent service "$MCP_NAME" "$MCP_NS"+}++mcpserver_current_ready() {+ local name=$1+ kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null |+ jq -e '+ .metadata.generation as $generation |+ any(+ .status.conditions[]?;+ .type == "Ready" and .status == "True" and+ (.observedGeneration // -1) == $generation+ )+ ' >/dev/null+}++mcpserver_current_ready_absent() {+ local name=$1 resource+ resource="$(kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null)" ||+ return 1+ jq -e '+ .metadata.generation as $generation |+ any(+ .status.conditions[]?;+ .type == "Ready" and .status == "True" and+ (.observedGeneration // -1) == $generation+ ) | not+ ' <<<"$resource" >/dev/null+}++fixture_runtime_and_ready_absent() {+ fixture_runtime_absent && mcpserver_current_ready_absent "$MCP_NAME"+}++context_policies_absent() {+ resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&+ resource_absent networkpolicy "$CONTEXT_EGRESS_POLICY" "$HOST_NS" &&+ resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS"+}++context_policies_converged() {+ local context_id=${1:-$CONTEXT_ID}+ local context_name=${2:-$CONTEXT_NAME}+ local server=${3:-$MCP_NAME}+ local ingress_policy="ctx-${context_id}-${server}"+ local host_egress_policy="${ingress_policy}-egress"+ local rpc_egress_policy="rpc-egress-${context_id}-${server}"+ kctl get networkpolicy "$ingress_policy" -n "$MCP_NS" -o json |+ jq -e \+ --arg contextId "$context_id" \+ --arg contextName "$context_name" \+ --arg server "$server" \+ --arg hostNs "$HOST_NS" \+ --arg rpcNs "$RPC_PROXY_NS" \+ --arg mcpNs "$MCP_NS" \+ --arg managedByLabel "$MANAGED_BY_LABEL" \+ --arg managedByValue "$MANAGED_BY_VALUE" \+ --arg contextLabel "$CONTEXT_LABEL" \+ --arg serverLabel "$MCP_SERVER_LABEL" \+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '+ .metadata.labels[$managedByLabel] == $managedByValue and+ .metadata.labels[$policyTypeLabel] == "context-allow" and+ .metadata.labels[$contextLabel] == $contextId and+ .metadata.labels[$serverLabel] == $server and+ .spec.podSelector.matchLabels[$serverLabel] == $server and+ .spec.policyTypes == ["Ingress"] and+ (.spec.ingress | length) == 1 and+ .spec.ingress[0].ports == [{port: 3000, protocol: "TCP"}] and+ (.spec.ingress[0].from | length) == 3 and+ any(.spec.ingress[0].from[];+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $hostNs and+ .podSelector.matchLabels[$managedByLabel] == $managedByValue and+ .podSelector.matchLabels[$contextLabel] == $contextName+ ) and+ any(.spec.ingress[0].from[];+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $rpcNs and+ .podSelector.matchLabels.app == "rpc-proxy"+ ) and+ any(.spec.ingress[0].from[];+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $mcpNs and+ .podSelector.matchLabels.app == "mcp-proxy"+ )+ ' >/dev/null || return 1+ kctl get networkpolicy "$host_egress_policy" -n "$HOST_NS" -o json |+ jq -e \+ --arg contextId "$context_id" \+ --arg contextName "$context_name" \+ --arg server "$server" \+ --arg mcpNs "$MCP_NS" \+ --arg managedByLabel "$MANAGED_BY_LABEL" \+ --arg managedByValue "$MANAGED_BY_VALUE" \+ --arg contextLabel "$CONTEXT_LABEL" \+ --arg serverLabel "$MCP_SERVER_LABEL" \+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '+ .metadata.labels[$managedByLabel] == $managedByValue and+ .metadata.labels[$policyTypeLabel] == "context-allow" and+ .metadata.labels[$contextLabel] == $contextId and+ .metadata.labels[$serverLabel] == $server and+ .spec.podSelector.matchLabels[$managedByLabel] == $managedByValue and+ .spec.podSelector.matchLabels[$contextLabel] == $contextName and+ .spec.policyTypes == ["Egress"] and+ (.spec.egress | length) == 1 and+ .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and+ (.spec.egress[0].to | length) == 1 and+ .spec.egress[0].to[0].namespaceSelector.matchLabels[+ "kubernetes.io/metadata.name"+ ] == $mcpNs and+ .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server+ ' >/dev/null || return 1+ kctl get networkpolicy "$rpc_egress_policy" -n "$RPC_PROXY_NS" -o json |+ jq -e \+ --arg contextId "$context_id" \+ --arg server "$server" \+ --arg mcpNs "$MCP_NS" \+ --arg managedByLabel "$MANAGED_BY_LABEL" \+ --arg managedByValue "$MANAGED_BY_VALUE" \+ --arg contextLabel "$CONTEXT_LABEL" \+ --arg serverLabel "$MCP_SERVER_LABEL" \+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '+ .metadata.labels[$managedByLabel] == $managedByValue and+ .metadata.labels[$policyTypeLabel] == "rpc-proxy-egress" and+ .metadata.labels[$contextLabel] == $contextId and+ .metadata.labels[$serverLabel] == $server and+ .spec.podSelector.matchLabels.app == "rpc-proxy" and+ .spec.policyTypes == ["Egress"] and+ (.spec.egress | length) == 1 and+ .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and+ (.spec.egress[0].to | length) == 1 and+ .spec.egress[0].to[0].namespaceSelector.matchLabels[+ "kubernetes.io/metadata.name"+ ] == $mcpNs and+ .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server+ ' >/dev/null+}++external_egress_policy_converged_with_protocol() {+ local protocol=$1+ kctl get networkpolicy -n "$MCP_NS" \+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \+ -o json |+ jq -e \+ --arg server "$MCP_NAME" \+ --arg cidr "$EXTERNAL_EGRESS_CIDR" \+ --arg managedByLabel "$MANAGED_BY_LABEL" \+ --arg managedByValue "$MANAGED_BY_VALUE" \+ --arg serverLabel "$MCP_SERVER_LABEL" \+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" \+ --arg protocol "$protocol" \+ --argjson port "$EXTERNAL_EGRESS_PORT" '+ (.items | length) == 1 and+ .items[0].metadata.labels[$managedByLabel] == $managedByValue and+ .items[0].metadata.labels[$policyTypeLabel] == "external-egress" and+ .items[0].metadata.labels[$serverLabel] == $server and+ .items[0].spec.podSelector.matchLabels[$serverLabel] == $server and+ .items[0].spec.policyTypes == ["Egress"] and+ (.items[0].spec.egress | length) == 1 and+ .items[0].spec.egress[0].ports == [{port: $port, protocol: $protocol}] and+ (.items[0].spec.egress[0].to | length) == 1 and+ .items[0].spec.egress[0].to[0].ipBlock == {cidr: $cidr}+ ' >/dev/null+}++external_egress_policy_converged() {+ external_egress_policy_converged_with_protocol "$EXTERNAL_EGRESS_PROTOCOL"+}++external_egress_policy_absent() {+ local policies+ policies="$(+ kctl get networkpolicy -n "$MCP_NS" \+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \+ -o name 2>/dev/null+ )" || return 1+ [ -z "$policies" ]+}++clean_window_netpol_log_negatives_hold() {+ # Family 1, fail-closed snapshot: read ONE authoritative log snapshot so a+ # kubectl logs failure cannot make the negative assertions pass vacuously.+ local pod=$1 logs+ [ -n "$pod" ] || return 1+ logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" \+ -c host-context-controller 2>/dev/null)" || return 1+ ! grep -Fq "$NETPOL_INITIAL_FAILED_MARKER" <<<"$logs" &&+ ! grep -Fq "$NETPOL_ADDITIVE_FAILED_MARKER" <<<"$logs" &&+ ! grep -Fq "$NETPOL_RETRY_SCHEDULED_MARKER" <<<"$logs" &&+ ! grep -Fq "${NETPOL_CONTEXT_FAILED_MARKER}${CONTEXT_NAME}:" <<<"$logs"+}++clean_window_runtime_and_ready_absent() {+ # Family 3, clean-window form. fixture_runtime_and_ready_absent requires the+ # McpServer object to EXIST (mcpserver_current_ready_absent fails on a missing+ # object); in clean windows the fixture may not exist at all, which is the+ # strongest form of absence. An API error still fails closed.+ fixture_runtime_absent || return 1+ if ! resource_absent mcpserver "$MCP_NAME" "$MCP_NS"; then+ mcpserver_current_ready_absent "$MCP_NAME" || return 1+ fi+ external_egress_policy_absent+}++fixture_runtime_converged_with_protocol() {+ local protocol=$1 desired available+ desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1+ available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1+ kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1+ mcpserver_current_ready "$MCP_NAME" || return 1+ external_egress_policy_converged_with_protocol "$protocol"+}++fixture_resources_absent() {+ fixture_mcp_runtime_absent &&+ context_policies_absent &&+ peer_fleet_runtime_absent &&+ peer_fleet_policies_absent+}++fixture_inputs_absent() {+ resource_absent context "$CONTEXT_NAME" "$MCP_NS" &&+ resource_absent mcpserver "$MCP_NAME" "$MCP_NS" && peer_fleet_inputs_absent+}++peer_fleet_inputs_absent() {+ local index+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ resource_absent context "${CONTEXT_FLEET_NAMES[index]}" "$MCP_NS" || return 1+ resource_absent mcpserver "${MCP_FLEET_NAMES[index]}" "$MCP_NS" || return 1+ done+}++peer_fleet_runtime_absent() {+ local index server+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ server="${MCP_FLEET_NAMES[index]}"+ resource_absent deployment "$server" "$MCP_NS" || return 1+ resource_absent service "$server" "$MCP_NS" || return 1+ done+}++peer_fleet_policies_absent() {+ local index server context context_policy external_policies+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ server="${MCP_FLEET_NAMES[index]}"+ context="${CONTEXT_FLEET_NAMES[index]}"+ context_policy="ctx-${context}-${server}"+ resource_absent networkpolicy "$context_policy" "$MCP_NS" || return 1+ resource_absent networkpolicy "${context_policy}-egress" "$HOST_NS" || return 1+ resource_absent networkpolicy "rpc-egress-${context}-${server}" "$RPC_PROXY_NS" || return 1+ external_policies="$(+ kctl get networkpolicy -n "$MCP_NS" \+ -l "${MCP_SERVER_LABEL}=${server},${POLICY_TYPE_LABEL}=external-egress" \+ -o name 2>/dev/null+ )" || return 1+ [ -z "$external_policies" ] || return 1+ done+}++peer_fleet_converged() {+ local index server context desired available+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ server="${MCP_FLEET_NAMES[index]}"+ context="${CONTEXT_FLEET_NAMES[index]}"+ desired="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1+ available="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1+ kctl get service "$server" -n "$MCP_NS" >/dev/null 2>&1 || return 1+ mcpserver_current_ready "$server" || return 1+ context_policies_converged "$context" "$context" "$server" || return 1+ done+}++baseline_policy_snapshot() {+ kctl get \+ "networkpolicy/deny-all-${MCP_NS}" \+ "networkpolicy/allow-dns-egress-${MCP_NS}" \+ networkpolicy/allow-host-context-controller-api \+ -n "$MCP_NS" -o json 2>/dev/null |+ jq -ceS \+ --arg managedBy "$MANAGED_BY_VALUE" \+ --arg deny "deny-all-${MCP_NS}" \+ --arg dns "allow-dns-egress-${MCP_NS}" '+ [.items[] | {+ name: .metadata.name,+ managedBy: .metadata.labels["clerum.io/managed-by"],+ policyType: .metadata.labels["clerum.io/policy-type"],+ spec+ }]+ | sort_by(.name)+ | select(length == 3)+ | select(all(.[];+ .managedBy == $managedBy and+ ((.name == $deny and .policyType == "default-deny") or+ (.name == $dns and .policyType == "infrastructure") or+ (.name == "allow-host-context-controller-api" and .policyType == "allow-api"))))+ '+}++baseline_policies_unchanged() {+ local current+ [ -n "$BASELINE_POLICY_SNAPSHOT" ] || return 1+ current="$(baseline_policy_snapshot)" || return 1+ [ "$current" = "$BASELINE_POLICY_SNAPSHOT" ]+}++context_watch_has_latest_spec() {+ kctl get context "$CONTEXT_NAME" -n "$MCP_NS" -o json |+ jq -e --arg server "$MCP_NAME" '.spec.mcpServers == [$server]' >/dev/null+}++mcp_egress_binding_is_udp() {+ kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json |+ jq -e --arg dns "$TARGET_DNS" '+ (.spec.egressBindings | length) == 1 and+ .spec.egressBindings[0].egressClass == "exact-host" and+ .spec.egressBindings[0].dns == $dns and+ .spec.egressBindings[0].port == 443 and+ .spec.egressBindings[0].protocol == "UDP"+ ' >/dev/null+}++create_peer_fleet() {+ local index server context+ # Arm cleanup before the first write so a partial apply cannot strand input+ # CRDs or leave HCC deliberately scaled down during failure recovery.+ PEER_FLEET_CREATED=1+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ server="${MCP_FLEET_NAMES[index]}"+ context="${CONTEXT_FLEET_NAMES[index]}"+ kctl apply -f - >/dev/null <<EOF+apiVersion: clerum.io/v1alpha1+kind: Context+metadata:+ name: ${context}+ namespace: ${MCP_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ contextId: ${context}+ description: Gate-owned peer Context for HCC readiness fleet coverage.+ mcpServers: [${server}]+---+apiVersion: clerum.io/v1alpha1+kind: McpServer+metadata:+ name: ${server}+ namespace: ${MCP_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ contextRef: ${context}+ description: Gate-owned peer McpServer for bounded HCC fleet coverage.+ image: clerum/mock-mcp-server:test+ imagePullPolicy: IfNotPresent+ transport:+ type: streamableHttp+ url: http://${server}.${MCP_NS}.svc.cluster.local:3000/mcp+ port: 3000+EOF+ done+}++fixture_converged() {+ local desired available+ desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1+ available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1+ kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1+ mcpserver_current_ready "$MCP_NAME" || return 1+ external_egress_policy_converged || return 1+ context_policies_converged+}++probe_hcc_final_context() {+ local probe_script+ probe_script="$(cat <<'NODE'+const http = require('http');+const context = process.argv[1];+const expected = process.argv[2];+function get(path) {+ return new Promise((resolve, reject) => {+ const request = http.get({+ host: '127.0.0.1',+ port: Number(process.env.HCC_E2E_PORT),+ path+ }, response => {+ let body = '';+ response.setEncoding('utf8');+ response.on('data', chunk => { body += chunk; });+ response.on('end', () => resolve({status: response.statusCode, body}));+ });+ request.setTimeout(5000, () => request.destroy(new Error('timeout')));+ request.once('error', reject);+ });+}+(async () => {+ const ready = await get('/ready');+ const scoped = await get('/api/v1/mcpservers/context/' + encodeURIComponent(context));+ const body = JSON.parse(scoped.body);+ const fixture = body.servers?.find(server => server.name === expected);+ if (ready.status !== 200 || JSON.parse(ready.body).ready !== true) {+ throw new Error('readiness regressed: ' + ready.status + ' ' + ready.body);+ }+ if (scoped.status !== 200 || !fixture || fixture.status?.ready !== true) {+ throw new Error('latest Context did not converge in discovery: ' + scoped.status + ' ' + scoped.body);+ }+ console.log(JSON.stringify({+ readyStatus: ready.status,+ scopedDiscoveryStatus: scoped.status,+ contextRef: body.contextRef,+ fixture: fixture.name,+ fixtureReady: fixture.status.ready+ }));+})().catch(error => { console.error(error.message); process.exit(1); });+NODE+)"+ kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" \+ node -e "$probe_script" "$CONTEXT_NAME" "$MCP_NAME"+}++probe_hcc_ready_pod() {+ local pod+ pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1+ kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" \+ node -e '+ const http = require("node:http");+ const request = http.get(+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},+ response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ let parsed;+ try { parsed = JSON.parse(body); } catch { process.exit(2); }+ process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);+ });+ }+ );+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));+ request.once("error", () => process.exit(4));+ ' >/dev/null 2>&1+}++probe_new_hcc_ready_endpoint() {+ local output first_line+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=pod-not-selected"+ [ -n "$NEW_HCC_POD" ] || return 1+ if output="$(+ kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" \+ node -e '+ const http = require("node:http");+ let finished = false;+ const finish = (code, category, detail = "") => {+ if (finished) return;+ finished = true;+ if (category) {+ console.error(`HCC_READY_PROBE category=${category}${detail}`);+ }+ process.exitCode = code;+ };+ const request = http.get(+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},+ response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ if (response.statusCode !== 200) {+ finish(3, "http-status", ` status=${response.statusCode ?? "unknown"}`);+ return;+ }+ let parsed;+ try {+ parsed = JSON.parse(body);+ } catch {+ finish(2, "json-invalid");+ return;+ }+ if (parsed?.ready !== true) {+ finish(3, "json-ready-false");+ return;+ }+ finish(0);+ });+ }+ );+ request.setTimeout(2000, () => {+ finish(4, "timeout");+ request.destroy();+ });+ request.once("error", () => finish(5, "transport"));+ ' 2>&1+ )"; then+ HCC_READY_PROBE_DIAGNOSTIC=""+ return 0+ fi+ first_line="${output%%$'\n'*}"+ case "$first_line" in+ "HCC_READY_PROBE category="*)+ printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s' "$first_line"+ ;;+ *)+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=kubectl-exec"+ ;;+ esac+ return 1+}++# Third, independent signal of the safety contract (M6): the authoritative pass+# must actually have MEASURED and REVOKED. The unit tests assert the readiness+# predicate over vi.fn() mocks, so a metric rename would leave them green; these+# family-name literals live in the gate, so a rename turns the REAL gate red.+# Pure adjudication over a /metrics scrape — unit-testable without a cluster.+safety_pass_metrics_are_certified() {+ local metrics=$1+ grep -Fq -- 'clerum_hcc_networkpolicy_safety_pass_duration_seconds' <<<"$metrics" || return 1+ # At least one policy revoked by an authoritative pass ($NF is the sample+ # value; tolerant of extra/reordered labels).+ awk '$0 ~ /^clerum_hcc_networkpolicy_safety_pass_policies_total\{/ && /operation="revoked"/ { if ($NF + 0 >= 1) found = 1 } END { exit(found ? 0 : 1) }' <<<"$metrics"+}++# Scrapes the running replacement pod's /metrics over the same in-pod node/http+# channel the /ready probes use, then adjudicates. The sub-0.5s window B4 leaves+# unobservable on /ready is covered here: the counter is monotonic.+safety_pass_metrics_certify() {+ local metrics+ metrics="$(kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" \+ node -e '+ const http = require("node:http");+ const request = http.get(+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/metrics"},+ response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ process.stdout.write(body);+ process.exit(response.statusCode === 200 ? 0 : 1);+ });+ }+ );+ request.setTimeout(2000, () => { request.destroy(); process.exit(1); });+ request.on("error", () => process.exit(1));+ ')" || return 1+ safety_pass_metrics_are_certified "$metrics"+}++hcc_ready_after_stale_policy_revoked() {+ # A single poll that sees 200 WHILE the stale policy is still present is a+ # safety-contract violation and must be FATAL, never retried: retrying lets a+ # later poll (policy already revoked) turn the violation into a false PASS.+ # rc=2 signals that fatal ordering violation to wait_until_fast; rc=1 is a+ # benign "not ready yet, keep polling". The sub-0.5s window where revocation+ # races the probe inside one poll stays unobservable on THIS signal — the+ # third signal (M6, /metrics) covers it.+ local policy_absent_before=0+ external_egress_policy_absent && policy_absent_before=1+ probe_new_hcc_ready_endpoint || return 1+ if external_egress_policy_absent; then+ [ "$policy_absent_before" -eq 1 ] && return 0+ # Revocation raced the probe inside this poll: the final state is clean but+ # this poll cannot adjudicate the order. Retry to adjudicate cleanly.+ return 1+ fi+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=ready-200-with-stale-policy-present"+ return 2+}++# Canonical, order-stable projection of the single external-egress policy for the+# fixture server: name, ownership labels, and full spec. Used to prove a+# same-intent restart RETAINS the identity-stable DNS allow byte-for-byte (B3+# retain-identity-stable contract), the inverse of the divergent revoke path.+external_egress_policy_snapshot() {+ kctl get networkpolicy -n "$MCP_NS" \+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \+ -o json 2>/dev/null |+ jq -ceS \+ --arg managedByLabel "$MANAGED_BY_LABEL" \+ --arg managedByValue "$MANAGED_BY_VALUE" \+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" \+ --arg serverLabel "$MCP_SERVER_LABEL" \+ --arg server "$MCP_NAME" '+ [.items[] | {+ name: .metadata.name,+ managedBy: .metadata.labels[$managedByLabel],+ policyType: .metadata.labels[$policyTypeLabel],+ serverLabel: .metadata.labels[$serverLabel],+ spec+ }]+ | select(length == 1)+ | .[0]+ | select(.managedBy == $managedByValue+ and .policyType == "external-egress"+ and .serverLabel == $server)+ '+}++# True when the external-egress policy is present and identical (spec ++# ownership) to the snapshot captured before the same-intent restart. An API+# error or a missing/renamed policy fails closed.+identity_stable_policy_retained() {+ local current+ [ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] || return 1+ current="$(external_egress_policy_snapshot)" || return 1+ [ -n "$current" ] || return 1+ [ "$current" = "$SAME_INTENT_EGRESS_SNAPSHOT" ]+}++hcc_ready_with_identity_stable_policy_retained() {+ # INVERSE of the pre-B3 revoke-before-ready contract. B3 retains an+ # identity-stable DNS allow across a same-intent restart with NO deny window and+ # certifies /ready=200 WHILE that allow is still present and unchanged. A single+ # poll that reaches /ready=200 with the identity-stable allow ABSENT from the+ # start is the retain regression (a deny window) and must be FATAL, never+ # retried: retrying would let a later poll — after the held-refresh timeout+ # legitimately fail-closed deletes the allow — mask the regression as a PASS.+ # rc=2 signals that fatal violation to wait_until_fast; rc=1 is a benign+ # "not ready yet, keep polling".+ local retained_before=0+ identity_stable_policy_retained && retained_before=1+ probe_new_hcc_ready_endpoint || return 1+ if identity_stable_policy_retained; then+ [ "$retained_before" -eq 1 ] && return 0+ # The allow reappeared within this poll: adjudicate cleanly on the next poll.+ return 1+ fi+ if [ "$retained_before" -eq 1 ]; then+ # The allow was present immediately before this probe and is gone now: a+ # legitimate fail-closed refresh timeout can race the probe inside one poll.+ # Retry to adjudicate — a real retain regression never presents the allow at+ # readiness, so a later poll still returns rc=2.+ return 1+ fi+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=identity-stable-policy-revoked"+ return 2+}++hcc_restore_is_verified() {+ local deployment+ deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1+ jq -e \+ --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \+ --argjson replicas "${ORIGINAL_REPLICAS:-1}" '+ .spec.replicas == $replicas and+ (.status.readyReplicas // 0) == $replicas and+ (.status.availableReplicas // 0) == $replicas and+ (.spec.template.spec.dnsPolicy // "ClusterFirst") == $dnsPolicy and+ (.spec.template.spec.dnsConfig // null) == null+ ' <<<"$deployment" >/dev/null || return 1+ probe_hcc_ready_pod+}++release_dns_blocker() {+ local release_script+ release_script="$(cat <<'NODE'+const http = require('http');+const request = http.request(+ {host: '127.0.0.1', port: 8090, path: '/release', method: 'POST'},+ response => {+ response.resume();+ response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));+ }+);+request.setTimeout(5000, () => request.destroy(new Error('timeout')));+request.once('error', error => { console.error(error.message); process.exit(3); });+request.end();+NODE+)"+ kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \+ node -e "$release_script"+}++hold_dns_blocker() {+ local hold_script+ DNS_HOLD_COUNT_AT_ARM="$(dns_hold_count)" ||+ return 1+ hold_script="$(cat <<'NODE'+const http = require('http');+const request = http.request(+ {host: '127.0.0.1', port: 8090, path: '/hold', method: 'POST'},+ response => {+ response.resume();+ response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));+ }+);+request.setTimeout(5000, () => request.destroy(new Error('timeout')));+request.once('error', error => { console.error(error.message); process.exit(3); });+request.end();+NODE+)"+ kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \+ node -e "$hold_script"+}++delete_fixture_runtime_leaving_dns_policy_for_safety() {+ local failed=0+ kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete service "$MCP_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ [ "$failed" = 0 ] && fixture_runtime_absent+}++delete_dns_blocker_fixture() {+ local failed=0+ kctl delete networkpolicy "$HCC_DNS_POLICY" "$DNS_BLOCKER_POLICY" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ [ "$failed" = 0 ]+}++delete_residual_fixture_runtime() {+ local failed=0+ kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete service "$MCP_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy "$CONTEXT_EGRESS_POLICY" -n "$HOST_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy "$RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${MCP_NAME}" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ delete_peer_fleet_runtime || failed=1+ [ "$failed" = 0 ]+}++delete_peer_fleet_runtime() {+ local index server context context_policy failed=0+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ server="${MCP_FLEET_NAMES[index]}"+ context="${CONTEXT_FLEET_NAMES[index]}"+ context_policy="ctx-${context}-${server}"+ kctl delete deployment "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \+ >/dev/null 2>&1 || failed=1+ kctl delete service "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \+ >/dev/null 2>&1 || failed=1+ kctl delete configmap "${server}-nginx-conf" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \+ >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${server}" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy "${context_policy}-egress" -n "$HOST_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete networkpolicy "rpc-egress-${context}-${server}" -n "$RPC_PROXY_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ done+ [ "$failed" = 0 ]+}++delete_peer_fleet_inputs() {+ local index failed=0+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do+ kctl delete context "${CONTEXT_FLEET_NAMES[index]}" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete mcpserver "${MCP_FLEET_NAMES[index]}" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ done+ [ "$failed" = 0 ]+}++suite_resources_absent() {+ local namespaced cluster_crds+ namespaced="$(kctl get deployment,service,networkpolicy -A \+ -l "e2e.clerum.io/suite=${SUITE_NAME}" -o name 2>/dev/null)" || return 1+ cluster_crds="$(+ kctl get mcpserver,context -A -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ -o name 2>/dev/null+ )" || return 1+ [ -z "$namespaced$cluster_crds" ]+}++print_repair_instructions() {+ cat >&2 <<EOF+HCC MCP readiness gate cleanup could not restore a verified clean state.+Context: ${E2E_KUBECONTEXT}+HCC: ${HCC_NS}/${HCC_DEPLOY}+DNS blocker: ${HCC_NS}/${DNS_BLOCKER_NAME}+Fixtures: ${MCP_NS}/${MCP_NAME}, ${MCP_NS}/${CONTEXT_NAME}++Inspect before changing anything:+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml+ kubectl --context=${E2E_KUBECONTEXT} get deployment,service,networkpolicy -A -l e2e.clerum.io/suite=${SUITE_NAME}+ kubectl --context=${E2E_KUBECONTEXT} get mcpserver,context -A -l e2e.clerum.io/suite=${SUITE_NAME}++Do not remove the HCC gate lock until HCC health and fixture absence are verified.++Restore the original HCC replica count before verifying rollout and /ready:+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=240s+EOF+}++cleanup() {+ local status=$? cleanup_failed=0 restore_ok=1 fixture_inputs_removed=1 restore_patch+ local clean_window_pod+ trap - EXIT+ set +e++ if [ "$HCC_MUTATED" = 1 ]; then+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1 ||+ restore_ok=0+ if [ "$restore_ok" = 1 ]; then+ wait_until 120 "HCC pods to stop before fixture cleanup" \+ hcc_pods_absent >/dev/null 2>&1 || restore_ok=0+ fi+ fi++ if [ "$CONTEXT_CREATED" = 1 ]; then+ kctl delete context "$CONTEXT_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||+ {+ cleanup_failed=1+ fixture_inputs_removed=0+ }+ fi+ if [ "$MCP_CREATED" = 1 ]; then+ kctl delete mcpserver "$MCP_NAME" -n "$MCP_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||+ {+ cleanup_failed=1+ fixture_inputs_removed=0+ }+ fi+ if [ "$PEER_FLEET_CREATED" = 1 ]; then+ delete_peer_fleet_inputs || {+ cleanup_failed=1+ fixture_inputs_removed=0+ }+ fi+ if ! wait_until 60 "gate-owned Context and McpServer inputs to disappear" \+ fixture_inputs_absent >/dev/null 2>&1; then+ cleanup_failed=1+ fixture_inputs_removed=0+ fi++ if [ "$HCC_MUTATED" = 1 ]; then+ if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then+ restore_patch="$(jq -cn --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \+ '{spec:{template:{spec:{dnsPolicy:$dnsPolicy,dnsConfig:null}}}}')"+ kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \+ -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ --timeout=240s >/dev/null 2>&1 || restore_ok=0+ if [ "$restore_ok" = 1 ]; then+ wait_until 60 "restored HCC DNS, replicas, Ready pod, and /ready endpoint" \+ hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0+ fi+ else+ restore_ok=0+ fi+ fi++ if [ "$restore_ok" = 1 ]; then+ if ! wait_until 180 "HCC to remove all MCP/Context fixture runtime resources" \+ fixture_resources_absent >/dev/null 2>&1; then+ cleanup_failed=1+ delete_residual_fixture_runtime || cleanup_failed=1+ fi+ if [ "$DNS_BLOCKER_CREATED" = 1 ]; then+ delete_dns_blocker_fixture || cleanup_failed=1+ fi+ wait_until 60 "all HCC MCP readiness suite resources to disappear" \+ suite_resources_absent >/dev/null 2>&1 || cleanup_failed=1++ # Clean window B: DNS restored and fixture inputs gone. The restored pod is+ # new (scale-0 preceded the restore), so its log is scoped to this window.+ clean_window_pod="$(running_hcc_pod)"+ if [ -n "$clean_window_pod" ] &&+ clean_window_netpol_log_negatives_hold "$clean_window_pod"; then+ clean_window_checks_run=$((clean_window_checks_run + 1))+ else+ fail "post-restore clean-window NetworkPolicy log negatives did not hold"+ cleanup_failed=1+ fi+ if fixture_mcp_runtime_absent; then+ clean_window_checks_run=$((clean_window_checks_run + 1))+ else+ fail "post-restore fixture runtime or external-egress policy survived"+ cleanup_failed=1+ fi+ if clean_window_runtime_and_ready_absent; then+ clean_window_checks_run=$((clean_window_checks_run + 1))+ else+ fail "post-restore McpServer Ready status or external-egress policy survived"+ cleanup_failed=1+ fi+ else+ cleanup_failed=1+ fi++ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then+ cleanup_failed=1+ fi++ if [ "$status" = 0 ]; then+ [ "${clean_window_checks_run:-0}" -gt 0 ] ||+ die "no clean-window negative checks ran (clean_window_checks_run=0)"+ fi++ if [ "$cleanup_failed" -ne 0 ]; then+ echo "HCC MCP readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2+ [ "$restore_ok" = 1 ] || print_repair_instructions+ [ "$status" = 0 ] && status=1+ fi+ if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then+ header "HCC McpServer/Context readiness gate passed"+ echo "$final_probe"+ fi+ exit "$status"+}+trap cleanup EXIT++header "HCC readiness during initial McpServer and Context convergence"++require_branch_owned_hcc_gate "$HCC_NS"+kctl get nodes -o json |+ jq -e --arg context "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||+ die "target context is not a minikube cluster"+suite_resources_absent ||+ die "stale ${SUITE_NAME} resources exist; inspect them before starting a new fault injection"+acquire_hcc_watch_gate_lock ||+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"+ok "branch helper, profile, exact HEAD/fingerprint/gate, and exclusive HCC lock verified"++# Clean window A: the DNS fault injection is not applied yet. The branch-deployed+# HCC must show a clean NetworkPolicy lane and zero fixture residue before the+# gate mutates anything.+clean_window_pod="$(running_hcc_pod)" && [ -n "$clean_window_pod" ] ||+ die "no Running HCC pod before fault injection"+clean_window_netpol_log_negatives_hold "$clean_window_pod" ||+ die "pre-injection HCC logs already contain NetworkPolicy-lane failure or retry markers"+clean_window_checks_run=$((clean_window_checks_run + 1))+fixture_mcp_runtime_absent ||+ die "pre-injection fixture runtime or external-egress policy already exists"+clean_window_checks_run=$((clean_window_checks_run + 1))+clean_window_runtime_and_ready_absent ||+ die "pre-injection McpServer Ready status or external-egress policy already exists"+clean_window_checks_run=$((clean_window_checks_run + 1))+ok "all three clean-window negative families hold before DNS fault injection"++ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.replicas}')"+[ "$ORIGINAL_REPLICAS" = 1 ] ||+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"+ORIGINAL_DNS_POLICY="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.dnsPolicy}')"+ORIGINAL_DNS_POLICY="${ORIGINAL_DNS_POLICY:-ClusterFirst}"+[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] ||+ die "HCC already has non-default dnsPolicy ${ORIGINAL_DNS_POLICY}; refusing a non-restorable fixture"+ORIGINAL_DNS_CONFIG="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |+ jq -c '.spec.template.spec.dnsConfig // null')"+[ "$ORIGINAL_DNS_CONFIG" = null ] ||+ die "HCC already has dnsConfig ${ORIGINAL_DNS_CONFIG}; refusing a non-restorable fixture"+HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"+[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"+HCC_PORT="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json | jq -r '+ .spec.template.spec.containers[]? |+ select(.name == "host-context-controller") as $container |+ (+ [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //+ [$container.ports[]? | select(.name == "http") | .containerPort][0] //+ empty+ )+')"+[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||+ die "could not resolve a valid HCC HTTP port from the Deployment"+UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"+[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-system/kube-dns ClusterIP"+running_dns_pods="$(kctl get pods -n kube-system -l k8s-app=kube-dns \+ --field-selector=status.phase=Running -o name)"+[ -n "$running_dns_pods" ] ||+ die "no Running kube-dns pod is available"++DNS_BLOCKER_CREATED=1+kctl apply -f - >/dev/null <<EOF+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ${DNS_BLOCKER_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ replicas: 1+ selector:+ matchLabels:+ app: ${DNS_BLOCKER_NAME}+ template:+ metadata:+ labels:+ app: ${DNS_BLOCKER_NAME}+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+ spec:+ automountServiceAccountToken: false+ terminationGracePeriodSeconds: 1+ containers:+ - name: dns-blocker+ image: ${HCC_IMAGE}+ imagePullPolicy: IfNotPresent+ command: [node, -e]+ args:+ - >-+ const dgram=require('node:dgram'),http=require('node:http'),+ net=require('node:net');+ const target=process.env.TARGET_DNS,upstream=process.env.UPSTREAM_DNS;+ const udp=dgram.createSocket('udp4');let released=process.env.START_RELEASED==='1';const held=[];+ function question(message){let offset=12,labels=[];while(offset<message.length){+ const length=message[offset++];if(length===0)break;if(offset+length>message.length)+ return null;labels.push(message.subarray(offset,offset+length).toString('ascii'));+ offset+=length;}if(offset+4>message.length)return null;return+ {name:labels.join('.'),type:message.readUInt16BE(offset),end:offset+4};}+ function answer(message,parsed){const header=Buffer.from(message.subarray(0,12));+ header.writeUInt16BE(0x8180,2);header.writeUInt16BE(1,6);+ header.writeUInt16BE(0,8);header.writeUInt16BE(0,10);+ const record=Buffer.from([0xc0,0x0c,0x00,0x01,0x00,0x01,0x00,0x00,+ 0x00,0x1e,0x00,0x04,93,184,216,34]);+ return Buffer.concat([header,message.subarray(12,parsed.end),record]);}+ function tcpFrame(message){const frame=Buffer.alloc(message.length+2);+ frame.writeUInt16BE(message.length,0);message.copy(frame,2);return frame;}+ function respond(entry){const response=answer(entry.message,entry.parsed);+ if(entry.transport==='udp'){udp.send(response,entry.rinfo.port,+ entry.rinfo.address);console.log('released DNS A '+target+' via UDP');return;}+ if(!entry.socket.destroyed){entry.socket.write(tcpFrame(response));+ console.log('released DNS A '+target+' via TCP');}}+ function forwardUdp(message,rinfo){const relay=dgram.createSocket('udp4');+ const timer=setTimeout(()=>relay.close(),5000);relay.once('message',response=>{+ clearTimeout(timer);udp.send(response,rinfo.port,rinfo.address);relay.close();});+ relay.once('error',()=>{clearTimeout(timer);relay.close();});+ relay.send(message,53,upstream);}+ function forwardTcp(message,socket){const relay=net.createConnection(+ {host:upstream,port:53},()=>relay.write(tcpFrame(message)));+ let pending=Buffer.alloc(0);relay.setTimeout(5000,()=>relay.destroy());+ relay.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);+ if(pending.length<2)return;const length=pending.readUInt16BE(0);+ if(pending.length<length+2)return;if(!socket.destroyed)+ socket.write(pending.subarray(0,length+2));relay.destroy();});+ relay.once('error',()=>relay.destroy());}+ function handle(message,entry){const parsed=question(message);+ if(parsed&&parsed.name===target&&parsed.type===1){entry.message=message;+ entry.parsed=parsed;if(released)respond(entry);else{held.push(entry);+ console.log('holding DNS A '+target+' via '+entry.transport.toUpperCase());}+ return;}if(entry.transport==='udp')forwardUdp(message,entry.rinfo);+ else forwardTcp(message,entry.socket);}+ let listeners=0;function listening(){listeners+=1;if(listeners===2)+ console.log('dns hold proxy ready for '+target+' over UDP and TCP');}+ udp.on('message',(message,rinfo)=>handle(message,{transport:'udp',rinfo}));+ udp.bind(8053,'0.0.0.0',listening);+ const tcp=net.createServer(socket=>{let pending=Buffer.alloc(0);+ socket.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);+ while(pending.length>=2){const length=pending.readUInt16BE(0);+ if(pending.length<length+2)return;const message=pending.subarray(2,length+2);+ pending=pending.subarray(length+2);handle(message,{transport:'tcp',socket});}});+ });tcp.listen(8053,'0.0.0.0',listening);+ http.createServer((request,response)=>{if(request.method==='POST'&&+ request.url==='/release'){released=true;while(held.length)respond(held.shift());+ response.writeHead(200);response.end('released');return;}if(request.method==='POST'&&+ request.url==='/hold'){released=false;response.writeHead(200);response.end('holding');return;}+ response.writeHead(404);response.end();}).listen(8090,'127.0.0.1');+ env:+ - name: TARGET_DNS+ value: ${TARGET_DNS}+ - name: START_RELEASED+ value: "1"+ - name: UPSTREAM_DNS+ value: ${UPSTREAM_DNS_IP}+ resources:+ requests: {cpu: 5m, memory: 20Mi}+ limits: {cpu: 75m, memory: 64Mi}+ securityContext:+ allowPrivilegeEscalation: false+ capabilities: {drop: [ALL]}+ readOnlyRootFilesystem: true+ runAsNonRoot: true+ runAsUser: 1000+ runAsGroup: 1000+ seccompProfile: {type: RuntimeDefault}+---+apiVersion: v1+kind: Service+metadata:+ name: ${DNS_BLOCKER_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ selector:+ app: ${DNS_BLOCKER_NAME}+ ports:+ - name: dns-udp+ protocol: UDP+ port: 53+ targetPort: 8053+ - name: dns-tcp+ protocol: TCP+ port: 53+ targetPort: 8053+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${DNS_BLOCKER_POLICY}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ podSelector:+ matchLabels:+ app: ${DNS_BLOCKER_NAME}+ policyTypes: [Ingress, Egress]+ ingress:+ - from:+ - podSelector:+ matchLabels:+ app: ${HCC_DEPLOY}+ ports:+ - {protocol: UDP, port: 8053}+ - {protocol: UDP, port: 53}+ - {protocol: TCP, port: 8053}+ - {protocol: TCP, port: 53}+ egress:+ - to:+ - namespaceSelector:+ matchLabels:+ kubernetes.io/metadata.name: kube-system+ podSelector:+ matchLabels:+ k8s-app: kube-dns+ ports:+ - {protocol: UDP, port: 53}+ - {protocol: TCP, port: 53}+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${HCC_DNS_POLICY}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ podSelector:+ matchLabels:+ app: ${HCC_DEPLOY}+ policyTypes: [Egress]+ egress:+ - to:+ - podSelector:+ matchLabels:+ app: ${DNS_BLOCKER_NAME}+ ports:+ - {protocol: UDP, port: 53}+ - {protocol: UDP, port: 8053}+ - {protocol: TCP, port: 53}+ - {protocol: TCP, port: 8053}+EOF+kctl rollout status deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \+ --timeout=90s >/dev/null || die "DNS blocker did not become ready"+wait_until 30 "DNS blocker process to listen" \+ dns_log_contains "dns hold proxy ready for ${TARGET_DNS}" ||+ die "DNS blocker process did not start"+DNS_BLOCKER_IP="$(kctl get service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \+ -o jsonpath='{.spec.clusterIP}')"+[ -n "$DNS_BLOCKER_IP" ] || die "DNS blocker Service has no ClusterIP"+ok "isolated DNS hold proxy is ready and forwards non-fixture UDP/TCP queries to CoreDNS"++[ "${clean_window_checks_run:-0}" -ge 3 ] ||+ die "pre-injection clean-window negatives did not run before the DNS mutation"++HCC_MUTATED=1+dns_patch="$(jq -cn --arg nameserver "$DNS_BLOCKER_IP" '+ {spec:{template:{spec:{+ dnsPolicy:"None",+ dnsConfig:{+ nameservers:[$nameserver],+ options:[+ {name:"ndots",value:"1"},+ {name:"timeout",value:"30"},+ {name:"attempts",value:"1"}+ ]+ }+ }}}}+')"+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \+ -p "$dns_patch" >/dev/null+kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||+ die "HCC did not become Ready with the deterministic DNS proxy"+wait_until 60 "HCC /ready endpoint with the released DNS proxy" probe_hcc_ready_pod ||+ die "HCC readiness was unavailable before the stale-policy fixture"++CONTEXT_CREATED=1+kctl apply -f - >/dev/null <<EOF+apiVersion: clerum.io/v1alpha1+kind: Context+metadata:+ name: ${CONTEXT_NAME}+ namespace: ${MCP_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ contextId: ${CONTEXT_ID}+ description: E2E HCC readiness Context updated while initial MCP convergence is blocked.+ mcpServers: []+EOF++MCP_CREATED=1+kctl apply -f - >/dev/null <<EOF+apiVersion: clerum.io/v1alpha1+kind: McpServer+metadata:+ name: ${MCP_NAME}+ namespace: ${MCP_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_LABEL}"+spec:+ contextRef: ${CONTEXT_NAME}+ description: E2E MCP fixture for initial fleet readiness.+ image: clerum/mock-mcp-server:test+ imagePullPolicy: IfNotPresent+ transport:+ type: streamableHttp+ url: http://${MCP_NAME}.${MCP_NS}.svc.cluster.local:3000/mcp+ port: 3000+ egressBindings:+ - dns: ${TARGET_DNS}+ port: 443+ protocol: ${INITIAL_EXTERNAL_EGRESS_PROTOCOL}+EOF++wait_until 180 "HCC to create the real initial TCP external-egress policy" \+ external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||+ die "HCC never created the initial external-egress policy for the stale-policy fixture"+ok "HCC created a real initial TCP external-egress policy"+BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)" ||+ die "could not capture canonical baseline NetworkPolicy ownership and specs"++# Same-intent safety phase: restart with the exact same desired binding. The+# safety pass must RETAIN the identity-stable DNS-derived policy without resolving+# DNS (B3 retain-identity-stable contract), certify global readiness with that+# allow still present — no deny window — and leave the affected runtime blocked+# for the additive lane. Retention-at-readiness is adjudicated promptly: a+# held-refresh timeout later fail-closed deletes the retained allow, so the proof+# must land before any refresh timeout.+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null+wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||+ die "existing HCC pod did not stop"+delete_fixture_runtime_leaving_dns_policy_for_safety ||+ die "could not remove the initial MCP runtime while retaining its external-egress policy"+external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||+ die "the initial TCP policy disappeared before the same-intent safety pass"+SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)" ||+ die "could not capture the identity-stable external-egress policy snapshot"+[ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] ||+ die "the identity-stable external-egress policy snapshot was empty before the same-intent restart"+create_peer_fleet || die "could not create the gate-owned MCP/Context peer fleet"+ok "created ${#MCP_FLEET_NAMES[@]} gate-owned MCP/Context fixtures before fleet bootstrap"++hold_dns_blocker || die "could not hold the next fixture DNS query"+DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||+ die "could not capture the DNS release baseline before the same-intent restart"+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null+wait_until 120 "same-intent HCC pod to enter Running" capture_hcc ||+ die "same-intent HCC pod did not start"+identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"+read -r HCC_UID HCC_RESTARTS <<<"$identity"+[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||+ die "could not capture same-intent HCC pod identity"+# Adjudicate retention-at-readiness FIRST — at the earliest possible moment —+# because the certifying safety pass retains WITHOUT resolving DNS, so /ready=200+# and the retained allow flip together. The separate additive refresh reaches the+# DNS hold and, on its timeout, fail-closed deletes the retained allow; latching+# the retained+Ready observation here beats that timeout.+wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained" \+ hcc_ready_with_identity_stable_policy_retained ||+ die "HCC did not certify readiness with the same-intent identity-stable DNS policy retained (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"+# Retention-at-readiness is latched. Corroborate promptly — present, unchanged,+# under a still-held DNS, with no restart and the baseline intact — before any+# held-refresh timeout can fail-closed delete the retained allow.+dns_has_not_released_since_hold ||+ die "the DNS blocker released before the same-intent retention assertion"+identity_stable_policy_retained ||+ die "the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)"+fixture_runtime_and_ready_absent ||+ die "the affected runtime or Ready status escaped the held same-intent DNS boundary"+hcc_identity_is_stable ||+ die "HCC restarted while the same-intent identity-stable DNS policy was retained"+baseline_policies_unchanged ||+ die "baseline safety NetworkPolicy ownership or specs changed during same-intent retention"+wait_until 30 "Kubernetes to corroborate same-intent HCC readiness" \+ hcc_kubernetes_readiness_is_exact ||+ die "Kubernetes did not corroborate same-intent HCC readiness"+wait_until 120 "the same-intent external-egress refresh to reach the DNS hold" \+ dns_hold_observed_since_arm ||+ die "the same-intent external-egress refresh never reached the DNS hold"+ok "same-intent safety retained the identity-stable DNS policy while HCC stayed Ready and runtime stayed blocked"++release_dns_blocker || die "could not release the same-intent DNS query"+wait_until 30 "DNS blocker to answer the same-intent query" \+ dns_released_since_hold ||+ die "DNS blocker did not answer the same-intent query"+wait_until 120 "current TCP external-egress policy after same-intent DNS release" \+ external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||+ die "the current TCP external-egress policy did not converge after DNS release"+wait_until 180 "same-intent runtime convergence after DNS release" \+ fixture_runtime_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||+ die "same-intent runtime did not converge after DNS release"++# Security phase: a protocol change invalidates the current TCP policy. The old+# TCP allow must be revoked even when DNS prevents constructing the UDP policy.+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null+wait_until 120 "same-intent HCC pod to stop" hcc_pods_absent ||+ die "same-intent HCC pod did not stop"+delete_fixture_runtime_leaving_dns_policy_for_safety ||+ die "could not remove the same-intent runtime before the divergent revision"+external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||+ die "the current TCP policy disappeared before the divergent safety pass"+hold_dns_blocker || die "could not hold the divergent fixture DNS query"+DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||+ die "could not capture the DNS release baseline before the divergent revision"+egress_patch="$(jq -cn --arg dns "$TARGET_DNS" \+ '{spec:{egressBindings:[{dns:$dns,port:443,protocol:"UDP"}]}}')"+kctl patch mcpserver "$MCP_NAME" -n "$MCP_NS" --type=merge \+ -p "$egress_patch" >/dev/null+wait_until 60 "McpServer API to expose the revised UDP external-egress binding" \+ mcp_egress_binding_is_udp ||+ die "McpServer external-egress revision did not persist"++kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null+wait_until 120 "replacement HCC pod to enter Running" capture_hcc ||+ die "replacement HCC pod did not start"+identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"+read -r HCC_UID HCC_RESTARTS <<<"$identity"+[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||+ die "could not capture replacement HCC pod identity"++wait_until 120 "fixture external-egress DNS query to reach the hold proxy" \+ dns_hold_observed_since_arm ||+ die "NetworkPolicy safety convergence never reached the divergent DNS hold"+wait_until_fast 120 "HCC readiness after revoking the divergent stale policy" \+ hcc_ready_after_stale_policy_revoked ||+ die "HCC did not become globally ready after revoking the divergent stale policy (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"+wait_until 30 "Kubernetes to corroborate HCC readiness during divergent DNS hold" \+ hcc_kubernetes_readiness_is_exact ||+ die "Kubernetes did not corroborate HCC readiness during divergent DNS hold"+dns_log_contains "holding DNS A ${TARGET_DNS}" ||+ die "DNS hold ended before the fail-closed revocation assertion"+if ! dns_has_not_released_since_hold; then+ die "DNS blocker released before the fail-closed revocation assertion"+fi+fixture_runtime_and_ready_absent ||+ die "fixture runtime or Ready status appeared while divergent DNS remained held"+external_egress_policy_absent ||+ die "a divergent external-egress policy survived the fail-closed revocation"+context_policies_absent ||+ die "Context policy existed before the Context watch update"+baseline_policies_unchanged ||+ die "baseline safety NetworkPolicy ownership or specs changed during safety convergence"+hcc_identity_is_stable ||+ die "HCC restarted while the MCP external-egress query was held"+wait_until_fast 30 "authoritative safety pass metrics to certify the revocation" \+ safety_pass_metrics_certify ||+ die "safety_pass_* metrics did not corroborate the stale-policy revocation (families renamed or pass never measured?)"+ok "/ready is 200 while the divergent stale policy and affected runtime remain absent"+ok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"+ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"++initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"+wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \+ hcc_log_contains "$initial_empty_context_log" ||+ die "initial NetworkPolicy pass never reconciled the fixture's empty Context"+wait_until 120 "a real divergent external-egress retry to be scheduled" \+ external_egress_retry_progress_is_observed ||+ die "the held divergent refresh did not schedule a real retry"+DNS_HOLD_COUNT_AT_RETRY_SCHEDULE="$(dns_hold_count)" ||+ die "could not snapshot DNS hold evidence after the divergent retry was scheduled"+wait_until 180 "the scheduled divergent external-egress retry to execute" \+ dns_retry_query_observed_since_schedule ||+ die "the scheduled divergent external-egress retry did not execute"+context_policies_absent ||+ die "Context policy appeared before the isolated MODIFIED event"++context_patch="$(jq -cn --arg server "$MCP_NAME" '{spec:{mcpServers:[$server]}}')"+kctl patch context "$CONTEXT_NAME" -n "$MCP_NS" --type=merge \+ -p "$context_patch" >/dev/null+wait_until 60 "Context API to expose the latest server membership" context_watch_has_latest_spec ||+ die "Context update did not persist"+wait_until 60 "HCC Context watch to observe the update during the blocked fleet pass" \+ hcc_log_contains "Context watch event: MODIFIED for ${CONTEXT_NAME}" ||+ die "HCC did not observe the real Context update while initial MCP convergence was blocked"+wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold" \+ context_policies_converged ||+ die "Context policies remained blocked behind the held McpServer convergence"+fixture_runtime_and_ready_absent ||+ die "McpServer runtime or Ready status escaped the deterministic DNS hold boundary"+external_egress_policy_absent ||+ die "a divergent external-egress policy reappeared while DNS remained held"+dns_log_contains "holding DNS A ${TARGET_DNS}" ||+ die "DNS hold evidence disappeared before Context policy convergence"+if ! dns_has_not_released_since_hold; then+ die "DNS blocker released the McpServer query before the Context isolation assertion"+fi+probe_new_hcc_ready_endpoint ||+ die "HCC lost global readiness while the affected runtime remained safely blocked (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"+hcc_kubernetes_readiness_is_exact ||+ die "Kubernetes lost HCC readiness during isolated Context convergence"+hcc_identity_is_stable ||+ die "HCC restarted while Context policies converged around the held MCP entity"+ok "the MODIFIED handler, not the initial empty-Context pass, converged all three policies"+ok "global readiness and Context progress continued while the affected runtime stayed blocked"++release_dns_blocker ||+ die "could not release the fixture DNS query"+wait_until 30 "DNS blocker to answer the held query" \+ dns_released_since_hold ||+ die "DNS blocker did not answer the held query"+wait_until 120 "current UDP external-egress policy after DNS release" \+ external_egress_policy_converged_with_protocol UDP ||+ die "the current UDP external-egress policy did not converge after DNS release"+probe_new_hcc_ready_endpoint ||+ die "HCC lost readiness while the released UDP policy converged (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"+wait_until 300 "McpServer runtime, status, and latest Context NetworkPolicies to converge" \+ fixture_converged ||+ die "McpServer/Context/NetworkPolicy convergence did not complete after DNS release"+wait_until 300 "all peer MCP/Context runtimes to converge after releasing the held primary egress" \+ peer_fleet_converged ||+ die "peer MCP/Context fleet did not converge after the held primary egress was released"+hcc_identity_is_stable ||+ die "HCC restarted instead of completing the released background convergence"+final_probe="$(probe_hcc_final_context)" ||+ die "HCC final Context-scoped discovery did not expose the converged fixture"+echo "$final_probe" | jq -e \+ --arg fixture "$MCP_NAME" --arg context "$CONTEXT_NAME" '+ .readyStatus == 200 and .scopedDiscoveryStatus == 200 and+ .fixture == $fixture and .fixtureReady == true and .contextRef == $context+ ' >/dev/null ||+ die "HCC final probe returned an invalid result: ${final_probe}"+baseline_policies_unchanged ||+ die "baseline safety NetworkPolicy ownership or specs changed before final convergence"+ok "DNS release created the current UDP policy and affected runtime without gating HCC readiness"+ok "the full worker-width-crossing peer fleet converged after releasing the held primary egress"+ok "Context ingress, mcp-host egress, and rpc-proxy egress policies reflect the latest watch state"+ok "HCC stayed Ready through divergent revocation while the affected runtime remained fail-closed"++header "HCC McpServer/Context assertions complete; restoring branch-owned runtime"diff --git a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh b/scripts/e2e/e2e-hcc-readiness-bootstrap.shnew file mode 100755index 000000000..eb485a590--- /dev/null+++ b/scripts/e2e/e2e-hcc-readiness-bootstrap.sh@@ -0,0 +1,785 @@+#!/usr/bin/env bash+# Proves that HCC exposes its real readiness and discovery endpoints while the+# initial Host fleet reconciliation is still demonstrably in progress.+#+# The gate temporarily routes HCC's control-api token issuance to an isolated+# in-cluster HTTP blocker, creates a gate-owned fleet while HCC is stopped, and+# restarts HCC. There are more fixture Hosts than two complete bounded worker+# waves, so the initial pass remains deterministically active even though each+# individual held token request has its own client timeout. No real Host or+# runtime-token Secret is modified.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+# shellcheck source=scripts/e2e/e2e-lib.sh+source "${SCRIPT_DIR}/e2e-lib.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"++[ -n "$E2E_KUBECONTEXT" ] || {+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2+ exit 1+}+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {+ echo "Refusing HCC readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2+ exit 1+}+require_safe_kube_context+[ "${E2E_HCC_READINESS_FAULT_INJECTION:-0}" = 1 ] || {+ echo "Set E2E_HCC_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC fault injection." >&2+ exit 1+}+command -v jq >/dev/null 2>&1 || {+ echo "jq is required" >&2+ exit 1+}++HCC_NS="${HCC_NS:-control-plane}"+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"+HCC_GATEWAY_DEPLOY="${HCC_GATEWAY_DEPLOY:-host-context-controller-api-gateway}"+HOST_NS="${MCP_HOST_NS:-mcp-host}"+MCP_NS="${MCP_SERVER_NS:-mcp-server}"+TOKEN_REQUEST_PATH="/api/v1/auth/mcp-host/${HOST_NS}/standalone/tokens"+TOKEN_REQUEST_LOG_PREFIX="holding-token POST ${TOKEN_REQUEST_PATH} host="+RUN_ID="$(date +%s)-$$"+BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-readiness-blocker-${RUN_ID}")"+BLOCKER_EGRESS_POLICY_NAME="$(truncate_rfc1123 "${BLOCKER_NAME}-egress")"+SUITE_NAME="hcc-readiness-bootstrap"+FIXTURE_HOST_PREFIX="$(truncate_rfc1123 "e2e-hcc-ready-${RUN_ID}")"+FIXTURE_CONTEXT="$(truncate_rfc1123 "e2e-hcc-ready-context-${RUN_ID}")"+FIXTURE_SECRET="$(truncate_rfc1123 "e2e-hcc-ready-llm-${RUN_ID}")"+START_MARKER='Starting initial Host background convergence'+COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'+FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'+ORIGINAL_CONTROL_API_BASE_URL=""+ORIGINAL_REPLICAS=""+HCC_PORT=""+HOST_RECONCILE_CONCURRENCY=""+FIXTURE_HOST_COUNT=0+FIXTURE_HOST_NAMES=()+HCC_MUTATED=0+BLOCKER_CREATED=0+FIXTURES_CREATED=0+# Read and mutated by the sourced compare-and-swap lock helper.+# shellcheck disable=SC2034+HCC_GATE_LOCK_ACQUIRED=0+# shellcheck disable=SC2034+HCC_GATE_LOCK_NAME=""+# shellcheck disable=SC2034+HCC_GATE_LOCK_UID=""+# shellcheck disable=SC2034+HCC_GATE_FINALIZATION_FAILURE=""++die() {+ if [ -n "${new_hcc_pod:-}" ]; then+ echo "Recent HCC logs from ${new_hcc_pod}:" >&2+ kctl logs "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller --tail=300 >&2 || true+ fi+ fail "$*"+ exit 1+}++wait_until() {+ local timeout=$1 description=$2+ shift 2+ local deadline now+ deadline=$(( $(date +%s) + timeout ))+ while :; do+ "$@" && return 0+ now="$(date +%s)"+ [ "$now" -lt "$deadline" ] || break+ sleep 1+ done+ echo "Timed out after ${timeout}s waiting for ${description}" >&2+ return 1+}++running_hcc_pod() {+ local rows+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ --field-selector=status.phase=Running \+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \+ 2>/dev/null)" || return 1+ awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"+}++hcc_pods_absent() {+ local pods+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||+ return 1+ [ -z "$pods" ]+}++hcc_logs() {+ local pod=$1+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller 2>/dev/null+}++hcc_log_contains() {+ local pod=$1 marker=$2 logs+ logs="$(hcc_logs "$pod")" || return 1+ hcc_log_snapshot_contains "$logs" "$marker"+}++blocker_holds_token_request() {+ local logs+ logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1+ hcc_log_snapshot_contains "$logs" \+ "${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-"+}++blocker_fixture_request_count() {+ local logs+ logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1+ awk -v prefix="${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-" '+ index($0, prefix) {+ host = substr($0, index($0, "host=") + length("host="))+ sub(/[[:space:]].*$/, "", host)+ if (!(host in seen)) {+ seen[host] = 1+ count += 1+ }+ }+ END { print count + 0 }+ ' <<<"$logs"+}++blocker_observed_more_than_one_worker_wave() {+ local count+ count="$(blocker_fixture_request_count)" || return 1+ [ "$count" -gt "$HOST_RECONCILE_CONCURRENCY" ]+}++initial_host_pass_is_active() {+ local pod=$1 logs+ logs="$(hcc_logs "$pod")" || return 1+ hcc_initial_pass_snapshot_is_active \+ "$logs" "$START_MARKER" "$COMPLETE_MARKER" "$FAIL_MARKER"+}++control_api_service_excludes_blocker() {+ kctl get endpointslice -n "$HCC_NS" -l 'kubernetes.io/service-name=control-api' -o json |+ jq -e --arg blocker "$BLOCKER_NAME" \+ 'all(.items[]?.endpoints[]?; .targetRef.name != $blocker)' >/dev/null+}++readiness_blocker_suite_absent() {+ local resources+ resources="$(+ kctl get deployment,service,networkpolicy -n "$HCC_NS" \+ -l 'e2e.clerum.io/hcc-readiness' -o name 2>/dev/null+ )" || return 1+ [ -z "$resources" ]+}++probe_hcc_ready_pod() {+ local pod+ pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1+ kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" \+ node -e '+ const http = require("node:http");+ const request = http.get(+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},+ response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ let parsed;+ try { parsed = JSON.parse(body); } catch { process.exit(2); }+ process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);+ });+ }+ );+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));+ request.once("error", () => process.exit(4));+ ' >/dev/null 2>&1+}++running_hcc_gateway_pod() {+ ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"+}++hcc_gateway_deployment_ready() {+ kctl get deployment "$HCC_GATEWAY_DEPLOY" -n "$HCC_NS" -o json |+ jq -e '+ .spec.replicas == 1 and+ .status.observedGeneration == .metadata.generation and+ (.status.updatedReplicas // 0) == 1 and+ (.status.readyReplicas // 0) == 1 and+ (.status.availableReplicas // 0) == 1 and+ (.status.unavailableReplicas // 0) == 0+ ' >/dev/null+}++hcc_gateway_local_health_ok() {+ local pod+ pod="$(running_hcc_gateway_pod)" || return 1+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \+ wget -qO- http://127.0.0.1:8081/health |+ jq -e '.status == "ok"' >/dev/null+}++hcc_gateway_ready_proxy_unavailable() {+ local pod output status+ pod="$(running_hcc_gateway_pod)" || return 1+ output="$(+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \+ wget -T 70 -t 1 -O /dev/null -S http://127.0.0.1:8081/ready 2>&1+ )" && return 1+ status="$(+ awk '+ /^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]([[:space:]]|$)/ {+ code = $2+ }+ END {+ if (code != "") print code+ }+ ' <<<"$output"+ )"+ case "$status" in+ 502|503|504) return 0 ;;+ *) return 1 ;;+ esac+}++hcc_gateway_remains_ready_without_hcc() {+ local duration=$1 deadline+ deadline=$(( $(date +%s) + duration ))+ while [ "$(date +%s)" -lt "$deadline" ]; do+ hcc_gateway_deployment_ready &&+ hcc_gateway_local_health_ok &&+ hcc_gateway_ready_proxy_unavailable &&+ hcc_gateway_deployment_ready &&+ hcc_gateway_local_health_ok ||+ return 1+ sleep 1+ done+}++hcc_gateway_ready_proxy_recovers() {+ local pod+ pod="$(running_hcc_gateway_pod)" || return 1+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \+ wget -qO- http://127.0.0.1:8081/ready |+ jq -e '.ready == true and .status == "ready"' >/dev/null+}++fixture_inputs_absent() {+ local hosts contexts secrets+ hosts="$(kctl get host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ -o name 2>/dev/null)" || return 1+ contexts="$(kctl get context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ -o name 2>/dev/null)" || return 1+ secrets="$(kctl get secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ -o name 2>/dev/null)" || return 1+ [ -z "${hosts}${contexts}${secrets}" ]+}++fixture_runtime_absent() {+ local host resources+ [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0+ for host in "${FIXTURE_HOST_NAMES[@]}"; do+ resources="$(+ kctl get deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \+ -o name 2>/dev/null+ )" || return 1+ [ -z "$resources" ] || return 1+ done+}++fixture_resources_absent() {+ fixture_inputs_absent && fixture_runtime_absent+}++hcc_restore_is_verified() {+ local deployment+ deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1+ jq -e \+ --arg controlApiBaseUrl "$ORIGINAL_CONTROL_API_BASE_URL" \+ --argjson replicas "${ORIGINAL_REPLICAS:-1}" '+ .spec.replicas == $replicas and+ .status.observedGeneration == .metadata.generation and+ (.status.updatedReplicas // 0) == $replicas and+ (.status.readyReplicas // 0) == $replicas and+ (.status.availableReplicas // 0) == $replicas and+ (.status.unavailableReplicas // 0) == 0 and+ any(.spec.template.spec.containers[]?;+ .name == "host-context-controller" and+ any(.env[]?;+ .name == "CONTROL_API_BASE_URL" and .value == $controlApiBaseUrl+ )+ )+ ' <<<"$deployment" >/dev/null || return 1+ probe_hcc_ready_pod && fixture_resources_absent+}++delete_blocker_fixture() {+ local failed=0+ kctl delete networkpolicy "$BLOCKER_EGRESS_POLICY_NAME" "$BLOCKER_NAME" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete service "$BLOCKER_NAME" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete deployment "$BLOCKER_NAME" -n "$HCC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ [ "$failed" = 0 ]+}++create_host_fixtures() {+ local index host+ FIXTURES_CREATED=1++ kctl apply -f - >/dev/null <<EOF || return 1+apiVersion: v1+kind: Secret+metadata:+ name: ${FIXTURE_SECRET}+ namespace: ${HOST_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_ID}"+type: Opaque+stringData:+ # Test-only placeholder: Host secret validation checks existence, while token+ # issuance is intentionally held before any mcp-host Deployment is created.+ OPENAI_API_KEY: "e2e-readiness-not-a-real-key"+---+apiVersion: clerum.io/v1alpha1+kind: Context+metadata:+ name: ${FIXTURE_CONTEXT}+ namespace: ${MCP_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_ID}"+spec:+ contextId: ${FIXTURE_CONTEXT}+ description: Gate-owned Context for HCC Host bootstrap readiness.+ mcpServers: []+EOF++ for ((index = 1; index <= FIXTURE_HOST_COUNT; index++)); do+ host="$(printf '%s-%02d' "$FIXTURE_HOST_PREFIX" "$index")"+ FIXTURE_HOST_NAMES+=("$host")+ kctl apply -f - >/dev/null <<EOF || return 1+apiVersion: clerum.io/v1alpha1+kind: Host+metadata:+ name: ${host}+ namespace: ${HOST_NS}+ labels:+ e2e.clerum.io/suite: ${SUITE_NAME}+ e2e.clerum.io/run: "${RUN_ID}"+spec:+ host: ${host}+ contextRef: ${FIXTURE_CONTEXT}+ secretRef: ${FIXTURE_SECRET}+ model:+ provider: openai+ name: e2e-readiness+EOF+ done+}++delete_host_fixtures() {+ local failed=0 host+ kctl delete host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1+ kctl delete context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"+ for host in "${FIXTURE_HOST_NAMES[@]}"; do+ kctl delete \+ deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1+ done+ [ "$failed" = 0 ] || return 1+ wait_until 60 "all gate-owned Host inputs and managed runtime resources to disappear" \+ fixture_resources_absent+}++print_repair_instructions() {+ cat >&2 <<EOF+HCC readiness bootstrap cleanup could not restore a verified clean state.+Context: ${E2E_KUBECONTEXT}+HCC: ${HCC_NS}/${HCC_DEPLOY}+Fixture Hosts: ${HOST_NS}/${FIXTURE_HOST_PREFIX}-*+Fixture Context: ${MCP_NS}/${FIXTURE_CONTEXT}+Fixture LLM Secret: ${HOST_NS}/${FIXTURE_SECRET}+Retained blocker: ${HCC_NS}/${BLOCKER_NAME}++Inspect before changing anything:+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml+ kubectl --context=${E2E_KUBECONTEXT} -n ${HOST_NS} get host,secret -l e2e.clerum.io/suite=${SUITE_NAME}+ kubectl --context=${E2E_KUBECONTEXT} -n ${MCP_NS} get context -l e2e.clerum.io/suite=${SUITE_NAME}+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/hcc-readiness++Do not remove the blocker or HCC gate lock until HCC configuration, rollout, Pod readiness,+/ready, and fixture absence are verified.++Restore the original HCC replica count before verifying rollout and /ready:+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s+EOF+}++cleanup() {+ local status=$? cleanup_failed=0 restore_ok=1 hcc_stopped=1+ trap - EXIT+ set +e++ # Remove every gate-owned input while HCC is stopped. Restarting HCC with a+ # partially deleted fixture fleet would let the real control-api path mutate+ # those leftovers and make cleanup non-deterministic.+ if [ "$HCC_MUTATED" = 1 ]; then+ hcc_stopped=0+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 \+ >/dev/null 2>&1 || restore_ok=0+ if [ "$restore_ok" = 1 ]; then+ wait_until 120 "HCC pods to stop before fixture deletion" \+ hcc_pods_absent >/dev/null 2>&1 || restore_ok=0+ fi+ [ "$restore_ok" = 1 ] && hcc_stopped=1+ fi++ if [ "$FIXTURES_CREATED" = 1 ]; then+ if [ "$hcc_stopped" = 1 ]; then+ delete_host_fixtures >/dev/null 2>&1 || restore_ok=0+ else+ restore_ok=0+ fi+ fi++ if [ "$HCC_MUTATED" = 1 ] && [ "$restore_ok" = 1 ] &&+ [ -n "$ORIGINAL_CONTROL_API_BASE_URL" ]; then+ kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ "CONTROL_API_BASE_URL=${ORIGINAL_CONTROL_API_BASE_URL}" >/dev/null 2>&1 ||+ restore_ok=0+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 ||+ restore_ok=0+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s \+ >/dev/null 2>&1 || restore_ok=0+ if [ "$restore_ok" = 1 ]; then+ wait_until 240 \+ "restored HCC config, rollout, Ready pod, /ready, and fixture absence" \+ hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0+ fi+ fi++ if [ "$restore_ok" = 1 ]; then+ if [ "$BLOCKER_CREATED" = 1 ]; then+ delete_blocker_fixture || cleanup_failed=1+ fi+ wait_until 60 "all HCC readiness blocker resources to disappear" \+ readiness_blocker_suite_absent >/dev/null 2>&1 || cleanup_failed=1+ else+ cleanup_failed=1+ fi++ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then+ cleanup_failed=1+ fi++ if [ "$cleanup_failed" -ne 0 ]; then+ echo "HCC readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2+ [ "$restore_ok" = 1 ] || print_repair_instructions+ [ "$status" = 0 ] && status=1+ fi+ if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then+ header "HCC readiness bootstrap gate passed"+ fi+ exit "$status"+}+trap cleanup EXIT++header "HCC readiness during initial Host fleet reconciliation"++require_branch_owned_hcc_gate "$HCC_NS"+ok "branch helper, profile, exact HEAD/fingerprint/gate verified"++kctl get nodes -o json |+ jq -e --arg context "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||+ die "target context is not a minikube cluster"+readiness_blocker_suite_absent ||+ die "stale HCC readiness blocker resources exist; inspect them before fault injection"+fixture_inputs_absent ||+ die "stale ${SUITE_NAME} Host, Context, or Secret fixtures exist; inspect them before fault injection"+acquire_hcc_watch_gate_lock ||+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"+ok "exclusive HCC fault-injection lock acquired"++HCC_DEPLOYMENT_JSON="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" ||+ die "could not read HCC Deployment"+ORIGINAL_REPLICAS="$(jq -r '.spec.replicas // empty' <<<"$HCC_DEPLOYMENT_JSON")"+[ "$ORIGINAL_REPLICAS" = 1 ] ||+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"+ORIGINAL_CONTROL_API_BASE_URL="$(jq -r '+ first(+ .spec.template.spec.containers[]? |+ select(.name == "host-context-controller") |+ .env[]? |+ select(.name == "CONTROL_API_BASE_URL") |+ .value+ ) // empty+' <<<"$HCC_DEPLOYMENT_JSON")"+[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ] ||+ die "HCC CONTROL_API_BASE_URL is missing"+HCC_PORT="$(jq -r '+ .spec.template.spec.containers[]? |+ select(.name == "host-context-controller") as $container |+ (+ [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //+ [$container.ports[]? | select(.name == "http") | .containerPort][0] //+ empty+ )+' <<<"$HCC_DEPLOYMENT_JSON")"+[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||+ die "could not resolve a valid HCC HTTP port from the Deployment"+HOST_RECONCILE_CONCURRENCY="$(jq -r '+ first(+ .spec.template.spec.containers[]? |+ select(.name == "host-context-controller") |+ .env[]? |+ select(.name == "HCC_HOST_FULL_RECONCILE_CONCURRENCY") |+ .value+ ) // "2"+' <<<"$HCC_DEPLOYMENT_JSON")"+[[ "$HOST_RECONCILE_CONCURRENCY" =~ ^[1-8]$ ]] ||+ die "HCC_HOST_FULL_RECONCILE_CONCURRENCY must be 1..8, found ${HOST_RECONCILE_CONCURRENCY}"+FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))+HCC_IMAGE="$(jq -r '+ first(+ .spec.template.spec.containers[]? |+ select(.name == "host-context-controller") |+ .image+ ) // empty+' <<<"$HCC_DEPLOYMENT_JSON")"+[ -n "$HCC_IMAGE" ] || die "could not resolve the HCC image"+ok "planned ${FIXTURE_HOST_COUNT} gate-owned Hosts for concurrency ${HOST_RECONCILE_CONCURRENCY}"++BLOCKER_CREATED=1+kctl apply -f - >/dev/null <<EOF+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ${BLOCKER_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+spec:+ replicas: 1+ selector:+ matchLabels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+ template:+ metadata:+ labels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+ spec:+ automountServiceAccountToken: false+ terminationGracePeriodSeconds: 1+ containers:+ - name: blocker+ image: ${HCC_IMAGE}+ imagePullPolicy: IfNotPresent+ env:+ - name: EXPECTED_TOKEN_PATH+ value: ${TOKEN_REQUEST_PATH}+ command: [node, -e]+ args:+ - >-+ require('http').createServer((request,response) => {+ if(request.method!=='POST'||request.url!==process.env.EXPECTED_TOKEN_PATH){+ console.log('rejecting '+request.method+' '+request.url);+ request.resume();+ request.on('end',()=>{response.statusCode=404;response.end()});+ return;+ }+ let body='';+ request.setEncoding('utf8');+ request.on('data',chunk=>{if(body.length<65536)body+=chunk;});+ request.on('end',()=>{+ let host='unknown';+ try{const parsed=JSON.parse(body);if(typeof parsed.host==='string')+ host=parsed.host;}catch{}+ console.log('holding-token '+request.method+' '+request.url+' host='+host);+ });+ }).listen(8090, '0.0.0.0');+ readinessProbe:+ tcpSocket: {port: 8090}+ periodSeconds: 1+ resources:+ requests: {cpu: 5m, memory: 16Mi}+ limits: {cpu: 50m, memory: 48Mi}+ securityContext:+ allowPrivilegeEscalation: false+ capabilities: {drop: [ALL]}+ readOnlyRootFilesystem: true+ runAsNonRoot: true+ runAsUser: 1000+ runAsGroup: 1000+ seccompProfile: {type: RuntimeDefault}+---+apiVersion: v1+kind: Service+metadata:+ name: ${BLOCKER_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+spec:+ selector:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+ ports:+ - name: http+ port: 8090+ targetPort: 8090+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${BLOCKER_EGRESS_POLICY_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+spec:+ podSelector:+ matchLabels:+ app: ${HCC_DEPLOY}+ policyTypes: [Egress]+ egress:+ - to:+ - podSelector:+ matchLabels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+ ports:+ - protocol: TCP+ port: 8090+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${BLOCKER_NAME}+ namespace: ${HCC_NS}+ labels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+spec:+ podSelector:+ matchLabels:+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"+ policyTypes: [Ingress]+ ingress:+ - from:+ - podSelector:+ matchLabels:+ app: ${HCC_DEPLOY}+ ports:+ - protocol: TCP+ port: 8090+EOF+kctl rollout status deployment "$BLOCKER_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||+ die "isolated token-issuance blocker did not become ready"+control_api_service_excludes_blocker ||+ die "isolated blocker was selected by the real control-api Service"+ok "isolated in-cluster blocker is ready"++HCC_MUTATED=1+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null+wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||+ die "existing HCC pod did not stop"+hcc_gateway_remains_ready_without_hcc 20 ||+ die "HCC API gateway did not remain Ready on local /health while proxied /ready was unavailable"+ok "API gateway remained Ready for a complete probe-failure window while /ready reflected HCC unavailability"+create_host_fixtures ||+ die "could not create the gate-owned Host fleet while HCC was stopped"+ok "created ${FIXTURE_HOST_COUNT} gate-owned Hosts without modifying any real Host or Secret"+kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ "CONTROL_API_BASE_URL=http://${BLOCKER_NAME}.${HCC_NS}.svc.cluster.local:8090" >/dev/null+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null++new_hcc_pod=""+capture_running_hcc() {+ new_hcc_pod="$(running_hcc_pod)"+ [ -n "$new_hcc_pod" ]+}+wait_until 120 "replacement HCC pod to enter Running" capture_running_hcc ||+ die "replacement HCC pod did not start"+wait_until 60 "initial Host reconciliation to start" \+ hcc_log_contains "$new_hcc_pod" "$START_MARKER" ||+ die "HCC never started the initial Host reconciliation"+wait_until 120 "a gate-owned Host token request to reach the blocker" blocker_holds_token_request ||+ die "initial Host reconciliation never reached the deterministic blocker"+wait_until 180 "more than one bounded Host worker wave to reach the blocker" \+ blocker_observed_more_than_one_worker_wave ||+ die "fixture fleet did not prove progress beyond the first bounded Host worker wave"+initial_host_pass_is_active "$new_hcc_pod" ||+ die "initial Host reconciliation failed or completed before the readiness assertion"+ok "initial Host reconciliation is active beyond one bounded worker wave"++probe_script="$(cat <<'NODE'+const http = require('http');+function get(path) {+ return new Promise((resolve, reject) => {+ const request = http.get(+ {host: '127.0.0.1', port: Number(process.env.HCC_E2E_PORT), path},+ response => {+ let body = '';+ response.setEncoding('utf8');+ response.on('data', chunk => { body += chunk; });+ response.on('end', () => resolve({status: response.statusCode, body}));+ }+ );+ request.setTimeout(5000, () => request.destroy(new Error('timeout')));+ request.once('error', reject);+ });+}+(async () => {+ const ready = await get('/ready');+ const readyBody = JSON.parse(ready.body);+ if (ready.status !== 200 || readyBody.status !== 'ready' || readyBody.ready !== true) {+ throw new Error('unexpected readiness response: ' + ready.status + ' ' + ready.body);+ }+ const discovery = await get('/api/v1/mcpservers');+ if (discovery.status !== 200) {+ throw new Error('unexpected discovery response: ' + discovery.status);+ }+ const discoveryBody = JSON.parse(discovery.body);+ if (!Array.isArray(discoveryBody.servers) || discoveryBody.contextRef !== '*') {+ throw new Error('discovery response does not match the live API contract');+ }+ console.log(JSON.stringify({readyStatus: ready.status, discoveryStatus: discovery.status,+ discoveredServers: discoveryBody.servers.length}));+})().catch(error => { console.error(error.message); process.exit(1); });+NODE+)"+probe_result="$(kctl exec "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" node -e "$probe_script")" ||+ die "real HCC readiness/discovery HTTP probe failed while the fleet pass was active"+echo "$probe_result" | jq -e \+ '.readyStatus == 200 and .discoveryStatus == 200 and (.discoveredServers | type == "number")' \+ >/dev/null ||+ die "HCC HTTP probe returned an invalid result: ${probe_result}"+hcc_gateway_ready_proxy_recovers ||+ die "HCC API gateway /ready did not recover after HCC became ready"+initial_host_pass_is_active "$new_hcc_pod" ||+ die "initial Host reconciliation failed or completed before the HTTP assertions finished"+ok "/ready returned 200 while the initial Host fleet pass remained active"+ok "API gateway /ready recovered without changing its local readiness"+ok "/api/v1/mcpservers returned a valid live discovery response during that same pass"++header "HCC readiness assertions passed; restoring branch-owned runtime"+echo "$probe_result"diff --git a/scripts/e2e/e2e-host-storm-gate.sh b/scripts/e2e/e2e-host-storm-gate.shindex 095bb8c93..1015dd276 100755--- a/scripts/e2e/e2e-host-storm-gate.sh+++ b/scripts/e2e/e2e-host-storm-gate.sh@@ -26,16 +26,14 @@# construction, so A1 is primarily a RECORDING of that evidence — it# can only fail if the harness itself degenerates into sequential# phases; the load-bearing falsifiable claims are A2-A8.-# A2 urgent queue-wait p95 <= 5s DURING the storm window, computed from-# the clerum_hcc_host_reconcile_queue_wait_seconds{lane="urgent"}-# histogram buckets (§14.1) across two counter windows: pre-kill pod-# delta (scrape B - scrape A) plus post-kill pod absolute (scrape C1,-# taken IMMEDIATELY after created-Host convergence so post-storm quiet-# samples cannot dilute the p95 — a fresh process starts at zero, and-# the C1 window includes replacement-pod recovery traffic). Samples in-# flight between scrape B and process death are unrecoverable (counter-# reset) and are recorded as a documented measurement gap, never-# silently backfilled.+# A2 urgent queue-wait coverage has two independent same-pod windows:+# (a) the supported wake window immediately before the HCC kill requires+# every completed urgent sample <= 5s, after proving the wake reconcile+# finished and the urgent lane returned idle; and+# (b) ten dedicated Host ADDED events issued only after the replacement+# HCC completed LIST→WATCH recovery use a D - C1 p95 delta. A slow wake+# can therefore never be diluted by later fast post-recovery probes, and+# neither window crosses the intentional process restart.# A3 no starvation: EVERY created Host's Deployment materializes within# STORM_MATERIALIZATION_BUDGET_S (20s, §14.2 "New Host# materialization") of that Host's OWN metadata.creationTimestamp,@@ -59,13 +57,16 @@# All-zero is legitimate (deletes fully handled by the direct# watch-DELETE path, which by design does not touch this counter) —# A5 remains the falsifiable cleanup proof.-# A7 fleet coalescing held (one active + one trailing slot), from-# clerum_hcc_host_fleet_requests_total (scrape C2): failed == 0 in-# both windows; on the post-kill pod started >= 1 (the recovery-# convergence pass ran), coalesced >= 1 (the recovery-triggered-# request landed against the active initial pass), and trailing <=-# coalesced (every trailing drain corresponds to at least one request-# that was queued behind the single pending slot).+# A7 recovered fleet convergence is bounded, from+# clerum_hcc_host_fleet_requests_total (scrape C2) plus the replacement+# pod's explicit initial-fleet COMPLETED log: failed == 0 in both windows,+# the exact replacement pod completes its startup fleet pass, and+# trailing <= coalesced. The pod identity, completion marker, and A4's+# successful recovery histogram form the causal fence; started is not+# capped because an independent request may legitimately begin after the+# recovery pass settles. Requiring a coalesced request here would invent+# a second overlapping signal; the concurrent-request coalescing contract+# is covered deterministically by hostFleetScheduler.test.ts.# A8 wake ingredient integrity: the woken Host leaves state=suspended and# reaches a Ready pod within STORM_WAKE_READY_BUDGET_S of the wake# POST. This is an ingredient-integrity bound (a silently no-op wake@@ -97,23 +98,26 @@## ----------------------------------------------------------------------# INPUTS (environment; fail-loud if missing/invalid)-# CONTEXT kubectl context. REFUSED unless it matches-# ^clerum-(test|codex-|detached-|claude-) — a local minikube-# profile. Any GKE / *prod* context is rejected outright.+# KUBECONTEXT / E2E_K8S_CONTEXT+# Exact generated branch-owned minikube context.+# MINIKUBE_PROFILE+# Must equal that context.+# E2E_EXPECTED_PRE_GATE_GATE+# Exact gate recorded by the branch-owned pre-gate sync.+# E2E_HCC_HOST_STORM_FAULT_INJECTION=1+# Explicit acknowledgement of the disruptive HCC pod restart.## OPTIONAL INPUTS# E2E_STORM_WAKE_HOST_REF suspended-wake target (default chatllm-stateless,# the seeded stateless Host — it must exist with# spec.lifecycle.stateless=true and be associated to the E2E# user; seed via scripts/e2e/seed-stateless-host.sh).-# EXTERNAL_REST_API_BASE_URL default http://127.0.0.1:8091-# RPC_PROXY_BASE_URL default http://127.0.0.1:8094# E2E_DEV_LOGIN_EMAIL default [email protected]-# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-admin123!}+# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-changeme123!}# STORM_CONTEXT_REF Context CR fixtures reference (default context1)# STORM_MODEL_PROVIDER / STORM_MODEL_NAME fixture model (default zai/glm-5.1)-# HCC_METRICS_LOCAL_PORT ephemeral scrape port (default 18082)-# KEEP_FIXTURES set to 1 to skip teardown (diagnosis only)+# CLERUM_PROFILE_PORTS_ENV optional explicit path to the selected branch+# profile's generated ports.env.## Prereqs (each a HARD FAIL with the concrete reason):# - branch HCC image deployed (clerum_hcc_host_delete_cleanup_total@@ -122,7 +126,10 @@# - external-rest-api + rpc-proxy reachable (make minikube-pf-all)## Usage:-# CONTEXT=clerum-test bash scripts/e2e/e2e-host-storm-gate.sh+# MINIKUBE_PROFILE=<profile> KUBECONTEXT=<profile> \+# E2E_EXPECTED_PRE_GATE_GATE=<gate> \+# E2E_HCC_HOST_STORM_FAULT_INJECTION=1 \+# bash scripts/e2e/e2e-host-storm-gate.sh## EXIT CODES# 0 every storm assertion passed and teardown left zero fixtures@@ -131,6 +138,17 @@# ======================================================================set -euo pipefail+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+# shellcheck source=scripts/e2e/e2e-lib.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/e2e-lib.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"+# ─── Constants mirrored from host-context-controller/src/{constants,config}.ts ──readonly MANAGED_BY_LABEL='clerum.io/managed-by'readonly MANAGED_BY_VALUE='host-context-controller'@@ -144,6 +162,10 @@ readonly CONTROL_NS='control-plane'readonly HCC_DEPLOY='host-context-controller'readonly HCC_PORT='8081' # containerPort http (deploy/base/control-plane)readonly HCC_METRICS_PATH='/metrics' # server.ts GET /metrics+readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'+readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'+# Shared HCC ownership/lock helpers use HCC_NS.+HCC_NS="$CONTROL_NS"# Owned-child kinds + namespaces of a per-Host bundle (mirrors# measure-host-bundle-reconcile.sh — includes per-Host NetworkPolicies).@@ -156,6 +178,7 @@ readonly BUNDLE_NAMESPACES="${MCP_HOST_NS} ${CHANNELS_NS} ${RPC_PROXY_NS}"# (`QUEUE_WAIT_METRIC="$QUEUE_WAIT_METRIC" python3 …`) would fail as a readonly# reassignment under `set -e` — export once here and drop the inline prefixes.readonly QUEUE_WAIT_METRIC='clerum_hcc_host_reconcile_queue_wait_seconds'+readonly IN_FLIGHT_METRIC='clerum_hcc_host_reconcile_in_flight'readonly WATCH_RECOVERY_METRIC='clerum_hcc_host_watch_recovery_seconds'readonly FLEET_METRIC='clerum_hcc_host_fleet_requests_total'readonly DELETE_CLEANUP_METRIC='clerum_hcc_host_delete_cleanup_total'@@ -164,11 +187,20 @@ export QUEUE_WAIT_METRIC WATCH_RECOVERY_METRIC FLEET_METRIC DELETE_CLEANUP_METRI# ─── Storm shape + budgets (each constant states its contract) ────────# Addendum 5.1: ">=10" created fixture Hosts, mixed stateful/stateless.readonly STORM_CREATE_COUNT=10+# A2's direct-watch probe is deliberately separate from the restart window:+# a cold-start LIST legitimately converges objects on the fleet lane. These+# ten ADDED events are emitted only once the replacement's LIST→WATCH recovery+# has succeeded, so their queue histogram is an unambiguous urgent-lane SLO.+readonly STORM_URGENT_PROBE_COUNT=10# Addendum 5.1: ">=3" pre-created fixture Hosts deleted mid-storm.readonly STORM_DELETE_COUNT=3# §14.2 "urgent new/wake queue wait": local hard gate <= 5s (histogram# bucket boundary le="5" exists in RECONCILE_LATENCY_BUCKETS).readonly STORM_URGENT_P95_BUDGET_S=5+# The queue value itself is adjudicated against the 5s hard gate. This larger+# deadline bounds collection of the completed reconcile evidence and produces+# a loud operational failure if the wake never settles or telemetry is absent.+readonly STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S=60# §14.2 "New Host materialization": Host CR -> Deployment <= 20s.readonly STORM_MATERIALIZATION_BUDGET_S=20# §14.2 "Watch recovery": recovery requested -> WATCH installed <= 15s@@ -240,26 +272,44 @@ print_results() {log "fairness and convergence under realistic concurrency, not API degradation."}-# ─── kubectl wrapper — EVERY call carries --context (CLAUDE.md mandate) ──-kctl() { kubectl --context "$CONTEXT" "$@"; }-# ─── Portable time helper (python3; mirrors measure script) ───────────now_ms() { python3 -c 'import time;print(int(time.time()*1000))'; }# ─── Strict local-profile context guard (fail-loud) ───────────────────require_storm_context() {- [[ -n "${CONTEXT:-}" ]] || die "CONTEXT is required (a local minikube profile context)." 2- case "$CONTEXT" in- *gke_*|*prod*)- die "Refusing: CONTEXT='${CONTEXT}' looks like a GKE/production context. Local profiles only." 2 ;;- esac- if [[ ! "$CONTEXT" =~ ^clerum-(test|codex-|detached-|claude-) ]]; then- die "Refusing: CONTEXT='${CONTEXT}' is not an allowed local profile. Must match ^clerum-(test|codex-|detached-|claude-)." 2- fi+ [[ -n "${E2E_KUBECONTEXT:-}" ]] ||+ die "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." 2+ is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||+ die "Refusing Host storm fault injection on non-branch context '${E2E_KUBECONTEXT}'." 2+ require_safe_kube_context ||+ die "context '${E2E_KUBECONTEXT}' is not an allowed local E2E context." 2if ! kctl get nodes -o name >/dev/null 2>&1; then- die "CONTEXT='${CONTEXT}' is not reachable (kubectl get nodes failed). Start the profile and its port-forwards." 1+ die "context '${E2E_KUBECONTEXT}' is not reachable (kubectl get nodes failed)." 1fi- log "context OK: ${CONTEXT}"+ kctl get nodes -o json | jq -e --arg context "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||+ die "target context is not the exact selected minikube profile '${E2E_KUBECONTEXT}'." 2+ log "branch-owned minikube context OK: ${E2E_KUBECONTEXT}"+}++load_branch_profile_endpoints() {+ local ports_env external_url rpc_url+ ports_env="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/ports.env}"+ [[ -r "$ports_env" ]] ||+ die "branch profile ports evidence is missing at ${ports_env}; refresh the branch-profile helper." 2+ external_url="$(profile_env_value EXTERNAL_REST_API_BASE_URL "$ports_env")"+ [[ -n "$external_url" ]] ||+ external_url="$(profile_env_value EXTERNAL_REST_API_URL "$ports_env")"+ rpc_url="$(profile_env_value RPC_PROXY_BASE_URL "$ports_env")"+ [[ -n "$rpc_url" ]] ||+ rpc_url="$(profile_env_value RPC_PROXY_URL "$ports_env")"+ [[ "$external_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||+ die "branch ports.env has no valid loopback External REST URL." 2+ [[ "$rpc_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||+ die "branch ports.env has no valid loopback RPC proxy URL." 2+ EXT_BASE="$external_url"+ RPC_BASE="$rpc_url"+ log "using branch-profile endpoints from ${ports_env}"}require_deps() {@@ -295,11 +345,12 @@ RUN_ID=''; WORK_DIR=''FIXTURE_SECRET=''STORM_CREATED_HOSTS=()STORM_DELETE_HOSTS=()+STORM_URGENT_PROBE_HOSTS=()WAKE_HOST_REF="${E2E_STORM_WAKE_HOST_REF:-chatllm-stateless}"-EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"-RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"+EXT_BASE=''+RPC_BASE=''DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"-DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-admin123!}}"+DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"STORM_CTX_REF="${STORM_CONTEXT_REF:-context1}"STORM_PROVIDER="${STORM_MODEL_PROVIDER:-zai}"STORM_MODEL="${STORM_MODEL_NAME:-glm-5.1}"@@ -311,6 +362,14 @@ T_CREATE_FIRST_MS=''; T_DELETE_MS=''; T_WAKE_MS=''; T_KILL_MS=''; T_CREATE_LAST_PF_PID=''HCC_ENV_SAVED=''TEARDOWN_DONE=0+TEARDOWN_CLEANUP_FAILED=0+HCC_RESTORE_OK=1+STORM_FIXTURES_CREATED=0+# Read and mutated by the sourced compare-and-swap lock helper.+HCC_GATE_LOCK_ACQUIRED=0+HCC_GATE_LOCK_NAME=''+HCC_GATE_LOCK_UID=''+HCC_GATE_FINALIZATION_FAILURE=''cleanup_portforward() {if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then@@ -340,10 +399,16 @@ restore_hcc_env() {_hcc_restore_args[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }log "Restoring HCC cadence env on deployment/${HCC_DEPLOY}"- kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1 || \- warn "failed to restore HCC env (manual check advised)"- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1 || \+ if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then+ warn "failed to restore HCC env (manual check required)"+ HCC_RESTORE_OK=0+ return 1+ fi+ if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; thenwarn "HCC rollout did not settle after env restore"+ HCC_RESTORE_OK=0+ return 1+ fiHCC_ENV_SAVED=''}@@ -358,10 +423,12 @@ restore_hcc_env_strict() {[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }log "Restoring HCC cadence env on deployment/${HCC_DEPLOY} (strict)"if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then+ HCC_RESTORE_OK=0fail "cadence restore FAILED — the profile is left with test cadences on deployment/${HCC_DEPLOY} (the EXIT trap retries best-effort; verify manually)"return 0fiif ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then+ HCC_RESTORE_OK=0fail "HCC rollout did not settle after the cadence restore — verify deployment/${HCC_DEPLOY} manually"return 0fi@@ -371,10 +438,11 @@ restore_hcc_env_strict() {# shellcheck disable=SC2329 # invoked via the EXIT trap belowcleanup_on_exit() {- local status=$?+ local status=$? cleanup_failed=0set +ecleanup_portforward- if [[ "$TEARDOWN_DONE" != '1' && -n "$RUN_ID" && "${KEEP_FIXTURES:-}" != '1' ]]; then+ if [[ "$TEARDOWN_DONE" != '1' && "$STORM_FIXTURES_CREATED" == '1' && -n "$RUN_ID" ]]; then+ cleanup_failed=1local sel="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"warn "abnormal exit — best-effort deletion of storm fixtures (${sel})"kctl -n "$MCP_HOST_NS" delete host -l "$sel" --wait=false >/dev/null 2>&1 || \@@ -382,7 +450,10 @@ cleanup_on_exit() {kctl -n "$MCP_HOST_NS" delete secret -l "$sel" --wait=false >/dev/null 2>&1 || \warn "best-effort fixture Secret deletion failed (manual check advised)"fi- restore_hcc_env+ restore_hcc_env || HCC_RESTORE_OK=0+ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$HCC_RESTORE_OK"; then+ status=1+ fiexit "$status"}trap cleanup_on_exit EXIT@@ -405,6 +476,9 @@ init_run() {for i in $(seq 1 "$STORM_DELETE_COUNT"); doSTORM_DELETE_HOSTS+=("$(printf 's791-%s-d%d' "$RUN_ID" "$i")")done+ for i in $(seq 1 "$STORM_URGENT_PROBE_COUNT"); do+ STORM_URGENT_PROBE_HOSTS+=("$(printf 's791-%s-u%02d' "$RUN_ID" "$i")")+ donelog "run-id ${RUN_ID}; work dir ${WORK_DIR}"log "create fixtures: ${STORM_CREATED_HOSTS[*]}"log "delete fixtures: ${STORM_DELETE_HOSTS[*]}"@@ -531,14 +605,30 @@ storm_diagnostics() {# different pod (e.g. a terminating one during a rollout) than the one the# identity guard checked; cross-pod counter deltas would false-PASS A4.scrape_hcc_metrics() {- local out="$1" pod="$2" lport="${HCC_METRICS_LOCAL_PORT:-18082}"- local url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"- kctl -n "$CONTROL_NS" port-forward "pod/${pod}" "${lport}:${HCC_PORT}" >"${WORK_DIR}/pf.log" 2>&1 &+ local out="$1" pod="$2" lport='' url pf_log+ pf_log="${WORK_DIR}/pf-${pod}.log"+ # Let kubectl allocate a process-owned loopback port. A fixed port is+ # machine-global and can silently collide with another branch/profile gate.+ kctl -n "$CONTROL_NS" port-forward --address=127.0.0.1 \+ "pod/${pod}" ":${HCC_PORT}" >"$pf_log" 2>&1 &PF_PID=$!# shellcheck disable=SC2329 # invoked by name via wait_until+ _pf_ready_probe() {+ kill -0 "$PF_PID" 2>/dev/null || return 1+ lport="$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> .*/\1/p' "$pf_log" | head -n1)"+ [[ "$lport" =~ ^[0-9]+$ ]]+ }+ # shellcheck disable=SC2329 # invoked by name via wait_until+ _pf_diag() { { echo '----- port-forward log -----'; cat "$pf_log" 2>/dev/null; } >&2; }+ if ! wait_until 30 1 "random HCC metrics port-forward" _pf_ready_probe _pf_diag; then+ cleanup_portforward+ return 1+ fi+ url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"+ # shellcheck disable=SC2329 # invoked by name via wait_until_scrape_probe() { curl -fsS --max-time 3 "$url" -o "$out" 2>/dev/null; }# shellcheck disable=SC2329 # invoked by name via wait_until- _scrape_diag() { { echo '----- port-forward log -----'; cat "${WORK_DIR}/pf.log" 2>/dev/null; } >&2; }+ _scrape_diag() { _pf_diag; }if ! wait_until 30 2 "HCC metrics scrape -> ${out}" _scrape_probe _scrape_diag; thencleanup_portforwardreturn 1@@ -551,6 +641,23 @@ wake_lifecycle_state() {kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true}+# shellcheck disable=SC2329 # invoked from the wait_until callback below+wake_handled_generation() {+ kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" \+ -o jsonpath='{.status.lifecycle.wakeHandledGeneration}' 2>/dev/null || true+}++# Correlate the public wake request with HCC's durable lifecycle status. This+# predicate is deliberately pure so the static gate test can exercise the+# same ordering contract used by the cluster E2E.+# shellcheck disable=SC2329 # invoked from the wait_until callback below+wake_generation_is_handled() {+ local requested="$1" handled="$2"+ [[ "$requested" =~ ^[1-9][0-9]*$ ]] || return 1+ [[ "$handled" =~ ^[0-9]+$ ]] || return 1+ (( handled >= requested ))+}+# shellcheck disable=SC2329 # invoked from _wake_converged_probe (runs via wait_until)wake_ready_pod_exists() {kctl -n "$MCP_HOST_NS" get pods -l "app=${WAKE_HOST_REF}" \@@ -589,19 +696,31 @@ mint_rpc_token() {# The wake route is the user-facing rpc-proxy endpoint Desktop uses;# rpc-proxy owns the control-api hop. Never kubectl scale.-WAKE_STATUS=''; WAKE_BODY=''+WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''post_wake() {- local resp+ local resp response_status response_generation+ WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''mint_rpc_token || return 1resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \"${RPC_BASE}/api/v1/rpc/hosts/${WAKE_HOST_REF}/wake" \-H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json') || {echo "wake POST failed at ${RPC_BASE}" >&2; return 1; }WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"- case "$WAKE_STATUS" in- 200|202|204) return 0 ;;- *) echo "wake POST returned HTTP ${WAKE_STATUS}: ${WAKE_BODY}" >&2; return 1 ;;- esac+ [[ "$WAKE_STATUS" == '202' ]] || {+ echo "wake POST returned HTTP ${WAKE_STATUS}, expected 202: ${WAKE_BODY}" >&2+ return 1+ }+ response_status="$(jq -r '.status // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"+ response_generation="$(jq -r '.wakeGeneration // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"+ [[ "$response_status" == 'wake-requested' ]] || {+ echo "wake POST returned unexpected status '${response_status:-<missing>}': ${WAKE_BODY}" >&2+ return 1+ }+ [[ "$response_generation" =~ ^[1-9][0-9]*$ ]] || {+ echo "wake POST returned invalid wakeGeneration '${response_generation:-<missing>}': ${WAKE_BODY}" >&2+ return 1+ }+ WAKE_GENERATION="$response_generation"}# ─── HCC cadence shortening (only when the wake target must suspend) ──@@ -654,8 +773,15 @@ _wake_target_diag() {require_depsrequire_storm_contextinit_run+[[ "${KEEP_FIXTURES:-0}" != '1' ]] ||+ die "KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result; refusing before mutation." 2+[[ "${E2E_HCC_HOST_STORM_FAULT_INJECTION:-0}" == '1' ]] ||+ die "Set E2E_HCC_HOST_STORM_FAULT_INJECTION=1 to acknowledge the temporary HCC pod restart." 2+require_branch_owned_hcc_gate "$CONTROL_NS"+load_branch_profile_endpointsheader "Prerequisites"+ok "branch helper, profile, exact HEAD/fingerprint/gate, and minikube node label verified"if kctl get deployment "$HCC_DEPLOY" -n "$CONTROL_NS" >/dev/null 2>&1; thenok "HCC deployment present in ${CONTROL_NS}"else@@ -695,12 +821,18 @@ elsefail "cannot mint RPC token for hostRef=${WAKE_HOST_REF} -- is ${DEV_EMAIL} associated to it? Seed: bash scripts/e2e/seed-stateless-host.sh"print_results; exit 1fi+if acquire_hcc_watch_gate_lock; then+ ok "exclusive shared HCC fault-injection lock acquired"+else+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"+fi# ======================================================================# Prep 1 — storm fixtures that will be DELETED mid-storm (bundles must# exist first: deleting a bundle-less Host proves nothing)# ======================================================================header "Prep — pre-create ${STORM_DELETE_COUNT} delete-target fixture Hosts"+STORM_FIXTURES_CREATED=1ensure_fixture_secretfor h in "${STORM_DELETE_HOSTS[@]}"; doapply_storm_host "$h" false@@ -748,6 +880,9 @@ SCRAPE_A="${WORK_DIR}/metrics-A.prom"SCRAPE_B="${WORK_DIR}/metrics-B.prom"SCRAPE_C1="${WORK_DIR}/metrics-C1.prom"SCRAPE_C2="${WORK_DIR}/metrics-C2.prom"+SCRAPE_D="${WORK_DIR}/metrics-D.prom"+SCRAPE_WAKE_PRE="${WORK_DIR}/metrics-wake-pre.prom"+SCRAPE_WAKE_POST="${WORK_DIR}/metrics-wake-post.prom"HCC_POD_A="$(hcc_pod_name)"if [[ -z "$HCC_POD_A" ]]; thenfail "no Running HCC pod found for baseline identity"; storm_diagnostics; print_results; exit 1@@ -781,22 +916,89 @@ for h in "${STORM_DELETE_HOSTS[@]}"; dodonelog "storm: ${STORM_DELETE_COUNT} mid-storm Host deletions issued"+# Isolate the supported wake on the pre-kill pod. Samples from the first half+# of Host creation may still settle inside this window; they are valid urgent+# traffic and the hard-gate assertion below requires every one of them, not+# only a percentile, to remain within budget.+HCC_POD_WAKE="$(hcc_pod_name)"+if [[ -z "$HCC_POD_WAKE" || "$HCC_POD_WAKE" != "$HCC_POD_A" ]]; then+ fail "storm: HCC pod changed before the wake metric baseline (${HCC_POD_A} -> ${HCC_POD_WAKE:-<none>})"+ storm_diagnostics; print_results; exit 1+fi+if ! scrape_hcc_metrics "$SCRAPE_WAKE_PRE" "$HCC_POD_WAKE"; then+ fail "storm: could not capture the same-pod wake metric baseline"+ print_results; exit 1+fi+T_WAKE_MS="$(now_ms)"if post_wake; then- log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS})"+ log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS}, generation ${WAKE_GENERATION})"elsefail "storm: mid-storm wake POST failed for ${WAKE_HOST_REF}"; print_results; exit 1fi-# Scrape B: last look at the pre-kill pod's counters, immediately before-# the kill (samples still in flight at process death are a recorded gap).+# Prove that this specific supported wake reached HCC: its response generation+# must be durably reflected in lifecycle status. Then require a new urgent+# queue sample and an idle urgent lane on the same pod. Prometheus collectors+# are read independently, so this deliberately uses a two-scrape fence instead+# of assuming cross-metric atomicity: once the first scrape observes the urgent+# lane idle, HCC's observe-before-in-flight-decrement ordering guarantees the+# wake observation is durable and therefore visible to the second scrape.+# shellcheck disable=SC2329 # invoked by name via wait_until+_wake_urgent_evidence_recorded() {+ local current_pod lifecycle_state handled_generation+ local fence_in_flight pre_count post_count in_flight+ current_pod="$(hcc_pod_name)"+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_WAKE" ]] || return 1+ lifecycle_state="$(wake_lifecycle_state)"+ [[ -n "$lifecycle_state" && "$lifecycle_state" != 'suspended' ]] || return 1+ handled_generation="$(wake_handled_generation)"+ wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation" || return 1+ scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1+ fence_in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_WAKE_POST")"+ awk -v active="$fence_in_flight" 'BEGIN { exit !(active == 0) }' || return 1++ # The wake histogram observation precedes the decrement seen by the fence.+ # Re-scrape so the adjudicated histogram cannot predate that observation.+ scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1+ pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_WAKE_PRE")"+ post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_WAKE_POST")"+ in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_WAKE_POST")"+ awk -v before="$pre_count" -v after="$post_count" -v active="$in_flight" \+ 'BEGIN { exit !((after - before) >= 1 && active == 0) }'+}+if wait_until "$STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S" 1 \+ "supported wake urgent reconcile + same-pod queue evidence" \+ _wake_urgent_evidence_recorded _wake_target_diag; then+ ok "A2 wake input: supported wake reconcile completed and published same-pod urgent queue evidence"+else+ fail "A2 wake input: wake did not complete with durable same-pod urgent queue evidence within ${STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S}s"+ print_results; exit 1+fi++# Capture A7's fleet-failure baseline immediately before the kill. The wake+# scrape remains the A2 wake-window oracle; a separate fresh scrape prevents+# fleet failures after wake adjudication from falling outside A7's delta.HCC_POD_B="$(hcc_pod_name)"if [[ "$HCC_POD_B" != "$HCC_POD_A" ]]; thenfail "storm: HCC pod changed unexpectedly before the kill (${HCC_POD_A} -> ${HCC_POD_B:-<none>}) — counter deltas would be invalid"storm_diagnostics; print_results; exit 1fiif ! scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"; then- fail "storm: could not scrape HCC metrics immediately before the kill"; print_results; exit 1+ fail "storm: could not capture the fresh pre-kill metric baseline"+ print_results; exit 1filog "storm: pre-kill scrape B captured (pod ${HCC_POD_B})"@@ -848,6 +1050,41 @@ elseprint_results; exit 1fi+# A7's causal business signal is the replacement pod's own completion log for+# its startup fleet reconciliation. A scheduled/started counter is insufficient:+# it can increment before the pass fails, stalls, or is superseded.+FLEET_COMPLETION_PROVEN=0+replacement_fleet_state=''+# shellcheck disable=SC2329 # invoked by name via wait_until+_replacement_fleet_completed_probe() {+ local logs current_pod+ current_pod="$(hcc_pod_name)"+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1+ logs="$(kctl -n "$CONTROL_NS" logs "pod/${HCC_POD_C}" \+ -c host-context-controller 2>/dev/null)" || return 1+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then+ replacement_fleet_state='failed'+ return 0+ fi+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then+ replacement_fleet_state='completed'+ return 0+ fi+ return 1+}+if ! wait_until "$STORM_CONVERGENCE_DEADLINE_S" 2 \+ "replacement HCC initial Host fleet reconciliation completion" \+ _replacement_fleet_completed_probe storm_diagnostics; then+ fail "A7: replacement pod did not complete its initial Host fleet reconciliation within ${STORM_CONVERGENCE_DEADLINE_S}s"+ print_results; exit 1+fi+if [[ "$replacement_fleet_state" != 'completed' ]]; then+ fail "A7: replacement pod's initial Host fleet reconciliation failed"+ storm_diagnostics; print_results; exit 1+fi+FLEET_COMPLETION_PROVEN=1+ok "A7 input: replacement pod logged completed initial Host fleet reconciliation"+# shellcheck disable=SC2329 # invoked by name via wait_until_all_created_deploys_probe() {local h@@ -880,6 +1117,68 @@ if ! scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"; thenfilog "scrape C1 captured (replacement pod ${HCC_POD_C})"+# ─── A2 input: direct-watch urgent probe after replacement LIST→WATCH ──+# The storm's second half is intentionally issued immediately after pod+# deletion, when a cold-start LIST may legitimately own those Hosts on the+# fleet lane. C1 proves the replacement has completed LIST→WATCH recovery;+# issue a fresh, independent set now so every observed sample is a direct+# watch ADDED admission on the urgent lane.+header "A2 input — urgent direct-watch probe (${STORM_URGENT_PROBE_COUNT} Hosts)"+for i in $(seq 1 "${#STORM_URGENT_PROBE_HOSTS[@]}"); do+ h="${STORM_URGENT_PROBE_HOSTS[$((i - 1))]}"+ if (( i % 2 == 0 )); then apply_storm_host "$h" true; else apply_storm_host "$h" false; fi+done+log "urgent probe: ${STORM_URGENT_PROBE_COUNT} Host ADDED events issued after replacement LIST→WATCH"++# shellcheck disable=SC2329 # invoked by name via wait_until+_urgent_probe_deploys_probe() {+ local h+ for h in "${STORM_URGENT_PROBE_HOSTS[@]}"; do+ kctl -n "$MCP_HOST_NS" get deployment "$h" >/dev/null 2>&1 || return 1+ done+}+if wait_until "$STORM_CONVERGENCE_DEADLINE_S" 3 \+ "all ${STORM_URGENT_PROBE_COUNT} urgent-probe Hosts' Deployments to exist" \+ _urgent_probe_deploys_probe storm_diagnostics; then+ ok "A2 input: every direct-watch urgent-probe Host materialized"+else+ fail "A2 input: urgent-probe Host materialization incomplete after ${STORM_CONVERGENCE_DEADLINE_S}s"+ print_results; exit 1+fi+HCC_POD_C_NOW="$(hcc_pod_name)"+if [[ -z "$HCC_POD_C_NOW" || "$HCC_POD_C_NOW" != "$HCC_POD_C" ]]; then+ fail "replacement HCC pod changed before urgent-probe scrape D (${HCC_POD_C} -> ${HCC_POD_C_NOW:-<none>}) — urgent counter window is invalid"+ storm_diagnostics; print_results; exit 1+fi++# Deployment existence can precede the end of reconcileCore, where the queue+# histogram is observed. Poll the metric itself instead of sleeping, otherwise+# a fast API server can make this gate race its own telemetry.+# shellcheck disable=SC2329 # invoked by name via wait_until+_urgent_probe_metrics_recorded() {+ local current_pod pre_count post_count+ current_pod="$(hcc_pod_name)"+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1+ scrape_hcc_metrics "$SCRAPE_D" "$HCC_POD_C" || return 1+ pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_C1")"+ post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }+ END { print total + 0 }+ ' "$SCRAPE_D")"+ awk -v before="$pre_count" -v after="$post_count" -v required="$STORM_URGENT_PROBE_COUNT" \+ 'BEGIN { exit !((after - before) >= required) }'+}+if ! wait_until 60 2 \+ "at least ${STORM_URGENT_PROBE_COUNT} direct-watch urgent metric samples" \+ _urgent_probe_metrics_recorded storm_diagnostics; then+ fail "urgent-probe reconciles did not publish ${STORM_URGENT_PROBE_COUNT} queue samples within 60s"+ print_results; exit 1+fi+log "scrape D captured (replacement pod ${HCC_POD_C}; direct-watch urgent window C1→D)"+# ─── A3: per-host materialization <= 20s from ITS OWN creation ───────header "A3 — per-host materialization (budget ${STORM_MATERIALIZATION_BUDGET_S}s, §14.2)"MAT_ROWS="${WORK_DIR}/materialization.rows"@@ -991,13 +1290,14 @@ if ! scrape_hcc_metrics "$SCRAPE_C2" "$HCC_POD_C"; thenfilog "scrape C2 captured (replacement pod ${HCC_POD_C}; A6/A7 counters settled post-A5)"-# One python pass performs the four metric assertions and emits one+# One python pass performs the five metric assertions and emits one# VERDICT|<name>|PASS/FAIL|<detail> line each. python exits nonzero only# when it cannot parse its inputs (a hard failure of the gate itself).METRIC_VERDICTS="${WORK_DIR}/metric-verdicts.txt"-if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" \+if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" SCRAPE_D="$SCRAPE_D" \+ SCRAPE_WAKE_PRE="$SCRAPE_WAKE_PRE" SCRAPE_WAKE_POST="$SCRAPE_WAKE_POST" \P95_BUDGET_S="$STORM_URGENT_P95_BUDGET_S" RECOVERY_BUDGET_S="$STORM_WATCH_RECOVERY_BUDGET_S" \- MIN_URGENT_SAMPLES="$STORM_CREATE_COUNT" \+ MIN_URGENT_SAMPLES="$STORM_URGENT_PROBE_COUNT" FLEET_COMPLETION_PROVEN="$FLEET_COMPLETION_PROVEN" \python3 >"$METRIC_VERDICTS" <<'PY'import os, re, sysfrom collections import defaultdict@@ -1033,6 +1333,9 @@ a, _ = load(os.environ["SCRAPE_A"])b, _ = load(os.environ["SCRAPE_B"])c1, _ = load(os.environ["SCRAPE_C1"])c2, c2_types = load(os.environ["SCRAPE_C2"])+d, _ = load(os.environ["SCRAPE_D"])+wake_pre, _ = load(os.environ["SCRAPE_WAKE_PRE"])+wake_post, _ = load(os.environ["SCRAPE_WAKE_POST"])def buckets(samples, metric, match):"""Cumulative histogram buckets {le: count} summed over matching series."""@@ -1057,30 +1360,60 @@ def le_key(le):verdicts = []-# ── A2: urgent queue-wait p95 <= budget across (B-A) + C1 ─────────────+# ── A2a: every completed urgent reconcile in the supported wake window ─qm = os.environ["QUEUE_WAIT_METRIC"]budget = float(os.environ["P95_BUDGET_S"])-window_note = "windows: pre-kill delta (B-A) + replacement pod C1, which includes replacement-pod recovery traffic"-pre_a = buckets(a, qm, {"lane": "urgent"})-pre_b = buckets(b, qm, {"lane": "urgent"})-post = buckets(c1, qm, {"lane": "urgent"})+wake_before = buckets(wake_pre, qm, {"lane": "urgent"})+wake_after = buckets(wake_post, qm, {"lane": "urgent"})+wake_delta = defaultdict(float)+wake_reset = False+for le in set(wake_before) | set(wake_after):+ delta = wake_after.get(le, 0.0) - wake_before.get(le, 0.0)+ if delta < 0:+ wake_reset = True+ wake_delta[le] = delta+if wake_reset:+ verdicts.append(("A2-wake-urgent-hard-gate", False,+ "negative bucket delta in the same-pod wake window"))+else:+ wake_total = wake_delta.get("+Inf", 0.0)+ wake_within = wake_delta.get(str(int(budget)), wake_delta.get(str(budget), 0.0))+ wake_dist = " ".join(+ f"le{le}={wake_delta[le]:.0f}" for le in sorted(wake_delta, key=le_key)+ )+ if wake_total < 1:+ verdicts.append(("A2-wake-urgent-hard-gate", False,+ "no completed urgent queue sample in the supported wake window"))+ elif wake_within != wake_total:+ verdicts.append(("A2-wake-urgent-hard-gate", False,+ f"{wake_total - wake_within:.0f} of {wake_total:.0f} urgent sample(s) exceeded "+ f"{budget:.0f}s in the supported wake window ({wake_dist})"))+ else:+ verdicts.append(("A2-wake-urgent-hard-gate", True,+ f"all {wake_total:.0f} urgent sample(s) <= {budget:.0f}s in the supported "+ f"same-pod wake window ({wake_dist})"))++# ── A2b: urgent queue-wait p95 across direct-watch window D-C1 ────────+window_note = "window: replacement-pod direct-watch probe delta (D-C1) after LIST→WATCH recovery"+pre = buckets(c1, qm, {"lane": "urgent"})+post = buckets(d, qm, {"lane": "urgent"})combined = defaultdict(float)reset = False-for le in set(pre_a) | set(pre_b) | set(post):- delta = pre_b.get(le, 0.0) - pre_a.get(le, 0.0)+for le in set(pre) | set(post):+ delta = post.get(le, 0.0) - pre.get(le, 0.0)if delta < 0:reset = True- combined[le] = delta + post.get(le, 0.0)+ combined[le] = deltaif reset:- verdicts.append(("A2-urgent-queue-wait-p95", False,- "negative bucket delta between scrapes A and B (unexpected counter reset)"))+ verdicts.append(("A2-direct-watch-urgent-p95", False,+ "negative bucket delta between scrapes C1 and D (unexpected counter reset)"))else:total = combined.get("+Inf", 0.0)min_samples = float(os.environ["MIN_URGENT_SAMPLES"])if total < min_samples:- verdicts.append(("A2-urgent-queue-wait-p95", False,- f"only {total:.0f} urgent samples in the storm window (need >= {min_samples:.0f} — "- "the storm did not exercise the urgent lane, cannot claim a p95)"))+ verdicts.append(("A2-direct-watch-urgent-p95", False,+ f"only {total:.0f} direct-watch urgent samples (need >= {min_samples:.0f} — "+ "the urgent probe did not exercise its declared lane)"))else:p95_ub = Nonefor le in sorted(combined, key=le_key):@@ -1089,11 +1422,10 @@ else:breakdist = " ".join(f"le{le}={combined[le]:.0f}" for le in sorted(combined, key=le_key))if p95_ub is not None and le_key(p95_ub) <= budget:- verdicts.append(("A2-urgent-queue-wait-p95", True,- f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note}; "- "gap: samples in flight between scrape B and process death are unrecoverable)"))+ verdicts.append(("A2-direct-watch-urgent-p95", True,+ f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note})"))else:- verdicts.append(("A2-urgent-queue-wait-p95", False,+ verdicts.append(("A2-direct-watch-urgent-p95", False,f"p95 bucket upper bound {p95_ub} exceeds {budget:.0f}s over {total:.0f} samples ({dist}; {window_note})"))# ── A4: watch recovery <= budget on the replacement pod (C1) ──────────@@ -1117,28 +1449,36 @@ else:verdicts.append(("A4-watch-recovery", True,f"{succ_total:.0f} recovery sample(s), all <= {bound}s (budget {rbudget}s)"))-# ── A7: fleet coalescing held (settled window C2) ─────────────────────+# ── A7: recovered fleet convergence remains bounded (settled C2) ─────fm = os.environ["FLEET_METRIC"]-pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")-post_started = counter(c2, fm, "result", "started")-post_coalesced = counter(c2, fm, "result", "coalesced")-post_trailing = counter(c2, fm, "result", "trailing")-post_failed = counter(c2, fm, "result", "failed")-problems = []-if pre_failed > 0 or post_failed > 0:- problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")-if post_started < 1:- problems.append("no fleet pass started on the replacement pod (recovery convergence missing)")-if post_coalesced < 1:- problems.append("no coalesced fleet request on the replacement pod (recovery request should land against the active initial pass)")-if post_trailing > post_coalesced:- problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")-detail = (f"started={post_started:.0f} coalesced={post_coalesced:.0f} "- f"trailing={post_trailing:.0f} failed={post_failed:.0f} (pre-kill failed delta={pre_failed:.0f})")-if problems:- verdicts.append(("A7-fleet-coalescing", False, "; ".join(problems) + f" [{detail}]"))+# Presence check (A6 pattern): counter() returns 0 for an absent family, so+# without this a renamed/unregistered FLEET_METRIC makes every count 0 and A7+# passes vacuously. The causal leg (FLEET_COMPLETION_PROVEN) alone cannot catch+# a metric rename — the metric itself must be registered on the running image.+if fm not in c2_types:+ verdicts.append(("A7-fleet-recovery-bounded", False,+ f"{fm} not registered on the running image — is the branch HCC image deployed?"))else:- verdicts.append(("A7-fleet-coalescing", True, detail))+ pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")+ post_started = counter(c2, fm, "result", "started")+ post_coalesced = counter(c2, fm, "result", "coalesced")+ post_trailing = counter(c2, fm, "result", "trailing")+ post_failed = counter(c2, fm, "result", "failed")+ problems = []+ if pre_failed != 0 or post_failed > 0:+ problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")+ if os.environ.get("FLEET_COMPLETION_PROVEN") != "1":+ problems.append("replacement-pod initial fleet completion was not causally proven")+ if post_trailing > post_coalesced:+ problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")+ detail = (f"completion-proven={os.environ.get('FLEET_COMPLETION_PROVEN', 'missing')} "+ f"started={post_started:.0f} coalesced={post_coalesced:.0f} "+ f"trailing={post_trailing:.0f} failed={post_failed:.0f} "+ f"(replacement pod; A4 recovery-success fenced; pre-kill failed delta={pre_failed:.0f})")+ if problems:+ verdicts.append(("A7-fleet-recovery-bounded", False, "; ".join(problems) + f" [{detail}]"))+ else:+ verdicts.append(("A7-fleet-recovery-bounded", True, detail))# ── A6: delete-cleanup counters consistent (settled window C2) ────────dm = os.environ["DELETE_CLEANUP_METRIC"]@@ -1178,8 +1518,8 @@ while IFS='|' read -r tag name verdict detail; dofail "${name}: ${detail}"fidone <"$METRIC_VERDICTS"-if [[ "$VERDICT_COUNT" -ne 4 ]]; then- fail "metric adjudication emitted ${VERDICT_COUNT}/4 verdicts — truncated output is a failure, not a pass"+if [[ "$VERDICT_COUNT" -ne 5 ]]; then+ fail "metric adjudication emitted ${VERDICT_COUNT}/5 verdicts — truncated output is a failure, not a pass"fi# ======================================================================@@ -1187,11 +1527,7 @@ fi# The profile and seeded data (including the wake target) stay.# ======================================================================header "Teardown — storm fixtures (${FIXTURE_RUN_KEY}=${RUN_ID})"-if [[ "${KEEP_FIXTURES:-}" == '1' ]]; then- warn "KEEP_FIXTURES=1 — leaving fixtures in place for diagnosis (run-id ${RUN_ID})"- TEARDOWN_DONE=1-else- STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"+STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"kctl -n "$MCP_HOST_NS" delete host -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \warn "teardown: label-selector Host deletion returned nonzero (leftover check below is authoritative)"kctl -n "$MCP_HOST_NS" delete secret -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \@@ -1214,6 +1550,7 @@ elseok "teardown: all storm fixture Hosts deleted via the supported CR path"elsefail "teardown: storm fixture Hosts still present after ${STORM_TEARDOWN_DEADLINE_S}s (run-id ${RUN_ID})"+ TEARDOWN_CLEANUP_FAILED=1fi# FAIL-CLOSED (B2): an unverifiable read counts as a leftover — the teardown@@ -1227,7 +1564,7 @@ elseteardown_converge_deadline=$((SECONDS + STORM_TEARDOWN_CONVERGE_S))while :; doTEARDOWN_LEFTOVER=0; teardown_orphan=0; teardown_unverifiable=0; teardown_report=''- for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do+ for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_URGENT_PROBE_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; doif ! leftovers="$(collect_children "$h")"; thenteardown_report+=" ${h}: kubectl read FAILED (unverifiable, fail-closed)"$'\n'TEARDOWN_LEFTOVER=$((TEARDOWN_LEFTOVER + 1)); teardown_unverifiable=1@@ -1280,14 +1617,19 @@ elseok "teardown: zero storm-fixture leftovers across ${BUNDLE_NAMESPACES} (profile and seeded data untouched)"elsefail "teardown: ${TEARDOWN_LEFTOVER} storm fixture subject(s) left resources behind (run-id ${RUN_ID}) — investigate before trusting results"+ TEARDOWN_CLEANUP_FAILED=1fi- TEARDOWN_DONE=1-fi+TEARDOWN_DONE=1# M3: on the success path a failed cadence restore must FAIL the gate# (restore_hcc_env_strict registers fail() and leaves HCC_ENV_SAVED for# the trap's best-effort retry); the EXIT trap keeps the lenient restore.restore_hcc_env_strict+if ! finalize_hcc_watch_gate_lock "$TEARDOWN_CLEANUP_FAILED" "$HCC_RESTORE_OK"; then+ fail "shared HCC fault-injection lock was retained fail-closed; follow the printed repair instructions"+else+ ok "exclusive shared HCC fault-injection lock finalized"+fiprint_resultsif [[ "$g_fail" -gt 0 ]]; thendiff --git a/scripts/e2e/e2e-wrc-hcc-contracts.sh b/scripts/e2e/e2e-wrc-hcc-contracts.shindex e0b4c5029..570f0f923 100755--- a/scripts/e2e/e2e-wrc-hcc-contracts.sh+++ b/scripts/e2e/e2e-wrc-hcc-contracts.sh@@ -271,6 +271,8 @@ spec:agent:provider: zaimodel: glm-4.7+ triggers:+ onDemand: {}steps:- id: tool-stepinstruction: "Synthetic status-contract step for toolsCalled args validation."@@ -296,7 +298,7 @@ elsefirm -f "$running_response_file"-if running_status_json="$(wait_for_running_workflowrecipe_status)"; then+if wait_for_running_workflowrecipe_status >/dev/null; thenok "WorkflowRecipe becomes active when workflow execution is running"elsefail "WorkflowRecipe did not become active on running workflow execution"@@ -427,7 +429,7 @@ elsefirm -f "$pending_response_file"-if pending_status_json="$(wait_for_pending_preserves_non_terminal_recipe_status)"; then+if wait_for_pending_preserves_non_terminal_recipe_status >/dev/null; thenok "WorkflowRecipe stays non-terminal when workflow execution is pending"elsefail "WorkflowRecipe did not stay non-terminal on pending workflow execution"diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.shindex 35a61e913..f72cbca00 100755--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh@@ -18,6 +18,8 @@ BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"+SOURCE_DEPLOYMENT="$SOURCE_ID"+BACKEND_DEPLOYMENT="$BACKEND_ID"CREATED=0cleanup() {@@ -28,9 +30,9 @@ cleanup() {--ignore-not-found --wait=false >/dev/null 2>&1 || status=1wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \>/dev/null 2>&1 || status=1- kctl delete deployment "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \+ kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \--ignore-not-found --wait=false >/dev/null 2>&1 || status=1- kctl delete service "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \+ kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \--ignore-not-found >/dev/null 2>&1 || status=1kctl delete networkpolicy -n "$SANDBOX_NS" \-l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \@@ -85,6 +87,21 @@ wait_internal_ready() {return 1}+wait_for_workload_instance() {+ local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance+ while [ "$elapsed" -lt "$timeout" ]; do+ instance="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \+ -o "jsonpath={.status.workloadInstances.${workload_id}}" 2>/dev/null || true)"+ if [ -n "$instance" ]; then+ printf '%s\n' "$instance"+ return 0+ fi+ sleep "$POLL_INTERVAL"+ elapsed=$((elapsed + POLL_INTERVAL))+ done+ return 1+}+policy_refs() {kctl get networkpolicy -A -l "$1" \-o go-template='{{range .items}}{{.metadata.namespace}}/{{.metadata.name}}{{"\n"}}{{end}}' \@@ -214,18 +231,26 @@ fiok "Fixture has no egressBindings shortcut"header "Phase 2 - WRC reconciliation"-wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID" "$TIMEOUT_POD" && ok "Source deployment ready" || {+SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID" "$TIMEOUT_POD")" || {+ fail "Source workload instance was not assigned"+ exit 1+}+BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")" || {+ fail "Backend workload instance was not assigned"+ exit 1+}+wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {fail "Source deployment not ready"exit 1}-wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID" "$TIMEOUT_POD" && ok "Backend deployment ready" || {+wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {fail "Backend deployment not ready"exit 1}wait_internal_ready "$TIMEOUT_POD"-resolved_target="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"-expected_target="http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"+resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"+expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {fail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"exit 1@@ -240,7 +265,7 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"header "Phase 4 - Positive packet flow"-positive_output="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- \+positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \ok "WRC source reached backend through inferred internal dependency" || {@@ -282,7 +307,7 @@ wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod}if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \--timeout="$CONNECT_TIMEOUT" --tries=1 \- "http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then+ "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; thenfail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"exit 1fidiff --git a/scripts/e2e/measure-host-bundle-reconcile.sh b/scripts/e2e/measure-host-bundle-reconcile.shindex 1eb66d287..c1f7ee5a7 100755--- a/scripts/e2e/measure-host-bundle-reconcile.sh+++ b/scripts/e2e/measure-host-bundle-reconcile.sh@@ -151,9 +151,10 @@ readonly DELPASS_RESURRECTION_SETTLE_S=30 # quiet window before re-asserting# termination grace. A child with NO deletionTimestamp never gets this# extension (the delete was never issued → immediate fail).readonly MEASURE_TEARDOWN_CONVERGE_S=120-# Exact log lines emitted by host-context-controller/src/k8sClient.ts for-# the startup full fleet pass (reason string 'initial Host reconciliation').-readonly HCC_PASS_STARTED_MARKER='Running initial Host reconciliation'+# The startup marker is emitted immediately before the cold-start Host+# inventory/watch recovery. COMPLETE/FAIL below retain the precise+# 'initial Host reconciliation' reason supplied to the fleet pass.+readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'@@ -964,9 +965,9 @@ phase_delete_during_pass() {_delpass_pass_active_probe() {local logslogs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then pass_state='completed'; return 0; fi- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_STARTED_MARKER"; then pass_state='active'; return 0; fi+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then pass_state='completed'; return 0; fi+ if grep -Fq -- "$HCC_PASS_STARTED_MARKER" <<<"$logs"; then pass_state='active'; return 0; fireturn 1}# shellcheck disable=SC2329 # invoked by name via wait_until@@ -1015,7 +1016,7 @@ phase_delete_during_pass() {fail "delete-during-pass: could not re-read HCC logs immediately after the delete — overlap cannot be verified (fail-closed)"return 1fi- if printf '%s' "$post_delete_logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$post_delete_logs"; thenfail "delete-during-pass: pass logged COMPLETED within the probe->delete window — overlap NOT established; re-run"return 1fi@@ -1027,8 +1028,8 @@ phase_delete_during_pass() {_delpass_pass_completed_probe() {local logslogs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi- printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER" && pass_state='completed'+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi+ grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs" && pass_state='completed'}if ! wait_until "$DELPASS_PASS_COMPLETE_DEADLINE_S" 2 "initial full pass completion after the delete" \_delpass_pass_completed_probe _delpass_logs_diag; thendiff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 0f9247a86..0186ed584 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -1012,7 +1012,7 @@ if [ "$RESET_DB" = true ]; then$KC -n control-plane rollout status deployment/trace-maintenance-worker --timeout=180s >/dev/nullfiif [ "$RESET_HCC_REPLICAS" -gt 0 ]; then- $KC -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null+ $KC -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/nullfiok "Control-api database and GFS roles converged after reset"elsediff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.shindex 71f036828..590c749c0 100755--- a/scripts/tests/test-converge-control-db-after-reset.sh+++ b/scripts/tests/test-converge-control-db-after-reset.sh@@ -51,7 +51,7 @@ case "$*" in*'rollout status deployment/workflow-recipes --timeout=180s') exit 0 ;;*'rollout status deployment/trace-maintenance-worker --timeout=180s') exit 0 ;;*'scale deployment/host-context-controller --replicas=4') exit 0 ;;- *'rollout status deployment/host-context-controller --timeout=180s') exit 0 ;;+ *'rollout status deployment/host-context-controller --timeout=900s') exit 0 ;;*'get pods -l app=workflow-recipes -o name') printf 'pod/workflow-old\n' ;;*'get pods -l app=trace-maintenance-worker -o name') printf 'pod/trace-old\n' ;;*'wait --for=delete pod -l app=workflow-recipes --timeout=180s') exit 0 ;;diff --git a/scripts/tests/test-hcc-api-gateway-readiness.sh b/scripts/tests/test-hcc-api-gateway-readiness.shnew file mode 100755index 000000000..ce2930e86--- /dev/null+++ b/scripts/tests/test-hcc-api-gateway-readiness.sh@@ -0,0 +1,184 @@+#!/usr/bin/env bash+# Functions under test are evaluated from the E2E source, so ShellCheck cannot+# see their use of the mock environment variables declared by each harness.+# shellcheck disable=SC2034+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+DEPLOYMENT="${ROOT}/deploy/base/control-plane/host-context-controller-api-gateway.yaml"+CONFIGMAP="${ROOT}/deploy/base/control-plane/configmaps.yaml"+BOOTSTRAP_E2E="${ROOT}/scripts/e2e/e2e-hcc-readiness-bootstrap.sh"++fail() {+ echo "FAIL: $1" >&2+ exit 1+}++readiness_probe="$(+ awk '+ /^[[:space:]]*readinessProbe:/ { in_probe = 1; next }+ in_probe && /^[[:space:]]*resources:/ { exit }+ in_probe { print }+ ' "${DEPLOYMENT}"+)"++if ! grep -Eq '^[[:space:]]*path:[[:space:]]*/health[[:space:]]*$' <<<"${readiness_probe}"; then+ fail "HCC API gateway readiness probe must use the gateway-local /health endpoint"+fi+echo "PASS: HCC API gateway readiness probe uses the gateway-local /health endpoint"++hcc_config="$(+ awk '+ BEGIN { RS = "---" }+ /name:[[:space:]]*host-context-controller-api-gateway/ { print; exit }+ ' "${CONFIGMAP}"+)"++if [[ -z "${hcc_config}" ]]; then+ fail "HCC API gateway nginx ConfigMap is missing"+fi++health_location="$(+ awk '+ /^[[:space:]]*location = \/health \{/ { in_location = 1 }+ in_location { print }+ in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }+ ' <<<"${hcc_config}"+)"++if [[ "${health_location}" != *'return 200'* ]] ||+ [[ "${health_location}" == *'proxy_pass'* ]]; then+ fail "HCC API gateway /health must remain a local nginx health response"+fi+echo "PASS: HCC API gateway /health remains local to nginx"++ready_location="$(+ awk '+ /^[[:space:]]*location = \/ready \{/ { in_location = 1 }+ in_location { print }+ in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }+ ' <<<"${hcc_config}"+)"++if [[ "${ready_location}" != *'proxy_pass http://host_context_controller_upstream;'* ]]; then+ fail "HCC API gateway /ready must continue proxying the HCC readiness contract"+fi+echo "PASS: HCC API gateway /ready continues proxying HCC readiness"++runtime_hold_function="$(+ sed -n '/^hcc_gateway_remains_ready_without_hcc() {$/,/^}$/p' "${BOOTSTRAP_E2E}"+)"+unavailable_probe_function="$(+ sed -n '/^hcc_gateway_ready_proxy_unavailable() {$/,/^}$/p' "${BOOTSTRAP_E2E}"+)"+gateway_pod_function="$(+ sed -n '/^running_hcc_gateway_pod() {$/,/^}$/p' "${BOOTSTRAP_E2E}"+)"+if [[ "${gateway_pod_function}" != *'ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"'* ]] ||+ [[ "${gateway_pod_function}" == *'running_pod_name'* ]]; then+ fail "gateway runtime proof must select a Ready pod through the shared E2E helper"+fi+echo "PASS: gateway runtime proof selects a Ready pod through the shared helper"++if [[ "${runtime_hold_function}" != *'hcc_gateway_deployment_ready'* ]] ||+ [[ "${runtime_hold_function}" != *'hcc_gateway_local_health_ok'* ]] ||+ [[ "${runtime_hold_function}" != *'hcc_gateway_ready_proxy_unavailable'* ]] ||+ ! grep -Fq 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}"; then+ fail "bootstrap E2E must hold HCC unavailable across a full gateway probe-failure window"+fi+echo "PASS: bootstrap E2E proves gateway readiness remains local while HCC is unavailable"++# Exercise the full conjunction with a bounded synthetic clock. Each required+# observation must independently fail the hold; this kills a permissive+# && -> || mutation without requiring a cluster or a real sleep.+gateway_hold_case() (+ local failed_observation=$1 clock_file+ clock_file="$(mktemp "${TMPDIR:-/tmp}/hcc-gateway-clock.XXXXXX")"+ printf '%s' 0 >"$clock_file"+ trap 'rm -f "$clock_file"' EXIT+ HCC_NS=control-plane+ date() {+ local calls+ calls="$(<"$clock_file")"+ calls=$((calls + 1))+ printf '%s' "$calls" >"$clock_file"+ [ "$calls" -le 2 ] && printf '%s\n' 0 || printf '%s\n' 1+ }+ sleep() { :; }+ hcc_gateway_deployment_ready() {+ [ "$failed_observation" != deployment ]+ }+ hcc_gateway_local_health_ok() {+ [ "$failed_observation" != health ]+ }+ hcc_gateway_ready_proxy_unavailable() {+ [ "$failed_observation" != proxy ]+ }+ eval "${runtime_hold_function}"+ hcc_gateway_remains_ready_without_hcc 1+)++if gateway_hold_case none &&+ ! gateway_hold_case deployment &&+ ! gateway_hold_case health &&+ ! gateway_hold_case proxy; then+ echo "PASS: gateway hold requires every readiness, health, and proxy-unavailable observation"+else+ fail "gateway hold can accept a failed readiness, health, or proxy observation"+fi++for unavailable_status in 502 503 504; do+ if (+ HCC_NS=control-plane+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }+ kctl() {+ printf ' HTTP/1.1 %s upstream unavailable\n' "${unavailable_status}"+ return 8+ }+ eval "${unavailable_probe_function}"+ hcc_gateway_ready_proxy_unavailable+ ); then+ echo "PASS: gateway hold accepts explicit upstream-unavailable HTTP ${unavailable_status}"+ else+ fail "gateway hold rejects expected upstream-unavailable HTTP ${unavailable_status}"+ fi+done++if (+ HCC_NS=control-plane+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }+ kctl() {+ printf '%s\n' 'error: unable to upgrade connection: container not found'+ return 1+ }+ eval "${unavailable_probe_function}"+ ! hcc_gateway_ready_proxy_unavailable+); then+ echo "PASS: gateway hold rejects kubectl or container transport failure"+else+ fail "gateway hold can mistake kubectl or container transport failure for upstream unavailability"+fi++if (+ HCC_NS=control-plane+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }+ kctl() {+ printf '%s\n' 'healthy upstream response'+ return 0+ }+ eval "${unavailable_probe_function}"+ ! hcc_gateway_ready_proxy_unavailable+); then+ echo "PASS: gateway hold rejects a successful proxied /ready response"+else+ fail "gateway hold can accept a successful proxied /ready response as unavailable"+fi++hold_line="$(grep -nF 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}" | cut -d: -f1)"+restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "${BOOTSTRAP_E2E}" | cut -d: -f1)"+recovery_line="$(grep -nF 'hcc_gateway_ready_proxy_recovers ||' "${BOOTSTRAP_E2E}" | cut -d: -f1)"+if [[ -z "${hold_line}" || -z "${restart_line}" || -z "${recovery_line}" ]] ||+ (( hold_line >= restart_line || restart_line >= recovery_line )); then+ fail "gateway unavailable/ready/recovery assertions must surround the HCC restart"+fi+echo "PASS: bootstrap E2E proves proxied /ready failure and recovery in causal order"diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.shindex 50104032d..379d362e6 100755--- a/scripts/tests/test-hcc-watch-recovery-gate.sh+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh@@ -1,21 +1,39 @@#!/usr/bin/env bash+# Values in the eval-based fixture harness are consumed by extracted gate+# functions, which static shell analysis cannot follow.+# shellcheck disable=SC2034set -uFAIL=0ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"SCRIPT_DIR="${ROOT}/scripts/e2e"-GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"+WATCH_GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"+READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-readiness-bootstrap.sh"+MCP_READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-mcp-context-readiness.sh"+HOST_BUNDLE_MEASURE="${SCRIPT_DIR}/measure-host-bundle-reconcile.sh"+HOST_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"+GATES=("$WATCH_GATE" "$READINESS_GATE" "$MCP_READINESS_GATE")LOCK_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"LOG_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"+FIXTURE_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"+HCC_K8S_CLIENT="${ROOT}/host-context-controller/src/k8sClient.ts"+HCC_EGRESS_COORDINATOR="$(+ printf '%s' "${ROOT}/host-context-controller/src/externalEgressConvergenceCoordinator.ts"+)"+HCC_NETWORK_POLICY_RECONCILER="$(+ printf '%s' "${ROOT}/host-context-controller/src/networkPolicyReconciler.ts"+)"+WATCH_GATE_TARGET="$(sed -n '/^test-e2e-hcc-communicationchannel-watch-recovery:/,/^$/p' "${ROOT}/Makefile")"MOCK_STATE_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-lock-test.XXXXXX")"MOCK_LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-log-test.XXXXXX")"+MOCK_PROFILE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hcc-profile-test.XXXXXX")"rm -f "$MOCK_STATE_FILE"-trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"' EXIT+trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"; rm -rf "$MOCK_PROFILE_ROOT"' EXITpass() { echo "PASS: $1"; }fail() { echo "FAIL: $1"; FAIL=1; }-for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; do+for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; doif bash -n "$script"; thenpass "$(basename "$script") has valid bash syntax"else@@ -23,12 +41,1282 @@ for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; dofidone+# A node carrying a minikube label from another profile must not satisfy the+# destructive-gate boundary. Each gate binds the label value to the explicit+# context instead of accepting any non-null label.+matching_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"branch-profile"}}}]}'+mismatched_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}}}]}'+if jq -e --arg context branch-profile \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \+ <<<"$matching_nodes" >/dev/null &&+ ! jq -e --arg context branch-profile \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \+ <<<"$mismatched_nodes" >/dev/null &&+ [ "$(grep -Fl 'minikube.k8s.io/name"] == $context' "${GATES[@]}" | wc -l | tr -d ' ')" = 3 ] &&+ ! grep -Fq 'minikube.k8s.io/name"] != null' "${GATES[@]}"; then+ pass "all destructive HCC gates bind the minikube node label to the explicit profile"+else+ fail "an HCC gate can accept a minikube node owned by another profile"+fi++# Literal source-code assertions.+# shellcheck disable=SC2016+if [[ "$WATCH_GATE_TARGET" == *'E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"'* ]] &&+ [[ "$WATCH_GATE_TARGET" == *'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)'* ]]; then+ pass "CommunicationChannel gate target propagates exact pre-gate and profile ownership"+else+ fail "CommunicationChannel gate target omits exact pre-gate or profile ownership"+fi++proxy_rollout_function="$(+ sed -n '/^print_hcc_proxy_rollout_diagnostics() {$/,/^}$/p' "$WATCH_GATE"+)"+# Literal source-code assertions.+# shellcheck disable=SC2016+if [[ "$proxy_rollout_function" == *'kctl get pods'* ]] &&+ [[ "$proxy_rollout_function" == *'--previous --tail=120'* ]] &&+ [[ "$proxy_rollout_function" == *'status.containerStatuses'* ]] &&+ [[ "$proxy_rollout_function" == *'// "unknown"'* ]] &&+ [[ "$proxy_rollout_function" != *'\\"unknown\\"'* ]] &&+ grep -Fq 'if ! kctl rollout status deployment "$HCC_DEPLOY"' "$WATCH_GATE" &&+ [ "$(grep -Fc 'print_hcc_proxy_rollout_diagnostics' "$WATCH_GATE")" = 2 ]; then+ pass "proxy rollout failure emits bounded HCC pod and previous-container diagnostics"+else+ fail "proxy rollout failure can discard the distinguishing HCC startup evidence"+fi++if (+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ if [ "$1" = get ] && [ "$2" = pods ] && [[ "$*" == *"-o jsonpath="* ]]; then+ printf '%s\n' hcc-pod+ elif [ "$1" = get ] && [ "$2" = pods ]; then+ printf '%s\n' '{"items":[{"metadata":{"name":"hcc-pod"},"status":{"phase":"Pending","containerStatuses":[{"name":"host-context-controller","ready":false,"restartCount":2,"state":{"waiting":{"reason":"CrashLoopBackOff"}},"lastState":{}}]}}]}'+ elif [ "$1" = logs ]; then+ printf '%s\n' 'bounded diagnostic log'+ else+ return 1+ fi+ }+ eval "$proxy_rollout_function"+ output="$(print_hcc_proxy_rollout_diagnostics 2>&1)"+ [[ "$output" == *"pod=hcc-pod phase=Pending"* ]] &&+ [[ "$output" != *"pod=null"* ]] &&+ [[ "$output" != *"phase=null"* ]]+); then+ pass "proxy rollout diagnostics retain the parent pod identity and phase"+else+ fail "proxy rollout diagnostics lose pod metadata inside container status iteration"+fi++# The proxy changes only the HCC client's target hostname. The HCC's+# allow-k8s-api-egress policies must retain the Kubernetes-injected API IP,+# which is the explicit fail-closed configuration seam for CIDR policy data.+# shellcheck disable=SC2016+if grep -Fq 'K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY"' "$WATCH_GATE" &&+ grep -Fq 'K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"' "$WATCH_GATE" &&+ grep -Fq -- '--arg api_cidr "$K8S_API_CIDR"' "$WATCH_GATE" &&+ grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr' "$WATCH_GATE" &&+ grep -Fq 'allow-k8s-api-egress-${MCP_NS}' "$WATCH_GATE" &&+ grep -Fq '[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ]' "$WATCH_GATE" &&+ grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS-' "$FIXTURE_HELPER"; then+ pass "proxy fault injection preserves an explicit valid API egress CIDR and restores it"+else+ fail "proxy fault injection can turn a DNS hostname into an invalid API egress CIDR"+fi++if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' "$READINESS_GATE" &&+ [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$READINESS_GATE")" = 1 ] &&+ ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$READINESS_GATE" &&+ ! grep -Fq 'cluster_fingerprint_file=' "$READINESS_GATE" &&+ ! grep -Fq 'profile_env=' "$READINESS_GATE"; then+ pass "readiness gate delegates ownership, HEAD, profile, fingerprint, and gate proof once"+else+ fail "readiness gate duplicates, bypasses, or splits the shared branch-owned proof"+fi++bootstrap_repair_function="$(+ sed -n '/^print_repair_instructions() {$/,/^}$/p' "$READINESS_GATE"+)"+mcp_repair_function="$(+ sed -n '/^print_repair_instructions() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+# Literal source-code assertions: every fail-closed repair path must restore+# the captured replica count before treating rollout health as repair proof.+# shellcheck disable=SC2016+if [[ "$bootstrap_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&+ [[ "$bootstrap_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=180s'* ]] &&+ [[ "$mcp_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&+ [[ "$mcp_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=240s'* ]] &&+ grep -Fq 'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}' "$WATCH_GATE" &&+ grep -Fq 'rollout status deployment/${HCC_DEPLOY} --timeout=180s' "$WATCH_GATE"; then+ pass "all HCC repair instructions restore the original replica count and verify rollout"+else+ fail "an HCC repair path can leave the controller scaled to zero"+fi++fixture_gate_function="$(sed -n '/^require_branch_owned_hcc_gate() {$/,/^}$/p' "$FIXTURE_HELPER")"+marker_get_count="$(+ grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' <<<"$fixture_gate_function"+)"+# Literal source-code assertions.+# shellcheck disable=SC2016+if [ "$marker_get_count" = 1 ] &&+ [[ "$fixture_gate_function" == *'-o json'* ]] &&+ [[ "$fixture_gate_function" == *'.data.worktreeId'* ]] &&+ [[ "$fixture_gate_function" == *'.data.gitHead'* ]] &&+ [[ "$fixture_gate_function" == *'<<<"$HCC_BRANCH_GATE_SYNC_MARKER"'* ]]; then+ pass "shared branch gate derives ownership and HEAD from one exported ConfigMap JSON snapshot"+else+ fail "shared branch gate can split ownership and HEAD across marker reads"+fi++# Literal source-code assertions.+# shellcheck disable=SC2016+if [[ "$fixture_gate_function" == *'expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"'* ]] &&+ [[ "$fixture_gate_function" == *'[ -n "$expected_branch" ]'* ]] &&+ [[ "$fixture_gate_function" == *'[ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ]'* ]] &&+ [[ "$fixture_gate_function" == *'profile_env="${E2E_BRANCH_PROFILE_ENV:-'* ]] &&+ [[ "$fixture_gate_function" == *'profile_env_value PROFILE'* ]] &&+ [[ "$fixture_gate_function" == *'profile_env_value REPO_DIR'* ]] &&+ [[ "$fixture_gate_function" == *'profile_env_value BRANCH'* ]] &&+ [[ "$fixture_gate_function" != *'profile_env_value SHA_SHORT'* ]] &&+ [[ "$fixture_gate_function" == *'profile_env_value DIRTY'* ]] &&+ [[ "$fixture_gate_function" == *'cluster_fingerprint_file='* ]] &&+ [[ "$fixture_gate_function" == *'.data.clusterFingerprint'* ]] &&+ [[ "$fixture_gate_function" == *'.data.gate'* ]] &&+ [[ "$fixture_gate_function" == *'E2E_EXPECTED_PRE_GATE_GATE'* ]]; then+ pass "shared branch gate owns the complete profile, fingerprint, and expected-gate proof"+else+ fail "shared branch gate omits a required profile, fingerprint, or expected-gate boundary"+fi++for gate in "${GATES[@]}"; do+ marker_reads="$(grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' "$gate")"+ if [ "$marker_reads" = 0 ] &&+ ! grep -Fq 'profile_env=' "$gate" &&+ ! grep -Fq 'cluster_fingerprint_file=' "$gate" &&+ grep -Fq 'require_branch_owned_hcc_gate' "$gate"; then+ pass "$(basename "$gate") delegates its branch-owned proof to the shared authority"+ else+ fail "$(basename "$gate") duplicates or bypasses the shared branch-owned proof"+ fi+done++run_branch_gate_case() (+ local mode=$1+ local repo_root="${MOCK_PROFILE_ROOT}/repo"+ local context="clerum-codex-profile-test-1234abcd"+ local head="1111111111111111111111111111111111111111"+ local short_head="11111111"+ local branch="codex/profile-test"+ local fingerprint="2222222222222222222222222222222222222222"+ local marker_fingerprint="$fingerprint"+ local expected_gate="full"+ local marker_gate="$expected_gate"+ local minikube_profile="$context"+ local profile="$context"+ local profile_repo="$repo_root"+ local profile_branch="$branch"+ local profile_sha="$short_head"+ local profile_dirty=false+ local mock_worktree_dirty=""+ local worktree_id+ worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"++ case "$mode" in+ success) ;;+ detached) branch="" ;;+ minikube-profile-missing) minikube_profile="" ;;+ minikube-profile) minikube_profile="clerum-other-profile-1234abcd" ;;+ profile-name) profile="clerum-other-profile-1234abcd" ;;+ profile-repo) profile_repo="${repo_root}-other" ;;+ profile-branch) profile_branch="codex/other" ;;+ reused-profile) profile_sha="aaaaaaaa" ;;+ profile-dirty) profile_dirty=true ;;+ dirty-worktree) mock_worktree_dirty=" M changed.ts" ;;+ fingerprint) marker_fingerprint="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;+ gate) marker_gate="targeted" ;;+ *) exit 98 ;;+ esac++ mkdir -p "${MOCK_PROFILE_ROOT}/state/${worktree_id}"+ printf '%s\n' "$fingerprint" >"${MOCK_PROFILE_ROOT}/state/${worktree_id}/cluster.sha"+ printf '%s\n' \+ "PROFILE=${profile}" \+ "REPO_DIR=${profile_repo}" \+ "BRANCH=${profile_branch}" \+ "SHA_SHORT=${profile_sha}" \+ "DIRTY=${profile_dirty}" >"${MOCK_PROFILE_ROOT}/profile.env"++ E2E_KUBECONTEXT="$context"+ MINIKUBE_PROFILE="$minikube_profile"+ E2E_BRANCH_PROFILE_ENV="${MOCK_PROFILE_ROOT}/profile.env"+ E2E_PRE_GATE_STATE_ROOT="${MOCK_PROFILE_ROOT}/state"+ E2E_EXPECTED_PRE_GATE_GATE="$expected_gate"+ HCC_NS=control-plane+ SCRIPT_DIR="${ROOT}/scripts/e2e"+ MOCK_MARKER="$(+ jq -cn --arg worktreeId "$worktree_id" --arg gitHead "$head" \+ --arg clusterFingerprint "$marker_fingerprint" --arg gate "$marker_gate" \+ '{data:{worktreeId:$worktreeId,gitHead:$gitHead,+ clusterFingerprint:$clusterFingerprint,gate:$gate}}'+ )"++ is_branch_scoped_e2e_context() { return 0; }+ die() { exit 97; }+ git() {+ local command=$3+ shift 3+ case "$command $*" in+ "rev-parse --show-toplevel") printf '%s\n' "$repo_root" ;;+ "status --porcelain --untracked-files=normal") printf '%s\n' "$mock_worktree_dirty" ;;+ "rev-parse HEAD") printf '%s\n' "$head" ;;+ "rev-parse --short=8 HEAD") printf '%s\n' "$short_head" ;;+ "branch --show-current") printf '%s\n' "$branch" ;;+ *) return 96 ;;+ esac+ }+ kctl() { printf '%s\n' "$MOCK_MARKER"; }+ # shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+ source "$FIXTURE_HELPER"+ require_branch_owned_hcc_gate "$HCC_NS"+)++if run_branch_gate_case success; then+ pass "shared branch gate accepts matching branch-profile evidence"+else+ fail "shared branch gate rejects matching branch-profile evidence"+fi+if run_branch_gate_case reused-profile; then+ pass "shared branch gate accepts a reused same-branch profile after an exact-head pre-gate sync"+else+ fail "shared branch gate rejects a reused same-branch profile despite exact-head pre-gate evidence"+fi+for case_name in detached minikube-profile-missing minikube-profile profile-name profile-repo profile-branch \+ profile-dirty dirty-worktree fingerprint gate; do+ if run_branch_gate_case "$case_name" >/dev/null 2>&1; then+ fail "shared branch gate accepted mismatched ${case_name} evidence"+ else+ pass "shared branch gate rejects mismatched ${case_name} evidence"+ fi+done++retry_progress_function="$(+ sed -n '/^external_egress_retry_progress_is_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+retry_query_function="$(+ sed -n '/^dns_retry_query_observed_since_schedule() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+if (+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ CONTEXT_NAME=e2e-context+ MOCK_HCC_LOG='+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout+[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms+'+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }+ eval "$retry_progress_function"+ external_egress_retry_progress_is_observed+); then+ pass "DNS failure is attributed to the dedicated external-egress retry lane"+else+ fail "the injected DNS hold cannot prove dedicated external-egress retry progress"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ CONTEXT_NAME=e2e-context+ MOCK_HCC_LOG='+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout+[K8s] Scheduling external egress retry 1/3 for McpServer "different-server" in 5000ms+[K8s] unrelated message for McpServer "e2e-held-server"+'+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }+ eval "$retry_progress_function"+ ! external_egress_retry_progress_is_observed+); then+ pass "external-egress retry evidence cannot combine a peer retry with the fixture server marker"+else+ fail "external-egress retry evidence can be assembled from different log lines or servers"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ CONTEXT_NAME=e2e-context+ MOCK_HCC_LOG='+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout+[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms+'+ for ((line = 0; line < 4096; line++)); do+ MOCK_HCC_LOG+=$'\n[K8s] unrelated high-volume log line'+ done+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }+ eval "$retry_progress_function"+ external_egress_retry_progress_is_observed+); then+ pass "an early matching retry line survives a high-volume log buffer without SIGPIPE ambiguity"+else+ fail "a valid early retry marker is lost in a high-volume log buffer"+fi+if (+ DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3+ dns_hold_count() { printf '%s\n' 4; }+ eval "$retry_query_function"+ dns_retry_query_observed_since_schedule+); then+ pass "a post-schedule DNS query proves the dedicated retry executed"+else+ fail "a fresh DNS query after scheduling does not prove retry execution"+fi+if (+ DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3+ dns_hold_count() { printf '%s\n' 3; }+ eval "$retry_query_function"+ ! dns_retry_query_observed_since_schedule+); then+ pass "a pre-schedule DNS query cannot satisfy retry execution"+else+ fail "stale DNS evidence can masquerade as retry execution"+fi+if grep -Fq '{name:"timeout",value:"30"}' "$MCP_READINESS_GATE" &&+ grep -Fq '{name:"attempts",value:"1"}' "$MCP_READINESS_GATE"; then+ pass "the DNS fixture disables resolver retransmits that could counterfeit retry execution"+else+ fail "the DNS fixture can mistake a resolver retransmit for a dedicated retry"+fi++# Full marker inventory: every runtime log marker any HCC gate consumes, paired+# with the producer statement that emits it. Both sides are matched as fixed+# source text, never as a rendered string: a template literal is pinned by its+# fixed prefix plus the variable that supplies the dynamic identity, so a+# marker that only ever exists rendered (`${config.hostNamespace}`,+# `${reason}`, `${type}`, `${snapshot.channels.length}`) still resolves without+# reporting false drift. Matching the consumer side too means a rename on+# either side breaks the pair instead of silently disarming a live gate: the+# marker no longer matches any log line, the count stays zero, and the gate+# passes on evidence it never observed.+# Entries are flat 4-tuples: consumer file, consumer text, producer file,+# producer text.+# shellcheck disable=SC2016+marker_bindings=(+ "$WATCH_GATE" 'CommunicationChannel watch ended;'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'++ "$WATCH_GATE" 'Starting CommunicationChannel watch'+ "$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'++ "$WATCH_GATE" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'+ "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'++ "$WATCH_GATE" 'cache recovery failed;'+ "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'++ "$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'+ "$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'++ "$WATCH_GATE" 'Listing all Hosts in namespace ${HOST_NS}'+ "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'++ "$WATCH_GATE" 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'++ "$WATCH_GATE" 'CommunicationChannel watch event: DELETED for ${FIXTURE_CHANNEL}'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'++ "$WATCH_GATE" 'CommunicationChannel watch event: ADDED for ${FIXTURE_CHANNEL}'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'++ "$LOG_HELPER" 'CommunicationChannel watch ended;'+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'++ "$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'+ "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'++ "$LOG_HELPER" 'Listing all Hosts in namespace '+ "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'++ "$LOG_HELPER" 'Host(s) for lifecycle after CommunicationChannel recovery'+ "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) for lifecycle after ${reason}'++ "$LOG_HELPER" 'Host(s) after Host watch recovery convergence'+ "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) after ${reason}'++ "$LOG_HELPER" 'Completed Host reconciliation after CommunicationChannel recovery$'+ "$HCC_K8S_CLIENT" "requestHostFleetReconcile('CommunicationChannel recovery'"++ "$LOG_HELPER" 'Completed Host reconciliation after Host watch recovery convergence$'+ "$HCC_K8S_CLIENT" "convergenceReason = 'Host watch recovery convergence'"++ "$LOG_HELPER" 'Periodic resync'+ "$HCC_K8S_CLIENT" "requestHostFleetReconcile('Periodic resync'"++ "$READINESS_GATE" "START_MARKER='Starting initial Host background convergence'"+ "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'++ "$READINESS_GATE" "COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'"+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'++ "$READINESS_GATE" "FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'"+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'++ "$READINESS_GATE" 'after initial Host reconciliation'+ "$HCC_K8S_CLIENT" "'initial Host reconciliation',"++ "$HOST_BUNDLE_MEASURE" "HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'"+ "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'++ "$HOST_BUNDLE_MEASURE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'++ "$HOST_BUNDLE_MEASURE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'++ "$HOST_STORM_GATE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'++ "$HOST_STORM_GATE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'++ "$MCP_READINESS_GATE" 'Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};'+ "$HCC_EGRESS_COORDINATOR" 'Initial external egress reconciliation failed for ${key};'++ "$MCP_READINESS_GATE" 'Scheduling external egress retry '+ "$HCC_EGRESS_COORDINATOR" 'Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}'++ "$MCP_READINESS_GATE" 'for McpServer \"${MCP_NAME}\"'+ "$HCC_EGRESS_COORDINATOR" 'for McpServer "${this.retryIntents.get(key)!.server.name}"'++ "$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'+ "$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'++ "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'+ "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('++ "$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'"+ "$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'++ "$MCP_READINESS_GATE" "NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'"+ "$HCC_K8S_CLIENT" 'Initial NetworkPolicy post-certification additive reconciliation failed:'++ "$MCP_READINESS_GATE" "NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'"+ "$HCC_K8S_CLIENT" 'Scheduling initial ${lane} background convergence retry'++ "$MCP_READINESS_GATE" "NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '"+ "$HCC_K8S_CLIENT" 'NetworkPolicy reconciliation failed for context ${context.name}:'+)+marker_drift=""+marker_binding_count=0+for ((marker_index = 0; marker_index < ${#marker_bindings[@]}; marker_index += 4)); do+ consumer_file="${marker_bindings[marker_index]}"+ consumer_marker="${marker_bindings[marker_index + 1]}"+ producer_file="${marker_bindings[marker_index + 2]}"+ producer_marker="${marker_bindings[marker_index + 3]}"+ marker_binding_count=$((marker_binding_count + 1))+ # `grep -Fq -- ""` matches every file, so an empty marker would disarm this+ # pair while still reporting a pass. Refuse it instead of searching for it.+ if [ -z "$consumer_marker" ] || [ -z "$producer_marker" ]; then+ marker_drift+=" marker pair at index ${marker_index} has an empty marker"$'\n'+ continue+ fi+ if ! grep -Fq -- "$consumer_marker" "$consumer_file"; then+ marker_drift+=" consumer $(basename "$consumer_file") no longer reads: ${consumer_marker}"$'\n'+ fi+ if ! grep -Fq -- "$producer_marker" "$producer_file"; then+ marker_drift+=" producer $(basename "$producer_file") no longer emits: ${producer_marker}"$'\n'+ fi+done+# The stride loop consumes 4 elements per pair, so an array whose length is not+# a multiple of 4 silently shifts every entry after the edit and leaves the tail+# unset. Assert the arity explicitly rather than relying on the pair count to+# happen to notice.+if [ $((${#marker_bindings[@]} % 4)) -eq 0 ] &&+ [ "$marker_binding_count" -eq 35 ] &&+ [ -z "$marker_drift" ]; then+ pass "all ${marker_binding_count} consumed HCC log markers stay bound to their runtime producers"+else+ printf '%s' "$marker_drift" >&2+ fail "a consumed HCC log marker drifted from its runtime producer (${marker_binding_count} pairs checked)"+fi++# Completeness fence: every static single-quoted *_MARKER defined in the e2e+# gates MUST be inventoried above as a consumer_marker, so a NEW marker cannot be+# born outside the drift fence (the pinned count only catches removals). The scan+# covers both the gates themselves and the _lib/ helpers they source — a marker+# born in a sourced helper is the same escape class. Scoped to single-quoted+# literals — a dynamic id like C_MARKER="refusal-c-${RUN_ID}" (double-quoted, a+# per-run test id, not a cross-file HCC log marker) is not the class this fence+# covers and is excluded by the quote style, not a hard-coded allow-list.+marker_is_inventoried() {+ local needle="$1" i+ for ((i = 1; i < ${#marker_bindings[@]}; i += 4)); do+ [ "${marker_bindings[i]}" = "$needle" ] && return 0+ done+ return 1+}+uninventoried_markers=""+while IFS= read -r marker_def; do+ [ -n "$marker_def" ] || continue+ marker_is_inventoried "$marker_def" ||+ uninventoried_markers+=" ${marker_def}"$'\n'+done < <(+ grep -rhoE "^[[:space:]]*(readonly[[:space:]]+)?[A-Z_]+_MARKER='[^']*'" \+ "${SCRIPT_DIR}"/*.sh "${SCRIPT_DIR}"/_lib/*.sh |+ sed -E "s/^[[:space:]]*(readonly[[:space:]]+)?//"+)+if [ -z "$uninventoried_markers" ]; then+ pass "every single-quoted *_MARKER defined in the e2e gates is inventoried in the drift fence"+else+ printf '%s' "$uninventoried_markers" >&2+ fail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"+fi++fixture_mcp_runtime_absent_function="$(+ sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+hcc_kubernetes_readiness_function="$(+ sed -n '/^hcc_kubernetes_readiness_is_exact() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+external_policy_protocol_function="$(+ sed -n '/^external_egress_policy_converged_with_protocol() {$/,/^}$/p' \+ "$MCP_READINESS_GATE"+)"+hcc_ready_after_revoke_function="$(+ sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+ready_probe_function="$(+ sed -n '/^probe_new_hcc_ready_endpoint() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"++run_ready_probe_failure_case() (+ local expected=$1+ MOCK_READY_PROBE_OUTPUT=$2+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ HCC_PORT=8080+ HCC_READY_PROBE_DIAGNOSTIC=""+ kctl() {+ printf '%s\n' "$MOCK_READY_PROBE_OUTPUT"+ return 1+ }+ eval "$ready_probe_function"+ ! probe_new_hcc_ready_endpoint &&+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = "HCC_READY_PROBE category=${expected}" ]+)++if run_ready_probe_failure_case timeout 'HCC_READY_PROBE category=timeout' &&+ run_ready_probe_failure_case transport 'HCC_READY_PROBE category=transport' &&+ run_ready_probe_failure_case 'http-status status=503' \+ 'HCC_READY_PROBE category=http-status status=503' &&+ run_ready_probe_failure_case json-ready-false \+ 'HCC_READY_PROBE category=json-ready-false'; then+ pass "ready probe preserves distinct timeout, transport, HTTP, and ready=false diagnostics"+else+ fail "ready probe collapses a failure boundary into an indiagnostic boolean"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ HCC_PORT=8080+ HCC_READY_PROBE_DIAGNOSTIC="stale"+ kctl() { return 0; }+ eval "$ready_probe_function"+ probe_new_hcc_ready_endpoint && [ -z "$HCC_READY_PROBE_DIAGNOSTIC" ]+); then+ pass "successful ready probe clears stale failure diagnostics"+else+ fail "successful ready probe retains a misleading prior diagnostic"+fi++# H6 gate-fix mutation (E4): the clean-window NetworkPolicy negative must FAIL+# against an injection-window log (moving it inside the injected window would+# false-fail — the scoping is load-bearing), PASS against a clean log, and fail+# closed when kubectl errors. The gate itself can't run (minikube churn), so+# this statically pins the predicate.+clean_window_netpol_negatives_function="$(+ sed -n '/^clean_window_netpol_log_negatives_hold() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+clean_window_marker_defs="$(grep -E "^NETPOL_[A-Z_]+_MARKER='" "$MCP_READINESS_GATE")"+run_clean_window_negative_case() (+ local mode=$1+ CONTEXT_NAME=alpha+ HCC_NS=control-plane+ eval "$clean_window_marker_defs"+ case $mode in+ injected)+ kctl() {+ printf '%s\n' \+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed: dns timeout' \+ '[K8s] Scheduling initial NetworkPolicy background convergence retry 1 in 5000ms'+ } ;;+ clean) kctl() { printf '%s\n' '[K8s] NetworkPolicy convergence certified'; } ;;+ api-error) kctl() { return 1; } ;;+ esac+ eval "$clean_window_netpol_negatives_function"+ clean_window_netpol_log_negatives_hold hcc-pod+)+if ! run_clean_window_negative_case injected &&+ run_clean_window_negative_case clean &&+ ! run_clean_window_negative_case api-error; then+ pass "clean-window NetworkPolicy negative fails inside the injected window, passes clean, and fails closed"+else+ fail "clean-window NetworkPolicy negative does not distinguish injected-window failures or fails open"+fi++# Static ordering fence: a clean-window negative invocation must exist in the+# MAIN BODY (after `trap cleanup EXIT`) and before the DNS mutation. cleanup()+# is defined earlier in the file than the mutation but runs on EXIT, so a plain+# "first line < mutation line" test would be satisfied by the cleanup-window+# invocation alone; excluding everything at or before the trap line pins the+# pre-injection (window A) invocation specifically.+trap_line="$(grep -n '^trap cleanup EXIT' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"+dns_mutation_line="$(grep -n '^HCC_MUTATED=1' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"+body_clean_window_negative_line="$(+ grep -n 'clean_window_netpol_log_negatives_hold "' "$MCP_READINESS_GATE" |+ awk -F: -v t="${trap_line:-0}" -v m="${dns_mutation_line:-0}" \+ '$1 > t && $1 < m { print $1; exit }'+)"+if [ -n "$body_clean_window_negative_line" ]; then+ pass "clean-window NetworkPolicy negative runs in the body before the DNS fault injection"+else+ fail "clean-window NetworkPolicy negative is missing from the pre-injection body window"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_NS=control-plane+ HCC_PORT=8080+ HCC_READY_PROBE_DIAGNOSTIC=""+ kctl() {+ printf '%s\n' 'untrusted kubectl output containing super-secret material'+ return 1+ }+ eval "$ready_probe_function"+ ! probe_new_hcc_ready_endpoint &&+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=kubectl-exec' ]+); then+ pass "ready probe replaces untrusted exec output with a bounded sanitized category"+else+ fail "ready probe can leak untrusted exec output into diagnostics"+fi+# Literal source-code assertions guarantee one request and bounded categories,+# without logging the response body, headers, URL, or transport error text.+if [ "$(grep -Fc 'kctl exec' <<<"$ready_probe_function")" = 1 ] &&+ [[ "$ready_probe_function" == *'finish(4, "timeout")'* ]] &&+ [[ "$ready_probe_function" == *'finish(5, "transport")'* ]] &&+ [[ "$ready_probe_function" == *'finish(3, "http-status"'* ]] &&+ [[ "$ready_probe_function" == *'finish(3, "json-ready-false")'* ]] &&+ [[ "$ready_probe_function" == *"printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s'"* ]] &&+ [[ "$ready_probe_function" != *'console.error(body'* ]] &&+ [[ "$ready_probe_function" != *'error.message'* ]]; then+ pass "ready probe uses one request and emits only bounded non-secret failure categories"+else+ fail "ready probe duplicates requests or exposes an unbounded diagnostic surface"+fi++if (+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ MCP_SERVER_LABEL=clerum.io/mcp-server+ POLICY_TYPE_LABEL=clerum.io/policy-type+ fixture_runtime_absent() { return 0; }+ kctl() { return 0; }+ eval "$fixture_mcp_runtime_absent_function"+ fixture_mcp_runtime_absent+); then+ pass "fixture absence predicate accepts an absent runtime and external-egress policy"+else+ fail "fixture absence predicate rejects an actually absent runtime"+fi+if (+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ MCP_SERVER_LABEL=clerum.io/mcp-server+ POLICY_TYPE_LABEL=clerum.io/policy-type+ fixture_runtime_absent() { return 0; }+ kctl() { printf '%s\n' 'networkpolicy.networking.k8s.io/stale-policy'; }+ eval "$fixture_mcp_runtime_absent_function"+ ! fixture_mcp_runtime_absent+); then+ pass "fixture absence predicate rejects a surviving external-egress policy"+else+ fail "fixture absence predicate can hide a surviving external-egress policy"+fi+if (+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ MCP_SERVER_LABEL=clerum.io/mcp-server+ POLICY_TYPE_LABEL=clerum.io/policy-type+ fixture_runtime_absent() { return 1; }+ kctl() { return 0; }+ eval "$fixture_mcp_runtime_absent_function"+ ! fixture_mcp_runtime_absent+); then+ pass "fixture absence predicate rejects a surviving MCP runtime"+else+ fail "fixture absence predicate can hide a surviving MCP runtime"+fi++if (+ NEW_HCC_POD=hcc-pod+ HCC_UID=pod-uid+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ case "$2" in+ pod)+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'+ ;;+ deployment)+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'+ ;;+ *) return 1 ;;+ esac+ }+ eval "$hcc_kubernetes_readiness_function"+ hcc_kubernetes_readiness_is_exact+); then+ pass "exact Kubernetes readiness accepts one current Ready HCC identity"+else+ fail "exact Kubernetes readiness rejects current Ready HCC evidence"+fi+# Kubelet reports an indeterminate pod as Ready=Unknown, not Ready=False. Only+# an Unknown fixture separates the exact `== "True"` readiness contract from a+# weakened `!= "False"` one, so this red case pins the exact comparison instead+# of the single status value both spellings already reject. Every other field is+# healthy, which leaves the Ready condition as the only reason to refuse.+if (+ NEW_HCC_POD=hcc-pod+ HCC_UID=pod-uid+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ case "$2" in+ pod)+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'+ ;;+ deployment)+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'+ ;;+ *) return 1 ;;+ esac+ }+ eval "$hcc_kubernetes_readiness_function"+ ! hcc_kubernetes_readiness_is_exact+); then+ pass "exact Kubernetes readiness rejects a current pod whose Ready condition is Unknown"+else+ fail "exact Kubernetes readiness can accept an indeterminate HCC pod"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_UID=pod-uid+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ case "$2" in+ pod)+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"False"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'+ ;;+ deployment)+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'+ ;;+ *) return 1 ;;+ esac+ }+ eval "$hcc_kubernetes_readiness_function"+ ! hcc_kubernetes_readiness_is_exact+); then+ pass "exact Kubernetes readiness rejects a current pod whose Ready condition is False"+else+ fail "exact Kubernetes readiness can accept a non-Ready HCC pod"+fi+# The container-level readiness clause has its own red case: a pod whose Ready+# condition is True while the host-context-controller container is not ready+# must still be refused, so deleting or weakening that clause cannot pass.+if (+ NEW_HCC_POD=hcc-pod+ HCC_UID=pod-uid+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ case "$2" in+ pod)+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":false}]}}'+ ;;+ deployment)+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'+ ;;+ *) return 1 ;;+ esac+ }+ eval "$hcc_kubernetes_readiness_function"+ ! hcc_kubernetes_readiness_is_exact+); then+ pass "exact Kubernetes readiness rejects a Ready pod whose HCC container is not ready"+else+ fail "exact Kubernetes readiness can accept an unready host-context-controller container"+fi+if (+ NEW_HCC_POD=hcc-pod+ HCC_UID=pod-uid+ HCC_NS=control-plane+ HCC_DEPLOY='host-context-controller'+ kctl() {+ case "$2" in+ pod)+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'+ ;;+ deployment)+ printf '%s\n' '{"metadata":{"generation":5},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'+ ;;+ *) return 1 ;;+ esac+ }+ eval "$hcc_kubernetes_readiness_function"+ ! hcc_kubernetes_readiness_is_exact+); then+ pass "exact Kubernetes readiness rejects a stale Deployment generation"+else+ fail "exact Kubernetes readiness can accept stale Deployment status"+fi++if (+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ MCP_SERVER_LABEL=clerum.io/mcp-server+ POLICY_TYPE_LABEL=clerum.io/policy-type+ MANAGED_BY_LABEL=app.kubernetes.io/managed-by+ MANAGED_BY_VALUE='host-context-controller'+ EXTERNAL_EGRESS_CIDR=203.0.113.10/32+ EXTERNAL_EGRESS_PORT=443+ kctl() {+ printf '%s\n' '{"items":[{"metadata":{"labels":{"app.kubernetes.io/managed-by":"host-context-controller","clerum.io/policy-type":"external-egress","clerum.io/mcp-server":"e2e-held-server"}},"spec":{"podSelector":{"matchLabels":{"clerum.io/mcp-server":"e2e-held-server"}},"policyTypes":["Egress"],"egress":[{"ports":[{"port":443,"protocol":"UDP"}],"to":[{"ipBlock":{"cidr":"203.0.113.10/32"}}]}]}}]}'+ }+ eval "$external_policy_protocol_function"+ external_egress_policy_converged_with_protocol UDP &&+ ! external_egress_policy_converged_with_protocol TCP+); then+ pass "external-egress convergence binds the exact current protocol"+else+ fail "external-egress convergence can accept a stale protocol"+fi++mcpserver_current_ready_function="$(+ sed -n '/^mcpserver_current_ready() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+mcpserver_current_ready_absent_function="$(+ sed -n '/^mcpserver_current_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+fixture_runtime_and_ready_absent_function="$(+ sed -n '/^fixture_runtime_and_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+fixture_runtime_converged_function="$(+ sed -n '/^fixture_runtime_converged_with_protocol() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+peer_fleet_converged_function="$(+ sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+fixture_converged_function="$(+ sed -n '/^fixture_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+if (+ MCP_NS=mcp-server+ GENERATION=7+ OBSERVED_GENERATION=7+ kctl() {+ printf '{"metadata":{"generation":%s},"status":{"conditions":[{"type":"Ready","status":"True","observedGeneration":%s}]}}\n' \+ "$GENERATION" "$OBSERVED_GENERATION"+ }+ eval "$mcpserver_current_ready_function"+ eval "$mcpserver_current_ready_absent_function"+ mcpserver_current_ready current &&+ ! mcpserver_current_ready_absent current &&+ OBSERVED_GENERATION=6 &&+ ! mcpserver_current_ready stale &&+ mcpserver_current_ready_absent stale+) &&+ [[ "$fixture_runtime_and_ready_absent_function" == *'mcpserver_current_ready_absent "$MCP_NAME"'* ]] &&+ [[ "$fixture_runtime_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]] &&+ [[ "$peer_fleet_converged_function" == *'mcpserver_current_ready "$server"'* ]] &&+ [[ "$fixture_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]]; then+ pass "McpServer runtime predicates accept only Ready status observed for the current generation"+else+ fail "McpServer runtime predicates can accept stale Ready status or bypass the canonical check"+fi++if (+ MCP_NS=mcp-server+ MCP_NAME=e2e-held-server+ fixture_runtime_absent() { return 0; }+ kctl() { return 1; }+ eval "$mcpserver_current_ready_absent_function"+ eval "$fixture_runtime_and_ready_absent_function"+ ! fixture_runtime_and_ready_absent+); then+ pass "McpServer Ready absence fails closed when the API read is unavailable"+else+ fail "an unavailable McpServer API read can masquerade as Ready absence"+fi++if (+ probe_new_hcc_ready_endpoint() { return 0; }+ external_egress_policy_absent() { return 0; }+ eval "$hcc_ready_after_revoke_function"+ hcc_ready_after_stale_policy_revoked+); then+ pass "divergent DNS hold permits global readiness only after stale-policy revocation"+else+ fail "divergent DNS hold rejects globally safe readiness evidence"+fi+if (+ HCC_READY_PROBE_DIAGNOSTIC=""+ probe_new_hcc_ready_endpoint() { return 0; }+ external_egress_policy_absent() { return 1; }+ eval "$hcc_ready_after_revoke_function"+ rc=0+ hcc_ready_after_stale_policy_revoked || rc=$?+ # A 200 observed WITH the stale policy present is a fatal ordering violation+ # (rc=2), not a benign retryable "not ready" (rc=1).+ [ "$rc" -eq 2 ] &&+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=ready-200-with-stale-policy-present' ]+); then+ pass "global readiness evidence flags a surviving divergent policy as a fatal rc=2 ordering violation"+else+ fail "global readiness can self-confirm (or only soft-fail) while a divergent policy survives"+fi+# The loop must PROPAGATE the fatal rc=2 immediately, not retry until a later+# clean poll turns the violation into a false PASS (the round-2 bug).+wait_until_fast_function="$(+ sed -n '/^wait_until_fast() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+if (+ HCC_READY_PROBE_DIAGNOSTIC=""+ POLL=0+ probe_new_hcc_ready_endpoint() { return 0; }+ # Stale policy present for the first polls, revoked only later. The buggy+ # retry loop would poll past the fatal first observation and PASS here.+ external_egress_policy_absent() { POLL=$((POLL + 1)); [ "$POLL" -gt 4 ]; }+ eval "$hcc_ready_after_revoke_function"+ eval "$wait_until_fast_function"+ rc=0+ wait_until_fast 10 "rc=2 propagation" hcc_ready_after_stale_policy_revoked || rc=$?+ # rc must be the fatal 2, and it must have stopped inside the first predicate+ # poll (which calls external_egress_policy_absent twice) — POLL<3 proves no+ # retry occurred.+ [ "$rc" -eq 2 ] && [ "$POLL" -lt 3 ]+); then+ pass "wait_until_fast propagates a fatal rc=2 on the first poll instead of retrying to a false PASS"+else+ fail "wait_until_fast retried past a fatal ordering violation to a later clean poll"+fi++# M6: the third contract signal adjudicates a real /metrics scrape. Extract the+# PURE predicate (the wrapper does kctl exec, untestable here) and feed it+# synthetic scrapes. The family-name literals are exactly what a metric rename+# would break — a rename leaves the vi.fn() unit tests green but must turn this+# and the real gate red.+safety_pass_metrics_predicate="$(+ sed -n '/^safety_pass_metrics_are_certified() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+[ -n "$safety_pass_metrics_predicate" ] ||+ fail "could not extract safety_pass_metrics_are_certified from the readiness gate"+metrics_with_revocation='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="listed"} 5+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 1'+metrics_without_family='clerum_hcc_some_other_family_total 3'+metrics_zero_revoked='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 0'+if (+ eval "$safety_pass_metrics_predicate"+ safety_pass_metrics_are_certified "$metrics_with_revocation"+); then+ pass "safety_pass metrics predicate certifies a scrape with >=1 revoked policy"+else+ fail "safety_pass metrics predicate rejected a valid certifying scrape"+fi+if (+ eval "$safety_pass_metrics_predicate"+ ! safety_pass_metrics_are_certified "$metrics_without_family"+); then+ pass "safety_pass metrics predicate rejects a scrape missing the duration family (rename guard)"+else+ fail "safety_pass metrics predicate passed with the safety_pass family absent"+fi+if (+ eval "$safety_pass_metrics_predicate"+ ! safety_pass_metrics_are_certified "$metrics_zero_revoked"+); then+ pass "safety_pass metrics predicate rejects a scrape with zero revoked policies"+else+ fail "safety_pass metrics predicate passed with zero policies revoked"+fi+if (+ probe_new_hcc_ready_endpoint() { return 1; }+ external_egress_policy_absent() { return 0; }+ eval "$hcc_ready_after_revoke_function"+ ! hcc_ready_after_stale_policy_revoked+); then+ pass "stale-policy revocation alone cannot substitute for global readiness"+else+ fail "divergent hold evidence can pass without HCC readiness"+fi++baseline_policy_snapshot_function="$(+ sed -n '/^baseline_policy_snapshot() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+baseline_policies_unchanged_function="$(+ sed -n '/^baseline_policies_unchanged() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+valid_baseline_policies='{"items":[+ {"metadata":{"name":"deny-all-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"default-deny"}},"spec":{"podSelector":{},"policyTypes":["Ingress","Egress"]}},+ {"metadata":{"name":"allow-dns-egress-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"infrastructure"}},"spec":{"podSelector":{},"policyTypes":["Egress"],"egress":[{"ports":[{"port":53,"protocol":"UDP"},{"port":53,"protocol":"TCP"}]}]}},+ {"metadata":{"name":"allow-host-context-controller-api","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"allow-api"}},"spec":{"podSelector":{"matchLabels":{"app":"host-context-controller"}},"policyTypes":["Ingress"]}}+]}'+if (+ MCP_NS=mcp-server+ MANAGED_BY_VALUE='host-context-controller'+ MOCK_POLICY_JSON="$valid_baseline_policies"+ kctl() { printf '%s\n' "$MOCK_POLICY_JSON"; }+ eval "$baseline_policy_snapshot_function"+ eval "$baseline_policies_unchanged_function"+ BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)"+ baseline_policies_unchanged &&+ MOCK_POLICY_JSON="$(jq -c '+ .items[0].metadata.labels["clerum.io/managed-by"] = "foreign-controller"+ ' <<<"$valid_baseline_policies")" &&+ ! baseline_policies_unchanged &&+ MOCK_POLICY_JSON="$(jq -c '.items[1].spec.egress[0].ports[0].port = 5353' \+ <<<"$valid_baseline_policies")" &&+ ! baseline_policies_unchanged+); then+ pass "baseline safety proof rejects ownership and spec drift under stable policy names"+else+ fail "baseline safety proof can pass after ownership or spec drift"+fi++peer_fleet_policies_absent_function="$(+ sed -n '/^peer_fleet_policies_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+fixture_resources_absent_function="$(+ sed -n '/^fixture_resources_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+delete_peer_fleet_runtime_function="$(+ sed -n '/^delete_peer_fleet_runtime() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+if (+ MCP_NS=mcp-server+ HOST_NS=mcp-host+ RPC_PROXY_NS=rpc-proxy+ MCP_SERVER_LABEL=clerum.io/mcpserver+ POLICY_TYPE_LABEL=clerum.io/policy-type+ MCP_FLEET_NAMES=(primary peer)+ CONTEXT_FLEET_NAMES=(primary-context peer-context)+ SURVIVOR_NS=""+ resource_absent() { [ "$3" != "$SURVIVOR_NS" ]; }+ kctl() { return 0; }+ eval "$peer_fleet_policies_absent_function"+ peer_fleet_policies_absent &&+ SURVIVOR_NS=mcp-host &&+ ! peer_fleet_policies_absent &&+ SURVIVOR_NS=rpc-proxy &&+ ! peer_fleet_policies_absent+) &&+ [[ "$fixture_resources_absent_function" == *'peer_fleet_policies_absent'* ]] &&+ [[ "$delete_peer_fleet_runtime_function" == *'"${context_policy}-egress"'* ]] &&+ [[ "$delete_peer_fleet_runtime_function" == *'"rpc-egress-${context}-${server}"'* ]]; then+ pass "peer fleet cleanup rejects surviving mcp-host or rpc-proxy policies"+else+ fail "peer fleet cleanup can release ownership with residual derived policies"+fi+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.shsource "$LOG_HELPER"HCC_LOG_BUFFER="$MOCK_LOG_FILE"START_TIME=2026-07-14T00:00:00ZHOST_NS=mcp-host+blocker_holds_function="$(sed -n '/^blocker_holds_token_request() {$/,/^}$/p' "$READINESS_GATE")"+blocker_count_function="$(sed -n '/^blocker_fixture_request_count() {$/,/^}$/p' "$READINESS_GATE")"+if (+ BLOCKER_NAME=readiness-blocker+ HCC_NS=control-plane+ FIXTURE_HOST_PREFIX=e2e-hcc-ready-test+ TOKEN_REQUEST_LOG_PREFIX='holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host='+ kctl() { cat "$MOCK_LOG_FILE"; }+ eval "$blocker_holds_function"+ eval "$blocker_count_function"+ printf '%s\n' \+ 'holding-token GET /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-01' \+ 'holding-token POST /wrong/path host=e2e-hcc-ready-test-02' \+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-04' \+ >"$MOCK_LOG_FILE"+ [ "$(blocker_fixture_request_count)" = 2 ] &&+ blocker_holds_token_request+); then+ pass "blocker evidence counts unique fixtures only on the canonical token issuance method and path"+else+ fail "wrong-method, wrong-path, or duplicate blocker traffic can satisfy the fleet evidence"+fi++create_fixtures_function="$(sed -n '/^create_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"+if (+ RUN_ID=fixture-failure+ SUITE_NAME=hcc-readiness-bootstrap+ HOST_NS=mcp-host+ MCP_NS=mcp-server+ FIXTURE_SECRET=e2e-secret+ FIXTURE_CONTEXT=e2e-context+ FIXTURE_HOST_PREFIX=e2e-host+ FIXTURE_HOST_COUNT=4+ FIXTURE_HOST_NAMES=()+ FIXTURES_CREATED=0+ KCTL_APPLY_COUNT=0+ kctl() {+ KCTL_APPLY_COUNT=$((KCTL_APPLY_COUNT + 1))+ cat >/dev/null+ [ "$KCTL_APPLY_COUNT" -ne 3 ]+ }+ eval "$create_fixtures_function"+ if create_host_fixtures; then+ exit 1+ fi+ [ "$KCTL_APPLY_COUNT" = 3 ]+); then+ pass "an intermediate Host apply failure aborts fixture creation even from an OR-list"+else+ fail "fixture creation can continue or report success after an intermediate apply failure"+fi++bootstrap_runtime_absent_function="$(+ sed -n '/^fixture_runtime_absent() {$/,/^}$/p' "$READINESS_GATE"+)"+bootstrap_delete_fixtures_function="$(+ sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE"+)"+# Literal source-code assertions for macOS bash 3.2, where expanding an empty+# array under set -u terminates the shell before cleanup can release its lock.+# shellcheck disable=SC2016+if [[ "$bootstrap_runtime_absent_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0'* ]] &&+ [[ "$bootstrap_delete_fixtures_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"'* ]]; then+ pass "bootstrap cleanup guards empty fixture arrays before bash 3.2 expansion"+else+ fail "bootstrap cleanup can abort on an empty fixture array before lock finalization"+fi++create_peer_fleet_function="$(+ sed -n '/^create_peer_fleet() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+peer_flag_line="$(grep -nF ' PEER_FLEET_CREATED=1' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"+peer_loop_line="$(grep -nF ' for ((index = 1;' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"+if [ -n "$peer_flag_line" ] && [ -n "$peer_loop_line" ] &&+ [ "$peer_flag_line" -lt "$peer_loop_line" ]; then+ pass "peer fleet cleanup is armed before the first partial apply can fail"+else+ fail "peer fleet cleanup is armed only after all applies complete"+fi++divergent_ready_function="$(+ sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+divergent_ready_line="$(+ grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+divergent_context_line="$(+ grep -nF 'wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+divergent_release_line="$(+ grep -nF 'release_dns_blocker ||' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1+)"+udp_policy_line="$(+ grep -nF 'wait_until 120 "current UDP external-egress policy after DNS release"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+# shellcheck disable=SC2016+if [[ "$divergent_ready_function" == *'probe_new_hcc_ready_endpoint || return 1'* ]] &&+ [[ "$divergent_ready_function" == *'external_egress_policy_absent'* ]] &&+ [ -n "$divergent_ready_line" ] && [ -n "$divergent_context_line" ] &&+ [ -n "$divergent_release_line" ] && [ -n "$udp_policy_line" ] &&+ [ "$divergent_ready_line" -lt "$divergent_context_line" ] &&+ [ "$divergent_context_line" -lt "$divergent_release_line" ] &&+ [ "$divergent_release_line" -lt "$udp_policy_line" ] &&+ ! grep -Fq 'probe_new_hcc_unready_endpoint' "$MCP_READINESS_GATE" &&+ ! grep -Fq '/ready is 503' "$MCP_READINESS_GATE"; then+ pass "divergent DNS hold proves global readiness, fail-closed runtime isolation, and post-release UDP convergence"+else+ fail "the MCP gate still couples global readiness to divergent DNS replacement"+fi++initial_tcp_policy_line="$(+ grep -nF 'wait_until 180 "HCC to create the real initial TCP external-egress policy"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+# B3 retain-identity-stable contract: a same-intent restart RETAINS the+# identity-stable DNS-derived allow (no deny window), certifies /ready with it+# still present, keeps the affected runtime blocked, and converges the additive+# TCP lane only AFTER the DNS hold releases — the inverse of the pre-B3+# revoke-before-ready ordering this gate previously asserted.+same_intent_snapshot_line="$(+ grep -nF 'SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)"' \+ "$MCP_READINESS_GATE" | head -1 | cut -d: -f1+)"+same_intent_retained_ready_line="$(+ grep -nF 'wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+same_intent_retain_guard_line="$(+ grep -nF 'the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+same_intent_runtime_blocked_line="$(+ grep -nF 'the affected runtime or Ready status escaped the held same-intent DNS boundary' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+same_intent_release_line="$(+ grep -nF 'release_dns_blocker || die "could not release the same-intent DNS query"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+same_intent_tcp_policy_line="$(+ grep -nF 'wait_until 120 "current TCP external-egress policy after same-intent DNS release"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+divergent_revoke_line="$(+ grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \+ "$MCP_READINESS_GATE" | cut -d: -f1+)"+if [ -n "$initial_tcp_policy_line" ] && [ -n "$same_intent_snapshot_line" ] &&+ [ -n "$same_intent_retained_ready_line" ] && [ -n "$same_intent_retain_guard_line" ] &&+ [ -n "$same_intent_runtime_blocked_line" ] && [ -n "$same_intent_release_line" ] &&+ [ -n "$same_intent_tcp_policy_line" ] && [ -n "$divergent_revoke_line" ] &&+ [ "$initial_tcp_policy_line" -lt "$same_intent_snapshot_line" ] &&+ [ "$same_intent_snapshot_line" -lt "$same_intent_retained_ready_line" ] &&+ [ "$same_intent_retained_ready_line" -lt "$same_intent_retain_guard_line" ] &&+ [ "$same_intent_retain_guard_line" -lt "$same_intent_runtime_blocked_line" ] &&+ [ "$same_intent_runtime_blocked_line" -lt "$same_intent_release_line" ] &&+ [ "$same_intent_release_line" -lt "$same_intent_tcp_policy_line" ] &&+ [ "$same_intent_tcp_policy_line" -lt "$divergent_revoke_line" ] &&+ grep -Fq 'hcc_ready_with_identity_stable_policy_retained' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'HCC readiness after revoking the same-intent DNS policy' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'a real same-intent external-egress retry to be scheduled' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'CERTIFIED_POLICY_IDENTITY' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'external_egress_policy_identity' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'UID/spec' "$MCP_READINESS_GATE"; then+ pass "same-intent safety retains the identity-stable DNS policy at readiness; additive TCP convergence follows DNS release"+else+ fail "the MCP readiness gate no longer proves B3 retain-identity-stable (or regressed to revoke-same-intent before readiness)"+fi+printf '%s\n' \'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \@@ -41,6 +1329,30 @@ elsefail "valid LIST-to-recovery ordering was rejected"fi+printf '%s\n' \+ '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] Listing all Hosts in namespace mcp-host' \+ '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \+ '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \+ '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"+if recovery_cycle_used_fresh_host_inventory 1 5; then+ pass "a causally ordered Host-watch pass may cover the recovered CC and fresh Host inventories"+else+ fail "a valid covering Host-watch recovery pass was rejected"+fi++printf '%s\n' \+ '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \+ '[K8s] Listing all Hosts in namespace mcp-host' \+ '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \+ '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \+ '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"+if recovery_cycle_used_fresh_host_inventory 1 5; then+ fail "a Host-watch pass selected before CC recovery satisfied the covering assertion"+else+ pass "covering Host-watch recovery requires recovered CC state before pass selection"+fi+printf '%s\n' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \@@ -82,19 +1394,272 @@ elsefail "the second recovery cycle was not isolated from the first"fi-acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock || die' "$GATE" | cut -d: -f1)"+large_log_tail="$(awk 'BEGIN {+ for (i = 1; i <= 20000; i++) {+ printf "trailing-log-line-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n", i+ }+}')"+large_active_snapshot="${START_MARKER:-Starting initial Host background convergence}+${large_log_tail}"+if hcc_log_snapshot_contains \+ "$large_active_snapshot" 'Starting initial Host background convergence'; then+ pass "buffered marker lookup survives a log tail larger than a pipe buffer"+else+ fail "large trailing logs can hide a marker from buffered lookup"+fi++if hcc_initial_pass_snapshot_is_active \+ "$large_active_snapshot" \+ 'Starting initial Host background convergence' \+ 'Completed Host reconciliation after initial Host reconciliation' \+ 'Host reconciliation after initial Host reconciliation failed'; then+ pass "Host pass snapshot is active only with START and no terminal marker"+else+ fail "valid active Host pass snapshot was rejected"+fi++for terminal_marker in \+ 'Completed Host reconciliation after initial Host reconciliation' \+ 'Host reconciliation after initial Host reconciliation failed'; do+ if hcc_initial_pass_snapshot_is_active \+ "${large_active_snapshot}+${terminal_marker}" \+ 'Starting initial Host background convergence' \+ 'Completed Host reconciliation after initial Host reconciliation' \+ 'Host reconciliation after initial Host reconciliation failed'; then+ fail "terminal Host marker '${terminal_marker}' can still satisfy the active-pass guard"+ else+ pass "terminal Host marker '${terminal_marker}' invalidates the active-pass guard"+ fi+done++readiness_log_function="$(+ sed -n '/^hcc_log_contains() {$/,/^}$/p' "$READINESS_GATE"+)"+running_pod_function="$(+ sed -n '/^running_hcc_pod() {$/,/^}$/p' "$READINESS_GATE"+)"+# Literal source-code assertions.+# shellcheck disable=SC2016+if [[ "$readiness_log_function" == *'logs="$(hcc_logs "$pod")"'* ]] &&+ [[ "$readiness_log_function" == *'hcc_log_snapshot_contains "$logs" "$marker"'* ]] &&+ [[ "$running_pod_function" == *'rows="$(kctl get pods'* ]] &&+ [[ "$running_pod_function" == *'<<<"$rows"'* ]]; then+ pass "readiness log and Running-pod selection buffer producers before early-exit consumers"+else+ fail "readiness log or Running-pod selection reintroduced a pipefail/SIGPIPE path"+fi++active_guard_function="$(+ sed -n '/^initial_host_pass_is_active() {$/,/^}$/p' "$READINESS_GATE"+)"+initial_host_startup_producer="$(+ sed -n \+ '/Starting initial Host background convergence/,/const resyncSec = config.hostResyncIntervalSec/p' \+ "$HCC_K8S_CLIENT"+)"+# The bootstrap guard's negative COMPLETE/FAIL checks are safe only while all+# three consumed markers remain bound to the runtime templates and the startup+# call supplies the exact reason that renders the terminal pair.+if grep -Fq 'Starting initial Host background convergence...' \+ "$HCC_K8S_CLIENT" &&+ grep -Fq 'Completed Host reconciliation after ${reason}' \+ "$HCC_K8S_CLIENT" &&+ grep -Fq 'Host reconciliation after ${reason} failed:' \+ "$HCC_K8S_CLIENT" &&+ [[ "$initial_host_startup_producer" == *'await this.recoverHostInventoryAndWatch('* ]] &&+ [[ "$initial_host_startup_producer" == *"'initial Host reconciliation',"* ]]; then+ pass "bootstrap START/COMPLETE/FAIL markers remain bound to the initial Host producer reason"+else+ fail "bootstrap Host pass markers drifted from their runtime producer or reason binding"+fi++# The destructive Host-bundle measurement uses the same cold-start transition+# as the bootstrap readiness gate. Keep its active-pass marker tied to the+# emitted producer rather than an obsolete reason-string log line.+if grep -Fq "readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'" \+ "$HOST_BUNDLE_MEASURE" &&+ grep -Fq 'Starting initial Host background convergence...' "$HCC_K8S_CLIENT"; then+ pass "Host-bundle measurement START marker remains bound to the cold-start producer"+else+ fail "Host-bundle measurement can wait for a stale HCC START marker"+fi+# shellcheck disable=SC2016+probe_result_line="$(grep -nF 'probe_result="$(kctl exec' "$READINESS_GATE" | cut -d: -f1)"+# shellcheck disable=SC2016+final_active_guard_line="$(+ grep -nF 'initial_host_pass_is_active "$new_hcc_pod"' "$READINESS_GATE" |+ tail -1 |+ cut -d: -f1+)"+# Literal source-code assertions.+# shellcheck disable=SC2016+if [[ "$active_guard_function" == *'"$COMPLETE_MARKER"'* ]] &&+ [[ "$active_guard_function" == *'"$FAIL_MARKER"'* ]] &&+ [ -n "$probe_result_line" ] &&+ [ -n "$final_active_guard_line" ] &&+ [ "$final_active_guard_line" -gt "$probe_result_line" ]; then+ pass "final readiness assertion rejects both failed and completed Host passes"+else+ fail "final readiness assertion does not enforce both terminal Host markers"+fi++cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$READINESS_GATE")"+fixture_cleanup_body="$(sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"+# shellcheck disable=SC2016+cleanup_stop_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0' \+ <<<"$cleanup_body" | head -1 | cut -d: -f1)"+cleanup_fixture_line="$(grep -nF 'delete_host_fixtures' <<<"$cleanup_body" | head -1 | cut -d: -f1)"+# shellcheck disable=SC2016+cleanup_restore_line="$(grep -nF 'kctl set env deployment/"$HCC_DEPLOY"' \+ <<<"$cleanup_body" | head -1 | cut -d: -f1)"# Literal source-code assertion.# shellcheck disable=SC2016-mutation_line="$(grep -nF 'kctl get deployment "$HCC_DEPLOY"' "$GATE" | head -1 | cut -d: -f1)"+if grep -Fq 'FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))' "$READINESS_GATE" &&+ grep -Fq 'kind: Host' "$READINESS_GATE" &&+ grep -Fq "host='+host" "$READINESS_GATE" &&+ grep -Fq 'blocker_fixture_request_count' "$READINESS_GATE" &&+ [[ "$fixture_cleanup_body" == *'clerum.io/managed-by=host-context-controller,clerum.io/host=${host}'* ]] &&+ [[ "$fixture_cleanup_body" == *'fixture_resources_absent'* ]] &&+ ! grep -Fq 'RUNTIME_SECRET' "$READINESS_GATE" &&+ ! grep -Fq 'HOST_REF' "$READINESS_GATE" &&+ [ -n "$cleanup_stop_line" ] &&+ [ -n "$cleanup_fixture_line" ] &&+ [ -n "$cleanup_restore_line" ] &&+ [ "$cleanup_stop_line" -lt "$cleanup_fixture_line" ] &&+ [ "$cleanup_fixture_line" -lt "$cleanup_restore_line" ]; then+ pass "readiness gate uses a multi-wave owned Host fleet and deletes it while HCC is stopped"+else+ fail "readiness gate can mutate a real Host Secret or restore HCC before fixture deletion"+fi++if [[ "$cleanup_body" == *'header "HCC readiness bootstrap gate passed"'* ]] &&+ grep -Fq 'header "HCC readiness assertions passed; restoring branch-owned runtime"' \+ "$READINESS_GATE" &&+ [ "$(grep -Fc 'header "HCC readiness bootstrap gate passed"' "$READINESS_GATE")" = 1 ]; then+ pass "readiness gate emits its final pass banner only after verified restoration and lock finalization"+else+ fail "readiness gate can announce a final pass before cleanup and ownership release finish"+fi++mcp_cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$MCP_READINESS_GATE")"+# shellcheck disable=SC2016+mcp_input_absence_line="$(+ grep -nF 'fixture_inputs_absent' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1+)"+# shellcheck disable=SC2016+mcp_restore_line="$(+ grep -nF 'restore_patch="$(jq -cn' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1+)"+if [[ "$mcp_cleanup_body" == *'fixture_inputs_removed=0'* ]] &&+ [[ "$mcp_cleanup_body" == *'if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then'* ]] &&+ [ -n "$mcp_input_absence_line" ] &&+ [ -n "$mcp_restore_line" ] &&+ [ "$mcp_input_absence_line" -lt "$mcp_restore_line" ]; then+ pass "MCP readiness cleanup keeps HCC stopped until fixture input deletion is verified"+else+ fail "MCP readiness cleanup can restart HCC after an unverified Context or McpServer deletion"+fi++mcp_finalize_line="$(+ grep -nF 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' \+ <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1+)"+mcp_pass_line="$(+ grep -nF 'header "HCC McpServer/Context readiness gate passed"' \+ <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1+)"+if [ "$(grep -Fc 'header "HCC McpServer/Context readiness gate passed"' \+ "$MCP_READINESS_GATE")" = 1 ] &&+ [ -n "$mcp_finalize_line" ] &&+ [ -n "$mcp_pass_line" ] &&+ [ "$mcp_finalize_line" -lt "$mcp_pass_line" ]; then+ pass "MCP readiness gate announces PASS only after cleanup and lock finalization"+else+ fail "MCP readiness gate can announce PASS before cleanup and lock finalization"+fi++wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$READINESS_GATE")"# Literal source-code assertion.# shellcheck disable=SC2016-if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] && [ "$acquire_line" -lt "$mutation_line" ] &&- grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1' "$GATE"; then- pass "gate acquires before HCC access and finalizes ownership from cleanup"+if [[ "$wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&+ ! grep -Fq -- '--for=delete' "$READINESS_GATE" &&+ ! grep -Fq 'port: 8081' "$READINESS_GATE" &&+ grep -Fq 'HCC_E2E_PORT' "$READINESS_GATE"; then+ pass "readiness gate uses wall-clock deadlines, absence waits, and the deployed HCC port"+else+ fail "readiness gate retains a command-latency, zero-pod wait, or hard-coded-port edge"+fi++mcp_wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$MCP_READINESS_GATE")"+mcp_port_probe_count="$(grep -Fc 'env "HCC_E2E_PORT=${HCC_PORT}"' "$MCP_READINESS_GATE")"+mcp_pod_absence_count="$(grep -Fc 'hcc_pods_absent' "$MCP_READINESS_GATE")"+# Literal source-code assertions.+# shellcheck disable=SC2016+if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \+ "$MCP_READINESS_GATE" &&+ [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$MCP_READINESS_GATE")" = 1 ] &&+ ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'cluster_fingerprint_file=' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'profile_env=' "$MCP_READINESS_GATE" &&+ [[ "$mcp_wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&+ grep -Fq 'hcc_pods_absent' "$MCP_READINESS_GATE" &&+ ! grep -Fq -- '--for=delete' "$MCP_READINESS_GATE" &&+ ! grep -Fq 'port: 8081' "$MCP_READINESS_GATE" &&+ [ "$mcp_port_probe_count" = 4 ] &&+ [ "$mcp_pod_absence_count" = 4 ]; then+ pass "MCP readiness gate shares exact-head ownership and uses bounded portable runtime probes"+else+ fail "MCP readiness gate duplicates ownership or retains a deadline, pod-absence, or port edge"+fi++# A single held McpServer proves the minimal failure, but this gate must also+# cross the controller's bounded worker width with independent, gate-owned+# McpServer/Context pairs. Keep this as a static contract so future edits do+# not silently reduce the fleet scenario back to one fixture. The fixture+# must exist while HCC is stopped, otherwise it would not exercise startup.+mcp_peer_create_line="$(grep -nF 'create_peer_fleet || die' "$MCP_READINESS_GATE" | cut -d: -f1)"+mcp_restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1)"+peer_fleet_converged_body="$(+ sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"+)"+if grep -Fq 'MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"' "$MCP_READINESS_GATE" &&+ grep -Fq '[ "$MCP_FLEET_SIZE" -gt 10 ]' "$MCP_READINESS_GATE" &&+ grep -Fq 'create_peer_fleet' "$MCP_READINESS_GATE" &&+ grep -Fq 'peer_fleet_converged' "$MCP_READINESS_GATE" &&+ [[ "$peer_fleet_converged_body" == *'mcpserver_current_ready "$server"'* ]] &&+ [[ "$peer_fleet_converged_body" == *'context_policies_converged "$context" "$context" "$server"'* ]] &&+ grep -Fq 'all peer MCP/Context runtimes to converge after releasing the held primary egress' "$MCP_READINESS_GATE" &&+ [ -n "$mcp_peer_create_line" ] && [ -n "$mcp_restart_line" ] &&+ [ "$mcp_peer_create_line" -lt "$mcp_restart_line" ]; then+ pass "MCP readiness gate crosses the bounded worker width with gate-owned MCP and Context peers"else- fail "gate lock is not held across the full HCC fault-injection window"+ fail "MCP readiness gate does not prove a fleet larger than the worker width"fi+for gate in "${GATES[@]}"; do+ acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock' "$gate" | tail -1 | cut -d: -f1)"+ case "$gate" in+ "$WATCH_GATE")+ # Literal source-code assertion.+ # shellcheck disable=SC2016+ mutation_line="$(grep -nF 'kctl patch deployment "$HCC_DEPLOY"' "$gate" | tail -1 | cut -d: -f1)"+ ;;+ *)+ mutation_line="$(grep -nF 'HCC_MUTATED=1' "$gate" | tail -1 | cut -d: -f1)"+ ;;+ esac+ # The grep pattern intentionally matches literal shell variable references.+ # shellcheck disable=SC2016+ if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] &&+ [ "$acquire_line" -lt "$mutation_line" ] &&+ grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' "$gate"; then+ pass "$(basename "$gate") acquires before mutation and finalizes ownership from cleanup"+ else+ fail "$(basename "$gate") does not hold the shared lock across its full fault-injection window"+ fi+done+kctl() {local action=$1 body expected_rv current_rv current_uid next_rvcase "$action" in@@ -138,7 +1703,7 @@ truncate_rfc1123() { printf '%.63s' "$1"; }source "$LOCK_HELPER"HCC_NS=control-plane-HCC_DEPLOY=host-context-controller+HCC_DEPLOY="host-context-controller"E2E_KUBECONTEXT=clerum-codex-lock-test-1234abcdRUN_ID=owner-1HCC_GATE_LOCK_ACQUIRED=0@@ -253,12 +1818,62 @@ elsefiunset MOCK_CREATE_WITHOUT_UID-cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$GATE" | cut -d: -f1)"-results_line="$(grep -nF ' print_results' "$GATE" | head -1 | cut -d: -f1)"+cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$WATCH_GATE" | cut -d: -f1)"+results_line="$(grep -nF ' print_results' "$WATCH_GATE" | head -1 | cut -d: -f1)"if [ -n "$cleanup_fail_line" ] && [ -n "$results_line" ] && [ "$cleanup_fail_line" -lt "$results_line" ]; thenpass "cleanup failure is counted before the final result summary"elsefail "cleanup can still print a false all-passed summary"fi+# --- H3: SIGPIPE/pipefail sweep on HCC pass-marker log probes -------------+# A `printf '%s[\n]' "$logs" | grep -Fq "$..._MARKER"` probe is a false-green+# trap: on a large log with the marker near the top, `grep -Fq` exits early,+# `printf` takes SIGPIPE, and under `set -o pipefail` the whole pipeline reports+# failure — so a PRESENT marker reads as MISSING (deterministic 3/3 repro in the+# round-2 storm gate). The safe idiom is a here-string: no producer to signal.+# grep -Fq -- "$MARKER" <<<"$logs"+#+# Scope: HCC pass-marker gates only (storm + measure). Explicitly OUT OF SCOPE+# (tracked separately — a different failure class, or a non-HCC gate outside+# this PR's blast radius):+# - e2e-gke-secrets.sh:291-294 grep -oE|wc leak COUNT: reads the whole log,+# no early-exit, so it is NOT the SIGPIPE class.+# - e2e-plugin-workload-sdk.sh SDK markers (non-HCC).+# - e2e-lib.sh / e2e-prod-lib.sh / e2e-channel-reader-per-host.sh (non-HCC).+HCC_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"+# shellcheck disable=SC2016+sigpipe_offenders="$(+ grep -nE 'printf[^|]*\| *grep -Fq[^<]*_MARKER' \+ "$HCC_STORM_GATE" "$HOST_BUNDLE_MEASURE" 2>/dev/null || true+)"+if [ -z "$sigpipe_offenders" ]; then+ pass "HCC pass-marker probes use here-strings, not pipefail/SIGPIPE pipes"+else+ fail "HCC pass-marker probe reintroduced a printf|grep -Fq SIGPIPE path:+${sigpipe_offenders}"+fi++# Corroborating repro of the mechanism the sweep guards against. The here-string+# form MUST find a marker on line 1 of a 1.6 MB log (hard invariant). The pipe+# form is expected to miss it under pipefail (SIGPIPE) — that miss is the bug,+# demonstrated, not a standing assertion, so a platform that does not reproduce+# it is not a suite failure: the static sweep above is the real enforcement.+h3_marker='HCC_PASS_COMPLETED: h3 repro sentinel'+h3_bulk="$(yes 'xxxxxxxxxxxxxxxx' | head -c 1600000)"+h3_log="${h3_marker}+${h3_bulk}"+if grep -Fq -- "$h3_marker" <<<"$h3_log"; then+ pass "here-string probe finds a marker on line 1 of a 1.6 MB log"+else+ fail "here-string probe MISSED a present marker on a large log (regression)"+fi+h3_pipe_found=0+( set -o pipefail; printf '%s\n' "$h3_log" | grep -Fq -- "$h3_marker" ) && h3_pipe_found=1+if [ "$h3_pipe_found" -eq 0 ]; then+ pass "pipe+pipefail probe misses the same marker (SIGPIPE repro confirmed)"+else+ pass "pipe probe did not SIGPIPE on this platform; static sweep still enforces the idiom"+fi+exit "$FAIL"diff --git a/scripts/tests/test-host-storm-metric-gate.sh b/scripts/tests/test-host-storm-metric-gate.shnew file mode 100755index 000000000..e5d994bb5--- /dev/null+++ b/scripts/tests/test-host-storm-metric-gate.sh@@ -0,0 +1,296 @@+#!/usr/bin/env bash+# shellcheck disable=SC2016 # literal source-contract assertions below+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-host-storm-gate.sh"+WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/host-storm-metric-test.XXXXXX")"+ADJUDICATOR="${WORK_DIR}/adjudicator.py"+trap 'rm -rf "$WORK_DIR"' EXIT++fail() {+ echo "FAIL: $*" >&2+ exit 1+}++extract_adjudicator() {+ awk '+ /python3 >"\$METRIC_VERDICTS" <<'\''PY'\''/ { capture = 1; next }+ capture && /^PY$/ { exit }+ capture { print }+ ' "$GATE" >"$ADJUDICATOR"+ [ -s "$ADJUDICATOR" ] || fail "could not extract host-storm metric adjudicator"+}++write_metrics() {+ local path=$1 le_05=$2 le_5=$3 le_15=$4 total=$5 fleet_started=${6:-0}+ local fleet_coalesced=${7:-0} fleet_trailing=${8:-0} fleet_failed=${9:-0}+ cat >"$path" <<EOF+# TYPE clerum_hcc_host_delete_cleanup_total counter+clerum_hcc_host_delete_cleanup_total{outcome="queued"} 0+clerum_hcc_host_delete_cleanup_total{outcome="confirmed"} 0+clerum_hcc_host_delete_cleanup_total{outcome="completed"} 0+clerum_hcc_host_delete_cleanup_total{outcome="retried"} 0+clerum_hcc_host_delete_cleanup_total{outcome="superseded"} 0+# TYPE clerum_hcc_host_fleet_requests_total counter+clerum_hcc_host_fleet_requests_total{result="started"} ${fleet_started}+clerum_hcc_host_fleet_requests_total{result="coalesced"} ${fleet_coalesced}+clerum_hcc_host_fleet_requests_total{result="trailing"} ${fleet_trailing}+clerum_hcc_host_fleet_requests_total{result="failed"} ${fleet_failed}+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="0.5"} ${le_05}+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="5"} ${le_5}+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="15"} ${le_15}+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="+Inf"} ${total}+EOF+}++extract_adjudicator++for scrape in a b c1 c2 d wake_pre wake_post; do+ write_metrics "${WORK_DIR}/${scrape}.prom" 0 0 0 0+done++cat >>"${WORK_DIR}/c1.prom" <<'EOF'+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="15"} 1+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="+Inf"} 1+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="failure",le="+Inf"} 0+EOF++write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 1++# The post-recovery direct-watch window is healthy.+write_metrics "${WORK_DIR}/d.prom" 10 10 10 10++# The wake window contains one sample over the 5s hard gate plus nineteen+# fast samples. A combined p95 would be green (19/20 == 95%), but the+# supported wake journey violated its binding urgent queue budget.+write_metrics "${WORK_DIR}/wake_post.prom" 19 19 20 20++run_adjudicator() {+ local wake_post=$1 fleet_completion_proven=${2:-1}+ SCRAPE_A="${WORK_DIR}/a.prom" \+ SCRAPE_B="${WORK_DIR}/b.prom" \+ SCRAPE_C1="${WORK_DIR}/c1.prom" \+ SCRAPE_C2="${WORK_DIR}/c2.prom" \+ SCRAPE_D="${WORK_DIR}/d.prom" \+ SCRAPE_WAKE_PRE="${WORK_DIR}/wake_pre.prom" \+ SCRAPE_WAKE_POST="$wake_post" \+ QUEUE_WAIT_METRIC=clerum_hcc_host_reconcile_queue_wait_seconds \+ WATCH_RECOVERY_METRIC=clerum_hcc_host_watch_recovery_seconds \+ FLEET_METRIC=clerum_hcc_host_fleet_requests_total \+ DELETE_CLEANUP_METRIC=clerum_hcc_host_delete_cleanup_total \+ P95_BUDGET_S=5 \+ RECOVERY_BUDGET_S=15 \+ MIN_URGENT_SAMPLES=10 \+ FLEET_COMPLETION_PROVEN="$fleet_completion_proven" \+ python3 "$ADJUDICATOR"+}++OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_post.prom")"++if ! grep -Fq 'VERDICT|A2-direct-watch-urgent-p95|PASS|' <<<"$OUTPUT" &&+ ! grep -Fq 'VERDICT|A2-urgent-queue-wait-p95|PASS|' <<<"$OUTPUT"; then+ fail "healthy post-recovery direct-watch window did not pass"+fi+grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|FAIL|' <<<"$OUTPUT" ||+ fail "slow wake window was diluted into a false PASS"++write_metrics "${WORK_DIR}/wake_healthy.prom" 20 20 20 20+HEALTHY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|PASS|' <<<"$HEALTHY_OUTPUT" ||+ fail "healthy wake window did not pass its hard gate"++# A7 is fenced to the replacement pod's explicit initial-fleet COMPLETED log.+# A second non-overlapping request can therefore be legitimate activity after+# recovery; an absolute started==1 assertion must not false-red it.+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2+TWO_STARTS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$TWO_STARTS_OUTPUT" ||+ fail "healthy replacement-pod recovery false-reds when a second sequential fleet request starts"++write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0+NO_START_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$NO_START_OUTPUT" ||+ fail "A7 still treats a scheduled counter as stronger than causal completion evidence"++NO_COMPLETION_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom" 0)"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$NO_COMPLETION_OUTPUT" ||+ fail "A7 false-passed without replacement-pod fleet completion evidence"++write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 0 0 1+FAILED_PASS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$FAILED_PASS_OUTPUT" ||+ fail "A7 false-passed a failed replacement-pod fleet pass"++write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 1 2+UNBOUNDED_TRAILING_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$UNBOUNDED_TRAILING_OUTPUT" ||+ fail "A7 false-passed trailing fleet work without a matching coalesced request"++# M8: without the FLEET_METRIC family registered on the running image, counter()+# returns 0 for every result label and A7 would pass vacuously (all-zero looks+# like "no failures"). Strip the family's # TYPE line so it drops out of+# c2_types (mirrors a metric rename / an image that never registered it) and+# assert A7 FAILs on the presence check, not vacuously passes.+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0+grep -v '^# TYPE clerum_hcc_host_fleet_requests_total ' "${WORK_DIR}/c2.prom" \+ >"${WORK_DIR}/c2.nofleet.prom"+mv "${WORK_DIR}/c2.nofleet.prom" "${WORK_DIR}/c2.prom"+MISSING_FAMILY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$MISSING_FAMILY_OUTPUT" ||+ fail "A7 vacuously passed with the FLEET_METRIC family absent from the scrape"+grep -Fq 'not registered on the running image' <<<"$MISSING_FAMILY_OUTPUT" ||+ fail "A7 failed for the wrong reason when the FLEET_METRIC family was absent"++WAKE_EVIDENCE_FUNCTION="$(+ sed -n '/^_wake_urgent_evidence_recorded() {$/,/^}$/p' "$GATE"+)"+WAKE_GENERATION_FUNCTION="$(+ sed -n '/^wake_generation_is_handled() {$/,/^}$/p' "$GATE"+)"+POST_WAKE_FUNCTION="$(+ sed -n '/^post_wake() {$/,/^}$/p' "$GATE"+)"++[[ -n "$WAKE_GENERATION_FUNCTION" ]] ||+ fail "runtime wake evidence has no generation-correlation predicate"++# Exercise the exact pure predicate embedded in the E2E gate. The requested+# generation is the business identifier returned by the public wake route;+# an older handled generation must never satisfy that request.+eval "$WAKE_GENERATION_FUNCTION"+if wake_generation_is_handled 7 6; then+ fail "wake generation predicate accepted an older handled generation"+fi+wake_generation_is_handled 7 7 ||+ fail "wake generation predicate rejected the exact handled generation"+wake_generation_is_handled 7 8 ||+ fail "wake generation predicate rejected a newer handled generation"+if wake_generation_is_handled 0 8 || wake_generation_is_handled invalid 8 ||+ wake_generation_is_handled 7 invalid; then+ fail "wake generation predicate accepted malformed generation evidence"+fi++# Literal source assertions intentionally match unexpanded shell variables.+# shellcheck disable=SC2016+if ! grep -Fq '"$current_pod" == "$HCC_POD_WAKE"' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq '"$lifecycle_state" != '\''suspended'\''' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq 'handled_generation="$(wake_handled_generation)"' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq 'metric="${IN_FLIGHT_METRIC}"' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq '(after - before) >= 1 && active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then+ fail "runtime wake evidence is not fenced to the same pod, requested generation, metric delta, and idle urgent lane"+fi+generation_check_line="$(+ grep -nF 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' \+ <<<"$WAKE_EVIDENCE_FUNCTION" | cut -d: -f1+)"+metrics_scrape_line="$(+ grep -nF 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \+ <<<"$WAKE_EVIDENCE_FUNCTION" | head -n1 | cut -d: -f1+)"+if [[ -z "$generation_check_line" || -z "$metrics_scrape_line" ]] ||+ (( generation_check_line >= metrics_scrape_line )); then+ fail "wake generation is not correlated before accepting shared urgent metrics"+fi+wake_scrape_count="$(+ grep -Fc 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \+ <<<"$WAKE_EVIDENCE_FUNCTION"+)"+if (( wake_scrape_count < 2 )) ||+ ! grep -Fq 'fence_in_flight=' <<<"$WAKE_EVIDENCE_FUNCTION" ||+ ! grep -Fq 'active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then+ fail "wake evidence lacks a completed-lane fence followed by a fresh metric scrape"+fi++# shellcheck disable=SC2016+if ! grep -Fq 'WAKE_GENERATION=' <<<"$POST_WAKE_FUNCTION" ||+ ! grep -Fq '"$WAKE_STATUS" == '\''202'\''' <<<"$POST_WAKE_FUNCTION" ||+ ! grep -Fq '.status // empty' <<<"$POST_WAKE_FUNCTION" ||+ ! grep -Fq '.wakeGeneration // empty' <<<"$POST_WAKE_FUNCTION"; then+ fail "public wake response is not validated as an accepted wake-generation contract"+fi++# A7 must start from a fresh scrape immediately before the kill. Reusing the+# earlier wake-evidence scrape creates an unobserved interval in which a fleet+# failure could occur and escape the pre/post recovery counter delta.+# shellcheck disable=SC2016+if ! grep -Fq 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" ||+ grep -Fq 'cp "$SCRAPE_WAKE_POST" "$SCRAPE_B"' "$GATE"; then+ fail "pre-kill scrape B is not freshly captured after wake adjudication"+fi+fresh_scrape_line="$(+ grep -nF 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" | cut -d: -f1+)"+kill_line="$(grep -nF 'T_KILL_MS="$(now_ms)"' "$GATE" | cut -d: -f1)"+if [[ -z "$fresh_scrape_line" || -z "$kill_line" ]] ||+ (( fresh_scrape_line >= kill_line )); then+ fail "fresh scrape B does not precede the HCC kill"+fi++# The destructive Host storm must use the same exact-HEAD/profile/fingerprint+# ownership contract and exclusive compare-and-swap lock as every HCC gate.+# shellcheck disable=SC2016+for required in \+ 'source "${SCRIPT_DIR}/e2e-lib.sh"' \+ 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \+ 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"' \+ 'require_branch_owned_hcc_gate "$CONTROL_NS"' \+ 'acquire_hcc_watch_gate_lock' \+ 'finalize_hcc_watch_gate_lock'; do+ grep -Fq "$required" "$GATE" ||+ fail "Host storm omits shared HCC gate contract: ${required}"+done+grep -Fq '.metadata.labels["minikube.k8s.io/name"] == $context' "$GATE" ||+ fail "Host storm does not bind the node's exact minikube profile label"+grep -Fq 'MINIKUBE_PROFILE must explicitly select context' \+ "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" ||+ fail "shared profile ownership helper no longer binds MINIKUBE_PROFILE"++# Branch-owned URLs must come from the selected profile's ports.env. Fixed+# machine-global service or metrics ports are a cross-worktree false-green risk.+grep -Fq 'CLERUM_PROFILE_PORTS_ENV' "$GATE" ||+ fail "Host storm does not load branch-profile port evidence"+grep -Fq 'profile_env_value EXTERNAL_REST_API_URL' "$GATE" ||+ fail "Host storm does not derive External REST from branch ports.env"+grep -Fq 'profile_env_value RPC_PROXY_URL' "$GATE" ||+ fail "Host storm does not derive RPC proxy from branch ports.env"+if grep -Eq '127\.0\.0\.1:(8091|8094|18082)' "$GATE"; then+ fail "Host storm retains a fixed machine-global service or metrics port"+fi+grep -Fq '"pod/${pod}" ":${HCC_PORT}"' "$GATE" ||+ fail "HCC metric scrape no longer asks kubectl for a process-owned random port"++# KEEP_FIXTURES may be useful interactively, but it cannot produce PASS evidence+# or leave a lock-free mutated profile that could later be mistaken for T2.+keep_guard_line="$(grep -nF 'KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result' "$GATE" | cut -d: -f1)"+branch_gate_line="$(grep -nF 'require_branch_owned_hcc_gate "$CONTROL_NS"' "$GATE" | cut -d: -f1)"+if [[ -z "$keep_guard_line" || -z "$branch_gate_line" ]] ||+ (( keep_guard_line >= branch_gate_line )); then+ fail "KEEP_FIXTURES is not rejected before branch verification and mutation"+fi+if grep -Fq 'leaving fixtures in place' "$GATE"; then+ fail "Host storm can still report through the fixture-retention path"+fi++# Completion must be read from the exact replacement pod and established+# before C1/C2 metric adjudication. Merely observing `started` is not proof.+FLEET_COMPLETION_FUNCTION="$(+ sed -n '/^_replacement_fleet_completed_probe() {$/,/^}$/p' "$GATE"+)"+grep -Fq '"$current_pod" == "$HCC_POD_C"' <<<"$FLEET_COMPLETION_FUNCTION" ||+ fail "fleet completion probe is not fenced to the replacement pod"+grep -Fq '"pod/${HCC_POD_C}"' <<<"$FLEET_COMPLETION_FUNCTION" ||+ fail "fleet completion probe reads a deployment aggregate instead of the replacement pod"+grep -Fq '$HCC_PASS_COMPLETED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||+ fail "fleet completion probe does not require the explicit COMPLETED marker"+grep -Fq '$HCC_PASS_FAILED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||+ fail "fleet completion probe does not fail loud on the explicit FAILED marker"+completion_wait_line="$(grep -nF '"replacement HCC initial Host fleet reconciliation completion"' "$GATE" | cut -d: -f1)"+c1_line="$(grep -nF 'scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"' "$GATE" | head -n1 | cut -d: -f1)"+if [[ -z "$completion_wait_line" || -z "$c1_line" ]] ||+ (( completion_wait_line >= c1_line )); then+ fail "replacement fleet completion is not causally established before post-recovery metrics"+fi++echo "PASS: host-storm oracle binds exact branch/profile ownership, shared fault locking, random endpoints, wake generation, and causal fleet completion"diff --git a/scripts/tests/test-lint-networkpolicies.sh b/scripts/tests/test-lint-networkpolicies.shindex 2400d8517..3153dd5dd 100755--- a/scripts/tests/test-lint-networkpolicies.sh+++ b/scripts/tests/test-lint-networkpolicies.sh@@ -3,7 +3,23 @@ set -euo pipefailROOT="$(cd "$(dirname "$0")/../.." && pwd)"TMP_DIR="$(mktemp -d)"-trap 'rm -rf "${TMP_DIR}"' EXIT++# The minikube k8s-api-ip patch is a machine-specific, gitignored file rendered+# at setup time. On a developer box with a live minikube it may already exist;+# this test shadows it with a dummy render below. Save the real one (if any) and+# restore it on EXIT so the test leaves the working tree exactly as it found it.+MINIKUBE_API_IP_PATCH=""+MINIKUBE_API_IP_PATCH_BACKUP=""+MINIKUBE_API_IP_PATCH_PREEXISTING=0+cleanup() {+ if [[ "${MINIKUBE_API_IP_PATCH_PREEXISTING}" -eq 1 ]]; then+ cp "${MINIKUBE_API_IP_PATCH_BACKUP}" "${MINIKUBE_API_IP_PATCH}"+ elif [[ -n "${MINIKUBE_API_IP_PATCH:-}" && -f "${MINIKUBE_API_IP_PATCH}" ]]; then+ rm -f "${MINIKUBE_API_IP_PATCH}"+ fi+ rm -rf "${TMP_DIR}"+}+trap cleanup EXITwrite_policy() {local path="$1"@@ -62,7 +78,7 @@ write_policy "${TMP_DIR}/safe-specific.yaml" ' - ports:protocol: TCPto:- ipBlock:- cidr: 203.0.113.10/32'+ cidr: 198.51.100.10/32'run_lint "${TMP_DIR}/safe-specific.yaml" >/dev/nullecho "PASS: lint-networkpolicies allows explicit port plus specific destination"@@ -135,8 +151,30 @@ ${PUBLIC_EXCEPT_BLOCK}"run_lint "${TMP_DIR}/public-with-excepts.yaml" >/dev/nullecho "PASS: lint-networkpolicies allows public-only exception set with explicit ports"-GCP_DEV_RENDERED="${TMP_DIR}/gcp-dev-rendered.yaml"-kubectl kustomize "${ROOT}/deploy/overlays/gcp-dev" >"${GCP_DEV_RENDERED}"+# The minikube overlay references patches/k8s-api-ip.yaml, which is rendered+# from a gitignored template at setup time (deploy/scripts/minikube-detect-k8s-api-ip.sh)+# and absent on a fresh checkout — so `kubectl kustomize` would abort with a+# "no such file" evalsymlink error before ever reaching a lint verdict. Render+# it inline here from the template with a placeholder CIDR, exactly as the+# single-writer CI step does (.github/workflows/ci-public.yml), so this test+# exercises the real rendered overlay instead of failing on a missing env file.+MINIKUBE_API_IP_TEMPLATE="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template"+MINIKUBE_API_IP_PATCH="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml"+if [[ ! -f "${MINIKUBE_API_IP_TEMPLATE}" ]]; then+ echo "FAIL: missing ${MINIKUBE_API_IP_TEMPLATE}; cannot render the minikube k8s-api-ip patch" >&2+ exit 1+fi+if [[ -f "${MINIKUBE_API_IP_PATCH}" ]]; then+ # A developer's real rendered patch is present — preserve it so the dummy+ # render below doesn't clobber it; the EXIT trap restores it verbatim.+ MINIKUBE_API_IP_PATCH_BACKUP="${TMP_DIR}/k8s-api-ip.yaml.orig"+ cp "${MINIKUBE_API_IP_PATCH}" "${MINIKUBE_API_IP_PATCH_BACKUP}"+ MINIKUBE_API_IP_PATCH_PREEXISTING=1+fi+sed 's#__K8S_API_IP__#10.96.0.1#g' "${MINIKUBE_API_IP_TEMPLATE}" >"${MINIKUBE_API_IP_PATCH}"++MINIKUBE_RENDERED="${TMP_DIR}/minikube-rendered.yaml"+kubectl kustomize "${ROOT}/deploy/overlays/minikube" >"${MINIKUBE_RENDERED}"RUBYOPT=--disable=gems ruby -ryaml -e 'forbidden = %w[allow-dns-egress-sandbox-ui deny-all-sandbox-ui]required = %w[sandbox-ui-static-deny-all sandbox-ui-static-dns-egress]@@ -151,7 +189,6 @@ RUBYOPT=--disable=gems ruby -ryaml -e 'found << name if forbidden.include?(name)if name == "sandbox-ui-static-dns-egress"saw_dns_peer = false- saw_dns_ipblock = falseArray(doc.dig("spec", "egress")).each_with_index do |rule, rule_idx|ports = Array(rule["ports"]).map { |port| [port.fetch("protocol", "TCP"), port["port"]] }.sortexpected_ports = [["TCP", 53], ["UDP", 53]]@@ -164,27 +201,20 @@ RUBYOPT=--disable=gems ruby -ryaml -e 'peers.each_with_index do |peer, peer_idx|ns_name = peer.dig("namespaceSelector", "matchLabels", "kubernetes.io/metadata.name")dns_selector = peer.dig("podSelector", "matchLabels", "k8s-app")- dns_ipblock = peer.dig("ipBlock", "cidr")if ns_name == "kube-system" && dns_selector == "kube-dns"saw_dns_peer = truenextend- if dns_ipblock == "203.0.113.10/32"- saw_dns_ipblock = true- next- end-- abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods or the example-dev kube-dns ClusterIP")+ abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods")endendabort("sandbox-ui static DNS egress must include a kube-system kube-dns peer") unless saw_dns_peer- abort("sandbox-ui static DNS egress must include the example-dev kube-dns ClusterIP") unless saw_dns_ipblockendendabort("sandbox-ui static slice must not render HCC-owned policies: #{found.join(", ")}") unless found.empty?missing = required - namesabort("sandbox-ui static slice must render renamed baseline policies: #{missing.join(", ")}") unless missing.empty?-' "${GCP_DEV_RENDERED}"-echo "PASS: gcp-dev sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and include kube-dns ClusterIP"+' "${MINIKUBE_RENDERED}"+echo "PASS: minikube sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and retain kube-dns-only DNS egress"diff --git a/scripts/tests/test-postgres-single-writer-strategy.sh b/scripts/tests/test-postgres-single-writer-strategy.shnew file mode 100755index 000000000..40174bf46--- /dev/null+++ b/scripts/tests/test-postgres-single-writer-strategy.sh@@ -0,0 +1,160 @@+#!/usr/bin/env bash+# Regression test for deploy/scripts/validate-postgres-single-writer-strategy.sh+#+# Captures the [R4-D] fix: a Deployment this repo DEFINES (deploy/base/<ns>/+# exists) that is ABSENT from a rendered overlay must FAIL LOUDLY, not be+# NOTICE-and-skipped. Before the fix the validator skipped any absent deploy,+# so a local overlay that stopped rendering control-plane/host-context-controller+# validated the single-writer rollout invariant over an empty set and still+# printed "validation passed" (a fail-open / false green).+#+# The CI step "Validate single-writer rollout invariant on rendered overlays"+# already runs the validator against the real minikube overlay (a happy-path+# check). This test is complementary: it fabricates the absent-deploy condition+# in a hermetic fixture so the fail-loud behavior itself is under test.+#+# The test builds a hermetic fixture ROOT_DIR, copies the validator into it so+# its ROOT_DIR ($(dirname)/../..) resolves to the fixture, and renders overlays+# OFFLINE with `kubectl kustomize` — the same path the validator uses. No cluster.+#+# Fail-loud / fail-fast: every assertion fails the suite immediately, and the+# reporter refuses to pass unless a non-zero number of assertions actually ran.+set -uo pipefail++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+VALIDATOR="${REPO_ROOT}/deploy/scripts/validate-postgres-single-writer-strategy.sh"++for tool in kubectl yq jq; do+ command -v "${tool}" >/dev/null 2>&1 ||+ { echo "FAIL: required tool '${tool}' is missing" >&2; exit 1; }+done+[ -f "${VALIDATOR}" ] ||+ { echo "FAIL: validator not found at ${VALIDATOR}" >&2; exit 1; }++TESTS_RUN=0+FAILURES=0+pass() { TESTS_RUN=$((TESTS_RUN + 1)); echo "ok - $1"; }+fail() { TESTS_RUN=$((TESTS_RUN + 1)); FAILURES=$((FAILURES + 1)); echo "FAIL - $1" >&2; }++WORK="$(mktemp -d "${TMPDIR:-/tmp}/single-writer-test.XXXXXX")"+trap 'rm -rf "${WORK}"' EXIT++ROOT="${WORK}/repo"+mkdir -p "${ROOT}/deploy/scripts"+cp "${VALIDATOR}" "${ROOT}/deploy/scripts/"+FIXTURE_VALIDATOR="${ROOT}/deploy/scripts/$(basename "${VALIDATOR}")"++# control-plane is a namespace this fixture repo DEFINES (base tree present).+# registry is intentionally NOT defined here -> it is FOREIGN (NOTICE-and-skip).+mkdir -p "${ROOT}/deploy/base/control-plane"+: >"${ROOT}/deploy/base/control-plane/.keep"++write_kustomization() { # dir, resource-file...+ local dir="$1"; shift+ {+ echo 'apiVersion: kustomize.config.k8s.io/v1beta1'+ echo 'kind: Kustomization'+ echo 'resources:'+ local r+ for r in "$@"; do echo " - ${r}"; done+ } >"${dir}/kustomization.yaml"+}++write_control_postgres() { # dir (valid stateful DB deploy: PVC + Recreate + replicas 1)+ cat >"$1/control-postgres.yaml" <<'YAML'+apiVersion: apps/v1+kind: Deployment+metadata:+ name: control-postgres+ namespace: control-plane+spec:+ replicas: 1+ strategy:+ type: Recreate+ template:+ spec:+ volumes:+ - name: data+ persistentVolumeClaim:+ claimName: control-postgres-data+YAML+}++write_hcc() { # dir (valid single-writer deploy: replicas 1 + Recreate)+ cat >"$1/host-context-controller.yaml" <<'YAML'+apiVersion: apps/v1+kind: Deployment+metadata:+ name: host-context-controller+ namespace: control-plane+spec:+ replicas: 1+ strategy:+ type: Recreate+ template:+ spec:+ containers:+ - name: host-context-controller+ image: example/hcc:test+YAML+}++# --- Overlay A: LOCAL single-writer deploy ABSENT (the fail-open trigger) ---+# Renders a valid control-postgres but OMITS host-context-controller.+OV_MISSING="${ROOT}/deploy/overlays/local-missing"+mkdir -p "${OV_MISSING}"+write_control_postgres "${OV_MISSING}"+write_kustomization "${OV_MISSING}" control-postgres.yaml++# --- Overlay B: every LOCAL deploy PRESENT and valid (foreign still skipped) ---+OV_OK="${ROOT}/deploy/overlays/all-local-present"+mkdir -p "${OV_OK}"+write_control_postgres "${OV_OK}"+write_hcc "${OV_OK}"+write_kustomization "${OV_OK}" control-postgres.yaml host-context-controller.yaml++run_validator() { # overlays-value ; prints combined output, returns exit code+ SINGLE_WRITER_OVERLAYS="$1" bash "${FIXTURE_VALIDATOR}" 2>&1+}++# 1) A locally-defined single-writer deploy absent from a rendered overlay MUST+# fail loudly (this is exactly what the pre-fix NOTICE-and-skip let pass).+out="$(run_validator "deploy/overlays/local-missing")"; rc=$?+if [ "${rc}" -ne 0 ] && printf '%s' "${out}" |+ grep -q "renders no Deployment control-plane/host-context-controller"; then+ pass "absent locally-defined single-writer deploy fails loudly (exit ${rc})"+else+ fail "absent local single-writer deploy did NOT fail loudly (exit ${rc}); output: ${out}"+fi++# 2) With every locally-defined deploy present and valid, a still-absent FOREIGN+# deploy (registry/registry-postgres) is NOTICE-and-skipped, and the script+# passes — the fix must not over-fail on foreign deploys.+out="$(run_validator "deploy/overlays/all-local-present")"; rc=$?+if [ "${rc}" -eq 0 ] &&+ printf '%s' "${out}" | grep -q "Single-writer rollout strategy validation passed" &&+ printf '%s' "${out}" | grep -q "does not render foreign Deployment registry/registry-postgres"; then+ pass "all local deploys present -> passes; absent foreign deploy is skipped"+else+ fail "expected pass with foreign-skip (exit ${rc}); output: ${out}"+fi++# 3) Zero overlays must not pass vacuously.+out="$(run_validator " ")"; rc=$?+if [ "${rc}" -ne 0 ] && printf '%s' "${out}" | grep -q "zero overlays"; then+ pass "whitespace-only overlay set fails loudly (no vacuous pass)"+else+ fail "empty overlay set did not fail loudly (exit ${rc}); output: ${out}"+fi++# --- Reporter (fail-loud / zero-tests-is-never-success) ---+echo "1..${TESTS_RUN}"+if [ "${TESTS_RUN}" -lt 3 ]; then+ echo "FAIL: expected at least 3 assertions, ran ${TESTS_RUN}" >&2+ exit 1+fi+if [ "${FAILURES}" -ne 0 ]; then+ echo "FAIL: ${FAILURES}/${TESTS_RUN} assertions failed" >&2+ exit 1+fi+echo "All ${TESTS_RUN} single-writer validator assertions passed"diff --git a/scripts/tests/test-wrc-hcc-contracts-gate.sh b/scripts/tests/test-wrc-hcc-contracts-gate.shnew file mode 100755index 000000000..a0d6180bc--- /dev/null+++ b/scripts/tests/test-wrc-hcc-contracts-gate.sh@@ -0,0 +1,28 @@+#!/usr/bin/env bash+set -u++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-wrc-hcc-contracts.sh"+FAIL=0++pass() { echo "PASS: $1"; }+fail() { echo "FAIL: $1"; FAIL=1; }++if bash -n "$GATE"; then+ pass "WRC-HCC contracts gate has valid bash syntax"+else+ fail "WRC-HCC contracts gate has invalid bash syntax"+fi++# The Phase 2 fixture has steps, so the admission policy requires an explicit+# trigger. Keep this immediately alongside the fixture's agent/steps contract+# so the E2E cannot regress into an admission-only false negative.+phase_two_fixture="$(sed -n '/^header "Phase 2 - WRC toolsCalled args status contract"$/,/^YAML$/p' "$GATE")"+if [[ "$phase_two_fixture" == *$' triggers:\n onDemand: {}'* ]] &&+ [[ "$phase_two_fixture" == *$' steps:'* ]]; then+ pass "WRC-HCC Phase 2 fixture declares its required on-demand trigger"+else+ fail "WRC-HCC Phase 2 fixture can be rejected before its status contract runs"+fi++exit "$FAIL"diff --git a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.shnew file mode 100755index 000000000..98f9a9177--- /dev/null+++ b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh@@ -0,0 +1,33 @@+#!/usr/bin/env bash+set -u++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"+FAIL=0++pass() { echo "PASS: $1"; }+fail() { echo "FAIL: $1"; FAIL=1; }++if bash -n "$GATE"; then+ pass "WRC internal-dependency gate has valid bash syntax"+else+ fail "WRC internal-dependency gate has invalid bash syntax"+fi++# WRC persists its generated workload resource names in status before+# materializing workloads. The gate must use that public CRD contract rather+# than assuming a workload ID is also a Deployment or Service name.+# shellcheck disable=SC2016+if grep -Fq 'wait_for_workload_instance() {' "$GATE" &&+ grep -Fq 'SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID"' "$GATE" &&+ grep -Fq 'BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID"' "$GATE" &&+ ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID"' "$GATE" &&+ ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID"' "$GATE" &&+ grep -Fq 'deploy/${SOURCE_DEPLOYMENT}' "$GATE" &&+ grep -Fq '${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local' "$GATE"; then+ pass "WRC gate resolves persisted workload instances before runtime assertions"+else+ fail "WRC gate assumes raw workload IDs are runtime resource names"+fi++exit "$FAIL"diff --git a/tests/e2e/integration/network-policies.test.ts b/tests/e2e/integration/network-policies.test.tsindex aba9302b2..07925e9db 100644--- a/tests/e2e/integration/network-policies.test.ts+++ b/tests/e2e/integration/network-policies.test.ts@@ -8,12 +8,50 @@* the NetworkPolicy is blocking traffic as expected. A successful response or* connection refused (port not listening) would indicate a policy gap.*/-import { describe, expect, it } from 'vitest'+import { beforeAll, describe, expect, it } from 'vitest'import { execFileSync } from 'child_process'// Short timeout for negative tests — we expect these to time outconst NP_TIMEOUT = 3+// ─── Cluster reachability gate ───────────────────────────────────────────────+// This suite drives kubectl against a live minikube cluster (Calico CNI + all+// services deployed). Mirrors the opt-in-skip convention already used by the+// sibling integration suites — channel-reader-via-api.test.ts and+// mcpCredentialRotation.helpers.ts (E2E_SKIP_IF_CLUSTER_UNREACHABLE) plus+// registry-pull-secret-runtime.test.ts (E2E_REQUIRE_CLUSTER / CI):+// - Default (no flag): fail-fast when the cluster is unreachable. A silent+// pass would hide a broken E2E setup, so the suite must NOT be neutralized+// unconditionally.+// - E2E_SKIP_IF_CLUSTER_UNREACHABLE=1: opt in to a clean skip for the tier+// that runs integration WITHOUT a cluster (T1).+// - E2E_REQUIRE_CLUSTER=1 or CI=true: force a hard failure even if the skip+// flag is also set — CI must never skip.+const SKIP_IF_UNREACHABLE = process.env.E2E_SKIP_IF_CLUSTER_UNREACHABLE === '1'+const REQUIRE_CLUSTER = process.env.E2E_REQUIRE_CLUSTER === '1' || process.env.CI === 'true'++let clusterReachable = false++/**+ * One-shot reachability probe: succeeds only when kubectl can talk to the API+ * server. This is a precondition probe (the established sibling idiom —+ * registry-pull-secret-runtime.test.ts's `kubectlOrNull`), NOT an error+ * swallow: its boolean result drives the fail-hard-or-skip decision in+ * `beforeAll`, which stays loud in the default and CI paths.+ */+function probeClusterReachable(): boolean {+ try {+ execFileSync('kubectl', ['get', 'ns', 'default', '-o', 'name'], {+ encoding: 'utf-8',+ timeout: 15_000,+ stdio: ['ignore', 'pipe', 'pipe'],+ })+ return true+ } catch {+ return false+ }+}+interface ConnectResult {reachable: booleanstatusCode?: number@@ -141,8 +179,32 @@ function testConnectivity(}describe('NetworkPolicy Enforcement (Calico)', () => {+ beforeAll(() => {+ clusterReachable = probeClusterReachable()+ if (clusterReachable) return++ const msg =+ '[network-policies] kubernetes cluster not reachable (kubectl get ns default failed)'+ if (REQUIRE_CLUSTER) {+ throw new Error(+ `${msg}. Refusing to skip: E2E_REQUIRE_CLUSTER/CI is set, and a silent skip here ` ++ 'would report success while asserting nothing about NetworkPolicy enforcement. ' ++ 'Bring up the cluster (e.g. `make minikube-setup`) or unset the flag.'+ )+ }+ if (SKIP_IF_UNREACHABLE) {+ console.log(`${msg} — tests will be skipped (E2E_SKIP_IF_CLUSTER_UNREACHABLE=1)`)+ return+ }+ throw new Error(+ `${msg}. This E2E suite requires a minikube cluster with Calico CNI and all services ` ++ 'deployed. Run `make minikube-setup` first, or set E2E_SKIP_IF_CLUSTER_UNREACHABLE=1 to skip.'+ )+ })+// ─── Prerequisite Check ──────────────────────────────────────────────- it('Calico CNI is active', () => {+ it('Calico CNI is active', (ctx) => {+ if (!clusterReachable) ctx.skip()const pods = execFileSync('kubectl',['get', 'pods', '-n', 'kube-system', '-l', 'k8s-app=calico-node', '--no-headers'],@@ -151,7 +213,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(pods).toContain('Running')})- it('deny-all policies exist in secured namespaces', () => {+ it('deny-all policies exist in secured namespaces', (ctx) => {+ if (!clusterReachable) ctx.skip()const namespaces = ['mcp-host', 'mcp-server', 'rpc-proxy', 'sandbox-recipes']for (const ns of namespaces) {let nps: string@@ -184,7 +247,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {// ─── POSITIVE TESTS (Allowed Traffic) ────────────────────────────────describe('Allowed traffic (positive)', () => {- it('control-api → registry-api:8085 (cross-namespace, registry access)', () => {+ it('control-api → registry-api:8085 (cross-namespace, registry access)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('control-plane','control-api',@@ -193,7 +257,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('control-ui → control-api:8090 (UI backend path)', () => {+ it('control-ui → control-api:8090 (UI backend path)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('control-plane','control-ui',@@ -202,7 +267,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('external-rest-api → profile-control-funnel:8080 (internal control path)', () => {+ it('external-rest-api → profile-control-funnel:8080 (internal control path)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('profiles','external-rest-api',@@ -211,12 +277,14 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('rpc-proxy → mcp-host:8080 (message relay)', () => {+ it('rpc-proxy → mcp-host:8080 (message relay)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('rpc-proxy', 'rpc-proxy', getMcpHostRuntimeUrl())expect(result.reachable).toBe(true)})- it('mcp-host can reach the HCC gateway readiness endpoint', () => {+ it('mcp-host can reach the HCC gateway readiness endpoint', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -230,7 +298,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {// ─── NEGATIVE TESTS (Blocked by Design) ──────────────────────────────describe('Blocked traffic (negative — deny-all enforced)', () => {- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', () => {+ it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -240,7 +309,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('mcp-host → control-api:8090 (no direct access to control-plane)', () => {+ it('mcp-host → control-api:8090 (no direct access to control-plane)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -250,11 +320,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', () => {+ it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', (ctx) => {+ if (!clusterReachable) ctx.skip()const podName = getChannelReaderPodName()if (!podName) {console.warn('SKIPPED: no running channel-reader pod in channels')- return+ ctx.skip()}const result = testConnectivity(@@ -266,7 +337,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('mcp-host → registry-api:8085 (no direct registry access)', () => {+ it('mcp-host → registry-api:8085 (no direct registry access)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -276,7 +348,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', () => {+ it('control-api → rpc-proxy:8094 (no direct control-plane hop)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('control-plane','control-api',@@ -286,7 +359,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', () => {+ it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('profiles','external-rest-api',@@ -296,19 +370,22 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {+ it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('control-plane', 'control-api', getMcpHostRuntimeUrl())expect(result.reachable).toBe(false)expect(result.error).toContain('timeout')})- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {+ it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('profiles', 'external-rest-api', getMcpHostRuntimeUrl())expect(result.reachable).toBe(false)expect(result.error).toContain('timeout')})- it('mcp-proxy → external-rest-api:3000 (no reverse path)', () => {+ it('mcp-proxy → external-rest-api:3000 (no reverse path)', (ctx) => {+ if (!clusterReachable) ctx.skip()const result = testConnectivity('mcp-server','mcp-proxy',@@ -348,11 +425,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {}}- it('sandbox-recipes → control-api:8090 (isolated sandbox)', () => {+ it('sandbox-recipes → control-api:8090 (isolated sandbox)', (ctx) => {+ if (!clusterReachable) ctx.skip()const podName = findExecPod('sandbox-recipes', 'app=db')if (!podName) {console.warn('SKIPPED: no running db pod in sandbox-recipes')- return+ ctx.skip()}// sandbox-recipes has deny-all; only DNS + HCC API egress allowedconst result = testConnectivity(@@ -364,11 +442,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', () => {+ it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', (ctx) => {+ if (!clusterReachable) ctx.skip()const podName = findExecPod('sandbox-recipes', 'app=db')if (!podName) {console.warn('SKIPPED: no running db pod in sandbox-recipes')- return+ ctx.skip()}const result = testConnectivity('sandbox-recipes',diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex 558c31760..9981d0aa7 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.66",+ "version": "0.1.65","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.66",+ "version": "0.1.65","devDependencies": {"@types/jsonwebtoken": "^9.0.10","@types/node": "^22.0.0",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex 19145d619..63c62d203 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.66",+ "version": "0.1.65","private": true,"type": "module","scripts": {