Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
Revert "Merge pull request #205 from evenfire-ai/codex/issue-204-hcc-readiness-decouple"
Alfredo Lopez562741148d50
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 08541e5bc..18daf933c 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -390,21 +390,6 @@ jobs:- name: Test HCC watch-recovery gate coordinationrun: bash scripts/tests/test-hcc-watch-recovery-gate.sh- - name: Test HCC API gateway readiness separation- run: bash scripts/tests/test-hcc-api-gateway-readiness.sh-- - name: Test host-storm metric adjudication- run: bash scripts/tests/test-host-storm-metric-gate.sh-- - name: Test WRC-to-HCC contracts- run: bash scripts/tests/test-wrc-hcc-contracts-gate.sh-- - name: Test WRC internal dependency NetworkPolicy contracts- run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh-- - name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)- run: bash scripts/tests/test-lint-networkpolicies.sh-- name: Test Plugin Workload SDK auth bootstrap policyrun: bash scripts/tests/test-plugin-workload-sdk-auth-bootstrap.sh@@ -414,24 +399,6 @@ jobs:- name: Test minikube detect-k8s-api-ip template renderingrun: bash scripts/tests/test-minikube-detect-k8s-api-ip.sh- - name: Validate single-writer rollout invariant on rendered overlays- run: |- # The k8s-api-ip patch is rendered from a gitignored template at setup- # time; render it with a dummy IP so `kubectl kustomize` can build the- # minikube overlay here. The gcp-* overlays live in evenfire-infra, so- # this repo validates only the minikube overlay it actually ships.- sed 's#__K8S_API_IP__#10.96.0.1#g' \- deploy/overlays/minikube/patches/k8s-api-ip.yaml.template \- > deploy/overlays/minikube/patches/k8s-api-ip.yaml- SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" \- bash deploy/scripts/validate-postgres-single-writer-strategy.sh-- - name: Test single-writer validator fails loud on an absent local deploy- run: bash scripts/tests/test-postgres-single-writer-strategy.sh-- - name: Test post-reset convergence isolates DB-dependent controllers- run: bash scripts/tests/test-converge-control-db-after-reset.sh-- name: Test release coordinates (writer, checker, floor decoupling)run: bash scripts/tests/test-release-coordinates.shdiff --git a/Makefile b/Makefileindex 107633d70..57f47ec5a 100644--- a/Makefile+++ b/Makefile@@ -934,20 +934,7 @@ test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R.PHONY: test-e2e-hcc-communicationchannel-watch-recoverytest-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC watch-recovery fault-injection gate@echo "Running HCC CommunicationChannel watch-recovery gate..."- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }- E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh--.PHONY: test-e2e-hcc-readiness-bootstrap-test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active- @echo "Running HCC initial-fleet readiness gate..."- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }- E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh--.PHONY: test-e2e-hcc-mcp-context-readiness-test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence- @echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }- E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh+ E2E_HCC_WATCH_FAULT_INJECTION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh.PHONY: test-e2e-stateless-multinodetest-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)diff --git a/deploy/base/control-plane/host-context-controller-api-gateway.yaml b/deploy/base/control-plane/host-context-controller-api-gateway.yamlindex 1c30ccfbc..961cebe48 100644--- a/deploy/base/control-plane/host-context-controller-api-gateway.yaml+++ b/deploy/base/control-plane/host-context-controller-api-gateway.yaml@@ -43,7 +43,7 @@ spec:timeoutSeconds: 5readinessProbe:httpGet:- path: /health+ path: /readyport: httpscheme: HTTPinitialDelaySeconds: 3diff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yamlindex 24a4c21dd..df8126a53 100644--- a/deploy/base/control-plane/host-context-controller.yaml+++ b/deploy/base/control-plane/host-context-controller.yaml@@ -13,33 +13,21 @@ metadata:app: host-context-controllerspec:replicas: 1- # HCC's safety model assumes a single NetworkPolicy writer: replicas:1 with no- # leader election (see statelessLifecycleExecutor.ts). RollingUpdate (the- # default) starts the new pod before terminating the old one, so every rollout- # would overlap two writers — during which the outgoing replica can win a- # conflict-retry replace and land its wider spec back over a policy the new- # replica just hardened and certified, with no NetworkPolicy resync to heal it- # and /ready still 200. Recreate terminates the old pod first, honouring the- # single-writer assumption through the rollout too.- strategy:- type: Recreate- # Recreate + replicas:1 means a botched rollout (new pod never Ready) leaves- # the control plane down until a manual `kubectl rollout undo` — the single-- # writer invariant forbids any overlap-based auto-recovery. This deadline is- # DETECTION ONLY: it flips Progressing=False (ProgressDeadlineExceeded) and- # fails `kubectl rollout status` for callers without their own --timeout; it- # does not roll back.+ # HCC's /ready stays 503 until its initial fleet reconciliation finishes, so time-to-Ready+ # scales with the McpServer/Context fleet rather than with process start. clerum-dev took+ # 651s on 2026-08-05 at 108 McpServers / 22 Contexts.#- # Value (origin/dev, 2026-08-05): HCC's /ready stays 503 until its initial fleet- # reconciliation finishes, so time-to-Ready scales with the McpServer/Context- # fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default- # (600s) governed the deploy because `kubectl rollout status` aborts the instant- # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's- # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s- # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s- # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the- # fix — #205 decouples readiness from fleet convergence, after which this should- # come back down.+ # Unset, this defaults to 600s, and that default is what actually governed the deploy:+ # `kubectl rollout status` aborts the instant the controller sets ProgressDeadlineExceeded,+ # so provision-gfs-runtime.sh's --timeout could never exceed the deadline remaining. The+ # clock also starts several steps earlier, at apply-inter-service-tokens.sh's `rollout+ # restart` of HCC, so the wait inherits only what is left. On 2026-08-05 that was 600-160+ # = 440s against a 480s timeout: the timeout raised in #202 never bound, and the deploy+ # failed 50s before HCC came Ready.+ #+ # 1200s covers the 900s wait plus a 300s head-start allowance. This is a mitigation, not+ # the fix — evenfire-ai/evenfire#205 decouples readiness from fleet convergence, after+ # which both numbers should come back down.progressDeadlineSeconds: 1200selector:matchLabels:@@ -103,8 +91,6 @@ spec:# now fails closed at startup on any interval > overlap/2.- name: HCC_EXTERNAL_EGRESS_RESYNC_SECvalue: "60"- - name: HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS- value: "5000"- name: HCC_HOST_K8S_REQUEST_TIMEOUT_MSvalue: "30000"# Bounded cross-Host concurrency for a full/lifecycle Host fleet pass.diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.shindex 404d61076..99f50d8bf 100755--- a/deploy/scripts/converge-control-db-after-reset.sh+++ b/deploy/scripts/converge-control-db-after-reset.sh@@ -183,7 +183,7 @@ if [ "$STATE_PHASE" = converged ]; thenfikc -n control-plane scale deployment/host-context-controller --replicas="$HCC_REPLICAS" >/dev/nullif [ "$HCC_REPLICAS" -gt 0 ]; then- kc -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null+ kc -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/nullfiassert_replacement_uiddelete_reset_state@@ -264,7 +264,7 @@ kc -n control-plane scale deployment/host-context-controller \--replicas="$HCC_REPLICAS" >/dev/nullif [ "$HCC_REPLICAS" -gt 0 ]; thenkc -n control-plane rollout status \- deployment/host-context-controller --timeout=900s >/dev/null+ deployment/host-context-controller --timeout=180s >/dev/nullfiassert_replacement_uidread_reset_state || fail "cannot reload reset state before completion"diff --git a/deploy/scripts/validate-postgres-single-writer-strategy.sh b/deploy/scripts/validate-postgres-single-writer-strategy.shindex ee90b96be..f1a77091c 100755--- a/deploy/scripts/validate-postgres-single-writer-strategy.sh+++ b/deploy/scripts/validate-postgres-single-writer-strategy.sh@@ -2,86 +2,40 @@set -euo pipefailROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-# Overlays to render. Defaults to the full production set; the gcp-* overlays-# live in keyper-labs/evenfire-infra, so this script's full coverage runs there.-# Override (space-separated) to validate only the overlays present in a given-# checkout — e.g. SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" in this repo.-read -r -a OVERLAYS <<<"${SINGLE_WRITER_OVERLAYS:-deploy/overlays/minikube deploy/overlays/gcp-dev deploy/overlays/gcp-prod}"+OVERLAYS=(+ "deploy/overlays/minikube"+ "deploy/overlays/gcp-dev"+ "deploy/overlays/gcp-prod"+)DATABASE_DEPLOYS=("control-plane/control-postgres""registry/registry-postgres")-# Deployments whose safety model assumes a single writer with NO leader election-# (see host-context-controller/src/statelessLifecycleExecutor.ts). For these the-# RENDERED manifest — not just deploy/base — must keep replicas<=1 and-# strategy=Recreate with no rollingUpdate residue, or a rollout reopens the-# two-writer NetworkPolicy overlap closed in PR #205. Unlike DATABASE_DEPLOYS-# these mount no PVC, so they need their own list rather than the PVC heuristic.-SINGLE_WRITER_DEPLOYS=(- "control-plane/host-context-controller"-)fail() {echo "ERROR: $*" >&2exit 1}-# A Deployment id ("namespace/name") is DEFINED in this repo when its namespace-# has a base manifest tree here (deploy/base/<namespace>/). For such a LOCAL-# deploy the rendered overlay MUST contain it: a silent disappearance would let-# the loops below validate the single-writer/database rollout invariant over an-# absent deploy (a fail-open). Deploys whose base lives only in the sibling-# evenfire-infra repo (e.g. registry/*) are FOREIGN here and are validated where-# they are defined, so for them an absence is a NOTICE-and-skip, not a failure.-deploy_defined_in_this_repo() {- local id="$1"- local namespace="${id%%/*}"- [ -d "${ROOT_DIR}/deploy/base/${namespace}" ]-}-command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"command -v jq >/dev/null 2>&1 || fail "jq is required"-command -v yq >/dev/null 2>&1 || fail "yq is required"--# Zero overlays means the loop below runs zero iterations and the script exits 0-# having validated nothing — a fail-open. An empty or whitespace-only-# SINGLE_WRITER_OVERLAYS must fail loudly, not pass vacuously.-[ "${#OVERLAYS[@]}" -gt 0 ] ||- fail "SINGLE_WRITER_OVERLAYS resolved to zero overlays — refusing to pass vacuously"for overlay in "${OVERLAYS[@]}"; do- # Render OFFLINE: `kubectl kustomize` builds the overlay client-side, and yq- # converts the multi-document YAML into a single JSON array for jq. The- # previous `kubectl create --dry-run=client` performed API discovery- # (kind -> resource), which needs a live API server and fails in CI with- # "unable to recognize STDIN: connection refused". This validation is a pure- # rendered-manifest lint and must never touch a cluster.rendered_json="$(cd "${ROOT_DIR}" &&kubectl kustomize "${overlay}" |- yq ea -o=json '[.]'+ kubectl create --dry-run=client --validate=false -f - -o json |+ jq -s '.')"for id in "${DATABASE_DEPLOYS[@]}"; donamespace="${id%%/*}"name="${id#*/}"- if ! deployment="$(+ deployment="$(jq -e --arg namespace "${namespace}" --arg name "${name}" \'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \<<<"${rendered_json}"- )"; then- # This overlay does not render ${id}. A deploy this repo DEFINES- # (deploy/base/<namespace>/ exists here) MUST be rendered by a local- # overlay: a silent disappearance would validate the rollout invariant over- # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and- # the registry Deployment live in the sibling evenfire-infra repo) are- # validated where they are defined, so for them a NOTICE-and-skip is right.- if deploy_defined_in_this_repo "${id}"; then- fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"- fi- echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2- continue- fi+ )" || fail "${overlay}: missing Deployment ${id}"pvc_count="$(jq '[.spec.template.spec.volumes[]? | select(.persistentVolumeClaim.claimName != null)] | length' \@@ -100,42 +54,6 @@ for overlay in "${OVERLAYS[@]}"; dofail "${overlay}: Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"fidone-- for id in "${SINGLE_WRITER_DEPLOYS[@]}"; do- namespace="${id%%/*}"- name="${id#*/}"- if ! deployment="$(- jq -e --arg namespace "${namespace}" --arg name "${name}" \- 'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \- <<<"${rendered_json}"- )"; then- # This overlay does not render ${id}. A deploy this repo DEFINES- # (deploy/base/<namespace>/ exists here) MUST be rendered by a local- # overlay: a silent disappearance would validate the rollout invariant over- # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and- # the registry Deployment live in the sibling evenfire-infra repo) are- # validated where they are defined, so for them a NOTICE-and-skip is right.- if deploy_defined_in_this_repo "${id}"; then- fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"- fi- echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2- continue- fi-- replicas="$(jq -r '.spec.replicas // 1' <<<"${deployment}")"- if [ "${replicas}" -gt 1 ]; then- fail "${overlay}: single-writer Deployment ${id} must keep replicas<=1 (no leader election), got ${replicas}"- fi-- strategy="$(jq -r '.spec.strategy.type // "RollingUpdate"' <<<"${deployment}")"- if [ "${strategy}" != "Recreate" ]; then- fail "${overlay}: single-writer Deployment ${id} must use strategy.type=Recreate, got ${strategy}"- fi-- if jq -e '.spec.strategy.rollingUpdate != null' >/dev/null <<<"${deployment}"; then- fail "${overlay}: single-writer Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"- fi- donedone-echo "Single-writer rollout strategy validation passed"+echo "Postgres single-writer rollout strategy validation passed"diff --git a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsxindex ff8667aa0..0b1007986 100644--- a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx+++ b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx@@ -52,7 +52,9 @@ function createDeferred<T>(): Deferred<T> {}function createSessionState(authenticated: boolean, me = HARNESS_ME) {- return authenticated ? { authenticated: true, me } : { authenticated: false, me: null }+ return authenticated+ ? { authenticated: true, me }+ : { authenticated: false, me: null }}function createCatalog(agentNames: string[]) {diff --git a/docs/architecture/hcc-readiness-flows.md b/docs/architecture/hcc-readiness-flows.mddeleted file mode 100644index 6a4e89ca9..000000000--- a/docs/architecture/hcc-readiness-flows.md+++ /dev/null@@ -1,90 +0,0 @@-# HCC readiness & external-egress — design note (PR #205)--> Design note for the readiness-decoupling and single-writer NetworkPolicy work in-> `host-context-controller` (HCC). Addresses the "no design/decision doc for the-> 500+-line external-egress coordinator and the certification barrier" review gap.----## 1. Readiness lifecycle — decoupled from initial fleet convergence--Before this work, `McpServerWatcher.start()` awaited full initial fleet-reconciliation, so a large MCP/Context/Host fleet could keep `provider.start()`-and `/ready` unavailable until every individual reconcile finished.--Now readiness is gated on an **authoritative, fail-closed safety boundary**, not on-full convergence:--1. **HCC start** — four watch lanes (Context ingress, mcp-host egress, rpc-proxy- egress, external egress) start. A transient LIST/WATCH failure on any lane enters- in-process recovery and **does not kill the process**; HCC simply stays unready.-2. **4-LIST inventory** — bounded inventory across the four lanes.-3. **Certify safety inventory** — the authoritative pass revokes any stale /- unprovable allow **before** recording a certificate (fail-closed).-4. **`/ready = 200`** — served **only** when the inventory is certified and no stale- allow survives.-5. **Additive convergence** — the remaining fleet converges **after** ready,- asynchronously, without blocking it.--## 2. Fail-closed fences--| Fence | Rule | Source |-|-------|------|--------|-| Readiness gate | `/ready = ready && providerAuthoritativeFn()`; constructor default `() => false`; a throwing gate → 503 | `server.ts`, `readinessGate.ts` |-| G1 — unconfigured authority | `currentWhenUnconfigured()` returns `false` + warns once; prod always wires the source, dev never constructs the reconciler | `reconciler.ts` |-| F1 — readiness-poll rebind | rebinds `isCurrent`/`server` on a superseding reconcile so the poll window does not self-destruct on a stale fence | `reconciler.ts` |-| Authority fence — `isCurrent` | **required** (not an optional/defaulted arg) on all three allow-granting reconcilers — `bindingPolicyReconciler`, `reconcileContext` (L2 context-allow), `reconcileExternalEgress` (L3 external-egress ALLOW). No `?? (() => true)` fail-open default: a caller cannot compile without supplying the fence (TS2554/TS2345) | `networkPolicyReconciler.ts`, `bindingPolicyReconciler.ts` |--The affirmative literal (`() => true`) is reachable only from the dev/test wiring;-every production path resolves the real authority function, and — because the fence-is a **required argument** on every allow-granting reconciler rather than a defaulted-option — no production caller can silently fall back to it.--## 3. DNS external-egress contract (B3) — retain / revoke decision table--A DNS-derived external-egress allow is compared against the desired policy-reconstructed from the **live** resolved CIDRs ("modulo cidr"). The decision:--| Observed change | Classification | Action | Rationale |-|-----------------|----------------|--------|-----------|-| identity unchanged (same protocol, port, selector, ownership; cidrs only) | identity-stable | **RETAIN** while Ready; refresh cidrs in the additive lane | revoking a healthy binding every pass created a guaranteed deny window |-| protocol drift (TCP→UDP) | identity-changed | **REVOKE before readiness** | old allow is unprovable/stale |-| spec-level port drift | identity-changed | **REVOKE before readiness** | " |-| ownership drift | identity-changed | **REVOKE before readiness** | " |-| zero-cidr (deny-in-disguise) | never retained | **REVOKE** | an empty cidr set is never a valid retained allow |--Revocation of an unprovable policy runs immediately (`cleanupExternalEgress`) under-the `isCurrent()` fence, **before** the safety inventory is certified — so `/ready`-never reports 200 with a stale divergent policy present. The unit suite pins both-directions (retain identity-stable; revoke on each drift), and the branch-owned-Minikube gate `e2e-hcc-mcp-context-readiness.sh` exercises both phases live.--## 4. ExternalEgressConvergenceCoordinator--The coordinator is the **single owner** of external-egress reconcile cadence-(replacing an inline timer in `k8sClient`):--- **Per-server FIFO + concurrency limiter** — a slow or never-resolving binding- cannot head-of-line block the control-plane event loop.-- **Sliding-window CIDR accumulation (#299)** — resolved IPs live for `TTL + overlap`;- large provider pools are covered by provider CIDRs rather than per-IP `/32`s. A- failed refresh deletes the allow (fail-closed), never widens it.-- **TTL-aware periodic resync (H2)** — `startPeriodicResync` re-arms after each pass- with a delay of `min(interval, TTL/2)` bounded below by a floor, so low-TTL hosts- refresh promptly. The timer is `.unref()`'d and re-programs **only after a pass- completes** (no overlap).-- **Fence** — startup rejects `interval > overlap/2`, so the accumulated window can- never lapse regardless of the configured cadence.-- **Observability** — `clerum_hcc_external_egress_retries_at_cap` gauges how many- servers are stuck retrying at the delay cap (a never-resolving DNS binding is- otherwise invisible: denied with `/ready` = 200 and no signal).--## 5. Deploy invariant — single writer--The HCC Deployment ships `strategy: Recreate`, `replicas: 1`, and-`progressDeadlineSeconds: 1200`: the stateless single-writer safety model has no-leader election, so two concurrently-running controllers must be impossible during a-rollout. The trade-off (full HCC control-plane downtime per rollout; a rollout whose-new pod never becomes Ready needs a manual `kubectl rollout undo`) is declared in the-manifest and asserted by the bootstrap gate. `validate-postgres-single-writer-strategy.sh`-enforces the invariant on the rendered manifest in CI.diff --git a/docs/architecture/pr205-before-after.png b/docs/architecture/pr205-before-after.pngdeleted file mode 100644index 524481e1f..000000000Binary files a/docs/architecture/pr205-before-after.png and /dev/null differdiff --git a/docs/architecture/pr205-before-after.svg b/docs/architecture/pr205-before-after.svgdeleted file mode 100644index 471f74cc3..000000000--- a/docs/architecture/pr205-before-after.svg+++ /dev/null@@ -1,71 +0,0 @@-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1620 880" font-family="Helvetica, Arial, sans-serif">-<defs>-<marker id="arr_2563eb" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#2563eb"/></marker>-<marker id="arr_dc2626" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#dc2626"/></marker>-<marker id="arr_16a34a" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#16a34a"/></marker>-<marker id="arr_d97706" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#d97706"/></marker>-<marker id="arr_64748b" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#64748b"/></marker>-</defs>-<style>text{font-family:Helvetica,Arial,sans-serif;}</style>-<rect x="0" y="0" width="1620" height="880" rx="0" fill="#ffffff" stroke="#ffffff" stroke-width="0" opacity="1.0"/>-<text x="810.0" y="46" font-size="27" fill="#0f172a" font-weight="700" text-anchor="middle">PR #205 — Decoupling HCC readiness: before vs after</text>-<text x="810.0" y="76" font-size="15" fill="#475569" font-weight="400" text-anchor="middle">The Host Context Controller's /ready endpoint stops waiting for full fleet convergence</text>-<rect x="30" y="100" width="760" height="660" rx="18" fill="#fff7f6" stroke="#fecaca" stroke-width="2" opacity="1.0"/>-<rect x="830" y="100" width="760" height="660" rx="18" fill="#f5fbf7" stroke="#bbf7d0" stroke-width="2" opacity="1.0"/>-<text x="410" y="136" font-size="19" fill="#dc2626" font-weight="700" text-anchor="middle">BEFORE · origin/dev (issue #204)</text>-<text x="1210" y="136" font-size="19" fill="#16a34a" font-weight="700" text-anchor="middle">AFTER · PR #205</text>-<rect x="100.0" y="168" width="600" height="58" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>-<text x="400" y="201.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>-<line x1="400" y1="226" x2="400" y2="279" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>-<rect x="100.0" y="282" width="600" height="128" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>-<text x="400" y="330.05" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">Waits to reconcile the ENTIRE initial fleet</text>-<text x="400" y="357.35" font-size="14" fill="#475569" font-weight="400" text-anchor="middle">108 McpServers · 22 Contexts · Hosts</text>-<text x="400" y="383.94" font-size="15" fill="#dc2626" font-weight="700" text-anchor="middle">/ready = 503 for the entire wait</text>-<rect x="636" y="300" width="132" height="92" rx="12" fill="#dc2626" stroke="#dc2626" stroke-width="0" opacity="1.0"/>-<text x="702" y="336" font-size="15" fill="#ffffff" font-weight="700" text-anchor="middle">BLOCKED</text>-<text x="702" y="362" font-size="22" fill="#ffffff" font-weight="700" text-anchor="middle">~651 s</text>-<line x1="400" y1="410" x2="400" y2="467" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>-<rect x="100.0" y="470" width="600" height="58" rx="12" fill="#f1f5f9" stroke="#64748b" stroke-width="2" opacity="1.0"/>-<text x="400" y="503.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">/ready = 200 (only after the ENTIRE fleet converges)</text>-<line x1="400" y1="528" x2="400" y2="585" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>-<rect x="100.0" y="588" width="600" height="76" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>-<text x="400" y="621.7" font-size="17" fill="#dc2626" font-weight="700" text-anchor="middle">rollout status TIMEOUT → deploy FAILS</text>-<text x="400" y="649.0" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">even though the HCC is healthy and progressing</text>-<text x="410" y="700" font-size="14" fill="#dc2626" font-weight="700" text-anchor="middle" font-style="italic">Readiness COUPLED to full fleet convergence</text>-<rect x="890.0" y="168" width="520" height="54" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>-<text x="1150" y="199.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>-<line x1="1150" y1="222" x2="1150" y2="265" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>-<rect x="890.0" y="268" width="520" height="50" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>-<text x="1150" y="297.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">BOUNDED inventory (bounded 4-LIST)</text>-<line x1="1150" y1="318" x2="1150" y2="357" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>-<rect x="890.0" y="360" width="520" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>-<text x="1150" y="391.89" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">Certifies authoritative fail-closed BARRIER</text>-<text x="1150" y="417.77" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">revokes stale / unprovable NetworkPolicies</text>-<text x="1150" y="442.22999999999996" font-size="13" fill="#d97706" font-weight="700" text-anchor="middle">BEFORE declaring Ready</text>-<line x1="1150" y1="452" x2="1150" y2="493" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>-<rect x="890.0" y="496" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>-<text x="1150" y="528.9300000000001" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">/ready = 200 (seconds)</text>-<line x1="1150" y1="552" x2="1150" y2="593" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>-<rect x="890.0" y="596" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>-<text x="1150" y="628.93" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">rollout OK → deploy OK</text>-<path d="M 1410.0 524 H 1452 V 596" fill="none" stroke="#d97706" stroke-width="3" stroke-dasharray="7 5" marker-end="url(#arr_d97706)"/>-<rect x="1336.0" y="596" width="232" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>-<text x="1452" y="629.44" font-size="13" fill="#0f172a" font-weight="700" text-anchor="middle">ADDITIVE fleet convergence</text>-<text x="1452" y="653.19" font-size="12" fill="#475569" font-weight="400" text-anchor="middle">async, AFTER Ready</text>-<text x="1452" y="675.5200000000001" font-size="11" fill="#475569" font-weight="400" text-anchor="middle">(the rest of the policies converge)</text>-<rect x="1362.0" y="357" width="200.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>-<text x="1462" y="373" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">isCurrent required (J3)</text>-<rect x="1382.0" y="389" width="160.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>-<text x="1462" y="405" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">B3 retain / revoke</text>-<rect x="1358.0" y="421" width="208.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>-<text x="1462" y="437" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">single-writer Recreate·1</text>-<text x="1150" y="700" font-size="14" fill="#16a34a" font-weight="700" text-anchor="middle" font-style="italic">Readiness DECOUPLED: gated on the safety barrier, not the fleet</text>-<rect x="300" y="812" width="26" height="20" rx="5" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>-<text x="336" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Startup step</text>-<rect x="506" y="812" width="26" height="20" rx="5" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>-<text x="542" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Block / failure</text>-<rect x="742" y="812" width="26" height="20" rx="5" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>-<text x="778" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Fast Ready / OK</text>-<rect x="978" y="812" width="26" height="20" rx="5" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>-<text x="1014" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Barrier / async branch</text>-</svg>\ No newline at end of filediff --git a/docs/architecture/pr205-hcc-flows.png b/docs/architecture/pr205-hcc-flows.pngdeleted file mode 100644index d0bbfa3d7..000000000Binary files a/docs/architecture/pr205-hcc-flows.png and /dev/null differdiff --git a/docs/architecture/pr205-hcc-flows.svg b/docs/architecture/pr205-hcc-flows.svgdeleted file mode 100644index adb22120f..000000000--- a/docs/architecture/pr205-hcc-flows.svg+++ /dev/null@@ -1,126 +0,0 @@-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 880" font-family="Inter, Segoe UI, Helvetica, Arial, sans-serif">-<defs>-<marker id="aBlue" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#2563eb"/></marker>-<marker id="aRose" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#dc2626"/></marker>-<marker id="aViolet" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#7c3aed"/></marker>-<marker id="aAmber" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#d97706"/></marker>-<marker id="aGreen" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#059669"/></marker>-<marker id="aGray" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#64748b"/></marker>-<filter id="sh" x="-5%" y="-6%" width="110%" height="122%"><feDropShadow dx="0" dy="1.5" stdDeviation="1.8" flood-color="#0f172a" flood-opacity="0.16"/></filter>-</defs>-<rect width="1280" height="880" fill="#ffffff"/>-<text x="40" y="36" font-size="20" font-weight="800" fill="#0f172a">host-context-controller — flujos que resuelve el PR #205</text>-<text x="40" y="57" font-size="12.5" fill="#64748b">decouple readiness from initial fleet convergence · seguridad fail-closed · gobernanza single-writer de NetworkPolicy</text>-<text x="40" y="92" font-size="13" font-weight="700" fill="#1d4ed8">1 · Lifecycle de readiness — DESACOPLADO de la convergencia de la flota</text>-<rect x="40.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>-<text x="147.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">HCC start</text>-<text x="147.0" y="146" text-anchor="middle" font-size="11" fill="#475569">4 watch lanes arrancan;</text>-<text x="147.0" y="162" text-anchor="middle" font-size="11" fill="#475569">una lane que falla → recovery</text>-<text x="147.0" y="178" text-anchor="middle" font-size="11" fill="#475569">in-process, NO mata el proceso</text>-<rect x="286.5" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>-<text x="393.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">4-LIST inventory</text>-<text x="393.5" y="146" text-anchor="middle" font-size="11" fill="#475569">Context ingress · mcp-host</text>-<text x="393.5" y="162" text-anchor="middle" font-size="11" fill="#475569">egress · rpc-proxy egress</text>-<text x="393.5" y="178" text-anchor="middle" font-size="11" fill="#475569">· external egress</text>-<rect x="533.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>-<text x="640.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Certificar safety inventory</text>-<text x="640.0" y="146" text-anchor="middle" font-size="11" fill="#475569">revoca allows stale/</text>-<text x="640.0" y="162" text-anchor="middle" font-size="11" fill="#475569">improbables ANTES de</text>-<text x="640.0" y="178" text-anchor="middle" font-size="11" fill="#475569">certificar (fail-closed)</text>-<rect x="779.5" y="104" width="214" height="92" rx="11" fill="#dbeafe" stroke="#1d4ed8" stroke-width="1.8" filter="url(#sh)"/>-<text x="886.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">/ready = 200</text>-<text x="886.5" y="146" text-anchor="middle" font-size="11" fill="#475569">solo si el inventory está</text>-<text x="886.5" y="162" text-anchor="middle" font-size="11" fill="#475569">certificado y ningún allow</text>-<text x="886.5" y="178" text-anchor="middle" font-size="11" fill="#475569">stale sobrevive</text>-<rect x="1026.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>-<text x="1133.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Additive convergence</text>-<text x="1133.0" y="146" text-anchor="middle" font-size="11" fill="#475569">la flota converge DESPUÉS</text>-<text x="1133.0" y="162" text-anchor="middle" font-size="11" fill="#475569">del ready — asíncrono,</text>-<text x="1133.0" y="178" text-anchor="middle" font-size="11" fill="#475569">sin bloquearlo</text>-<line x1="254.0" y1="150.0" x2="286.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>-<rect x="229.05" y="130.0" width="82.4" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="270.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">inventariar</text>-<line x1="500.5" y1="150.0" x2="533.0" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>-<rect x="478.75" y="130.0" width="76.0" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="516.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">certificar</text>-<line x1="747.0" y1="150.0" x2="779.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>-<rect x="738.05" y="130.0" width="50.400000000000006" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="763.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">/ready</text>-<line x1="993.5" y1="150.0" x2="1026.0" y2="150.0" stroke="#64748b" stroke-width="2.4" stroke-dasharray="6,4" marker-end="url(#aGray)" stroke-linecap="round"/>-<rect x="946.15" y="130.0" width="127.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="1009.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#475569">async · post-ready</text>-<text x="40" y="275" font-size="13" font-weight="700" fill="#b91c1c">2 · Fences fail-closed (R3 · R4 · J3)</text>-<rect x="40.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>-<text x="230.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Readiness gate — default () => false</text>-<text x="230.0" y="326" text-anchor="middle" font-size="11" fill="#475569">/ready = ready && providerAuthoritativeFn()</text>-<text x="230.0" y="342" text-anchor="middle" font-size="11" fill="#475569">throw → 503 · nunca un 200 equivocado</text>-<rect x="450.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>-<text x="640.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">G1 — deny sin inventory-authority</text>-<text x="640.0" y="326" text-anchor="middle" font-size="11" fill="#475569">currentWhenUnconfigured() → false + warn</text>-<text x="640.0" y="342" text-anchor="middle" font-size="11" fill="#475569">dev nunca construye el reconciler</text>-<rect x="860.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>-<text x="1050.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">F1 — rebind del readiness-poll</text>-<text x="1050.0" y="326" text-anchor="middle" font-size="11" fill="#475569">re-liga isCurrent/server en reconciles</text>-<text x="1050.0" y="342" text-anchor="middle" font-size="11" fill="#475569">superpuestos → la ventana no se autodestruye</text>-<text x="40" y="421" font-size="13" font-weight="700" fill="#6d28d9">3 · Egress externo DNS — contrato B3 (retener/revocar) + sliding-window #299</text>-<rect x="40" y="438" width="224" height="80" rx="11" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>-<text x="152.0" y="461" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">DNS binding observado</text>-<text x="152.0" y="480" text-anchor="middle" font-size="11" fill="#475569">resolveExternalEgressDns</text>-<text x="152.0" y="496" text-anchor="middle" font-size="11" fill="#475569">timeout 5s + fence isCurrent</text>-<polygon points="372,436 472,478 372,520 272,478" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>-<text x="372" y="475" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">¿identity</text>-<text x="372" y="491" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">unchanged?</text>-<line x1="264" y1="478" x2="272" y2="478" stroke="#7c3aed" stroke-width="2.6" marker-end="url(#aViolet)" stroke-linecap="round"/>-<rect x="540" y="414" width="306" height="60" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>-<text x="693.0" y="437" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">RETAIN — allow identity-stable</text>-<text x="693.0" y="456" text-anchor="middle" font-size="11" fill="#475569">mismo protocolo/puerto/ownership/cidrs</text>-<text x="693.0" y="472" text-anchor="middle" font-size="11" fill="#475569">· HCC sigue Ready (módulo cidr)</text>-<rect x="540" y="496" width="306" height="60" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>-<text x="693.0" y="519" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">REVOKE antes de readiness</text>-<text x="693.0" y="538" text-anchor="middle" font-size="11" fill="#475569">drift TCP→UDP · port · ownership · zero-cidr</text>-<text x="693.0" y="554" text-anchor="middle" font-size="11" fill="#475569">→ unprovable → cleanup fail-closed</text>-<path d="M 472,470 H 506.0 V 444 H 540" fill="none" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linejoin="round"/>-<rect x="423.6" y="415" width="88.80000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="468" y="427" text-anchor="middle" font-size="10" font-weight="700" fill="#047857">sí · retiene</text>-<path d="M 472,486 H 506.0 V 526 H 540" fill="none" stroke="#dc2626" stroke-width="2.6" marker-end="url(#aRose)" stroke-linejoin="round"/>-<rect x="417.2" y="523" width="101.60000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="468" y="535" text-anchor="middle" font-size="10" font-weight="700" fill="#b91c1c">drift · revoca</text>-<rect x="892" y="430" width="348" height="88" rx="11" fill="#fffbeb" stroke="#d97706" stroke-width="1.8" filter="url(#sh)"/>-<text x="1066.0" y="453" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ExternalEgressConvergenceCoordinator</text>-<text x="1066.0" y="472" text-anchor="middle" font-size="11" fill="#475569">resync periódico TTL-aware (≤ TTL/2, floor)</text>-<text x="1066.0" y="488" text-anchor="middle" font-size="11" fill="#475569">sliding-window accumulate · provider CIDRs</text>-<text x="1066.0" y="504" text-anchor="middle" font-size="11" fill="#475569">fence interval ≤ overlap/2 · limiter</text>-<line x1="846" y1="444" x2="892" y2="460" stroke="#d97706" stroke-width="2.6" marker-end="url(#aAmber)" stroke-linecap="round"/>-<rect x="821.4" y="425" width="95.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>-<text x="869" y="437" text-anchor="middle" font-size="10" font-weight="700" fill="#b45309">add (aditiva)</text>-<text x="40" y="575" font-size="13" font-weight="700" fill="#047857">4 · Seam de rotación #223 (preservado) & invariante de deploy single-writer</text>-<rect x="40" y="584" width="236" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>-<text x="158.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">validateSecret()</text>-<text x="158.0" y="626" text-anchor="middle" font-size="11" fill="#475569">sha256(canonicalJSON(secret.data))</text>-<rect x="306" y="584" width="306" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>-<text x="459.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ensureDeployment(rev, isCurrent)</text>-<text x="459.0" y="626" text-anchor="middle" font-size="11" fill="#475569">el fence de #205 se propaga aquí</text>-<rect x="642" y="584" width="300" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>-<text x="792.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">pod annotation</text>-<text x="792.0" y="626" text-anchor="middle" font-size="11" fill="#475569">clerum.io/credentials-revision → rueda 1 vez</text>-<line x1="276" y1="617" x2="306" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>-<line x1="612" y1="617" x2="642" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>-<rect x="972" y="584" width="268" height="66" rx="11" fill="#f8fafc" stroke="#475569" stroke-width="1.8" filter="url(#sh)"/>-<text x="1106.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Deploy single-writer</text>-<text x="1106.0" y="626" text-anchor="middle" font-size="11" fill="#475569">strategy: Recreate · replicas 1</text>-<text x="1106.0" y="642" text-anchor="middle" font-size="11" fill="#475569">· progressDeadlineSeconds 1200</text>-<text x="40" y="700" font-size="11.5" font-weight="700" fill="#334155">Leyenda</text>-<rect x="118" y="688" width="14" height="14" rx="3" fill="#2563eb" opacity="0.9"/>-<text x="137" y="700" font-size="11" fill="#475569">lifecycle readiness</text>-<rect x="282.6" y="688" width="14" height="14" rx="3" fill="#dc2626" opacity="0.9"/>-<text x="301.6" y="700" font-size="11" fill="#475569">fence fail-closed / revoke</text>-<rect x="492.0" y="688" width="14" height="14" rx="3" fill="#7c3aed" opacity="0.9"/>-<text x="511.0" y="700" font-size="11" fill="#475569">egress DNS (B3)</text>-<rect x="631.0" y="688" width="14" height="14" rx="3" fill="#d97706" opacity="0.9"/>-<text x="650.0" y="700" font-size="11" fill="#475569">coordinator #299</text>-<rect x="776.4" y="688" width="14" height="14" rx="3" fill="#059669" opacity="0.9"/>-<text x="795.4" y="700" font-size="11" fill="#475569">retain / rotación #223</text>-<rect x="960.2" y="688" width="14" height="14" rx="3" fill="#475569" opacity="0.9"/>-<text x="979.2" y="700" font-size="11" fill="#475569">invariante deploy</text>-<text x="40" y="858" font-size="10.5" fill="#94a3b8">PR #205 · evenfire-ai/evenfire · el /ready certifica una frontera de seguridad autoritativa ANTES de que la flota converja — la convergencia restante es aditiva y asíncrona</text>-</svg>\ No newline at end of filediff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex 0ae6125bb..e1227184a 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.7.0',externalRestApiVersion: '0.1.80',- rpcProxyVersion: '0.1.65',+ rpcProxyVersion: '0.1.64',desktopVersion: '0.7.0',minimumDesktopVersion: '0.1.252',}diff --git a/host-context-controller/README.md b/host-context-controller/README.mdindex 169458690..8ee40e981 100644--- a/host-context-controller/README.md+++ b/host-context-controller/README.md@@ -39,8 +39,6 @@ When an `McpServer` CRD is created, modified, or deleted, the host-context-contr- **MODIFIED** — Updates the Deployment and Service to match the CRD spec (env vars, image, ports, resources, etc.).- **DELETED** — Removes the Deployment and Service.-Remote egress-proxy McpServers keep the author's `spec.image` as desired-state input; HCC no longer rewrites it to the platform proxy image. The former `canonicalizeRemoteEgressProxyImage` step and its `ImageCanonicalized` status condition were removed (PR #205 readiness-safety cleanup): canonicalization mutated the observed CRD cache object and bumped the resource generation, retiring the in-flight reconcile before it could create the runtime resources. The platform-owned proxy image is instead selected at Deployment build time (`buildDeployment`), leaving the CRD spec untouched.-The reconciler builds Deployments with labels used for pod selection and network policy targeting:| Label | Value | Purpose |@@ -99,21 +97,7 @@ Servers removed from a Context lose their policies on the next reconcile; deleti- **`exact-host`** (default) — a CIDR binding, or a public DNS hostname resolved to `/32` ipBlocks, on one declared port/protocol. CIDRs and resolved IPs are rejected if they overlap `PUBLIC_EGRESS_EXCEPT_CIDRS` (RFC1918, CGNAT, loopback, link-local incl. cloud metadata, documentation, benchmarking, multicast, reserved ranges). Hostnames like `localhost`, `metadata.goog`, `*.internal`, `*.svc`, `*.cluster.local` are rejected outright.- **`public-web`** — `0.0.0.0/0` on TCP 80/443 with `PUBLIC_EGRESS_EXCEPT_CIDRS` carved out via `except`.-Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS-derived allows are never retained across a failed refresh: HCC revokes the unprovable policy before certifying global readiness, while only the affected runtime remains blocked until DNS recovers. Each lookup has a bounded deadline.--HCC readiness certifies authoritative inventory and revocation safety, not-completion of positive policy creation. A renamed or newly requested allow may-therefore remain unavailable after HCC becomes Ready until its additive-reconciliation succeeds. HCC never keeps an obsolete allow merely to avoid-that availability gap: when old and new ownership cannot be proven equivalent,-revocation wins and only the affected runtime remains fail-closed.--During an in-place release, HCC is also the controller that stamps-`CONTEXT_MAPPER_HOST_IMAGE` onto managed mcp-host workloads. The new HCC-producer is consequently running before it replaces those consumers with the-new mcp-host image. For compatibility, an omitted `status.authoritative` field-retains the legacy fail-closed meaning; only an explicit `authoritative: false`-asks a new mcp-host to preserve an unchanged live connection.+Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS answers are re-resolved periodically (`HCC_EXTERNAL_EGRESS_RESYNC_SEC`, default 300s).### Recipe binding policies@@ -121,8 +105,7 @@ McpServers carrying a `clerum.io/recipe-bindings` annotation (set for WorkflowRe### Reconciliation Lifecycle-- **Startup safety** — Ensures L0/L1 defaults, revokes stale or orphaned context-allow, rpc-proxy-egress, and external-egress policies from authoritative Context/McpServer inventories, then certifies readiness safety.-- **Startup additive convergence** — Reconciles Context allows after certification. The dedicated external-egress coordinator is the sole owner of McpServer egress creation, per-server startup gating, retry, and periodic DNS refresh; a failed binding blocks only that server's runtime.+- **Startup** — Ensures L0/L1 defaults, reconciles all Contexts and all McpServer egress bindings, then deletes orphaned context-allow policies in the MCP server namespace, plus orphaned rpc-proxy-egress and external-egress policies, whose owning CRD no longer exists (`fullReconcile`).- **Context ADDED/MODIFIED/DELETED** — Creates, updates, or removes the L2 policy set for that context.- **McpServer ADDED/MODIFIED/DELETED** — Reconciles L3 external egress (before the workload Deployment is created) and recipe binding policies.@@ -284,17 +267,16 @@ a permissive policy — see the NetworkPolicy and GFS tables:### MCP server provisioning-| Variable | Default | Description |-| -------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------- |-| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |-| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |-| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |-| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync interval (seconds). `0` disables periodic refresh; watch events still reconcile. |-| `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS` | `5000` | Per-hostname DNS lookup deadline in milliseconds. |+| Variable | Default | Description |+| -------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------- |+| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |+| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |+| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |+| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync (seconds), so DNS changes converge without a watch event. `0` disables. |### mcp-host provisioningdiff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 9bcfc9c0f..e9b28c157 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.223",+ "version": "0.1.170","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.223",+ "version": "0.1.170","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 42fe8f398..17c98b81a 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.223",+ "version": "0.1.170","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.tsindex f4e15bf35..f8d6857cb 100644--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts@@ -119,7 +119,6 @@ describe('connector credentials revision (issue #223)', () => {data: { 'api-key': stored('old-key') },})reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -221,7 +220,6 @@ describe('generation-aware rollout readiness (issue #223)', () => {vi.clearAllMocks()coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -324,7 +322,6 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',vi.useFakeTimers()coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -551,111 +548,3 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)})})--describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()- let reconciler: McpServerReconciler- // The inventory-authority generation IS mcpWatchGeneration in production- // (k8sClient wiring): a routine McpServer watch recovery bumps it.- let authorityGeneration: number- let currentServer: McpServerCRD-- // A rollout mid-flight (total 2 vs updated 1) that never converges on its own.- const midRollout = {- metadata: { generation: 2, labels: {} },- spec: { replicas: 1 },- status: {- observedGeneration: 2,- replicas: 2,- updatedReplicas: 1,- readyReplicas: 1,- availableReplicas: 1,- unavailableReplicas: 0,- },- }-- beforeEach(() => {- vi.clearAllMocks()- vi.useFakeTimers()- authorityGeneration = 1- currentServer = makeServer()- coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(coreApi),- customApi: asCustomApi(customApi),- })- reconciler.setInventoryAuthority(() => ({ known: true, generation: authorityGeneration }))- reconciler.setResolveCurrentServer(name => (name === 'linear' ? currentServer : undefined))- })-- afterEach(() => {- vi.useRealTimers()- })-- it('rebinds the fence so a converged rollout is recorded after a watch-generation bump (T1)', async () => {- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)- // R1 arms the window under authority generation 1 (timer pending).- await reconciler.reconcile(makeServer())- // A routine McpServer watch recovery bumps the inventory-authority generation.- authorityGeneration = 2- // R2, current under generation 2, finds R1's pending window and rebinds it.- await reconciler.reconcile(makeServer())- // The rollout converges before the (single) pending tick fires.- appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))- await vi.advanceTimersByTimeAsync(5000)-- // Without the rebind, R1's tick evaluates its stale generation-1 fence,- // self-deletes, and DeploymentReady=True is never written.- expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')- expect(- (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls.size- ).toBe(0)- })-- it('control: without the generation bump the rollout converges the same way (T2)', async () => {- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)- await reconciler.reconcile(makeServer())- await reconciler.reconcile(makeServer()) // no bump- appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))- await vi.advanceTimersByTimeAsync(5000)-- expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')- })-- it('rebind does NOT reset the attempt budget across the bump (E2 guard, T3)', async () => {- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout) // never converges- await reconciler.reconcile(makeServer())- await vi.advanceTimersByTimeAsync(2 * 5000) // 2 attempts consumed- authorityGeneration = 2- await reconciler.reconcile(makeServer()) // rebind — must NOT reset attempts- await vi.advanceTimersByTimeAsync(21 * 5000) // 2 + 21 = 23 attempts, still short of 24-- expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)- await vi.advanceTimersByTimeAsync(5000) // the 24th tick- expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(true)- })-- it('does NOT adopt a same-name recreate — a new uid gets a fresh window (T4)', async () => {- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)- await reconciler.reconcile(makeServer())- const polls = (reconciler as unknown as { readinessPolls: Map<string, { attempts: number }> })- .readinessPolls- const firstWindow = polls.get('linear')- await vi.advanceTimersByTimeAsync(12 * 5000)-- // Same name, NEW uid: identity retirement (beginStatusTracking), not a- // rebind, must own this — a rebind adopting the recreate would carry the- // old window's spent budget.- const recreated = { ...makeServer(), uid: 'uid-linear-9999' }- currentServer = recreated- await reconciler.reconcile(recreated)-- const secondWindow = polls.get('linear')- expect(secondWindow).not.toBe(firstWindow)- expect(secondWindow?.attempts).toBe(0)- })-})diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.tsindex fa30ec7a5..a4b425ae7 100644--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts@@ -89,7 +89,6 @@ describe('McpServer generated pod hardening', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.tsindex 044452efd..09c0b5b84 100644--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts@@ -87,7 +87,6 @@ function buildReconciler(customApi: MockCustomApi): McpServerReconciler {return new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -172,14 +171,7 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {const calls = customApi.patchNamespacedCustomObjectStatus.mock.callsconst lastCall = calls[calls.length - 1]?.[0]const patch = lastCall?.body- const statusAdd = Array.isArray(patch)- ? patch.find(- candidate =>- candidate?.op === 'add' &&- (candidate.path === '/status' || candidate.path === '/status/conditions')- )- : undefined- const value = statusAdd?.value+ const value = Array.isArray(patch) ? patch[0]?.value : undefinedreturn Array.isArray(value) ? value : (value?.conditions ?? [])}@@ -676,6 +668,105 @@ describe('McpServerReconciler remote egress proxy', () => {})})+ describe('canonicalizeRemoteEgressProxyImage', () => {+ it('patches stale remote McpServer spec.image to the platform egress proxy image', async () => {+ const staleRemote = cloneServer(REMOTE_SERVER, {+ image: LEGACY_REMOTE_EGRESS_IMAGE,+ })++ await (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)++ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({+ group: 'clerum.io',+ version: 'v1alpha1',+ namespace: 'mcp-server',+ plural: 'mcpservers',+ name: 'mcp-sentry-remote',+ body: {+ spec: {+ image: 'clerum/nginx-egress-proxy:0.1.0',+ },+ },+ }),+ expect.objectContaining({+ middleware: expect.any(Array),+ })+ )+ expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')+ expect(lastPatchedStatusConditions(customApi)).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: 'ImageCanonicalized',+ status: 'True',+ reason: 'RemoteEgressProxyImageMatches',+ message: 'Remote McpServer spec.image matches the platform egress proxy image',+ }),+ ])+ )+ })++ it('does not patch spec for a remote McpServer already using the platform image', async () => {+ const currentRemote = cloneServer(REMOTE_SERVER)++ await (reconciler as any).canonicalizeRemoteEgressProxyImage(currentRemote)++ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()+ expect(lastPatchedStatusConditions(customApi)).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: 'ImageCanonicalized',+ status: 'True',+ reason: 'RemoteEgressProxyImageMatches',+ message: 'Remote McpServer spec.image matches the platform egress proxy image',+ }),+ ])+ )+ })++ it('does not patch local McpServers even when their image differs', async () => {+ const localServer = cloneServer(LOCAL_SERVER, {+ image: 'clerum/custom-local-mcp:1.2.3',+ })++ await (reconciler as any).canonicalizeRemoteEgressProxyImage(localServer)++ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(localServer.spec.image).toBe('clerum/custom-local-mcp:1.2.3')+ })++ it('does not throw and records status when declarative image patching fails', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const staleRemote = cloneServer(REMOTE_SERVER, {+ image: LEGACY_REMOTE_EGRESS_IMAGE,+ })+ customApi.patchNamespacedCustomObject.mockRejectedValueOnce(new Error('rbac denied'))++ await expect(+ (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)+ ).resolves.toBeUndefined()++ expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)+ expect(warn).toHaveBeenCalledWith(+ expect.stringContaining('Failed to canonicalize remote McpServer'),+ expect.any(Error)+ )+ expect(lastPatchedStatusConditions(customApi)).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: 'ImageCanonicalized',+ status: 'False',+ reason: 'RemoteEgressProxyImagePatchFailed',+ message:+ 'Failed to patch remote McpServer spec.image to the platform egress proxy image',+ }),+ ])+ )+ warn.mockRestore()+ })+ })+// ─── Test 3: buildDeployment for local servers ─────────────────────────describe('buildDeployment for local servers', () => {@@ -744,12 +835,16 @@ describe('McpServerReconciler remote egress proxy', () => {expect(cmIndex).toBeLessThan(deployIndex)})- it('should use the platform image without rewriting a divergent remote desired image', async () => {+ it('should canonicalize stale remote image before syncing ConfigMap and Deployment', async () => {const callOrder: string[] = []const staleRemote = cloneServer(REMOTE_SERVER, {image: LEGACY_REMOTE_EGRESS_IMAGE,})+ customApi.patchNamespacedCustomObject.mockImplementation(async () => {+ callOrder.push('canonicalizeImage')+ return {}+ })coreApi.createNamespacedConfigMap.mockImplementation(async () => {callOrder.push('createConfigMap')return {}@@ -765,28 +860,23 @@ describe('McpServerReconciler remote egress proxy', () => {await reconciler.reconcile(staleRemote)- expect(callOrder).toEqual(['createConfigMap', 'createService', 'createDeployment'])- expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(- expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),- expect.anything()- )- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(+ expect(callOrder).toEqual([+ 'canonicalizeImage',+ 'createConfigMap',+ 'createService',+ 'createDeployment',+ ])+ expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')+ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(expect.objectContaining({- body: expect.objectContaining({- spec: expect.objectContaining({- template: expect.objectContaining({- spec: expect.objectContaining({- containers: [- expect.objectContaining({- name: 'egress-proxy',- image: 'clerum/nginx-egress-proxy:0.1.0',- }),- ],- }),- }),- }),- }),+ body: {+ spec: {+ image: 'clerum/nginx-egress-proxy:0.1.0',+ },+ },+ }),+ expect.objectContaining({+ middleware: expect.any(Array),}))})@@ -1025,12 +1115,6 @@ describe('McpServerReconciler remote egress proxy', () => {})describe('reconcileDelete for remote servers', () => {- beforeEach(() => {- reconciler.setInventoryAuthority(() => ({ known: true, generation: 1 }))- reconciler.setResolveCurrentServer(() => undefined)- customApi.getNamespacedCustomObject.mockRejectedValue(make404Error())- })-it('should clean up ConfigMap via deleteResources', async () => {mockHccOwnedRuntimeReads(appsApi, coreApi)@@ -1052,14 +1136,11 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should clear status tracking after delete', async () => {- let current: McpServerCRD | undefined = REMOTE_SERVER- reconciler.setResolveCurrentServer(() => current)// First reconcile to populate statusawait reconciler.reconcile(REMOTE_SERVER)expect(reconciler.getStatus('mcp-sentry-remote').deployed).toBe(true)// Then delete- current = undefinedawait reconciler.reconcileDelete('mcp-sentry-remote', 'mcp-server')const status = reconciler.getStatus('mcp-sentry-remote')expect(status.deployed).toBe(false)diff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.tsindex 38b7c571f..b3d979815 100644--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts@@ -91,7 +91,6 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -123,19 +122,6 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {const container = capturedContainer(appsApi)expect(container.imagePullPolicy).toBe('IfNotPresent')})-- it('does not mutate a watched desired spec while building the sanitized Deployment', async () => {- const server = makeServer({- imagePullPolicy: 'Always',- security: { runAsUser: 0, addCapabilities: ['CHOWN', 'SYS_ADMIN'] },- })- const originalSpec = structuredClone(server.spec)-- await reconciler.reconcile(server)-- expect(capturedContainer(appsApi).imagePullPolicy).toBe('IfNotPresent')- expect(server.spec).toEqual(originalSpec)- })})// ════════════════════════════════════════════════════════════════════diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.tsindex 48d6fe191..b2fd2bc8f 100644--- a/host-context-controller/src/bindingPolicyReconciler.test.ts+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts@@ -2,14 +2,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'-function deferred<T = void>() {- let resolve!: (value: T | PromiseLike<T>) => void- const promise = new Promise<T>(res => {- resolve = res- })- return { promise, resolve }-}-function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -48,9 +40,7 @@ describe('BindingPolicyReconciler', () => {const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]it('creates egress + ingress policies for a binding', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {- isCurrent: () => true,- })+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')// 2 calls: 1 egress + 1 ingressexpect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)@@ -85,9 +75,7 @@ describe('BindingPolicyReconciler', () => {})it('applies binding-allow labels to all policies', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {- isCurrent: () => true,- })+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body@@ -101,43 +89,12 @@ describe('BindingPolicyReconciler', () => {it('handles 409 conflict with create-or-replace pattern', async () => {mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {- isCurrent: () => true,- })+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()})- it('does not replace or start another apply after its authority lease is retired', async () => {- const readStarted = deferred()- const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()- mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {- readStarted.resolve()- return releaseRead.promise- })- let current = true-- const reconcile = reconciler.reconcileBindings(- 'my-recipe',- singleBinding,- 'redis-mcp',- 'redis-mcp',- { isCurrent: () => current }- )- await readStarted.promise-- current = false- releaseRead.resolve({ metadata: { resourceVersion: '1' } })- await reconcile-- // The first create was authorized and returned 409 without mutating. Once- // the lease is retired, no replacement or second policy apply may start.- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-it('cleanupBindings deletes policies in both namespaces', async () => {mockApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],@@ -151,65 +108,8 @@ describe('BindingPolicyReconciler', () => {expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalled()})- it('does not delete listed policies when a same-name recipe is recreated during LIST', async () => {- const bothListsStarted = deferred()- const releaseLists = deferred<{- items: Array<{ metadata: { name: string } }>- }>()- let listCalls = 0- mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {- listCalls += 1- if (listCalls === 2) bothListsStarted.resolve()- return releaseLists.promise- })- let recipeRecreated = false- const deleteAllowed = vi.fn(async () => !recipeRecreated)-- const cleanup = reconciler.cleanupBindings('my-recipe', { deleteAllowed })- await bothListsStarted.promise-- recipeRecreated = true- releaseLists.resolve({- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],- })- await cleanup-- expect(deleteAllowed).toHaveBeenCalledTimes(2)- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('rejects cleanup when listing binding policies fails', async () => {- mockApi.listNamespacedNetworkPolicy.mockRejectedValue(new Error('transient LIST failure'))-- await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(- 'Failed to clean up binding policies for recipe "my-recipe"'- )- })-- it('rejects cleanup when deleting a binding policy fails with a non-404 error', async () => {- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],- })- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('transient DELETE failure'))-- await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(- 'Failed to clean up binding policies for recipe "my-recipe"'- )- })-- it('treats a 404 while deleting binding policies as successful cleanup', async () => {- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],- })- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })-- await expect(reconciler.cleanupBindings('my-recipe')).resolves.toBeUndefined()- })-it('uses _from for ingress rules (K8s client ObjectSerializer maps _from → from)', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {- isCurrent: () => true,- })+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')const calls = mockApi.createNamespacedNetworkPolicy.mock.callsconst ingressCall = calls.find(@@ -226,9 +126,7 @@ describe('BindingPolicyReconciler', () => {})it('uses correct pod selectors and namespace-aware placement', async () => {- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {- isCurrent: () => true,- })+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')const calls = mockApi.createNamespacedNetworkPolicy.mock.callsconst egressCall = calls.find(@@ -298,8 +196,7 @@ describe('BindingPolicyReconciler', () => {'my-recipe',singleBinding,'redis-mcp',- 'my-recipe-redis-mcp',- { isCurrent: () => true }+ 'my-recipe-redis-mcp')const calls = mockApi.createNamespacedNetworkPolicy.mock.callsdiff --git a/host-context-controller/src/bindingPolicyReconciler.ts b/host-context-controller/src/bindingPolicyReconciler.tsindex 0281e69f6..b0486d63c 100644--- a/host-context-controller/src/bindingPolicyReconciler.ts+++ b/host-context-controller/src/bindingPolicyReconciler.ts@@ -37,18 +37,6 @@ export interface BindingDef {protocol?: 'TCP' | 'UDP'}-export interface BindingPolicyReconcileOptions {- // Required (fail-closed): every caller must supply the authority predicate so- // a retired reconcile pass cannot keep mutating binding NetworkPolicies. There- // is no assume-current default — an omitted predicate is a compile error, not- // a silent fail-open (the old `?? (() => true)`) nor a silent no-op.- isCurrent: () => boolean-}--export interface BindingPolicyCleanupOptions {- deleteAllowed?: () => Promise<boolean>-}-export class BindingPolicyReconciler {private networkingApi: k8s.NetworkingV1Apiprivate mcpServerNamespace: string@@ -72,86 +60,70 @@ export class BindingPolicyReconciler {recipeName: string,bindings: BindingDef[],mcpWorkloadName: string,- mcpServerName = mcpWorkloadName,- options: BindingPolicyReconcileOptions+ mcpServerName = mcpWorkloadName): Promise<void> {- const isCurrent = options.isCurrent- if (!isCurrent()) returnconsole.log(`[BindingNP] Reconciling ${bindings.length} binding(s) for recipe "${recipeName}"`)- // Sequence writes so authority can be rechecked between every Kubernetes- // effect. A retired pass must not start the next binding policy mutation.- for (const binding of bindings) {- if (!isCurrent()) return- const protocol = binding.protocol ?? 'TCP'+ await Promise.all(+ bindings.map(async binding => {+ const protocol = binding.protocol ?? 'TCP'- // Determine namespaces based on which workload is the MCP server.- // The MCP workload (with transport) lives in mcp-server namespace,- // the other workload lives in sandbox-recipes namespace.- const fromIsMcp = binding.from === mcpWorkloadName- const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE- const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace- const fromPodSelector: Record<string, string> = fromIsMcp- ? { [MCPSERVER_LABEL]: mcpServerName }- : { app: binding.from }- const toPodSelector: Record<string, string> = fromIsMcp- ? { app: binding.to }- : { [MCPSERVER_LABEL]: mcpServerName }+ // Determine namespaces based on which workload is the MCP server.+ // The MCP workload (with transport) lives in mcp-server namespace,+ // the other workload lives in sandbox-recipes namespace.+ const fromIsMcp = binding.from === mcpWorkloadName+ const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE+ const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace+ const fromPodSelector: Record<string, string> = fromIsMcp+ ? { [MCPSERVER_LABEL]: mcpServerName }+ : { app: binding.from }+ const toPodSelector: Record<string, string> = fromIsMcp+ ? { app: binding.to }+ : { [MCPSERVER_LABEL]: mcpServerName }- const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')- const egressPolicy = this.buildEgressPolicy(- egressName,- recipeName,- fromPodSelector,- toPodSelector,- fromNs,- toNs,- binding.port,- protocol- )- const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')- const ingressPolicy = this.buildIngressPolicy(- ingressName,- recipeName,- fromPodSelector,- toPodSelector,- fromNs,- toNs,- binding.port,- protocol- )+ const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')+ const egressPolicy = this.buildEgressPolicy(+ egressName,+ recipeName,+ fromPodSelector,+ toPodSelector,+ fromNs,+ toNs,+ binding.port,+ protocol+ )+ const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')+ const ingressPolicy = this.buildIngressPolicy(+ ingressName,+ recipeName,+ fromPodSelector,+ toPodSelector,+ fromNs,+ toNs,+ binding.port,+ protocol+ )- if (!isCurrent()) return- await this.applyPolicy(egressName, egressPolicy, fromNs, isCurrent)- if (!isCurrent()) return- await this.applyPolicy(ingressName, ingressPolicy, toNs, isCurrent)- }+ // Egress and ingress target different namespaces — safe to parallelize+ await Promise.all([+ this.applyPolicy(egressName, egressPolicy, fromNs),+ this.applyPolicy(ingressName, ingressPolicy, toNs),+ ])+ })+ )}/*** Delete all binding-scoped NetworkPolicies for a recipe.*/- async cleanupBindings(- recipeName: string,- options: BindingPolicyCleanupOptions = {}- ): Promise<void> {+ async cleanupBindings(recipeName: string): Promise<void> {console.log(`[BindingNP] Cleaning up binding policies for recipe "${recipeName}"`)- // LIST operations are read-only and namespace-independent. Each resulting- // delete is separately authorized after its LIST completes.- const results = await Promise.allSettled([- this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE, options.deleteAllowed),- this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace, options.deleteAllowed),+ // Delete from both namespaces in parallel (independent operations)+ await Promise.all([+ this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE),+ this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace),])- const failures = results.flatMap(result =>- result.status === 'rejected' ? [result.reason] : []- )- if (failures.length > 0) {- throw new AggregateError(- failures,- `Failed to clean up binding policies for recipe "${recipeName}"`- )- }}// ─── Policy Builders ────────────────────────────────────────────@@ -270,49 +242,43 @@ export class BindingPolicyReconciler {private async applyPolicy(name: string,policy: k8s.V1NetworkPolicy,- namespace: string,- isCurrent: () => boolean+ namespace: string): Promise<void> {- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]', isCurrent)+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]')}- private async deletePoliciesForRecipe(- recipeName: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {- let response: k8s.V1NetworkPolicyList+ private async deletePoliciesForRecipe(recipeName: string, namespace: string): Promise<void> {try {- response = await this.networkingApi.listNamespacedNetworkPolicy({+ const response = await this.networkingApi.listNamespacedNetworkPolicy({namespace,labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=binding-allow,${RECIPE_LABEL}=${recipeName}`,})++ await Promise.all(+ (response.items || []).map(async policy => {+ const policyName = policy.metadata?.name+ if (!policyName) {+ console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)+ return+ }+ try {+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404) {+ console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)+ }+ }+ })+ )} catch (error) {console.error(`[BindingNP] Failed to list policies for recipe "${recipeName}" in ${namespace}:`,error)- throw error- }-- for (const policy of response.items || []) {- const policyName = policy.metadata?.name- if (!policyName) {- console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)- continue- }- try {- if (deleteAllowed && !(await deleteAllowed())) return- await this.networkingApi.deleteNamespacedNetworkPolicy({- name: policyName,- namespace,- })- console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)- } catch (error: unknown) {- if (getErrorCode(error) === 404) continue- console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)- throw error- }}}}diff --git a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts b/host-context-controller/src/config.externalEgressDnsTimeout.test.tsdeleted file mode 100644index 7396924b9..000000000--- a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts+++ /dev/null@@ -1,46 +0,0 @@-import { afterEach, describe, expect, it, vi } from 'vitest'--const originalEnv = process.env--async function loadConfig(raw: string | undefined) {- vi.resetModules()- process.env = { ...originalEnv }- if (raw === undefined) {- delete process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS- } else {- process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS = raw- }- return import('./config')-}--afterEach(() => {- process.env = originalEnv- vi.resetModules()-})--describe('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS', () => {- it.each([undefined, '', ' '])('defaults to 5000 for %s', async raw => {- const { config } = await loadConfig(raw)-- expect(config.externalEgressDnsResolveTimeoutMs).toBe(5_000)- })-- it.each([- ['1', 1],- ['5000', 5_000],- ['2147483647', 2_147_483_647],- ])('accepts the positive bounded integer %s', async (raw, expected) => {- const { config } = await loadConfig(raw)-- expect(config.externalEgressDnsResolveTimeoutMs).toBe(expected)- })-- it.each(['0', '-1', '1.5', '5000junk', 'NaN', 'Infinity', '2147483648'])(- "fails configuration validation for '%s'",- async raw => {- await expect(loadConfig(raw)).rejects.toThrow(- 'HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer'- )- }- )-})diff --git a/host-context-controller/src/config.ts b/host-context-controller/src/config.tsindex 162ccd06e..7b7632b75 100644--- a/host-context-controller/src/config.ts+++ b/host-context-controller/src/config.ts@@ -197,10 +197,6 @@ export interface Config {// so accumulated IPs never expire between refreshes (issue #299).externalEgressResyncIntervalSec: number- // Deadline for one DNS resolution attempt. A silent resolver cannot block- // safety convergence indefinitely.- externalEgressDnsResolveTimeoutMs: number-// Grace kept after a DNS TTL before an accumulated /32 may expire (seconds).// The sliding window that fixes #299: entries live for TTL + overlap, so a// provider that rotates a single A record still resolves through the union of@@ -249,17 +245,6 @@ function getEnvInt(key: string, defaultValue: number): number {return isNaN(parsed) ? defaultValue : parsed}-export function parseExternalEgressDnsTimeoutMs(raw: string | undefined): number {- if (raw === undefined || raw.trim() === '') return 5_000- const parsed = Number(raw)- if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 2_147_483_647) {- throw new Error(- `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer no greater than 2147483647, got '${raw}'`- )- }- return parsed-}-/*** Loud integer env parser for the external-egress knobs (audit R1-L1). Unlike* getEnvInt (parseInt), this rejects non-canonical values instead of silently@@ -729,9 +714,6 @@ export const config: Config = {// they expire, closing the #299 rotation gap. 0 disables (reintroduces the// stale-snapshot risk; only for operators who explicitly opt out).externalEgressResyncIntervalSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_RESYNC_SEC', 60),- externalEgressDnsResolveTimeoutMs: parseExternalEgressDnsTimeoutMs(- getEnv('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS')- ),externalEgressOverlapSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_OVERLAP_SEC', 300),externalEgressRefreshFloorSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_REFRESH_FLOOR_SEC',diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.tsdeleted file mode 100644index 7a9258139..000000000--- a/host-context-controller/src/deploymentStrategy.test.ts+++ /dev/null@@ -1,66 +0,0 @@-import { describe, expect, it } from 'vitest'-import { loadAll } from 'js-yaml'-import { readFileSync } from 'node:fs'-import path from 'node:path'--/**- * HCC's safety model assumes a single writer: replicas:1 with NO leader- * election (see statelessLifecycleExecutor.ts — "Running >1 replica risks two- * controllers racing"). The default Deployment strategy is RollingUpdate with- * maxSurge>0, which starts the new pod before the old one exits — a guaranteed- * two-replica overlap on every rollout. During that window the outgoing replica- * can win a conflict-retry replace (applyNetworkPolicy → replaceWithConflictRetry)- * and land its wider spec back over a policy the new replica just hardened and- * certified. There is no NetworkPolicy watch or periodic resync to heal it, and- * the new replica's safety fence only sees failures in its own process — so- * /ready stays 200 over a stale allow. Recreate closes the overlap, which is the- * only way to honour the documented single-writer assumption during a rollout.- */-describe('host-context-controller Deployment strategy', () => {- const docs = loadAll(- readFileSync(- path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),- 'utf8'- )- ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>-- const deployment = docs.find(- d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'- )-- it('defines the host-context-controller Deployment', () => {- expect(deployment).toBeDefined()- })-- it('runs a single replica', () => {- expect(deployment?.spec?.replicas).toBe(1)- })-- it('uses Recreate so a rollout never overlaps two writers', () => {- // RollingUpdate (the default) would start the new pod before terminating the- // old one; with no leader election that races two NetworkPolicy writers.- const strategy = deployment?.spec?.strategy as { type?: string } | undefined- expect(strategy?.type).toBe('Recreate')- })-- it('carries no rollingUpdate block under Recreate', () => {- // A leftover spec.strategy.rollingUpdate is invalid under Recreate and, if a- // future edit flipped only `type` back, would quietly reintroduce surge.- const strategy = deployment?.spec?.strategy as { rollingUpdate?: unknown } | undefined- expect(strategy?.rollingUpdate).toBeUndefined()- })-- it('bounds rollout-stall detection above the worst legitimate startup', () => {- // Detection only, not recovery: Recreate + replicas:1 means a botched- // rollout is healed operationally (kubectl rollout undo), never by K8s.- // This deadline just flips Progressing=False / fails `kubectl rollout- // status` deterministically for observers without their own --timeout. It- // must exceed the worst legitimate startup, which — until #205 fully- // decouples readiness from fleet convergence — scales with the McpServer/- // Context fleet: origin/dev observed 651s on clerum-dev (108 McpServers /- // 22 Contexts) on 2026-08-05, so 1200s (900s wait + 300s head-start) is the- // merged mitigation. Comes back down once readiness no longer waits on the- // fleet.- expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)- })-})diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.tsdeleted file mode 100644index ac8a5d926..000000000--- a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts+++ /dev/null@@ -1,968 +0,0 @@-import { afterEach, describe, expect, it, vi } from 'vitest'-import {- ExternalEgressConvergenceCoordinator,- type ExternalEgressRetryHandle,- type ExternalEgressWatchEventType,-} from './externalEgressConvergenceCoordinator'-import { externalEgressRetriesAtCap } from './metrics'-import type { McpServerCRD } from './types'--function deferred<T = void>() {- let resolve!: (value: T | PromiseLike<T>) => void- let reject!: (reason?: unknown) => void- const promise = new Promise<T>((resolvePromise, rejectPromise) => {- resolve = resolvePromise- reject = rejectPromise- })- return { promise, resolve, reject }-}--function server(- name: string,- generation = 1,- options: { bindings?: boolean; namespace?: string } = {}-): McpServerCRD {- return {- name,- namespace: options.namespace ?? 'mcp-server',- uid: `${name}-uid`,- generation,- spec: {- contextRef: 'default',- image: `clerum/${name}:v${generation}`,- transport: { type: 'streamableHttp', port: 3000 },- egressBindings:- options.bindings === false ? undefined : [{ dns: `${name}.example`, port: 443 }],- },- }-}--type CoordinatorOverrides = Partial<- ConstructorParameters<typeof ExternalEgressConvergenceCoordinator>[0]->--function coordinator(overrides: CoordinatorOverrides = {}) {- const servers = new Map<string, McpServerCRD>()- const mutate = vi.fn(async () => undefined)- const replay = vi.fn(async () => undefined)- const instance = new ExternalEgressConvergenceCoordinator({- listServers: () => [...servers.values()],- getCurrentServer: name => servers.get(name),- inventoryAuthoritative: () => true,- sameDesiredRevision: (left, right) =>- left.uid === right.uid &&- left.generation === right.generation &&- JSON.stringify(left.spec) === JSON.stringify(right.spec),- enqueue: async (_selected, work) => work(),- mutate,- replay,- externalEgressRefreshMinTtlMs: () => Infinity,- ...overrides,- })- return { instance, mutate, replay, servers }-}--async function flushMicrotasks(): Promise<void> {- await Promise.resolve()- await Promise.resolve()- await Promise.resolve()-}--describe('ExternalEgressConvergenceCoordinator', () => {- afterEach(() => {- vi.useRealTimers()- vi.restoreAllMocks()- })-- it.each([- ['presence', 'DELETED' as const, undefined],- ['revision', 'MODIFIED' as const, server('redis-tools', 2)],- ])(- 'lets an existing timer consume the latest %s intent and resets its attempt ladder',- async (_change, nextType, nextServer) => {- vi.useFakeTimers()- const initial = server('redis-tools', 1)- const state = new Map<string, McpServerCRD>([[initial.name, initial]])- const replayed: Array<{ type: ExternalEgressWatchEventType; server: McpServerCRD }> = []- let instance!: ExternalEgressConvergenceCoordinator- instance = coordinator({- getCurrentServer: name => state.get(name),- replay: async (type, selected) => {- replayed.push({ type, server: selected })- instance.scheduleRetry(type, selected)- },- }).instance-- instance.scheduleRetry('ADDED', initial)- if (nextServer) state.set(nextServer.name, nextServer)- else state.delete(initial.name)- instance.scheduleRetry(nextType, nextServer ?? initial)-- await vi.advanceTimersByTimeAsync(5000)- expect(replayed).toEqual([- {- type: nextType,- server: expect.objectContaining({- generation: nextServer?.generation ?? initial.generation,- }),- },- ])-- await vi.advanceTimersByTimeAsync(4999)- expect(replayed).toHaveLength(1)- await vi.advanceTimersByTimeAsync(1)- expect(replayed).toHaveLength(2)- instance.stop()- }- )-- it('continues retrying at the capped delay until the replay succeeds', async () => {- vi.useFakeTimers()- const selected = server('web-search')- let instance!: ExternalEgressConvergenceCoordinator- let attempts = 0- instance = coordinator({- getCurrentServer: () => selected,- replay: async (type, current, retry) => {- attempts += 1- if (attempts < 5) {- instance.scheduleRetry(type, current)- return- }- retry.complete()- },- }).instance-- instance.scheduleRetry('ADDED', selected)- await vi.advanceTimersByTimeAsync(5000)- await vi.advanceTimersByTimeAsync(15000)- await vi.advanceTimersByTimeAsync(30000)- expect(attempts).toBe(3)- await vi.advanceTimersByTimeAsync(29999)- expect(attempts).toBe(3)- await vi.advanceTimersByTimeAsync(1)- expect(attempts).toBe(4)- await vi.advanceTimersByTimeAsync(29999)- expect(attempts).toBe(4)- await vi.advanceTimersByTimeAsync(1)-- expect(attempts).toBe(5)- await vi.advanceTimersByTimeAsync(30000)- expect(attempts).toBe(5)- instance.stop()- })-- it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {- vi.useFakeTimers()- const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')- const selected = server('web-search')- let instance!: ExternalEgressConvergenceCoordinator- let succeed = false- instance = coordinator({- getCurrentServer: () => selected,- replay: async (type, current, retry) => {- if (succeed) {- retry.complete()- return- }- instance.scheduleRetry(type, current)- },- }).instance-- instance.scheduleRetry('ADDED', selected)- // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.- await vi.advanceTimersByTimeAsync(5000)- expect(setSpy).not.toHaveBeenCalled()-- // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.- await vi.advanceTimersByTimeAsync(15000)- expect(setSpy).toHaveBeenLastCalledWith(1)-- // Once the replay finally succeeds, the capped server is cleared back to 0.- succeed = true- await vi.advanceTimersByTimeAsync(30000)- expect(setSpy).toHaveBeenLastCalledWith(0)-- instance.stop()- setSpy.mockRestore()- })-- it('never clears a pending full replay after an egress-only success', async () => {- vi.useFakeTimers()- const selected = server('web-search')- const { instance, replay, servers } = coordinator()- servers.set(selected.name, selected)-- instance.scheduleRetry('ADDED', selected)- const retry = await instance.reconcile('MODIFIED', selected)- expect(retry).toBeDefined()-- await vi.advanceTimersByTimeAsync(5000)- expect(replay).toHaveBeenCalledOnce()- instance.stop()- })-- it('clears a retry only after its complete full-replay handle finishes', async () => {- vi.useFakeTimers()- const selected = server('web-search')- let instance!: ExternalEgressConvergenceCoordinator- const replays: ExternalEgressRetryHandle[] = []- instance = coordinator({- getCurrentServer: () => selected,- replay: async (type, current, retry) => {- replays.push(retry)- const completion = await instance.reconcile(type, current, { retry })- expect(completion).toBe(retry)- retry.complete()- },- }).instance-- instance.scheduleRetry('ADDED', selected)- await vi.advanceTimersByTimeAsync(5000)- await vi.advanceTimersByTimeAsync(30000)-- expect(replays).toHaveLength(1)- instance.stop()- })-- it('prevents a stale retry handle from clearing a newer intent', async () => {- vi.useFakeTimers()- const first = server('web-search', 1)- const second = server('web-search', 2)- const state = new Map<string, McpServerCRD>([[first.name, first]])- const handles: ExternalEgressRetryHandle[] = []- const replayed: McpServerCRD[] = []- const firstReplayStarted = deferred()- const releaseFirstReplay = deferred()- const { instance } = coordinator({- getCurrentServer: name => state.get(name),- replay: async (_type, current, retry) => {- replayed.push(current)- handles.push(retry)- if (replayed.length === 1) {- firstReplayStarted.resolve()- await releaseFirstReplay.promise- }- },- })-- instance.scheduleRetry('ADDED', first)- await vi.advanceTimersByTimeAsync(5000)- await firstReplayStarted.promise- expect(handles).toHaveLength(1)-- state.set(second.name, second)- instance.scheduleRetry('MODIFIED', second)- handles[0].complete()- releaseFirstReplay.resolve()- await flushMicrotasks()- await vi.advanceTimersByTimeAsync(5000)-- expect(replayed).toEqual([- expect.objectContaining({ generation: 1 }),- expect.objectContaining({ generation: 2 }),- ])- instance.stop()- })-- it('keeps retry, resync, and startup concurrency lanes independent', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const retryRelease = deferred()- const activeRetries = deferred()- let retryCount = 0- const resyncServer = server('resync-server')- const startupServer = server('startup-server')- const canonical = new Map<string, McpServerCRD>([- [resyncServer.name, resyncServer],- [startupServer.name, startupServer],- ])- const mutations: string[] = []- const { instance } = coordinator({- listServers: () => [resyncServer],- getCurrentServer: name => canonical.get(name),- mutate: async (_type, selected) => {- mutations.push(selected.name)- },- replay: async () => {- retryCount += 1- if (retryCount === 10) activeRetries.resolve()- await retryRelease.promise- },- })-- for (let index = 0; index < 10; index += 1) {- const selected = server(`retry-${index}`, 1, { bindings: false })- instance.scheduleRetry('ADDED', selected)- }- await vi.advanceTimersByTimeAsync(5000)- await activeRetries.promise-- const resync = instance.runResync()- const startup = instance.prepareStartupGates([startupServer])- await Promise.all([resync, startup.waitFor(startupServer)])-- expect(mutations).toEqual(expect.arrayContaining(['resync-server', 'startup-server']))- retryRelease.resolve()- instance.stop()- })-- it('queues at most one latest-wins retry job per key behind a saturated limiter', async () => {- vi.useFakeTimers()- const blockersRelease = deferred()- const tenBlockersStarted = deferred()- const finalReplayStarted = deferred()- const initial = server('churning-server', 1)- const second = server('churning-server', 2)- const final = server('churning-server', 3)- const state = new Map<string, McpServerCRD>([[initial.name, initial]])- const targetEnqueues: McpServerCRD[] = []- const targetReplays: McpServerCRD[] = []- let blockerCount = 0- const { instance } = coordinator({- getCurrentServer: name => state.get(name),- enqueue: async (selected, work) => {- if (selected.name === initial.name) targetEnqueues.push(selected)- await work()- },- replay: async (_type, selected, retry) => {- if (selected.name.startsWith('limiter-blocker-')) {- blockerCount += 1- if (blockerCount === 10) tenBlockersStarted.resolve()- await blockersRelease.promise- return- }- targetReplays.push(selected)- retry.complete()- finalReplayStarted.resolve()- },- })-- for (let index = 0; index < 10; index += 1) {- instance.scheduleRetry('DELETED', server(`limiter-blocker-${index}`))- }- await vi.advanceTimersByTimeAsync(5000)- await tenBlockersStarted.promise-- instance.scheduleRetry('ADDED', initial)- await vi.advanceTimersByTimeAsync(5000)- state.set(second.name, second)- instance.scheduleRetry('MODIFIED', second)- await vi.advanceTimersByTimeAsync(5000)- state.set(final.name, final)- instance.scheduleRetry('MODIFIED', final)- await vi.advanceTimersByTimeAsync(5000)-- expect(targetEnqueues).toHaveLength(0)- blockersRelease.resolve()- await finalReplayStarted.promise- await flushMicrotasks()- await flushMicrotasks()-- expect(targetEnqueues).toHaveLength(1)- expect(targetReplays).toEqual([expect.objectContaining({ generation: 3 })])- instance.stop()- })-- it('re-arms a retry scheduled by an active replay only after that key leaves its slot', async () => {- vi.useFakeTimers()- const selected = server('slow-retry')- const firstReplayStarted = deferred()- const releaseFirstReplay = deferred()- let instance!: ExternalEgressConvergenceCoordinator- let replayCount = 0- instance = coordinator({- getCurrentServer: () => selected,- replay: async (type, current, retry) => {- replayCount += 1- if (replayCount === 1) {- instance.scheduleRetry(type, current)- firstReplayStarted.resolve()- await releaseFirstReplay.promise- return- }- retry.complete()- },- }).instance-- instance.scheduleRetry('ADDED', selected)- await vi.advanceTimersByTimeAsync(5000)- await firstReplayStarted.promise-- await vi.advanceTimersByTimeAsync(15000)- expect(replayCount).toBe(1)-- releaseFirstReplay.resolve()- await flushMicrotasks()- await vi.advanceTimersByTimeAsync(14999)- expect(replayCount).toBe(1)- await vi.advanceTimersByTimeAsync(1)- expect(replayCount).toBe(2)- instance.stop()- })-- it('cancels a retry waiter on stop before limiter capacity becomes available', async () => {- vi.useFakeTimers()- const blockersRelease = deferred()- const tenBlockersStarted = deferred()- const queued = server('queued-before-stop')- let blockerCount = 0- const enqueuedTargets: string[] = []- const { instance } = coordinator({- enqueue: async (selected, work) => {- if (selected.name === queued.name) enqueuedTargets.push(selected.name)- await work()- },- replay: async (_type, selected) => {- if (!selected.name.startsWith('stop-blocker-')) return- blockerCount += 1- if (blockerCount === 10) tenBlockersStarted.resolve()- await blockersRelease.promise- },- })-- for (let index = 0; index < 10; index += 1) {- instance.scheduleRetry('DELETED', server(`stop-blocker-${index}`))- }- await vi.advanceTimersByTimeAsync(5000)- await tenBlockersStarted.promise- instance.scheduleRetry('DELETED', queued)- await vi.advanceTimersByTimeAsync(5000)-- instance.stop()- blockersRelease.resolve()- await flushMicrotasks()- await flushMicrotasks()-- expect(enqueuedTargets).toEqual([])- })-- it('resolves the current canonical server after periodic jitter', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0.5)- const stale = server('web-search', 1)- const current = server('web-search', 2)- const state = new Map<string, McpServerCRD>([[stale.name, stale]])- const reconciled: McpServerCRD[] = []- const { instance } = coordinator({- listServers: () => [stale],- getCurrentServer: name => state.get(name),- mutate: async (_type, selected) => {- reconciled.push(selected)- },- })-- const resync = instance.runResync()- await flushMicrotasks()- state.set(current.name, current)- await vi.advanceTimersByTimeAsync(2500)- await resync-- expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])- instance.stop()- })-- it('returns one global single-flight promise for overlapping resync requests', async () => {- vi.spyOn(Math, 'random').mockReturnValue(0)- const selected = server('web-search')- const release = deferred()- const mutationStarted = deferred()- const { instance, servers } = coordinator({- mutate: async () => {- mutationStarted.resolve()- await release.promise- },- })- servers.set(selected.name, selected)-- const first = instance.runResync()- const overlapping = instance.runResync()- expect(overlapping).toBe(first)- await mutationStarted.promise-- release.resolve()- await first- instance.stop()- })-- it('skips a resync key whose external-egress mutation is already in flight', async () => {- vi.spyOn(Math, 'random').mockReturnValue(0)- const selected = server('web-search')- const release = deferred()- const mutationStarted = deferred()- const mutate = vi.fn(async () => {- mutationStarted.resolve()- await release.promise- })- const { instance, servers } = coordinator({ mutate })- servers.set(selected.name, selected)-- const active = instance.reconcile('MODIFIED', selected)- await mutationStarted.promise- await instance.runResync()-- expect(mutate).toHaveBeenCalledOnce()- release.resolve()- await active- instance.stop()- })-- it('resolves canonical state after entering the keyed resync queue', async () => {- vi.spyOn(Math, 'random').mockReturnValue(0)- const stale = server('web-search', 1)- const current = server('web-search', 2)- const state = new Map<string, McpServerCRD>([[stale.name, stale]])- const queueEntered = deferred()- const releaseQueue = deferred()- const reconciled: McpServerCRD[] = []- const { instance } = coordinator({- listServers: () => [stale],- getCurrentServer: name => state.get(name),- enqueue: async (_selected, work) => {- queueEntered.resolve()- await releaseQueue.promise- await work()- },- mutate: async (_type, selected) => {- reconciled.push(selected)- },- })-- const resync = instance.runResync()- await queueEntered.promise- state.set(current.name, current)- releaseQueue.resolve()- await resync-- expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])- instance.stop()- })-- it('keeps a startup gate false when desired revision changes during egress', async () => {- const selected = server('web-search', 1)- const current = server('web-search', 2)- const state = new Map<string, McpServerCRD>([[selected.name, selected]])- const mutationStarted = deferred()- const releaseMutation = deferred()- let selectedLease: (() => boolean) | undefined- const { instance } = coordinator({- getCurrentServer: name => state.get(name),- mutate: async (_type, _server, options) => {- selectedLease = options.isCurrent- mutationStarted.resolve()- await releaseMutation.promise- },- })-- const gates = instance.prepareStartupGates([selected])- await mutationStarted.promise- expect(selectedLease?.()).toBe(true)- state.set(current.name, current)- expect(selectedLease?.()).toBe(false)- releaseMutation.resolve()-- await expect(gates.waitFor(selected)).resolves.toBe(false)- instance.stop()- })-- it('keeps a failed startup DNS gate isolated and recovers it through the dedicated retry', async () => {- vi.useFakeTimers()- const selected = server('dns-held-server')- const dnsFailure = new Error('dns resolution failed')- const mutate = vi.fn().mockRejectedValueOnce(dnsFailure).mockResolvedValue(undefined)- const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)- let instance!: ExternalEgressConvergenceCoordinator- const built = coordinator({- mutate,- replay: async (type, current, retry) => {- const completion = await instance.reconcile(type, current, { retry })- completion?.complete()- },- })- instance = built.instance- built.servers.set(selected.name, selected)-- const gates = instance.prepareStartupGates([selected])- await expect(gates.waitFor(selected)).resolves.toBe(false)- expect(mutate).toHaveBeenCalledOnce()- expect(error).toHaveBeenCalledWith(- `[K8s] Initial external egress reconciliation failed for ${selected.namespace}/${selected.name}; ` +- 'runtime reconciliation will stay blocked until retry succeeds:',- dnsFailure- )-- await vi.advanceTimersByTimeAsync(5000)- expect(mutate).toHaveBeenCalledTimes(2)- await vi.advanceTimersByTimeAsync(30000)- expect(mutate).toHaveBeenCalledTimes(2)- instance.stop()- })-- it('retires a direct mutation lease when desired revision changes in place', async () => {- const selected = server('web-search', 1)- const replacement = server('web-search', 2)- const mutationStarted = deferred()- const releaseMutation = deferred()- let selectedLease: (() => boolean) | undefined- const { instance, servers } = coordinator({- mutate: async (_type, _server, options) => {- selectedLease = options.isCurrent- mutationStarted.resolve()- await releaseMutation.promise- },- })- servers.set(selected.name, selected)-- const mutation = instance.reconcile('MODIFIED', selected)- await mutationStarted.promise- expect(selectedLease?.()).toBe(true)-- servers.set(replacement.name, replacement)- expect(selectedLease?.()).toBe(false)- releaseMutation.resolve()- await mutation- instance.stop()- })-- it('does not start a second mutation after stop releases an in-flight wait', async () => {- const firstMutation = deferred()- const selected = server('web-search')- const mutate = vi.fn(async () => firstMutation.promise)- const { instance, servers } = coordinator({ mutate })- servers.set(selected.name, selected)-- const first = instance.reconcile('MODIFIED', selected)- await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())- const waiting = instance.reconcile('MODIFIED', selected)- instance.stop()- firstMutation.resolve()- await Promise.all([first, waiting])-- expect(mutate).toHaveBeenCalledOnce()- })-- it('rechecks per-key in-flight ownership when multiple waiters resume together', async () => {- const selected = server('web-search')- const releases = [deferred(), deferred(), deferred()]- const starts = [deferred(), deferred(), deferred()]- let active = 0- let maxActive = 0- let call = 0- const mutate = vi.fn(async () => {- const index = call++- active += 1- maxActive = Math.max(maxActive, active)- starts[index].resolve()- await releases[index].promise- active -= 1- })- const { instance, servers } = coordinator({ mutate })- servers.set(selected.name, selected)-- const first = instance.reconcile('MODIFIED', selected)- await starts[0].promise- const second = instance.reconcile('MODIFIED', selected)- const third = instance.reconcile('MODIFIED', selected)-- try {- releases[0].resolve()- await starts[1].promise- await flushMicrotasks()- expect(maxActive).toBe(1)- } finally {- for (const release of releases) release.resolve()- await Promise.all([first, second, third])- instance.stop()- }- })-- it('rechecks a caller lease after waiting for an in-flight mutation', async () => {- const firstMutation = deferred()- const selected = server('web-search')- let current = true- const mutate = vi.fn(async () => firstMutation.promise)- const { instance, servers } = coordinator({ mutate })- servers.set(selected.name, selected)-- const first = instance.reconcile('MODIFIED', selected)- await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())- const waiting = instance.reconcile('MODIFIED', selected, {- isCurrent: () => current,- })- current = false- firstMutation.resolve()- await Promise.all([first, waiting])-- expect(mutate).toHaveBeenCalledOnce()- instance.stop()- })-- it('keeps a retry pending while inventory authority is unavailable', async () => {- vi.useFakeTimers()- const selected = server('web-search')- let authoritative = false- const { instance, replay, servers } = coordinator({- inventoryAuthoritative: () => authoritative,- })- servers.set(selected.name, selected)-- instance.scheduleRetry('ADDED', selected)- await vi.advanceTimersByTimeAsync(5000)- expect(replay).not.toHaveBeenCalled()-- authoritative = true- await vi.advanceTimersByTimeAsync(14_999)- expect(replay).not.toHaveBeenCalled()- await vi.advanceTimersByTimeAsync(1)- expect(replay).toHaveBeenCalledOnce()- instance.stop()- })-- it('skips periodic resync while inventory authority is unavailable', async () => {- const selected = server('web-search')- let authoritative = false- const { instance, mutate, servers } = coordinator({- inventoryAuthoritative: () => authoritative,- })- servers.set(selected.name, selected)-- await instance.runResync()- expect(mutate).not.toHaveBeenCalled()-- authoritative = true- await instance.runResync()- expect(mutate).toHaveBeenCalledOnce()- instance.stop()- })-- it('does not resume periodic work from jitter after stop', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0.5)- const selected = server('web-search')- const { instance, mutate, servers } = coordinator()- servers.set(selected.name, selected)-- const resync = instance.runResync()- await flushMicrotasks()- instance.stop()- await vi.advanceTimersByTimeAsync(2500)- await resync-- expect(mutate).not.toHaveBeenCalled()- })-- it('does not start an eleventh startup mutation after stop releases the limiter', async () => {- const release = deferred()- const tenStarted = deferred()- let active = 0- const selected = Array.from({ length: 11 }, (_, index) => server(`startup-${index}`))- const state = new Map(selected.map(item => [item.name, item]))- const mutate = vi.fn(async () => {- active += 1- if (active === 10) tenStarted.resolve()- await release.promise- })- const { instance } = coordinator({- getCurrentServer: name => state.get(name),- mutate,- })-- const gates = instance.prepareStartupGates(selected)- await tenStarted.promise- instance.stop()- release.resolve()- await Promise.all(selected.map(item => gates.waitFor(item)))-- expect(mutate).toHaveBeenCalledTimes(10)- })-- it('passes DELETE authority through to the watcher mutation without evaluating it', async () => {- const selected = server('web-search')- const deleteAllowed = vi.fn(async () => false)- const mutate = vi.fn(async () => undefined)- const { instance } = coordinator({ mutate })-- await instance.reconcile('DELETED', selected, { deleteAllowed })-- expect(mutate).toHaveBeenCalledWith('DELETED', selected, {- deleteAllowed,- isCurrent: expect.any(Function),- })- expect(deleteAllowed).not.toHaveBeenCalled()- instance.stop()- })-- it('owns periodic resync scheduling and stops it immediately', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const selected = server('web-search')- const { instance, mutate, servers } = coordinator()- servers.set(selected.name, selected)-- instance.startPeriodicResync(1, 1)- await vi.advanceTimersByTimeAsync(1000)- expect(mutate).toHaveBeenCalledOnce()-- instance.stop()- await vi.advanceTimersByTimeAsync(5000)- expect(mutate).toHaveBeenCalledOnce()- })-- it('warns once when periodic resync is disabled (intervalSec <= 0) and arms no timer', async () => {- vi.useFakeTimers()- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- const selected = server('web-search')- const { instance, mutate, servers } = coordinator()- servers.set(selected.name, selected)-- instance.startPeriodicResync(0, 5)- // No timer armed: advancing time never triggers a resync pass.- await vi.advanceTimersByTimeAsync(60_000)- expect(mutate).not.toHaveBeenCalled()-- expect(warnSpy).toHaveBeenCalledTimes(1)- expect(warnSpy).toHaveBeenCalledWith(- expect.stringContaining('External egress periodic DNS resync disabled')- )-- // Called again while still disabled: no duplicate warning (once only).- instance.startPeriodicResync(-1, 5)- expect(warnSpy).toHaveBeenCalledTimes(1)-- instance.stop()- })-- it('does not warn about a disabled resync when it is enabled (intervalSec > 0)', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- const selected = server('web-search')- const { instance, servers } = coordinator()- servers.set(selected.name, selected)-- instance.startPeriodicResync(1, 1)- expect(warnSpy).not.toHaveBeenCalledWith(- expect.stringContaining('External egress periodic DNS resync disabled')- )-- instance.stop()- })-- it('arms one resync at a time, rescheduling only after each pass completes (H2)', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const selected = server('web-search')- const release: Array<() => void> = []- const mutate = vi.fn(- () =>- new Promise<void>(resolve => {- release.push(resolve)- })- )- const { instance, servers } = coordinator({ mutate })- servers.set(selected.name, selected)-- instance.startPeriodicResync(10, 5)- // First pass fires at the configured interval (10s) and then blocks.- await vi.advanceTimersByTimeAsync(10_000)- expect(mutate).toHaveBeenCalledOnce()-- // Complete the pass off the fixed grid (at t=17s). The successor is armed- // relative to completion (t=27s), NOT on a fixed 20s/30s interval grid.- await vi.advanceTimersByTimeAsync(7_000)- release[0]()- await flushMicrotasks()-- // A fixed setInterval would fire a second pass at t=20s; the self-- // rescheduling timer must not, because it only re-armed at completion.- await vi.advanceTimersByTimeAsync(3_000)- expect(mutate).toHaveBeenCalledOnce()-- // The successor fires 10s after completion (t=27s).- await vi.advanceTimersByTimeAsync(7_000)- expect(mutate).toHaveBeenCalledTimes(2)-- instance.stop()- release.forEach(resolve => resolve())- })-- it('advances the next periodic resync to <= observed TTL/2 (H2, issue #299)', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const selected = server('web-search')- // The mutation observes a low DNS TTL, ratcheting the coordinator's source- // down; scheduleNext reads it AFTER the pass and must advance to <= TTL/2.- let minObservedTtlMs = Infinity- const mutate = vi.fn(async () => {- minObservedTtlMs = 20_000- })- const { instance, servers } = coordinator({- mutate,- externalEgressRefreshMinTtlMs: () => minObservedTtlMs,- })- servers.set(selected.name, selected)-- // Interval 60s, floor 5s. No TTL observed yet, so the first pass fires at 60s.- instance.startPeriodicResync(60, 5)- await vi.advanceTimersByTimeAsync(59_999)- expect(mutate).not.toHaveBeenCalled()- await vi.advanceTimersByTimeAsync(1)- expect(mutate).toHaveBeenCalledOnce()-- // TTL/2 = 10s now governs the next delay, far below the 60s backstop.- await vi.advanceTimersByTimeAsync(9_999)- expect(mutate).toHaveBeenCalledOnce()- await vi.advanceTimersByTimeAsync(1)- expect(mutate).toHaveBeenCalledTimes(2)-- instance.stop()- })-- it('logs and absorbs a rejected periodic resync pass, then re-arms (no unhandledRejection)', async () => {- vi.useFakeTimers()- vi.spyOn(Math, 'random').mockReturnValue(0)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)- // Capture any process-level unhandledRejection: the pre-fix code left the- // rejected `void runResync()` promise unhandled, which crashes modern Node.- const unhandled: unknown[] = []- const onUnhandled = (reason: unknown): void => {- unhandled.push(reason)- }- process.on('unhandledRejection', onUnhandled)-- const selected = server('web-search')- // Reject OUTSIDE the per-server try/catch in runResyncCore (that catch only- // wraps the mutation). A rejecting `enqueue` propagates through the worker to- // Promise.all -> runResyncCore -> runResync, so the whole pass rejects.- const enqueue = vi.fn(async () => {- throw new Error('resync enqueue boom')- })- const { instance, servers } = coordinator({ enqueue })- servers.set(selected.name, selected)-- try {- instance.startPeriodicResync(10, 5)- // First pass fires at 10s and rejects.- await vi.advanceTimersByTimeAsync(10_000)- await flushMicrotasks()-- // The rejection is logged by the fix's .catch (RED without it: no such log).- expect(errorSpy).toHaveBeenCalledWith(- '[K8s] External egress periodic resync pass failed:',- expect.any(Error)- )- expect(enqueue).toHaveBeenCalledTimes(1)-- // The loop RE-ARMS despite the failure: the successor fires 10s later.- await vi.advanceTimersByTimeAsync(10_000)- await flushMicrotasks()- expect(enqueue).toHaveBeenCalledTimes(2)-- // The fix handles the rejection, so nothing escaped as unhandled.- expect(unhandled).toEqual([])- } finally {- process.off('unhandledRejection', onUnhandled)- instance.stop()- }- })-})diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.tsdeleted file mode 100644index 6a8a19db2..000000000--- a/host-context-controller/src/externalEgressConvergenceCoordinator.ts+++ /dev/null@@ -1,607 +0,0 @@-import { externalEgressRetriesAtCap } from './metrics'-import type { McpServerCRD } from './types'--const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const-const EXTERNAL_EGRESS_MAX_CONCURRENCY = 10-const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000--export type ExternalEgressWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'--export type ExternalEgressRetryHandle = {- isCurrent: () => boolean- complete: () => void-}--type ExternalEgressRetryIntent = {- sequence: number- type: ExternalEgressWatchEventType- server: McpServerCRD-}--type ExternalEgressMutationOptions = {- deleteAllowed?: () => Promise<boolean>- isCurrent: () => boolean-}--type ExternalEgressReconcileOptions = {- deleteAllowed?: () => Promise<boolean>- retry?: ExternalEgressRetryHandle- isCurrent?: () => boolean-}--type ExternalEgressCoordinatorDependencies = {- listServers: () => McpServerCRD[]- getCurrentServer: (name: string) => McpServerCRD | undefined- inventoryAuthoritative: () => boolean- sameDesiredRevision: (left: McpServerCRD, right: McpServerCRD) => boolean- enqueue: (server: McpServerCRD, work: () => Promise<void>) => Promise<void>- mutate: (- type: ExternalEgressWatchEventType,- server: McpServerCRD,- options: ExternalEgressMutationOptions- ) => Promise<void>- replay: (- type: ExternalEgressWatchEventType,- server: McpServerCRD,- retry: ExternalEgressRetryHandle- ) => Promise<void>- // Smallest DNS TTL (ms) observed across external-egress resolutions, or- // Infinity if none. startPeriodicResync reads it to advance the next resync to- // <= TTL/2 (H2, issue #299).- externalEgressRefreshMinTtlMs: () => number-}--export type ExternalEgressStartupGates = {- waitFor: (server: Pick<McpServerCRD, 'name' | 'namespace'>) => Promise<boolean>-}--class AsyncConcurrencyLimiter {- private active = 0- private readonly waiters: Array<(admitted: boolean) => void> = []- private closed = false-- constructor(private readonly limit: number) {}-- async run<T>(work: () => Promise<T>): Promise<T | undefined> {- if (this.closed) return undefined- if (this.active >= this.limit) {- const admitted = await new Promise<boolean>(resolve => this.waiters.push(resolve))- if (!admitted) return undefined- } else {- this.active += 1- }- try {- return await work()- } finally {- this.release()- }- }-- cancelPending(): void {- if (this.closed) return- this.closed = true- for (const resume of this.waiters.splice(0)) {- resume(false)- }- }-- private release(): void {- this.active -= 1- if (this.closed) return- const resume = this.waiters.shift()- if (!resume) return- // Transfer the released permit before waking the waiter. Reserving it here- // prevents a new caller from entering between wake-up and its microtask.- this.active += 1- resume(true)- }-}--function delay(ms: number): Promise<void> {- return new Promise(resolve => setTimeout(resolve, ms))-}--/**- * External-egress resync delay (ms), H2 (issue #299). The configured interval is- * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a- * rotating low-TTL host is sampled every rotation. The floor is the hard lower- * bound to avoid a hot-loop. Pure and unit-testable; consumed by- * {@link ExternalEgressConvergenceCoordinator.startPeriodicResync}.- */-export function externalEgressResyncDelayMs(- intervalSec: number,- floorSec: number,- minObservedTtlMs = Infinity-): number {- const floorMs = floorSec * 1000- const configuredMs = intervalSec * 1000- const ttlAwareMs = Number.isFinite(minObservedTtlMs)- ? Math.min(configuredMs, minObservedTtlMs / 2)- : configuredMs- return Math.max(ttlAwareMs, floorMs)-}--/**- * Owns temporal coordination for external-egress convergence. Kubernetes- * authority and mutations remain behind callbacks supplied by the watcher.- */-export class ExternalEgressConvergenceCoordinator {- private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>()- private readonly retryAttempts = new Map<string, number>()- private readonly retryIntents = new Map<string, ExternalEgressRetryIntent>()- private retrySequence = 0- private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)- private readonly retrySlots = new Map<string, object>()- // Keys whose retry backoff is currently pinned at the capped (maximum) delay.- // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a- // binding stuck retrying forever is observable, not silently denied.- private readonly retriesAtCap = new Set<string>()- private readonly inFlight = new Map<string, Promise<void>>()- private resyncInFlight: Promise<void> | null = null- private resyncTimer: ReturnType<typeof setTimeout> | null = null- private resyncDisabledWarned = false- private stopped = false-- constructor(private readonly dependencies: ExternalEgressCoordinatorDependencies) {}-- startPeriodicResync(intervalSec: number, floorSec: number): void {- if (this.stopped || this.resyncTimer) return- if (intervalSec <= 0) {- // Disabled. Warn once (mirrors the Host/SFS/GFS resync lanes) so the- // operational consequence is visible: without periodic resync the- // accumulated external-egress DNS allows are never refreshed, so a- // binding whose upstream DNS rotates (issue #299) can drift until an- // unrelated McpServer event triggers reconciliation.- if (!this.resyncDisabledWarned) {- this.resyncDisabledWarned = true- console.warn(- '[K8s] External egress periodic DNS resync disabled; accumulated DNS allows will not ' +- 'be refreshed, so a binding whose upstream DNS rotates (issue #299) can drift until ' +- 'another McpServer event triggers reconciliation.'- )- }- return- }- // Self-rescheduling (not a fixed setInterval) so the delay can advance to- // <= observed TTL/2 (H2, issue #299), and rescheduling AFTER each pass- // completes guarantees no overlapping resyncs. runResync still dedupes any- // in-flight pass; the floor is the hard lower bound against a hot-loop.- const scheduleNext = (): void => {- if (this.stopped) return- const delayMs = externalEgressResyncDelayMs(- intervalSec,- floorSec,- this.dependencies.externalEgressRefreshMinTtlMs()- )- this.resyncTimer = setTimeout(() => {- // Re-arm FIRST (.finally) so a rejected pass still schedules its- // successor — the "re-arms after completion" property must survive a- // failure. Then .catch logs and absorbs the rejection so a rejecting- // pass (or a throwing scheduleNext) never becomes an unhandledRejection,- // which crashes the process on modern Node and would silently kill the- // periodic resync loop (reopening the #299 DNS-drift window).- void this.runResync()- .finally(scheduleNext)- .catch(error =>- console.error('[K8s] External egress periodic resync pass failed:', error)- )- }, delayMs)- this.resyncTimer.unref?.()- }- scheduleNext()- console.log(- `[K8s] External egress periodic DNS resync enabled (every ${intervalSec}s, advancing to <= TTL/2)`- )- }-- runResync(): Promise<void> {- if (this.resyncInFlight) return this.resyncInFlight- const run = this.runResyncCore()- this.resyncInFlight = run- const settle = () => {- if (this.resyncInFlight === run) this.resyncInFlight = null- }- void run.then(settle, settle)- return run- }-- prepareStartupGates(- servers: McpServerCRD[],- isCurrent: () => boolean = this.dependencies.inventoryAuthoritative- ): ExternalEgressStartupGates {- const gates = new Map<string, Promise<boolean>>()- const selectedServers = servers- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)- .sort((left, right) => this.key(left).localeCompare(this.key(right)))-- if (selectedServers.length > 0) {- console.log(- `[K8s] Reconciling external egress before startup runtime reconciliation for ` +- `${selectedServers.length} McpServer(s)`- )- }-- // Startup admission is intentionally independent from retry and periodic- // resync capacity, so a saturated recovery lane cannot head-of-line block- // initial runtime gates for unrelated servers.- const limiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)- for (const selected of selectedServers) {- const key = this.key(selected)- const gate = limiter- .run(async () => {- if (this.stopped || !isCurrent()) return false- let ready = false- await this.dependencies.enqueue(selected, async () => {- if (this.stopped || !isCurrent()) return- const current = this.dependencies.getCurrentServer(selected.name)- if (- !current ||- current.namespace !== selected.namespace ||- !this.dependencies.sameDesiredRevision(selected, current)- ) {- return- }- try {- // Startup is an egress-only gate. Any pre-existing retry intent is- // completed only by its full replay, never by this mutation.- await this.reconcile('MODIFIED', current, { isCurrent })- const latest = this.dependencies.getCurrentServer(selected.name)- ready =- !this.stopped &&- isCurrent() &&- latest !== undefined &&- latest.namespace === selected.namespace &&- this.dependencies.sameDesiredRevision(selected, latest)- } catch (error) {- console.error(- `[K8s] Initial external egress reconciliation failed for ${key}; ` +- 'runtime reconciliation will stay blocked until retry succeeds:',- error- )- this.scheduleRetry('MODIFIED', current)- }- })- return ready- })- .then(result => result ?? false)- // Side-attach a logger so a rejection is never *unhandled* (which crashes- // the process on modern Node) if fullReconcile aborts before waitFor- // awaits this gate. This does NOT swallow the failure: gates.set stores- // the original `gate`, so the awaited path in fullReconcile still rejects- // and fails the pass loudly — this only covers the unawaited case.- gate.catch(error => {- console.error(- `[K8s] External egress startup gate for ${key} rejected before it was awaited:`,- error- )- })- gates.set(key, gate)- }-- return {- waitFor: server => gates.get(this.key(server)) ?? Promise.resolve(true),- }- }-- async reconcile(- type: ExternalEgressWatchEventType,- server: McpServerCRD,- options: ExternalEgressReconcileOptions = {}- ): Promise<ExternalEgressRetryHandle | undefined> {- if (!this.reconcileIsCurrent(type, server, options)) return undefined- const key = this.key(server)- let existing = this.inFlight.get(key)- while (existing) {- console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)- await existing- if (!this.reconcileIsCurrent(type, server, options)) return undefined- existing = this.inFlight.get(key)- }-- if (!this.reconcileIsCurrent(type, server, options)) return undefined- const completion = options.retry ?? this.currentRetryHandle(key)- const run = this.dependencies.mutate(type, server, {- deleteAllowed: options.deleteAllowed,- isCurrent: () => this.reconcileIsCurrent(type, server, options),- })- this.inFlight.set(key, run)- try {- await run- return completion- } finally {- if (this.inFlight.get(key) === run) {- this.inFlight.delete(key)- }- }- }-- scheduleRetry(type: string, server: McpServerCRD): void {- if (!this.isWatchEventType(type) || this.stopped) return-- const cached = this.dependencies.getCurrentServer(server.name)- const retryType: ExternalEgressWatchEventType =- cached && cached.namespace === server.namespace- ? type === 'ADDED' && this.dependencies.sameDesiredRevision(server, cached)- ? 'ADDED'- : 'MODIFIED'- : 'DELETED'- const retryServer = cached && cached.namespace === server.namespace ? cached : server- const key = this.key(server)- const previousIntent = this.retryIntents.get(key)- if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {- this.retryAttempts.delete(key)- // A new target restarts the backoff ladder, so the key is no longer capped.- this.clearRetryAtCap(key)- }-- const intent: ExternalEgressRetryIntent = {- sequence: ++this.retrySequence,- type: retryType,- server: retryServer,- }- this.retryIntents.set(key, intent)- // A timer or queued/active slot already owning the key consumes the newest- // intent when it next enters the retry and per-server lanes.- if (this.retryTimers.has(key) || this.retrySlots.has(key)) return- this.armRetryTimer(key)- }-- private armRetryTimer(key: string): void {- if (- this.stopped ||- this.retryTimers.has(key) ||- this.retrySlots.has(key) ||- !this.retryIntents.has(key)- ) {- return- }- const attempt = Math.min(- (this.retryAttempts.get(key) ?? 0) + 1,- EXTERNAL_EGRESS_RETRY_DELAYS_MS.length- )- const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]- this.retryAttempts.set(key, attempt)- const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length- if (capped) this.markRetryAtCap(key)- console.warn(- `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +- `${capped ? ' (capped)' : ''} ` +- `for McpServer "${this.retryIntents.get(key)!.server.name}" in ${delayMs}ms`- )-- const timer = setTimeout(() => {- if (this.retryTimers.get(key) !== timer) return- this.retryTimers.delete(key)- this.queueRetry(key)- }, delayMs)- this.retryTimers.set(key, timer)- }-- private queueRetry(key: string): void {- if (this.stopped || !this.retryIntents.has(key)) return- if (this.retrySlots.has(key)) {- // A timer can overlap work admitted through another lane. The durable- // slot owns the latest intent and rearms it on exit if it remains live.- return- }-- const slot = {}- this.retrySlots.set(key, slot)- void this.retryLimiter- .run(async () => {- if (- this.stopped ||- this.retrySlots.get(key) !== slot ||- !this.dependencies.inventoryAuthoritative()- ) {- return- }- // Do not capture intent before the limiter. Churn while capacity is- // saturated is represented by one slot and resolved latest-wins here.- const admittedIntent = this.retryIntents.get(key)- if (!admittedIntent) return- await this.dependencies.enqueue(admittedIntent.server, async () => {- if (- this.stopped ||- this.retrySlots.get(key) !== slot ||- !this.dependencies.inventoryAuthoritative()- ) {- return- }- // The keyed queue may also wait. Resolve the intent again so only the- // latest desired revision or absence is replayed.- const currentIntent = this.retryIntents.get(key)- if (!currentIntent) return- await this.replayIntent(currentIntent)- })- })- .catch(error => {- if (!this.stopped) {- console.error(`[K8s] External egress retry execution failed for ${key}:`, error)- }- })- .finally(() => {- if (this.retrySlots.get(key) !== slot) return- this.retrySlots.delete(key)- // A replay may schedule its successor while still active, or a newer- // intent may arrive after this slot selected work. Rearm only after the- // key leaves the slot so one key never accumulates limiter backlog.- if (!this.stopped && this.retryIntents.has(key) && !this.retryTimers.has(key)) {- this.armRetryTimer(key)- }- })- }-- stop(): void {- if (this.stopped) return- this.stopped = true- this.retryLimiter.cancelPending()- if (this.resyncTimer) {- clearTimeout(this.resyncTimer)- this.resyncTimer = null- }- for (const timer of this.retryTimers.values()) {- clearTimeout(timer)- }- this.retryTimers.clear()- this.retryAttempts.clear()- this.retryIntents.clear()- this.retrySlots.clear()- if (this.retriesAtCap.size > 0) {- this.retriesAtCap.clear()- externalEgressRetriesAtCap.set(0)- }- this.inFlight.clear()- }-- private async runResyncCore(): Promise<void> {- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return- const servers = this.dependencies- .listServers()- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)- .sort((left, right) => this.key(left).localeCompare(this.key(right)))- if (servers.length === 0) return-- console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)- let index = 0- const workers = Array.from(- { length: Math.min(EXTERNAL_EGRESS_MAX_CONCURRENCY, servers.length) },- async () => {- while (!this.stopped && this.dependencies.inventoryAuthoritative()) {- const server = servers[index++]- if (!server) return- const key = this.key(server)- if (this.inFlight.has(key)) {- console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)- continue- }- const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)- if (jitterMs > 0) await delay(jitterMs)- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return-- await this.dependencies.enqueue(server, async () => {- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return- // The fleet snapshot selects work only. Desired state is resolved- // after jitter and after entering the per-server FIFO lane.- const current = this.dependencies.getCurrentServer(server.name)- if (- !current ||- current.namespace !== server.namespace ||- (current.spec.egressBindings?.length ?? 0) === 0- ) {- return- }- try {- const retryIntent = this.retryIntents.get(key)- if (retryIntent) {- await this.replayIntent(retryIntent)- return- }- // Periodic refresh is egress-only and therefore cannot complete- // a retry intent that may appear while the mutation is running.- await this.reconcile('MODIFIED', current, {- isCurrent: this.dependencies.inventoryAuthoritative,- })- } catch (error) {- console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)- this.scheduleRetry('MODIFIED', current)- }- })- }- }- )- await Promise.all(workers)- }-- private async replayIntent(intent: ExternalEgressRetryIntent): Promise<void> {- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return- const handle = this.retryHandle(intent)- if (!handle.isCurrent()) return-- const current = this.dependencies.getCurrentServer(intent.server.name)- const type: ExternalEgressWatchEventType =- current && current.namespace === intent.server.namespace ? 'MODIFIED' : 'DELETED'- const server = type === 'DELETED' ? intent.server : current!- await this.dependencies.replay(type, server, handle)- }-- private reconcileIsCurrent(- type: ExternalEgressWatchEventType,- server: McpServerCRD,- options: ExternalEgressReconcileOptions- ): boolean {- const callerIsCurrent =- !this.stopped &&- (!options.retry || options.retry.isCurrent()) &&- (!options.isCurrent || options.isCurrent())- if (!callerIsCurrent || type === 'DELETED') return callerIsCurrent-- const current = this.dependencies.getCurrentServer(server.name)- return (- current !== undefined &&- current.namespace === server.namespace &&- this.dependencies.sameDesiredRevision(server, current)- )- }-- private currentRetryHandle(key: string): ExternalEgressRetryHandle | undefined {- const intent = this.retryIntents.get(key)- return intent ? this.retryHandle(intent) : undefined- }-- private retryHandle(intent: ExternalEgressRetryIntent): ExternalEgressRetryHandle {- return {- isCurrent: () =>- !this.stopped &&- this.retryIntents.get(this.key(intent.server))?.sequence === intent.sequence,- complete: () => {- if (this.retryIntents.get(this.key(intent.server))?.sequence !== intent.sequence) return- this.clearRetry(intent.server)- },- }- }-- private clearRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {- const key = this.key(server)- const timer = this.retryTimers.get(key)- if (timer) {- clearTimeout(timer)- this.retryTimers.delete(key)- }- this.retryAttempts.delete(key)- this.retryIntents.delete(key)- this.clearRetryAtCap(key)- }-- private markRetryAtCap(key: string): void {- if (this.retriesAtCap.has(key)) return- this.retriesAtCap.add(key)- externalEgressRetriesAtCap.set(this.retriesAtCap.size)- }-- private clearRetryAtCap(key: string): void {- if (!this.retriesAtCap.delete(key)) return- externalEgressRetriesAtCap.set(this.retriesAtCap.size)- }-- private sameRetryTarget(- previous: ExternalEgressRetryIntent,- type: ExternalEgressWatchEventType,- server: McpServerCRD- ): boolean {- const previousPresent = previous.type !== 'DELETED'- const nextPresent = type !== 'DELETED'- return (- previousPresent === nextPresent &&- this.dependencies.sameDesiredRevision(previous.server, server)- )- }-- private key(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {- return `${server.namespace}/${server.name}`- }-- private isWatchEventType(type: string): type is ExternalEgressWatchEventType {- return type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED'- }-}diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.tsindex d2fd7965b..c9cde6bcb 100644--- a/host-context-controller/src/gfsReconciler.test.ts+++ b/host-context-controller/src/gfsReconciler.test.ts@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'import type * as k8s from '@kubernetes/client-node'import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'import { GfsFactoryConfig } from './k8s/gfsFactory'-import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'+import type { GlobalFileSystemCRD } from './types'const config: GfsFactoryConfig = {gfsNamespace: 'gfs',@@ -42,12 +42,11 @@ class FakeApi implements GfsK8sApi {netpols: string[] = []pvcs: string[] = []deleted: string[] = []- statuses: GlobalFileSystemStatus[] = []+ statuses: { phase?: string; serviceUrl?: string }[] = []writerNeedsUpdate = truewriterAvailable = truewriterAvailabilitySequence: boolean[] = []failPvc = false- statusFailuresRemaining = 0async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {if (this.failPvc) throw new Error('simulated PVC apply failure')@@ -102,11 +101,11 @@ class FakeApi implements GfsK8sApi {async deletePvc(name: string): Promise<void> {this.deleted.push(`pvc/${name}`)}- async patchStatus(_name: string, _ns: string, status: GlobalFileSystemStatus): Promise<void> {- if (this.statusFailuresRemaining > 0) {- this.statusFailuresRemaining--- throw new Error('simulated status patch failure')- }+ async patchStatus(+ _name: string,+ _ns: string,+ status: { phase?: string; serviceUrl?: string }+ ): Promise<void> {this.statuses.push(status)}}@@ -219,59 +218,6 @@ describe('GfsReconciler.reconcile', () => {expect(seed.calls).toBe(1)})- it('does not rewrite unchanged status but still publishes a later phase transition', async () => {- const api = new FakeApi()- api.writerNeedsUpdate = false- const reconciler = new GfsReconciler(api, config, undefined, 0, 0)-- await reconciler.reconcile(gfs)- const observedReady = { ...gfs, status: api.statuses[0] }- await reconciler.reconcile(observedReady)-- expect(api.statuses).toHaveLength(1)- expect(api.statuses[0]?.phase).toBe('Ready')-- api.writerAvailable = false- await reconciler.reconcile(observedReady)-- expect(api.statuses.map(status => status.phase)).toEqual(['Ready', 'Initializing'])- })-- it('publishes status for a same-name recreation even when the DELETE watch was missed', async () => {- const api = new FakeApi()- const reconciler = new GfsReconciler(api, config)-- await reconciler.reconcile(gfs)- await reconciler.reconcile({ ...gfs, status: api.statuses[0] })- await reconciler.reconcile({ ...gfs, status: undefined })-- expect(api.statuses).toHaveLength(2)- })-- it('repairs status that was removed externally after HCC observed its prior write', async () => {- const api = new FakeApi()- const reconciler = new GfsReconciler(api, config)-- await reconciler.reconcile(gfs)- await reconciler.reconcile({ ...gfs, status: api.statuses[0] })- await reconciler.reconcile({ ...gfs, status: undefined })-- expect(api.statuses).toHaveLength(2)- expect(api.statuses.at(-1)?.phase).toBe('Ready')- })-- it('retries an unchanged status when the previous patch failed', async () => {- const api = new FakeApi()- api.statusFailuresRemaining = 1- const reconciler = new GfsReconciler(api, config)-- await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')- await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()-- expect(api.statuses).toHaveLength(1)- expect(api.statuses[0]?.phase).toBe('Ready')- })-it('reports Initializing and does NOT seed when the writer is not yet available', async () => {const api = new FakeApi()api.writerAvailable = false@@ -319,15 +265,4 @@ describe('GfsReconciler.reconcileDelete', () => {expect(api.deleted).toContain('pvc/gfs-drive')expect(api.deleted.at(-1)).toBe('pvc/gfs-drive') // bytes removed LAST})-- it('publishes status again after delete clears the prior write suppression', async () => {- const api = new FakeApi()- const reconciler = new GfsReconciler(api, config)-- await reconciler.reconcile(gfs)- await reconciler.reconcileDelete(gfs)- await reconciler.reconcile(gfs)-- expect(api.statuses).toHaveLength(2)- })})diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.tsindex 0462a12e6..de5e29ea2 100644--- a/host-context-controller/src/gfsReconciler.ts+++ b/host-context-controller/src/gfsReconciler.ts@@ -95,27 +95,6 @@ export class GfsReconciler {} while (true)}- private async writeStatus(- gfs: GlobalFileSystemCRD,- status: GlobalFileSystemStatus- ): Promise<void> {- // Compare against status observed on this exact LIST/watch object, not- // process memory keyed only by name. A local name cache could suppress the- // repair of a recreated object when its DELETE event was missed, or of- // status that another actor removed. Conversely, an identical observed- // status proves this is our own status-only MODIFIED event and closes the- // patch -> watch -> reconcile loop without another API write.- if (- gfs.status?.phase === status.phase &&- gfs.status?.pvcName === status.pvcName &&- gfs.status?.serviceName === status.serviceName &&- gfs.status?.serviceUrl === status.serviceUrl- ) {- return- }- await this.api.patchStatus(gfs.name, gfs.namespace, status)- }-async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {const ns = this.config.gfsNamespaceconst writerDeployment = buildDeployment(gfs, this.config, 'writer')@@ -156,7 +135,7 @@ export class GfsReconciler {}const phase = writerReady ? 'Ready' : 'Initializing'- await this.writeStatus(gfs, {+ await this.api.patchStatus(gfs.name, gfs.namespace, {phase,pvcName: pvcName(gfs),serviceName: serviceName(gfs),diff --git a/host-context-controller/src/hostFleetScheduler.test.ts b/host-context-controller/src/hostFleetScheduler.test.tsdeleted file mode 100644index 0f6817cfa..000000000--- a/host-context-controller/src/hostFleetScheduler.test.ts+++ /dev/null@@ -1,284 +0,0 @@-import { afterEach, describe, expect, it, vi } from 'vitest'-import {- type HostFleetReconcileRequest,- type HostFleetRequestResult,- HostFleetScheduler,-} from './hostFleetScheduler'--function deferred<T = void>() {- let resolve!: (value: T | PromiseLike<T>) => void- let reject!: (reason?: unknown) => void- const promise = new Promise<T>((resolvePromise, rejectPromise) => {- resolve = resolvePromise- reject = rejectPromise- })- return { promise, resolve, reject }-}--describe('HostFleetScheduler', () => {- afterEach(() => {- vi.useRealTimers()- })-- it('merges newer work into one trailing pass with full coverage and latest inputs', async () => {- const activePass = deferred()- const requests: HostFleetReconcileRequest[] = []- const results: HostFleetRequestResult[] = []- const scheduler = new HostFleetScheduler({- perform: async request => {- requests.push({ ...request })- if (requests.length === 1) await activePass.promise- },- recordRequest: result => results.push(result),- })-- const firstGeneration = scheduler.beginLifecycleTransition()- scheduler.bumpInputRevision()- const active = scheduler.request('active lifecycle', firstGeneration, 'lifecycle')- await vi.waitFor(() => expect(requests).toHaveLength(1))-- scheduler.bumpInputRevision()- const pendingFull = scheduler.request('pending full', firstGeneration, 'full')- const latestGeneration = scheduler.beginLifecycleTransition()- scheduler.bumpInputRevision()- const latestLifecycle = scheduler.request('latest lifecycle', latestGeneration, 'lifecycle')-- activePass.resolve()- await Promise.all([active, pendingFull, latestLifecycle])-- expect(requests).toEqual([- {- reason: 'active lifecycle',- mode: 'lifecycle',- ccLifecycleGeneration: firstGeneration,- inputRevision: 1,- },- {- reason: 'latest lifecycle',- mode: 'full',- ccLifecycleGeneration: latestGeneration,- inputRevision: 3,- },- ])- expect(results).toEqual(['started', 'coalesced', 'coalesced', 'trailing'])- })-- it('carries the known lifecycle generation when merging a generation-less full request', async () => {- const activePass = deferred()- const requests: HostFleetReconcileRequest[] = []- const scheduler = new HostFleetScheduler({- perform: async request => {- requests.push({ ...request })- if (requests.length === 1) await activePass.promise- },- recordRequest: vi.fn(),- })- const generation = scheduler.beginLifecycleTransition()- const active = scheduler.request('active')- await vi.waitFor(() => expect(requests).toHaveLength(1))- scheduler.bumpInputRevision()- const pendingLifecycle = scheduler.request('lifecycle', generation, 'lifecycle')- scheduler.bumpInputRevision()- const pendingFull = scheduler.request('full')-- activePass.resolve()- await Promise.all([active, pendingLifecycle, pendingFull])-- expect(requests[1]).toMatchObject({- reason: 'full',- mode: 'full',- ccLifecycleGeneration: generation,- inputRevision: 2,- })- })-- it('reuses active work only when its mode, revision, and lifecycle generation cover the request', async () => {- const activePass = deferred()- const requests: HostFleetReconcileRequest[] = []- const scheduler = new HostFleetScheduler({- perform: async request => {- requests.push({ ...request })- if (requests.length === 1) await activePass.promise- },- recordRequest: vi.fn(),- })- const generation = scheduler.beginLifecycleTransition()- scheduler.bumpInputRevision()-- const active = scheduler.request('full', generation, 'full')- await vi.waitFor(() => expect(requests).toHaveLength(1))- const covered = scheduler.request('covered lifecycle', generation, 'lifecycle')-- activePass.resolve()- await Promise.all([active, covered])-- expect(requests).toHaveLength(1)- })-- it('rejects stale or already-applied lifecycle work without suppressing a current full pass', async () => {- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(- async () => undefined- )- const scheduler = new HostFleetScheduler({- perform,- recordRequest: vi.fn(),- })- const firstGeneration = scheduler.beginLifecycleTransition()- scheduler.markLifecycleApplied(firstGeneration)-- await scheduler.request('already applied', firstGeneration, 'lifecycle')- await scheduler.request('current full', firstGeneration, 'full')- scheduler.beginLifecycleTransition()- await scheduler.request('stale full', firstGeneration, 'full')-- expect(perform).toHaveBeenCalledOnce()- expect(perform).toHaveBeenCalledWith(- expect.objectContaining({ reason: 'current full', mode: 'full' })- )- })-- it('does not let a rejected pass poison later work', async () => {- const failure = new Error('transient failure')- const perform = vi- .fn<(request: HostFleetReconcileRequest) => Promise<void>>()- .mockRejectedValueOnce(failure)- .mockResolvedValueOnce(undefined)- const scheduler = new HostFleetScheduler({- perform,- recordRequest: vi.fn(),- })-- await expect(scheduler.request('fails')).rejects.toThrow(failure)- await expect(scheduler.request('recovers')).resolves.toBeUndefined()-- expect(perform).toHaveBeenCalledTimes(2)- })-- it('releases active and queued waiters on stop without starting queued work', async () => {- const activePass = deferred()- const requests: HostFleetReconcileRequest[] = []- const scheduler = new HostFleetScheduler({- perform: async request => {- requests.push({ ...request })- await activePass.promise- },- recordRequest: vi.fn(),- })-- const active = scheduler.request('active')- await vi.waitFor(() => expect(requests).toHaveLength(1))- scheduler.bumpInputRevision()- const queued = scheduler.request('queued')-- scheduler.stop()- await Promise.all([active, queued, scheduler.request('after stop')])- activePass.resolve()- await Promise.resolve()-- expect(requests).toHaveLength(1)- })-- it('tracks lifecycle state and cancels obsolete retries', async () => {- vi.useFakeTimers()- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(- async () => undefined- )- const scheduler = new HostFleetScheduler({- perform,- recordRequest: vi.fn(),- })- const firstGeneration = scheduler.beginLifecycleTransition()-- expect(scheduler.currentLifecycleGeneration).toBe(firstGeneration)- expect(scheduler.appliedLifecycleGeneration).toBe(-1)-- scheduler.scheduleLifecycleRetry({- reason: 'failed lifecycle',- mode: 'lifecycle',- ccLifecycleGeneration: firstGeneration,- inputRevision: 0,- })- await vi.advanceTimersByTimeAsync(4999)- expect(perform).not.toHaveBeenCalled()-- const secondGeneration = scheduler.beginLifecycleTransition()- await vi.advanceTimersByTimeAsync(1)- expect(perform).not.toHaveBeenCalled()-- scheduler.scheduleLifecycleRetry({- reason: 'current lifecycle',- mode: 'lifecycle',- ccLifecycleGeneration: secondGeneration,- inputRevision: 0,- })- scheduler.markLifecycleApplied(secondGeneration)- await vi.advanceTimersByTimeAsync(5000)-- expect(scheduler.appliedLifecycleGeneration).toBe(secondGeneration)- expect(perform).not.toHaveBeenCalled()- })-- it('does not let a stale applied marker clear the current generation retry', async () => {- vi.useFakeTimers()- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(- async () => undefined- )- const scheduler = new HostFleetScheduler({- perform,- recordRequest: vi.fn(),- })- const staleGeneration = scheduler.beginLifecycleTransition()- const currentGeneration = scheduler.beginLifecycleTransition()- scheduler.scheduleLifecycleRetry({- reason: 'current lifecycle',- mode: 'lifecycle',- ccLifecycleGeneration: currentGeneration,- inputRevision: 0,- })-- scheduler.markLifecycleApplied(staleGeneration)- await vi.advanceTimersByTimeAsync(5000)-- expect(perform).toHaveBeenCalledWith(- expect.objectContaining({ ccLifecycleGeneration: currentGeneration })- )- })-- it('retries an unapplied current lifecycle generation with bounded backoff', async () => {- vi.useFakeTimers()- const requests: HostFleetReconcileRequest[] = []- const scheduler = new HostFleetScheduler({- perform: async request => {- requests.push({ ...request })- },- recordRequest: vi.fn(),- })- const generation = scheduler.beginLifecycleTransition()- const failedRequest: HostFleetReconcileRequest = {- reason: 'failed lifecycle',- mode: 'lifecycle',- ccLifecycleGeneration: generation,- inputRevision: 0,- }-- scheduler.scheduleLifecycleRetry(failedRequest)- await vi.advanceTimersByTimeAsync(5000)- scheduler.scheduleLifecycleRetry(failedRequest)- await vi.advanceTimersByTimeAsync(14999)- expect(requests).toHaveLength(1)- await vi.advanceTimersByTimeAsync(1)-- expect(requests).toEqual([- expect.objectContaining({- reason: 'CommunicationChannel lifecycle convergence retry',- mode: 'lifecycle',- ccLifecycleGeneration: generation,- }),- expect.objectContaining({- reason: 'CommunicationChannel lifecycle convergence retry',- mode: 'lifecycle',- ccLifecycleGeneration: generation,- }),- ])- })-})diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.tsdeleted file mode 100644index d872d6cad..000000000--- a/host-context-controller/src/hostFleetScheduler.ts+++ /dev/null@@ -1,246 +0,0 @@-const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000] as const--export type HostFleetReconcileMode = 'full' | 'lifecycle'--export type HostFleetReconcileRequest = {- reason: string- mode: HostFleetReconcileMode- ccLifecycleGeneration?: number- inputRevision: number-}--type PendingHostFleetReconcile = HostFleetReconcileRequest & {- promise: Promise<void>- resolve: () => void-}--type ActiveHostFleetReconcile = HostFleetReconcileRequest & {- promise: Promise<void>-}--export type HostFleetRequestResult = 'started' | 'coalesced' | 'trailing'--type HostFleetSchedulerDependencies = {- perform: (request: HostFleetReconcileRequest) => Promise<void>- recordRequest: (result: HostFleetRequestResult) => void-}--function hostFleetRequestCovers(- active: HostFleetReconcileRequest,- requested: HostFleetReconcileRequest-): boolean {- const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'- if (!modeCovered) return false- if (active.inputRevision !== requested.inputRevision) return false- if (requested.ccLifecycleGeneration === undefined) return true- return active.ccLifecycleGeneration === requested.ccLifecycleGeneration-}--function mergeHostFleetRequests(- pending: HostFleetReconcileRequest,- requested: HostFleetReconcileRequest-): HostFleetReconcileRequest {- const pendingGeneration = pending.ccLifecycleGeneration- const requestedGeneration = requested.ccLifecycleGeneration- return {- reason: requested.reason,- mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',- inputRevision: Math.max(pending.inputRevision, requested.inputRevision),- ccLifecycleGeneration:- pendingGeneration === undefined- ? requestedGeneration- : requestedGeneration === undefined- ? pendingGeneration- : Math.max(pendingGeneration, requestedGeneration),- }-}--/**- * Owns temporal coordination for Host fleet convergence. Kubernetes reads and- * mutations stay behind `perform`, so this scheduler only serializes requests,- * coalesces a single trailing pass, and manages lifecycle retry/shutdown state.- */-export class HostFleetScheduler {- private ccLifecycleGeneration = 0- private ccAppliedLifecycleGeneration = -1- private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null- private ccFleetRetryAttempt = 0- private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null- private hostFleetReconcilePending: PendingHostFleetReconcile | null = null- private hostFleetInputRevision = 0- private resolveHostFleetShutdown: () => void = () => {}- private readonly hostFleetShutdown = new Promise<void>(resolve => {- this.resolveHostFleetShutdown = resolve- })- private stopped = false-- constructor(private readonly dependencies: HostFleetSchedulerDependencies) {}-- get currentLifecycleGeneration(): number {- return this.ccLifecycleGeneration- }-- get appliedLifecycleGeneration(): number {- return this.ccAppliedLifecycleGeneration- }-- beginLifecycleTransition(): number {- this.ccLifecycleGeneration += 1- this.ccFleetRetryAttempt = 0- this.clearLifecycleRetry()- return this.ccLifecycleGeneration- }-- bumpInputRevision(): number {- this.hostFleetInputRevision += 1- return this.hostFleetInputRevision- }-- markLifecycleApplied(generation: number): void {- if (generation !== this.ccLifecycleGeneration) return- this.ccAppliedLifecycleGeneration = generation- this.ccFleetRetryAttempt = 0- this.clearLifecycleRetry()- }-- scheduleLifecycleRetry(request: HostFleetReconcileRequest): void {- const generation = request.ccLifecycleGeneration- if (generation === undefined) return- if (- this.stopped ||- generation !== this.ccLifecycleGeneration ||- generation === this.ccAppliedLifecycleGeneration ||- this.ccFleetRetryTimer- ) {- return- }-- const delayIndex = Math.min(- this.ccFleetRetryAttempt,- COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1- )- const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]- this.ccFleetRetryAttempt += 1- this.ccFleetRetryTimer = setTimeout(() => {- this.ccFleetRetryTimer = null- if (- this.stopped ||- generation !== this.ccLifecycleGeneration ||- generation === this.ccAppliedLifecycleGeneration- ) {- return- }- void this.request(- 'CommunicationChannel lifecycle convergence retry',- generation,- request.mode- )- }, retryDelay)- }-- /**- * Cache transitions, periodic resyncs, and retries share one active pass and- * at most one merged trailing pass. Each caller waits only for work covering- * its request, keeping startup bounded under a sustained retry stream.- */- request(- reason: string,- ccLifecycleGeneration?: number,- mode?: HostFleetReconcileMode- ): Promise<void> {- if (this.stopped) return Promise.resolve()- const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')- if (- ccLifecycleGeneration !== undefined &&- (ccLifecycleGeneration !== this.ccLifecycleGeneration ||- (requestMode === 'lifecycle' &&- ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))- ) {- return Promise.resolve()- }-- const request: HostFleetReconcileRequest = {- reason,- mode: requestMode,- ccLifecycleGeneration,- inputRevision: this.hostFleetInputRevision,- }- const active = this.hostFleetReconcileInFlight- if (!active) {- this.dependencies.recordRequest('started')- return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))- }- if (hostFleetRequestCovers(active, request)) {- this.dependencies.recordRequest('coalesced')- return this.waitForHostFleetOrShutdown(active.promise)- }-- if (!this.hostFleetReconcilePending) {- let resolve!: () => void- const promise = new Promise<void>(resolvePromise => {- resolve = resolvePromise- })- this.hostFleetReconcilePending = {- ...request,- promise,- resolve,- }- } else {- const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)- this.hostFleetReconcilePending.reason = merged.reason- this.hostFleetReconcilePending.mode = merged.mode- this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration- this.hostFleetReconcilePending.inputRevision = merged.inputRevision- }- this.dependencies.recordRequest('coalesced')- return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)- }-- stop(): void {- if (this.stopped) return- this.stopped = true- this.resolveHostFleetShutdown()- this.hostFleetReconcilePending?.resolve()- this.hostFleetReconcilePending = null- this.clearLifecycleRetry()- }-- private clearLifecycleRetry(): void {- if (!this.ccFleetRetryTimer) return- clearTimeout(this.ccFleetRetryTimer)- this.ccFleetRetryTimer = null- }-- private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {- return Promise.race([coverage, this.hostFleetShutdown])- }-- private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {- const reconcileRequest: HostFleetReconcileRequest = {- reason: request.reason,- mode: request.mode,- ccLifecycleGeneration: request.ccLifecycleGeneration,- inputRevision: request.inputRevision,- }- const promise = this.dependencies.perform(reconcileRequest)- const active: ActiveHostFleetReconcile = { ...reconcileRequest, promise }- this.hostFleetReconcileInFlight = active-- const settle = () => {- if (this.hostFleetReconcileInFlight !== active) return- this.hostFleetReconcileInFlight = null- const pending = this.hostFleetReconcilePending- this.hostFleetReconcilePending = null- if (!pending) return- if (this.stopped) {- pending.resolve()- return- }- this.dependencies.recordRequest('trailing')- const trailing = this.startHostFleetReconcile(pending)- void trailing.then(pending.resolve, pending.resolve)- }- void promise.then(settle, settle)- return promise- }-}diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.tsindex 53625ff43..495a4bc38 100644--- a/host-context-controller/src/hostReconciler.test.ts+++ b/host-context-controller/src/hostReconciler.test.ts@@ -943,42 +943,6 @@ describe('HostReconciler cleanup authority gate', () => {expect(del).not.toHaveBeenCalled()})- it('defers orphan deletion when authority changes before serializer admission', async () => {- const getObj = vi.fn().mockRejectedValue({ code: 404 })- const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })- const authority = { known: true, generation: 3 }- reconciler.setHostWatchAuthority(() => ({ ...authority }))- reconciler.setResolveCurrentHost(() => undefined)- const internals = reconciler as unknown as {- lifecycle: {- serializeByHost(name: string, task: () => Promise<void>): Promise<void>- }- listManagedHostDeployments(): Promise<k8s.V1Deployment[]>- deleteHostRuntimeResources(name: string, namespace: string): Promise<void>- }- vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([- orphanDeployment('authority-race-host'),- ])- const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = internals.lifecycle.serializeByHost('authority-race-host', async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- })- await blockerStarted.promise-- const cleanup = reconciler.fullReconcile([])- await vi.waitFor(() => expect(getObj).toHaveBeenCalledOnce())- authority.known = false- authority.generation = 4- releaseBlocker.resolve(undefined)- await occupied- await cleanup-- expect(del).not.toHaveBeenCalled()- })-it('defers an orphan whose fresh authoritative read fails', async () => {const getObj = vi.fn().mockRejectedValue({ code: 503 })const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })@@ -1032,687 +996,3 @@ describe('HostReconciler cleanup authority gate', () => {expect(del).not.toHaveBeenCalled()})})--describe('HostReconciler Host inventory mutation authority', () => {- function wireAuthoritativeHost(- reconciler: HostReconciler,- host: HostCRD,- authorityRef: { current: { known: boolean; generation: number } }- ): void {- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => authorityRef.current)- }-- function stubPreDeploymentReconcileEffects(reconciler: HostReconciler): void {- vi.spyOn(reconciler as any, 'validateHostSecret').mockResolvedValue({ ok: true })- for (const method of [- 'ensureHostServiceAccount',- 'ensureHostRole',- 'ensureHostRoleBinding',- 'ensurePvc',- 'ensureService',- 'ensureMcpHostIngressNetworkPolicy',- 'ensureMcpHostGfsEgressNetworkPolicy',- 'ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy',- 'ensureRpcProxyMcpHostIngressNetworkPolicy',- 'ensureRpcProxyHostEgressNetworkPolicy',- 'ensureDesktopNetworkPolicy',- 'ensureChannelReaderEgressNetworkPolicy',- 'ensureWorkflowApprovalReaderHostEgressNetworkPolicy',- ]) {- vi.spyOn(reconciler as any, method).mockResolvedValue(undefined)- }- }-- function runtimeTokenScopeWithoutChannels(host: HostCRD): string {- return (- HostReconciler as unknown as {- runtimeTokenScopeHash(selected: HostCRD, hasChannelIngress: boolean): string- }- ).runtimeTokenScopeHash(host, false)- }-- it('rejects a queued reconcile when Host authority changes before admission', async () => {- const reconciler = makeBasicReconciler()- const host = makeHost({ name: 'authority-race' })- const authority = { current: { known: true, generation: 7 } }- wireAuthoritativeHost(reconciler, host, authority)- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const pending = reconciler.reconcile(host)- authority.current = { known: false, generation: 8 }- releaseBlocker.resolve(undefined)- await occupied-- await expect(pending).rejects.toThrow(/Host inventory authority/)- expect(reconcileCore).not.toHaveBeenCalled()- })-- it('rejects a queued fleet worker after Context authority retires and recovers', async () => {- const reconciler = makeBasicReconciler()- const hostA = makeHost({ name: 'context-authority-a' })- const hostB = makeHost({ name: 'context-authority-b' })- const hosts = new Map([- [hostA.name, hostA],- [hostB.name, hostB],- ])- reconciler.setResolveCurrentHost(name => hosts.get(name))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 7 }))- const mutationAuthority = {- current: {- known: true,- hostGeneration: 7,- contextGeneration: 11,- },- }- ;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- vi.spyOn(reconciler as any, 'collectHostCleanupFailures').mockResolvedValue([])- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = (reconciler as any).lifecycle.serializeByHost(hostB.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const fleet = reconciler.fullReconcile([hostA, hostB])- await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))-- // Context retired and recovered to a fresh LIST -> WATCH generation while- // Host authority remained continuously known: a host-only lease would miss- // this ABA transition and admit Host B with stale derived mounts.- mutationAuthority.current = {- known: true,- hostGeneration: 7,- contextGeneration: 13,- }- releaseBlocker.resolve(undefined)- await occupied-- await expect(fleet).rejects.toBeInstanceOf(HostFleetReconcileError)- expect(reconcileCore).toHaveBeenCalledTimes(1)- expect(reconcileCore).not.toHaveBeenCalledWith(hostB, expect.any(Function))- })-- it('retires a Host reconcile when its Context is superseded within the same watch', async () => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'context-revision-race', contextRef: 'context-a' }),- uid: 'context-revision-race-uid',- generation: 5,- }- const contextV1 = { name: 'context-a', spec: { sharedFileSystems: [] } }- const contextV2 = {- name: 'context-a',- spec: {- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],- },- }- let selectedContext: unknown = contextV1- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostMutationAuthority(() => ({- known: true,- hostGeneration: 7,- contextGeneration: 11,- }))- reconciler.setResolveHostMutationDependencies(() => [selectedContext])-- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return { ok: true }- })- const ensureServiceAccount = vi- .spyOn(reconciler as any, 'ensureHostServiceAccount')- .mockResolvedValue(undefined)-- // R1-M2: a benign mid-pass supersession must NOT be reported as a hard- // reconcile failure. The single reconcile() catch used to fire- // enqueueControllerError('reconcile_exception') → controller_error telemetry- // (status:'failed') AND an administrative 'failed' outcome to control-api for- // these retirements. Spy (observe-only) to prove the benign branch emits- // neither, while still re-throwing so callers treat it as a retire.- const enqueueControllerError = vi.spyOn(reconciler as any, 'enqueueControllerError')- const enqueueAdministrativeOutcome = vi.spyOn(reconciler as any, 'enqueueAdministrativeOutcome')-- const pending = reconciler.reconcile(host)- await secretReadStarted.promise- selectedContext = contextV2- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toThrow(/Host mutation dependency changed/)- expect(ensureServiceAccount).not.toHaveBeenCalled()- expect(enqueueControllerError).not.toHaveBeenCalled()- expect(enqueueAdministrativeOutcome).not.toHaveBeenCalledWith(- expect.anything(),- 'failed',- expect.anything()- )- })-- it('keeps a Host reconcile admitted when a watch replaces dependencies with deep-equal values', async () => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'deep-equal-dependency', contextRef: 'context-a' }),- uid: 'deep-equal-dependency-uid',- generation: 5,- }- let selectedContext: unknown = {- name: 'context-a',- spec: {- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],- },- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostMutationAuthority(() => ({- known: true,- hostGeneration: 7,- contextGeneration: 11,- }))- reconciler.setResolveHostMutationDependencies(() => [selectedContext])-- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return { ok: true }- })- const admitted = new Error('deep-equal dependency admitted')- const ensureServiceAccount = vi- .spyOn(reconciler as any, 'ensureHostServiceAccount')- .mockRejectedValue(admitted)-- const pending = reconciler.reconcile(host)- await secretReadStarted.promise- selectedContext = structuredClone(selectedContext)- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toBe(admitted)- expect(ensureServiceAccount).toHaveBeenCalled()- })-- it.each([- [- 'referenced SharedFileSystem revision',- { name: 'documents', spec: { storageClassName: 'fast-v1' } },- { name: 'documents', spec: { storageClassName: 'fast-v2' } },- ],- ['referenced SharedFileSystem mountability', true, false],- [- 'Host-scoped CommunicationChannel revision',- { name: 'telegram', spec: { hostRef: 'dependency-race' } },- {- name: 'telegram',- spec: {- hostRef: 'dependency-race',- credentialsSecretRef: { name: 'telegram-v2' },- },- },- ],- ])('retires a Host reconcile when its %s changes', async (_label, original, replacement) => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'dependency-race', contextRef: 'context-a' }),- uid: 'dependency-race-uid',- generation: 5,- }- const context = { name: 'context-a', spec: { sharedFileSystems: [] } }- let selectedDependency: unknown = original- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostMutationAuthority(() => ({- known: true,- hostGeneration: 7,- contextGeneration: 11,- }))- reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])-- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return { ok: true }- })- const ensureServiceAccount = vi- .spyOn(reconciler as any, 'ensureHostServiceAccount')- .mockResolvedValue(undefined)-- const pending = reconciler.reconcile(host)- await secretReadStarted.promise- selectedDependency = replacement- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toThrow(/Host mutation dependency changed/)- expect(ensureServiceAccount).not.toHaveBeenCalled()- })-- it('does not delete runtime after a Secret rejection when Host authority changes in flight', async () => {- const reconciler = makeBasicReconciler()- const host = { ...makeHost({ name: 'secret-authority-race' }), uid: 'host-uid', generation: 3 }- const authority = { current: { known: true, generation: 7 } }- wireAuthoritativeHost(reconciler, host, authority)- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return {- ok: false,- reason: 'SecretNotFound',- message: 'secret missing in stale spec',- }- })- const deleteRuntime = vi- .spyOn(reconciler as any, 'deleteHostRuntimeResources')- .mockResolvedValue(undefined)-- const pending = reconciler.reconcile(host)- await secretReadStarted.promise- authority.current = { known: false, generation: 8 }- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toThrow(/Host inventory authority/)- expect(deleteRuntime).not.toHaveBeenCalled()- })-- it.each([- [- 'UID',- (host: HostCRD) => ({ ...host, uid: 'replacement-host-uid', generation: 1 }),- /Host identity/,- ],- [- 'spec generation',- (host: HostCRD) => ({- ...host,- generation: (host.generation ?? 0) + 1,- spec: { ...host.spec, secretRef: 'replacement-secret' },- }),- /Host spec generation/,- ],- ] as const)(- 'does not delete runtime after a Secret rejection when the Host %s changes in flight',- async (_, replaceCurrent, expectedError) => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'secret-identity-race' }),- uid: 'host-uid',- generation: 3,- }- let current: HostCRD = host- const authority = { current: { known: true, generation: 7 } }- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return {- ok: false,- reason: 'SecretNotFound',- message: 'secret missing in stale spec',- }- })- const deleteRuntime = vi- .spyOn(reconciler as any, 'deleteHostRuntimeResources')- .mockResolvedValue(undefined)-- const pending = reconciler.reconcile(host)- await secretReadStarted.promise- current = replaceCurrent(host)- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toThrow(expectedError)- expect(deleteRuntime).not.toHaveBeenCalled()- }- )-- it('revalidates the Host spec after token provisioning and before Deployment mutation', async () => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'deployment-generation-race' }),- uid: 'host-uid',- generation: 3,- }- let current = host- const authority = { current: { known: true, generation: 11 } }- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- stubPreDeploymentReconcileEffects(reconciler)- const tokenProvisionStarted = deferred()- const releaseTokenProvision = deferred()- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {- tokenProvisionStarted.resolve(undefined)- await releaseTokenProvision.promise- return { revision: 'stale-revision', scopeHash: 'stale-scope' }- })- const ensureDeployment = vi- .spyOn(reconciler as any, 'ensureDeployment')- .mockResolvedValue(undefined)-- const pending = reconciler.reconcile(host)- await tokenProvisionStarted.promise- current = {- ...host,- generation: 4,- spec: { ...host.spec, contextRef: 'replacement-context' },- }- releaseTokenProvision.resolve(undefined)-- await expect(pending).rejects.toThrow(/Host spec generation/)- expect(ensureDeployment).not.toHaveBeenCalled()- })-- it('revalidates the Host spec at the final Deployment create-state boundary', async () => {- const appsApi = createMockAppsApi()- const reconciler = new HostReconciler({} as k8s.KubeConfig, {- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(createMockCoreApi()),- networkingApi: asNetworkingApi(createMockNetworkingApi()),- rbacApi: asRbacApi(createMockRbacApi()),- })- const host = {- ...makeHost({ name: 'deployment-create-race' }),- uid: 'host-uid',- generation: 3,- }- let current = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))- stubPreDeploymentReconcileEffects(reconciler)- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({- revision: 'runtime-revision',- scopeHash: runtimeTokenScopeWithoutChannels(host),- })- let channelIngressReads = 0- vi.spyOn(reconciler as any, 'hasChannelIngress').mockImplementation(() => {- channelIngressReads += 1- if (channelIngressReads === 2) {- current = {- ...host,- generation: 4,- spec: { ...host.spec, contextRef: 'replacement-context' },- }- }- return false- })-- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)-- expect(channelIngressReads).toBe(2)- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()- expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()- })-- it('revalidates the Host spec after a Deployment create conflict and before replace', async () => {- const appsApi = createMockAppsApi()- const reconciler = new HostReconciler({} as k8s.KubeConfig, {- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(createMockCoreApi()),- networkingApi: asNetworkingApi(createMockNetworkingApi()),- rbacApi: asRbacApi(createMockRbacApi()),- })- const host = {- ...makeHost({ name: 'deployment-replace-race' }),- uid: 'host-uid',- generation: 3,- }- let current = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))- stubPreDeploymentReconcileEffects(reconciler)- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({- revision: 'runtime-revision',- scopeHash: runtimeTokenScopeWithoutChannels(host),- })- vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })- appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {- current = {- ...host,- generation: 4,- spec: { ...host.spec, contextRef: 'replacement-context' },- }- return { metadata: { resourceVersion: 'rv-existing' } }- })-- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)-- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()- })-- it('revalidates the Host spec before retrying a conflicted Deployment replace', async () => {- const appsApi = createMockAppsApi()- const reconciler = new HostReconciler({} as k8s.KubeConfig, {- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(createMockCoreApi()),- networkingApi: asNetworkingApi(createMockNetworkingApi()),- rbacApi: asRbacApi(createMockRbacApi()),- })- const host = {- ...makeHost({ name: 'deployment-retry-race' }),- uid: 'host-uid',- generation: 3,- }- let current = host- let deploymentReads = 0- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))- stubPreDeploymentReconcileEffects(reconciler)- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({- revision: 'runtime-revision',- scopeHash: runtimeTokenScopeWithoutChannels(host),- })- vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })- appsApi.readNamespacedDeployment.mockImplementation(async () => {- deploymentReads += 1- if (deploymentReads === 2) {- current = {- ...host,- generation: 4,- spec: { ...host.spec, contextRef: 'replacement-context' },- }- }- return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }- })- appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })- const random = vi.spyOn(Math, 'random').mockReturnValue(0)-- try {- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)- } finally {- random.mockRestore()- }-- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)- })-- it('resolves the latest authoritative Host object at reconcile admission', async () => {- const reconciler = makeBasicReconciler()- const stale = { ...makeHost({ name: 'latest-host' }), generation: 1 }- const current = {- ...stale,- generation: 2,- spec: { ...stale.spec, contextRef: 'current-context' },- }- const authority = { current: { known: true, generation: 7 } }- wireAuthoritativeHost(reconciler, current, authority)- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)-- await reconciler.reconcile(stale)-- expect(reconcileCore).toHaveBeenCalledOnce()- expect(reconcileCore).toHaveBeenCalledWith(- expect.objectContaining({- generation: 2,- spec: expect.objectContaining({ contextRef: 'current-context' }),- }),- expect.any(Function)- )- })-- it('does not patch channel-reader after Host authority changes during revision resolution', async () => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'channel-authority-race' }),- uid: 'channel-authority-race-uid',- generation: 3,- }- const authority = { current: { known: true, generation: 9 } }- wireAuthoritativeHost(reconciler, host, authority)- const revisionStarted = deferred()- const releaseRevision = deferred<string>()- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(- async () => {- revisionStarted.resolve(undefined)- return releaseRevision.promise- }- )- const patchDeployment = vi- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')- .mockResolvedValue(undefined)-- const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)- await revisionStarted.promise- authority.current = { known: false, generation: 10 }- releaseRevision.resolve('resolved-revision')- await pending-- expect(patchDeployment).not.toHaveBeenCalled()- })-- it.each([- ['UID', (host: HostCRD) => ({ ...host, uid: 'replacement-channel-reader-uid' })],- [- 'spec generation',- (host: HostCRD) => ({- ...host,- generation: (host.generation ?? 0) + 1,- spec: { ...host.spec, contextRef: 'replacement-context' },- }),- ],- ] as const)(- 'does not patch channel-reader when the Host %s changes during revision resolution',- async (_, replaceCurrent) => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'channel-host-race' }),- uid: 'channel-host-race-uid',- generation: 3,- }- let current: HostCRD = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 9 }))- const revisionStarted = deferred()- const releaseRevision = deferred<string>()- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(- async () => {- revisionStarted.resolve(undefined)- return releaseRevision.promise- }- )- const patchDeployment = vi- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')- .mockResolvedValue(undefined)-- const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)- await revisionStarted.promise- current = replaceCurrent(host)- releaseRevision.resolve('resolved-revision')- await pending-- expect(patchDeployment).not.toHaveBeenCalled()- }- )-- it('serializes a fresh channel-reader revision behind an older in-flight Host reconcile', async () => {- const reconciler = makeBasicReconciler()- const host = {- ...makeHost({ name: 'channel-revision-order' }),- uid: 'channel-revision-order-uid',- generation: 4,- }- const authority = { current: { known: true, generation: 9 } }- wireAuthoritativeHost(reconciler, host, authority)- const staleReconcileStarted = deferred()- const releaseStaleReconcile = deferred()- const order: string[] = []- let liveRevision = ''-- vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async () => {- order.push('reconcile-a:start')- staleReconcileStarted.resolve(undefined)- await releaseStaleReconcile.promise- liveRevision = 'revision-a'- order.push('reconcile-a:write')- })- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue(- 'revision-b'- )- const patchDeployment = vi- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')- .mockImplementation(async (request: any) => {- const { body } = request- liveRevision = body.spec.template.metadata.annotations['clerum.io/credentials-revision']- order.push('revision-b:patch')- })-- const staleReconcile = reconciler.reconcile(host)- await staleReconcileStarted.promise- const freshRevision = reconciler.patchChannelReaderRevisionAnnotation(host.name)-- // Give B the opportunity to patch before A is released. Without a shared- // Host lane, B writes first and A deterministically overwrites it.- await Promise.resolve()- await Promise.resolve()- releaseStaleReconcile.resolve(undefined)- await Promise.all([staleReconcile, freshRevision])-- expect(liveRevision).toBe('revision-b')- expect(patchDeployment).toHaveBeenCalledOnce()- expect(order).toEqual(['reconcile-a:start', 'reconcile-a:write', 'revision-b:patch'])- })-- it('rejects a queued delete when Host authority changes before admission', async () => {- const reconciler = makeBasicReconciler()- const host = makeHost({ name: 'delete-authority-race' })- const authority = { current: { known: true, generation: 11 } }- reconciler.setResolveCurrentHost(() => undefined)- reconciler.setHostWatchAuthority(() => authority.current)- const deleteRuntime = vi- .spyOn(reconciler as any, 'deleteHostRuntimeResources')- .mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const pending = reconciler.reconcileDelete(host.name, host.namespace)- authority.current = { known: false, generation: 12 }- releaseBlocker.resolve(undefined)- await occupied-- await expect(pending).rejects.toThrow(/Host inventory authority/)- expect(deleteRuntime).not.toHaveBeenCalled()- })-})diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 9e2f236d5..5730e5553 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -1,7 +1,6 @@import * as k8s from '@kubernetes/client-node'import { IntOrString } from '@kubernetes/client-node/dist/types.js'import { createHash, randomUUID } from 'crypto'-import { isDeepStrictEqual } from 'node:util'import * as path from 'path'import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'import { config } from './config'@@ -44,7 +43,7 @@ import {resolveStatelessImagePullPolicy,} from './statelessDeployment'import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'-import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'+import { StatelessLifecycleExecutor } from './statelessLifecycleExecutor'import {CommunicationChannelCRD,HostCRD,@@ -79,74 +78,6 @@ export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'/** Immutable Host watch authority snapshot captured at a full-pass boundary. */export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }-/**- * Immutable lease for every inventory that contributes to a Host mutation.- * Host resources come from the Host watch; mounted SharedFileSystems come from- * the Context watch. Keeping both generations prevents a false-stable- * Host-only lease when Context retires and recovers during queued work.- */-export type HostMutationAuthoritySnapshot = {- known: boolean- hostGeneration: number- contextGeneration: number-}--class HostInventoryAuthorityUnavailableError extends Error {- constructor(- action: string,- captured: HostMutationAuthoritySnapshot,- current: HostMutationAuthoritySnapshot- ) {- super(- `Host inventory authority changed before ${action} admission ` +- `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` +- `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` +- `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`- )- this.name = 'HostInventoryAuthorityUnavailableError'- }-}--class HostMutationIdentityChangedError extends Error {- constructor(action: string, hostName: string) {- super(`Host identity changed before ${action} admission for "${hostName}"`)- this.name = 'HostMutationIdentityChangedError'- }-}--class HostMutationSpecRevisionChangedError extends Error {- constructor(action: string, hostName: string) {- super(`Host spec generation changed before ${action} admission for "${hostName}"`)- this.name = 'HostMutationSpecRevisionChangedError'- }-}--class HostMutationDependencyChangedError extends Error {- constructor(action: string, hostName: string) {- super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)- this.name = 'HostMutationDependencyChangedError'- }-}--/**- * The four benign mid-pass supersession signals. revalidateHostMutationBoundary()- * throws these when the watch observes a newer spec/uid/authority-generation while- * a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass- * is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT- * `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts- * synthesizes name-equivalent plain Errors for the same races.- */-const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([- 'HostInventoryAuthorityUnavailableError',- 'HostMutationIdentityChangedError',- 'HostMutationSpecRevisionChangedError',- 'HostMutationDependencyChangedError',-])--function isBenignSupersessionError(error: unknown): boolean {- return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)-}-/*** The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's* owned bundle is permitted ONLY when watch authority is known, the watch@@ -447,12 +378,6 @@ export class HostReconciler {* snapshot for reconcile, and cleanup stays fail-closed (authority unknown).*/private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null- /**- * H2: late-bound cache reflector. Wired by McpServerWatcher next to- * setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect- * only onto the caller's own object (legacy contract).- */- private reflectHostOutcome: ReflectHostOutcomeFn | null = null/*** Snapshot the current Host watch authority + generation. Wired by* McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never@@ -462,27 +387,6 @@ export class HostReconciler {known: false,generation: 0,})- /**- * Positive Host mutations depend on Host plus Context authority. The default- * preserves the standalone HostReconciler contract by deriving a Host-only- * lease; McpServerWatcher wires the complete composite lease.- */- private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {- const host = this.hostWatchAuthority()- return {- known: host.known,- hostGeneration: host.generation,- contextGeneration: 0,- }- }- /**- * Exact revisions of the cross-resource inputs selected by a Host. Watch- * generations fence LIST -> WATCH authority, but normal MODIFIED events stay- * in the same generation. The watcher supplies the selected Context,- * referenced SharedFileSystems plus mountability, and Host-scoped- * CommunicationChannels so unrelated fleet changes do not retire this lease.- */- private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = nullconstructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {this.appsApi =@@ -516,10 +420,7 @@ export class HostReconciler {now: this.now,countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),- reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),- prepareHostMutationAdmission: (action, host) =>- this.prepareHostMutationAdmission(action, host),- reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),+ reconcileCore: host => this.reconcileCore(host),onLifecycleStatusCommitted: (host, lifecycle) => {const occurredAt = this.now().toISOString()this.infrastructureTelemetryReporter?.enqueueHealthTransition({@@ -585,15 +486,6 @@ export class HostReconciler {this.resolveCurrentHost = fn}- /**- * H2: late-bound setter for the guarded cache reflector. Wired by- * McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern- * mirrors setResolveCurrentHost.- */- setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {- this.reflectHostOutcome = fn- }-/*** Late-bound setter for the Host watch authority/generation snapshot. Wired by* McpServerWatcher so orphan cleanup can require known authority and a stable@@ -603,103 +495,6 @@ export class HostReconciler {this.hostWatchAuthority = fn}- setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {- this.hostMutationAuthority = fn- }-- setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {- this.resolveHostMutationDependencies = fn- }-- /**- * A standalone HostReconciler receives an explicit Host object and has no- * watch-backed cache to fence. Once a live-cache resolver is wired (the- * production McpServerWatcher path), every mutation captures and revalidates- * Host LIST -> WATCH authority at its actual serializer admission boundary.- */- private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {- return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()- }-- private requireHostMutationAuthority(- action: string,- captured: HostMutationAuthoritySnapshot | null- ): void {- if (captured === null) return- const current = this.hostMutationAuthority()- if (- !captured.known ||- !current.known ||- captured.hostGeneration !== current.hostGeneration ||- captured.contextGeneration !== current.contextGeneration- ) {- throw new HostInventoryAuthorityUnavailableError(action, captured, current)- }- }-- /**- * Heartbeat writers own their serializer inside StatelessLifecycleExecutor.- * Capture authority here at dispatch, then let the returned closure re-check- * it and resolve the current same-UID Host when that serializer admits the- * work. This preserves one queue per Host without a nested-lock deadlock.- */- private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {- const capturedAuthority = this.captureHostMutationAuthority()- const capturedDependencies = this.resolveHostMutationDependencies?.(requested)- return this.makeHostMutationAdmission(- action,- requested,- capturedAuthority,- capturedDependencies- )- }-- private makeHostMutationAdmission(- action: string,- requested: HostCRD,- capturedAuthority: HostMutationAuthoritySnapshot | null,- capturedDependencies: readonly unknown[] | undefined- ): () => HostCRD {- return () => {- this.requireHostMutationAuthority(action, capturedAuthority)- if (this.resolveCurrentHost === null) return requested- const current = this.resolveCurrentHost(requested.name)- if (- current === undefined ||- requested.uid === undefined ||- current.uid === undefined ||- requested.uid !== current.uid- ) {- throw new HostMutationIdentityChangedError(action, requested.name)- }- if (- requested.generation === undefined ||- current.generation === undefined ||- requested.generation !== current.generation- ) {- throw new HostMutationSpecRevisionChangedError(action, requested.name)- }- if (this.resolveHostMutationDependencies !== null) {- const currentDependencies = this.resolveHostMutationDependencies(current)- const sameDependencies =- capturedDependencies !== undefined &&- capturedDependencies.length === currentDependencies.length &&- capturedDependencies.every((value, index) =>- isDeepStrictEqual(value, currentDependencies[index])- )- if (!sameDependencies) {- throw new HostMutationDependencyChangedError(action, requested.name)- }- }- // McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation- // bodies historically reflect status into their admitted object for- // same-pass decisions, so detach the working snapshot whenever a live- // cache resolver is wired. Standalone callers without a resolver retain- // the legacy caller-visible reflection contract.- return structuredClone(current)- }- }-/*** Late-bound setter for the host-ref lookup used by* `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced@@ -2297,52 +2092,29 @@ export class HostReconciler {* Host CRD is present).*/async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {- const capturedAuthority = this.captureHostMutationAuthority()const depName = `channel-reader-${hostName}`try {- await this.lifecycle.serializeByHost(hostName, async () => {- const action = `channel-reader "${hostName}" revision patch`- this.requireHostMutationAuthority(action, capturedAuthority)-- const resolveCurrentHost = this.resolveCurrentHost- const admittedHost = resolveCurrentHost?.(hostName)- if (resolveCurrentHost !== null && admittedHost === undefined) return- const revalidate = admittedHost- ? this.makeHostMutationAdmission(- action,- admittedHost,- capturedAuthority,- this.resolveHostMutationDependencies?.(admittedHost)- )- : () => this.requireHostMutationAuthority(action, capturedAuthority)-- revalidate()- const revision = await this.computeChannelReaderRevisionForHost(hostName)- // The revision calculation can await multiple Secret reads. Revalidate- // Host LIST/WATCH authority plus UID and generation immediately before- // entering the Kubernetes mutation, while still holding this Host lane.- revalidate()- const patchBody = {- spec: {- template: {- metadata: {- annotations: { 'clerum.io/credentials-revision': revision },- },+ const revision = await this.computeChannelReaderRevisionForHost(hostName)+ const patchBody = {+ spec: {+ template: {+ metadata: {+ annotations: { 'clerum.io/credentials-revision': revision },},},+ },+ }+ await this.appsApi.patchNamespacedDeployment(+ { name: depName, namespace: config.channelsNamespace, body: patchBody },+ {+ middleware: [+ k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),+ ],}- await this.appsApi.patchNamespacedDeployment(- { name: depName, namespace: config.channelsNamespace, body: patchBody },- {- middleware: [- k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),- ],- }- )- console.log(- `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`- )- })+ )+ console.log(+ `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`+ )} catch (err) {if (getErrorCode(err) === 404) returnconsole.error(`[HostReconciler] Failed to reconcile ${depName} revision:`, err)@@ -3777,37 +3549,16 @@ export class HostReconciler {async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {const dispatchedAt = Date.now()- const capturedAuthority = this.captureHostMutationAuthority()- const capturedDependencies = this.resolveHostMutationDependencies?.(host)return this.lifecycle.serializeByHost(host.name, async () => {- this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)- const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host- if (!admittedHost) return- const revalidate = this.makeHostMutationAdmission(- `Host "${host.name}" reconcile`,- admittedHost,- capturedAuthority,- capturedDependencies- )const admittedAt = Date.now()hostReconcileInFlight.inc({ lane: source })try {- await this.reconcileCore(admittedHost, revalidate)- this.enqueueReconcileOutcome(admittedHost)+ await this.reconcileCore(host)+ this.enqueueReconcileOutcome(host)this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)} catch (error) {- if (isBenignSupersessionError(error)) {- // A newer spec/uid/authority-generation superseded this pass mid-flight;- // a fresh reconcile runs after. Model it as a withdrawal (superseded),- // NOT a hard failure — do NOT emit controller_error/reconcile_exception- // or an administrative 'failed' outcome. Callers already treat the- // rethrow as a retire (reconcileDelete → 'superseded', watch callers- // only console.error), so the rethrow is preserved.- this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)- throw error- }- this.enqueueControllerError(admittedHost, 'reconcile_exception', error)- this.enqueueReconcileOutcome(admittedHost)+ this.enqueueControllerError(host, 'reconcile_exception', error)+ this.enqueueReconcileOutcome(host)this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)throw error} finally {@@ -3818,7 +3569,7 @@ export class HostReconciler {private observeReconcileLatency(source: HostReconcileSource,- outcome: 'success' | 'error' | 'superseded',+ outcome: 'success' | 'error',dispatchedAt: number,admittedAt: number): void {@@ -3832,19 +3583,7 @@ export class HostReconciler {)}- private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {- // A per-Host serializer orders work, but it does not invalidate an older- // reconcile when the watch cache observes a newer spec while this body is- // awaiting Kubernetes or issuer I/O. Re-check the captured authority,- // identity, and spec generation at every material mutation boundary.- // Individual API writes remain atomic at the Kubernetes resource level;- // these checks prevent subsequent stale writes from spanning an observed- // generation change.- const revalidateHostMutationBoundary = (): void => {- revalidate?.()- }-- revalidateHostMutationBoundary()+ private async reconcileCore(host: HostCRD): Promise<void> {// Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles// are serialized PER HOST (serializeByHost), so a pending wake for THIS// Host must not wait behind token issuance, NetworkPolicies or the@@ -3853,7 +3592,6 @@ export class HostReconciler {// funnels through this same method, so a watch event dropped on disconnect// is recovered here.const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true- revalidateHostMutationBoundary()// Track whether this host has desktop enabledconst isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)@@ -3864,18 +3602,11 @@ export class HostReconciler {}const secretResult = await this.validateHostSecret(host)- revalidateHostMutationBoundary()if (!secretResult.ok) {if (secretResult.reason === 'SecretNotFound' ||secretResult.reason === 'SecretAccessDenied') {- // This branch deletes every ephemeral runtime resource. The Secret- // decision may have blocked on the API while Host watch authority or- // the Host UID/spec generation changed. Revalidate at the destructive- // commit boundary so a stale secretRef can never tear down the current- // runtime; the recovered/current Host reconcile will decide again.- revalidateHostMutationBoundary()await this.deleteHostRuntimeResources(host.name, host.namespace, {deleteWorkspacePvc: false,})@@ -3896,11 +3627,8 @@ export class HostReconciler {// Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,// otherwise the kubelet can't mount the SA token and the pod will// crash-loop until RBAC is created.- revalidateHostMutationBoundary()await this.ensureHostServiceAccount(host)- revalidateHostMutationBoundary()await this.ensureHostRole(host)- revalidateHostMutationBoundary()await this.ensureHostRoleBinding(host)let mounts: ResolvedSfsMount[] = []@@ -3913,12 +3641,10 @@ export class HostReconciler {// will inject them.console.error(`[HostReconciler] Failed to resolve context mounts for "${host.name}":`, err)}- revalidateHostMutationBoundary()// Stateless lifecycle (Stage 2): assess enable/reject and persist the// durable state to the Host status subresource BEFORE building the// Deployment, so replicas derive from the same assessment.let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)- revalidateHostMutationBoundary()// Drained-pre-scale guard (Stage 4.3): a pending wake must abort the// suspension IMMEDIATELY before replicas:0 derives from this assessment// — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null@@ -3926,17 +3652,14 @@ export class HostReconciler {// pass (the periodic resync retries) rather than scaling to 0 on stale// data.const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)- revalidateHostMutationBoundary()if (guardedLifecycle === null) {return}lifecycle = guardedLifecycle- revalidateHostMutationBoundary()const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(host,lifecycle)- revalidateHostMutationBoundary()if (lifecycleStatusCommitted) {this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {status: 'committed',@@ -3947,9 +3670,7 @@ export class HostReconciler {})}- revalidateHostMutationBoundary()await this.ensurePvc(host)- revalidateHostMutationBoundary()await this.ensureService(host)// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and@@ -3965,31 +3686,23 @@ export class HostReconciler {// surface it via HostRuntimeStatus instead of swallowing it.const npFailures: string[] = []try {- revalidateHostMutationBoundary()await this.ensureMcpHostIngressNetworkPolicy(host)- revalidateHostMutationBoundary()await this.ensureMcpHostGfsEgressNetworkPolicy(host)- revalidateHostMutationBoundary()await this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure mcp-host NP for "${host.name}":`, err)npFailures.push(`mcp-host NP: ${(err as Error).message}`)}try {- revalidateHostMutationBoundary()await this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)- revalidateHostMutationBoundary()await this.ensureRpcProxyHostEgressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure rpc-proxy host NP for "${host.name}":`, err)npFailures.push(`rpc-proxy NP: ${(err as Error).message}`)}- revalidateHostMutationBoundary()await this.ensureDesktopNetworkPolicy(host)try {- revalidateHostMutationBoundary()await this.ensureChannelReaderEgressNetworkPolicy(host)- revalidateHostMutationBoundary()await this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)} catch (err) {console.error(`[HostReconciler] Failed to ensure channels egress NP for "${host.name}":`, err)@@ -4004,10 +3717,8 @@ export class HostReconciler {)if (bootstrapLifecycle !== lifecycle) {lifecycle = bootstrapLifecycle- revalidateHostMutationBoundary()await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)}- revalidateHostMutationBoundary()// Bootstrap captures the scope contract used for issuance. The Deployment// guard below compares that contract with the live channel cache after this@@ -4016,28 +3727,23 @@ export class HostReconciler {forceFreshForWake,targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})- revalidateHostMutationBoundary()// replaceWithConflictRetry may wait and re-read after a 409, so each body is// rebuilt from a stable lifecycle plus token-scope pair before mutation.const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {- revalidateHostMutationBoundary()const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(host.name,lifecycle)if (deploymentLifecycle !== lifecycle) {lifecycle = deploymentLifecycle- revalidateHostMutationBoundary()await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)}- revalidateHostMutationBoundary()return lifecycle.effective}const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {for (let attempt = 1; attempt <= 3; attempt++) {- revalidateHostMutationBoundary()const currentScopeHash = HostReconciler.runtimeTokenScopeHash(host,this.hasChannelIngress(host)@@ -4049,7 +3755,6 @@ export class HostReconciler {targetSuspended:lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})- revalidateHostMutationBoundary()const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(host,this.hasChannelIngress(host)@@ -4069,7 +3774,6 @@ export class HostReconciler {const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {for (let attempt = 1; attempt <= 3; attempt++) {- revalidateHostMutationBoundary()await resolveDeploymentLifecycle()await ensureCurrentRuntimeTokenScope()const effective = await resolveDeploymentLifecycle()@@ -4078,7 +3782,6 @@ export class HostReconciler {this.hasChannelIngress(host))if (runtimeTokenProvision.scopeHash === currentScopeHash) {- revalidateHostMutationBoundary()return {lifecycle: effective,runtimeTokenRevision: runtimeTokenProvision.revision,@@ -4095,7 +3798,6 @@ export class HostReconciler {)}- revalidateHostMutationBoundary()await this.ensureDeployment(host,mounts,@@ -4103,7 +3805,6 @@ export class HostReconciler {lifecycle.effective,resolveDeploymentState)- revalidateHostMutationBoundary()const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'if (suspended) {@@ -4112,7 +3813,6 @@ export class HostReconciler {this.lifecycle.markHostNotSuspended(host.name)}const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)- revalidateHostMutationBoundary()// If any per-Host NP failed to apply, mark the host as degraded:// deployed remains true (the pod is in place), but ready=false +// a message that names the missing security boundary. Operators see@@ -4148,7 +3848,6 @@ export class HostReconciler {() => this.reconcileChannelReaderDeployment(host),]) {try {- revalidateHostMutationBoundary()await reconcileResource()} catch (err) {channelReaderFailures.push(err)@@ -4169,7 +3868,6 @@ export class HostReconciler {// Best-effort — checkChannelReaderStatus never throws.const ccCount = this.countCommunicationChannels(host.name)const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)- revalidateHostMutationBoundary()if (channelReaderFailures.length > 0) {// Overlay the reconcile error onto the status message so both signals// are visible (the deploy error may explain why the Deployment is missing).@@ -4204,21 +3902,18 @@ export class HostReconciler {* that path, so cache presence at admission is the correct fence. This* mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.*- * Callers that omit `opts` drop only the extra `skipIf` term; the- * resolveCurrentHost supersede fence still gates the delete.+ * Callers that omit `opts` keep the previous unconditional behavior exactly.*/async reconcileDelete(name: string,namespace: string,opts?: { skipIf?: () => boolean }): Promise<void> {- const capturedAuthority = this.captureHostMutationAuthority()// A DELETED event must not race an in-flight create/update for the same// Host: route it through the same per-Host serializer as reconcile(). The// global convergence tail no longer provides this ordering.await this.lifecycle.serializeByHost(name, async () => {- this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)- if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {+ if (opts?.skipIf?.()) {// Same vocabulary the watch path already reports for this condition// (reconcileHostWatchEvent), so the skip is observable, not silent.hostDeleteCleanupTotal.inc({ outcome: 'superseded' })@@ -4365,15 +4060,6 @@ export class HostReconciler {}try {await this.lifecycle.serializeByHost(hostName, async () => {- // The fresh 404 and the cache omission are only destructive- // authority while the same LIST -> WATCH generation remains live.- // Revalidate inside the serializer: queued Host work can otherwise- // outlive that generation and delete a same-name recreation whose- // ADDED event has not reached the retired cache.- if (!authorityValid()) {- hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })- return- }// F2 / #827 TOCTOU: a same-name Host recreated between the fresh read// above and admission into this serializer updates the current cache// (its ADDED event sets the cache before its reconcile runs, and thatdiff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex e45095c41..2aa0ccea4 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -1,19 +1,13 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import * as http from 'http'import { HostFleetReconcileError } from './hostReconciler'import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'-import { externalEgressResyncDelayMs } from './externalEgressConvergenceCoordinator'import {McpServerWatcher,- getContext,+ externalEgressResyncDelayMs,listAllCommunicationChannels,- listAllGlobalFileSystems,listAllHosts,- listAllSharedFileSystems,} from './k8sClient'-import { registry } from './metrics'-import { ContextMapperServer } from './server'-import type { HostCRD, McpServerCRD } from './types'+import type { HostCRD } from './types'function deferred<T = void>(): {promise: Promise<T>@@ -29,116 +23,19 @@ function deferred<T = void>(): {return { promise, resolve, reject }}-async function flushMicrotasks(turns = 8): Promise<void> {- for (let turn = 0; turn < turns; turn += 1) {- await Promise.resolve()- }-}--async function readInitialConvergenceMetric(- name:- | 'clerum_hcc_initial_convergence_retries_total'- | 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',- lane: 'McpServer' | 'NetworkPolicy'-): Promise<number> {- const metric = registry.getSingleMetric(name)- if (!metric) throw new Error(`${name} is not registered`)- const snapshot = await metric.get()- return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0-}--async function requestReadyOverHttp(server: ContextMapperServer): Promise<{- statusCode: number | undefined- body: string-}> {- const listener = (server as unknown as { server: http.Server | null }).server- const address = listener?.address()- if (!address || typeof address === 'string') {- throw new Error('HCC readiness listener did not bind a TCP port')- }-- return new Promise((resolve, reject) => {- const request = http.get(- { host: '127.0.0.1', port: address.port, path: '/ready' },- response => {- let body = ''- response.setEncoding('utf8')- response.on('data', chunk => {- body += chunk- })- response.on('end', () => resolve({ statusCode: response.statusCode, body }))- }- )- request.once('error', reject)- })-}--function stubAuthoritativeInventoryWatch(- watcher: McpServerWatcher,- kind: 'McpServer' | 'Context',- effect?: () => void | Promise<void>-) {- const method = kind === 'McpServer' ? 'startMcpServerWatch' : 'startContextWatch'- const generationField = kind === 'McpServer' ? 'mcpWatchGeneration' : 'contextWatchGeneration'- const requestField = kind === 'McpServer' ? 'mcpWatchRequest' : 'ctxWatchRequest'- return vi.spyOn(watcher as any, method).mockImplementation(async () => {- const generation = ((watcher as any)[generationField] as number) + 1- ;(watcher as any)[generationField] = generation- ;(watcher as any)[requestField] = { abort: vi.fn() }- await effect?.()- return generation- })-}--function markHostInventoryAuthoritative(watcher: McpServerWatcher): void {- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextCacheSynced = true-}--function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {- ;(watcher as any).mcpServerCacheSynced = true-}--function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {- ;(watcher as any).networkPolicyRevocationContextGeneration = (- watcher as any- ).contextWatchGeneration- ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration- ;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision- ;(watcher as any).networkPolicyRevocationServerRevision = (- watcher as any- ).mcpServerDesiredRevision-}--function newContextAuthoritativeWatcher(): McpServerWatcher {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- return watcher-}-const mocks = vi.hoisted(() => {const listNamespacedCustomObject = vi.fn()- const getNamespacedCustomObject = vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)- const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)- const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined- options?.onAuthoritativeRevocationComplete?.()- })+ const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)const serverFullReconcile = vi.fn().mockResolvedValue(undefined)const hostFullReconcile = vi.fn().mockResolvedValue(undefined)const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })const hostListCallOptions = vi.fn()- const sharedFileSystemListCallOptions = vi.fn()const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)return {- hasCertifiedSafetyInventory,listNamespacedCustomObject,- getNamespacedCustomObject,ensureDefaultPolicies,netPolFullReconcile,serverFullReconcile,@@ -147,15 +44,12 @@ const mocks = vi.hoisted(() => {sfsFullReconcile,watch,hostListCallOptions,- sharedFileSystemListCallOptions,createAdministrativeOutcomeReporter,}})-// Kept at externalEgressResyncIntervalSec: 0 in every test (reset in afterEach):-// the external-egress periodic resync timer now lives in the convergence-// coordinator and is unit-tested in externalEgressConvergenceCoordinator.test.ts,-// so the k8sClient integration tests leave it disabled to avoid a stray timer.+// Mutable so a test can enable the external-egress resync timer (R1-M4); every+// other test keeps externalEgressResyncIntervalSec: 0 (reset in afterEach).const mockConfig = vi.hoisted(() => ({devMode: false,namespace: 'mcp-server',@@ -196,12 +90,9 @@ vi.mock('@kubernetes/client-node', () => {return {listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {if (request.plural === 'hosts') mocks.hostListCallOptions(options)- if (request.plural === 'sharedfilesystems')- mocks.sharedFileSystemListCallOptions(options)const response = await mocks.listNamespacedCustomObject(request)if (- request.plural &&- ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&+ request.plural === 'hosts' &&response &&typeof response === 'object' &&!('metadata' in response)@@ -211,17 +102,11 @@ vi.mock('@kubernetes/client-node', () => {// production missing-resourceVersion failure path.return {...response,- metadata: {- resourceVersion:- request.plural === 'hosts'- ? 'test-host-collection-rv'- : `test-${request.plural}-collection-rv`,- },+ metadata: { resourceVersion: 'test-host-collection-rv' },}}return response},- getNamespacedCustomObject: mocks.getNamespacedCustomObject,}}return {}@@ -246,8 +131,6 @@ vi.mock('./reconciler', () => ({reconcile = vi.fn()reconcileDelete = vi.fn()getStatus = vi.fn()- setInventoryAuthority = vi.fn()- setResolveCurrentServer = vi.fn()},}))@@ -270,28 +153,6 @@ vi.mock('./hostReconciler', () => ({// and the watch-authority snapshot into HostReconciler at construction.setResolveCurrentHost = vi.fn()setHostWatchAuthority = vi.fn()- setHostMutationAuthority = vi.fn()- // H2: capture the injected uid-guarded cache reflector so Test C can invoke it.- _reflectHostOutcomeFn:- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)- | null = null- setReflectHostOutcome = vi.fn().mockImplementation(function (- this: {- _reflectHostOutcomeFn:- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)- | null- },- fn: (name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void- ) {- this._reflectHostOutcomeFn = fn- })- _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null = null- setResolveHostMutationDependencies = vi.fn().mockImplementation(function (- this: { _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null },- fn: (host: HostCRD) => readonly unknown[]- ) {- this._hostMutationDependenciesFn = fn- })// Stores the injected callback so tests can invoke it directly._countCommunicationChannelsFn: ((host: string) => number) | null = nullsetCountCommunicationChannels = vi.fn().mockImplementation(function (@@ -311,26 +172,16 @@ vi.mock('./hostReconciler', () => ({// B2: mock the CC cache-sync setter so McpServerWatcher constructor succeeds.setIsCommunicationChannelCacheSynced = vi.fn()patchChannelReaderRevisionAnnotation = vi.fn().mockResolvedValue(undefined)- reconcileChannelReaderRevision = vi.fn().mockResolvedValue(undefined)},}))vi.mock('./networkPolicyReconciler', () => ({- sameContextDesiredRevision: (- expected: { name: string; namespace: string; spec: unknown },- current: { name: string; namespace: string; spec: unknown }- ) =>- expected.name === current.name &&- expected.namespace === current.namespace &&- JSON.stringify(expected.spec) === JSON.stringify(current.spec),NetworkPolicyReconciler: class {ensureDefaultPolicies = mocks.ensureDefaultPoliciesfullReconcile = mocks.netPolFullReconcilereconcileExternalEgress = vi.fn()reconcileContext = vi.fn()- reconcileDeleteContext = vi.fn()cleanupExternalEgress = vi.fn()- hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory},}))@@ -348,7 +199,6 @@ vi.mock('./sharedFileSystemReconciler', () => ({reconcileDelete = vi.fn()getStatus = vi.fn()setListContextRefs = vi.fn()- isMountable = vi.fn().mockReturnValue(false)},}))@@ -363,7 +213,7 @@ describe('McpServerWatcher startup', () => {mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'mcpservers') return { items: [] }- if (plural === 'contexts') return { items: [] }+ if (plural === 'contexts') throw new Error('context discovery failed')if (plural === 'hosts') return { items: [] }if (plural === 'sharedfilesystems') return { items: [] }if (plural === 'communicationchannels')@@ -374,24 +224,60 @@ describe('McpServerWatcher startup', () => {afterEach(() => {vi.useRealTimers()- vi.restoreAllMocks()- // Certified is the resting state. vi.clearAllMocks() in beforeEach clears- // calls but not implementations, so a test that degrades this getter and- // fails before its own restore would silently re-gate readiness for every- // later test in the file. Restore it structurally rather than by- // per-test discipline.- mocks.hasCertifiedSafetyInventory.mockReset().mockReturnValue(true)- mockConfig.externalEgressResyncIntervalSec = 0 // keep periodic resync off by default+ mockConfig.externalEgressResyncIntervalSec = 0 // R1-M4: keep the timer off by defaultmocks.ensureDefaultPolicies.mockReset().mockResolvedValue(undefined)- mocks.netPolFullReconcile.mockReset().mockImplementation(async (...args: unknown[]) => {- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined- options?.onAuthoritativeRevocationComplete?.()- })+ mocks.netPolFullReconcile.mockReset().mockResolvedValue(undefined)mocks.serverFullReconcile.mockReset().mockResolvedValue(undefined)mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)mocks.sfsFullReconcile.mockReset().mockResolvedValue(undefined)})+ // R1-M4 (zach88): the self-rescheduling resync timer (only the pure delay+ // helper was covered). Assert the behavioral invariants the comments promise:+ // arms once, reschedules only AFTER a pass resolves, and stop() ends the chain.+ it('R1-M4: external-egress resync timer arms once, reschedules after each pass, and stops', async () => {+ vi.useFakeTimers()+ mockConfig.externalEgressResyncIntervalSec = 60+ const watcher = new McpServerWatcher()+ for (const m of [+ 'startMcpServerWatch',+ 'startContextWatch',+ 'startSharedFileSystemWatch',+ 'startGlobalFileSystemWatch',+ 'startCommunicationChannelWatch',+ ]) {+ vi.spyOn(watcher as any, m).mockImplementation(async () => undefined)+ }+ // Control each resync pass's completion so we can prove the reschedule waits.+ let pending = deferred()+ const runSpy = vi+ .spyOn(watcher as any, 'runExternalEgressResync')+ .mockImplementation(() => pending.promise)++ await watcher.start()+ expect(runSpy).not.toHaveBeenCalled() // not until the first delay elapses++ await vi.advanceTimersByTimeAsync(60_000)+ expect(runSpy).toHaveBeenCalledTimes(1) // armed once, fired once++ // The NEXT pass must not be scheduled until this one resolves.+ await vi.advanceTimersByTimeAsync(60_000)+ expect(runSpy).toHaveBeenCalledTimes(1) // still 1 — no overlap while pending++ pending.resolve()+ await Promise.resolve()+ pending = deferred()+ await vi.advanceTimersByTimeAsync(60_000)+ expect(runSpy).toHaveBeenCalledTimes(2) // rescheduled after the pass resolved++ // stop() ends the chain — no further fires.+ watcher.stop()+ pending.resolve()+ await Promise.resolve()+ await vi.advanceTimersByTimeAsync(60_000 * 5)+ expect(runSpy).toHaveBeenCalledTimes(2)+ })+it('passes the governed tracing switch to the administrative reporter factory', () => {new McpServerWatcher()@@ -400,83 +286,35 @@ describe('McpServerWatcher startup', () => {})})- it('ensures baseline policies before opening any inventory watch', async () => {+ it('ensures default policies even when initial Context discovery fails', async () => {const watcher = new McpServerWatcher()+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ // #281 (Task 9): startCommunicationChannelWatch is wired into start() — stub to prevent the real watch from firing.+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )await watcher.start()expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(1)- expect(mocks.ensureDefaultPolicies.mock.invocationCallOrder[0]).toBeLessThan(- mocks.watch.mock.invocationCallOrder[0]- )- await vi.waitFor(() =>- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {- runEffect: expect.any(Function),- })- )+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])expect(mocks.hostFullReconcile).toHaveBeenCalledWith([])expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([])- await watcher.stop()})- it('loads the SharedFileSystem inventory before scheduling the first Host fleet pass', async () => {- const sharedFileSystemsListed = deferred<{ items: unknown[] }>()+ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'sharedfilesystems') return sharedFileSystemsListed.promise- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'sfs-ordering-cc-rv' }, items: [] }- }+ if (plural === 'hosts') return { items: [] }return { items: [] }})- const watcher = new McpServerWatcher()- // Sampling "not yet called" after a fixed number of microtask turns cannot- // distinguish "correctly waiting" from "scheduled but not drained yet".- // Record what the first pass actually observed instead: the invariant is- // that it never runs against an unpopulated SharedFileSystem cache, because- // that is what writes mount-less templates and rerolls the fleet.- let sfsCacheSizeAtFirstPass: number | undefined- mocks.hostFullReconcile.mockImplementation(async () => {- if (sfsCacheSizeAtFirstPass === undefined) {- sfsCacheSizeAtFirstPass = (watcher as any).sharedFileSystems.size- }- })- const start = watcher.start()-- try {- // Await the whole startup first. That is the positive signal that the- // cold-start Host recovery already reached the point where it decides- // whether to schedule the fleet pass, so the absence assertion below- // cannot pass merely because nothing has been drained yet.- await start- await flushMicrotasks()- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()- sharedFileSystemsListed.resolve({- items: [{ metadata: { name: 'initial-sfs', namespace: 'mcp-host' }, spec: {} }],- })- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())- expect(sfsCacheSizeAtFirstPass).toBe(1)- expect((watcher as any).sharedFileSystems.get('initial-sfs')).toEqual(- expect.objectContaining({ name: 'initial-sfs' })- )- } finally {- sharedFileSystemsListed.resolve({ items: [] })- await start.catch(() => undefined)- await watcher.stop()- }- })-- it('lists the SharedFileSystem inventory through the deadline-bearing client', async () => {- // The cold-start Host fleet pass waits on this inventory. Bounding the wait- // is not enough: an unbounded request stays hung, so the socket leaks and- // `startSharedFileSystemWatch()` — which runs after this await — never- // starts. Only the deadline-bearing client aborts the request itself, and- // it is observable because it appends its deadline middleware to the call.- mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })-- await listAllSharedFileSystems()+ await expect(listAllHosts()).resolves.toEqual([])- expect(mocks.sharedFileSystemListCallOptions).toHaveBeenCalledWith(+ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(expect.objectContaining({middleware: [expect.objectContaining({ pre: expect.any(Function) })],middlewareMergeStrategy: 'append',@@ -484,4142 +322,125 @@ describe('McpServerWatcher startup', () => {)})- it('schedules the first Host fleet pass when the SharedFileSystem watch never starts', async () => {- // `k8s.Watch` carries no transport deadline — client-node overwrites its own- // AbortSignal.timeout with a bare controller signal — so awaiting the watch- // start inside the inventory promise re-strands the cold-start fleet pass- // that bounding the LIST was meant to protect.- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'sfs-watch-stall-cc-rv' }, items: [] }- }- return { items: [] }- })- mocks.watch.mockImplementation(async (path: string) => {- if (path.endsWith('/sharedfilesystems')) return new Promise(() => undefined)- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- const start = watcher.start()-- try {- await start- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())- } finally {- await watcher.stop()- await start.catch(() => undefined)- }- })-- it('schedules the first Host fleet pass when the SharedFileSystem inventory fails', async () => {- // The inventory LIST runs on the deadline-bearing client, so an apiserver- // that never answers is aborted and surfaces here as a rejection. The Host- // fleet must still converge: stranding it would leave every Host- // unreconciled behind an already-certified readiness.- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'sharedfilesystems') throw new Error('sharedfilesystems LIST aborted')- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'sfs-abort-cc-rv' }, items: [] }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- const start = watcher.start()-- try {- await start- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())- } finally {- await watcher.stop()- await start.catch(() => undefined)- }- })-- it('retains Context uid and generation from the authoritative initial snapshot', async () => {+ it('preserves Host UID from the Kubernetes inventory boundary', async () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'contexts') {+ if (plural === 'hosts') {return {items: [{metadata: {- name: 'identity-context',- namespace: 'mcp-server',- uid: 'context-uid-1',- generation: 7,+ name: 'uid-host',+ namespace: 'mcp-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,},- spec: { contextId: 'identity-context', mcpServers: [] },+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },},],}}- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'context-identity-cc-rv' }, items: [] }- }return { items: [] }})- const watcher = new McpServerWatcher()-- try {- await watcher.start()- expect((watcher as any).contexts.get('identity-context')).toEqual(- expect.objectContaining({ uid: 'context-uid-1', generation: 7 })- )- } finally {- await watcher.stop()- }- })- it('fails safe bootstrap and keeps readiness unavailable when baseline NetworkPolicies fail', async () => {- const policyError = new Error('baseline NetworkPolicies unavailable')- mocks.ensureDefaultPolicies.mockRejectedValueOnce(policyError)- const watcher = new McpServerWatcher()- const server = new ContextMapperServer(watcher, 0)- await server.start()- const bootstrap = watcher.start().then(() => server.setReady(true))-- try {- await expect(bootstrap).rejects.toThrow(policyError)- const ready = await requestReadyOverHttp(server)- expect(ready.statusCode).toBe(503)- expect(JSON.parse(ready.body)).toEqual({ status: 'starting', ready: false })- expect(mocks.watch).not.toHaveBeenCalled()- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- } finally {- await server.stop()- await watcher.stop()- }+ await expect(listAllHosts()).resolves.toEqual([+ expect.objectContaining({+ name: 'uid-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,+ }),+ ])})- it.each(['LIST', 'WATCH'] as const)(- 'stays live and recovers readiness after a transient Host %s bootstrap failure',- async failureBoundary => {- vi.useFakeTimers()- let hostListAttempts = 0- let hostWatchAttempts = 0- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- hostListAttempts += 1- if (failureBoundary === 'LIST' && hostListAttempts === 1) {- throw new Error('host discovery temporarily unavailable')- }- return { metadata: { resourceVersion: 'host-recovery-rv' }, items: [] }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- }- )- mocks.watch.mockImplementation(async path => {- if (path.endsWith('/hosts')) {- hostWatchAttempts += 1- if (failureBoundary === 'WATCH' && hostWatchAttempts === 1) {- throw new Error('host watch temporarily unavailable')- }- }- return { abort: vi.fn() }- })- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()-- try {- await expect(watcher.start()).resolves.toBeUndefined()- server.setReady(true)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()-- await vi.advanceTimersByTimeAsync(5000)- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())-- expect(hostListAttempts).toBe(2)- expect(hostWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)- expect((watcher as any).ccAppliedLifecycleGeneration).toBe(1)- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)- } finally {- await server.stop()- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- }- )-- it.each(['LIST', 'WATCH'] as const)(- 'stays live and recovers readiness after a transient McpServer %s bootstrap failure',- async failureBoundary => {- vi.useFakeTimers()- let mcpServerListAttempts = 0- let mcpServerWatchAttempts = 0- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') {- mcpServerListAttempts += 1- if (failureBoundary === 'LIST' && mcpServerListAttempts === 1) {- throw new Error('mcpserver discovery temporarily unavailable')- }- return { metadata: { resourceVersion: 'mcpserver-recovery-rv' }, items: [] }- }- if (plural === 'hosts') {- return { metadata: { resourceVersion: 'host-rv' }, items: [] }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- }- )- mocks.watch.mockImplementation(async path => {- if (path.endsWith('/mcpservers')) {- mcpServerWatchAttempts += 1- if (failureBoundary === 'WATCH' && mcpServerWatchAttempts === 1) {- throw new Error('mcpserver watch temporarily unavailable')- }- }- return { abort: vi.fn() }- })- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()- // The McpServer lane runs for real (LIST + WATCH) so its bootstrap- // failure and recovery exercise production code. Context stays- // authoritative so readiness can be promoted once the lane recovers.- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()-- try {- await expect(watcher.start()).resolves.toBeUndefined()- server.setReady(true)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- expect((watcher as any).mcpServerCacheRecoveryTimer).not.toBeNull()-- await vi.advanceTimersByTimeAsync(5000)- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())-- expect(mcpServerListAttempts).toBe(2)- expect(mcpServerWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)- } finally {- await server.stop()- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- }- )-- it.each(['LIST', 'WATCH'] as const)(- 'stays live and recovers readiness after a transient Context %s bootstrap failure',- async failureBoundary => {- vi.useFakeTimers()- let contextListAttempts = 0- let contextWatchAttempts = 0- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural }: { plural: string }) => {- if (plural === 'contexts') {- contextListAttempts += 1- if (failureBoundary === 'LIST' && contextListAttempts === 1) {- throw new Error('context discovery temporarily unavailable')- }- return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }- }- if (plural === 'hosts') {- return { metadata: { resourceVersion: 'host-rv' }, items: [] }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- }- )- mocks.watch.mockImplementation(async path => {- if (path.endsWith('/contexts')) {- contextWatchAttempts += 1- if (failureBoundary === 'WATCH' && contextWatchAttempts === 1) {- throw new Error('context watch temporarily unavailable')- }- }- return { abort: vi.fn() }- })- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()- // The Context lane runs for real (LIST + WATCH) so its bootstrap- // failure and recovery exercise production code. McpServer stays- // authoritative so readiness can be promoted once the lane recovers.- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()-- try {- await expect(watcher.start()).resolves.toBeUndefined()- server.setReady(true)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- expect((watcher as any).contextCacheRecoveryTimer).not.toBeNull()-- await vi.advanceTimersByTimeAsync(5000)- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())-- expect(contextListAttempts).toBe(2)- expect(contextWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)- } finally {- await server.stop()- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }+ it('reconciles startup external egress before runtime full reconciliation', async () => {+ const eventLog: string[] = []+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ spec: {+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ },}- )- it('cancels a pending Host bootstrap recovery when the watcher stops', async () => {- vi.useFakeTimers()- let hostListAttempts = 0mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- hostListAttempts += 1- throw new Error('host inventory remains unavailable')- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }return { items: [] }})- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ mocks.ensureDefaultPolicies.mockImplementation(async () => {+ eventLog.push('defaults')+ })+ mocks.serverFullReconcile.mockImplementation(async () => {+ eventLog.push('runtime')+ })+const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementation(async () => {+ eventLog.push('egress')+ })+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )- try {- await watcher.start()- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()+ await watcher.start()- await watcher.stop()- await vi.advanceTimersByTimeAsync(5000)+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'web-search' }),+ ])- expect(hostListAttempts).toBe(1)- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()- } finally {- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }+ watcher.stop()})- it('defers every Host-dependent effect while a bootstrap LIST lacks WATCH authority', async () => {+ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {vi.useFakeTimers()- const staleHost = {- metadata: {- name: 'stale-bootstrap-host',- namespace: 'mcp-host',- generation: 1,- uid: 'stale-bootstrap-host-uid',- },+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },spec: {- host: 'stale-bootstrap-host',- contextRef: 'bootstrap-context',- secretRef: 'bootstrap-secret',- model: { provider: 'openai', name: 'gpt-4o-mini' },+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],},}+mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- return {- metadata: { resourceVersion: 'stale-bootstrap-rv' },- items: [staleHost],- }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }return { items: [] }})- mocks.watch.mockImplementation(async path => {- if (path.endsWith('/hosts')) {- throw new Error('host watch remains unavailable')- }- return { abort: vi.fn() }- })++ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('dns resolution failed'))const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- const hostReconciler = (watcher as any).hostReconciler-- try {- await watcher.start()- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)-- expect(watcher.getHost('stale-bootstrap-host')).toBeUndefined()- const ccWatchCallback = (watcher as any).getCommunicationChannelWatchCallback()- await ccWatchCallback('ADDED', {- metadata: { name: 'bootstrap-channel', namespace: 'channels' },- spec: { hostRef: 'stale-bootstrap-host' },- })- ;(watcher as any).contexts.set('bootstrap-context', {- name: 'bootstrap-context',- namespace: 'mcp-server',- spec: {- contextId: 'bootstrap-context',- sharedFileSystems: [{ name: 'bootstrap-sfs', mountPath: '/shared' }],- },- })- await (watcher as any).reconcileHostsReferencingSfs('bootstrap-sfs')- await (watcher as any).reconcileHostsReferencingContext('bootstrap-context')- ;(watcher as any).ccCacheSynced = false- await (watcher as any).reconcileChannelReaderRevision('bootstrap-channel-secret', 'channels')-- expect(hostReconciler.reconcile).not.toHaveBeenCalled()- expect(hostReconciler.patchChannelReaderRevisionAnnotation).not.toHaveBeenCalled()- expect(hostReconciler.reconcileChannelReaderRevision).not.toHaveBeenCalled()- expect(warnSpy).toHaveBeenCalledWith(- '[K8s] Deferring CommunicationChannel Secret "channels/bootstrap-channel-secret" Host convergence; CommunicationChannel cache is not authoritative'- )- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()- } finally {- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- })-- it('defers Host fleet convergence until both Host and Context inventories are authoritative', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextCacheSynced = false- ;(watcher as any).hosts.set('context-dependent-host', {- name: 'context-dependent-host',- namespace: 'mcp-host',- spec: {- host: 'context-dependent-host',- contextRef: 'current-context',- secretRef: 'host-secret',- },- })-- await (watcher as any).performHostFleetReconcileOnce({- reason: 'Context authority test',- mode: 'full',- })-- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()- ;(watcher as any).contextCacheSynced = true- await (watcher as any).performHostFleetReconcileOnce({- reason: 'Context authority recovered',- mode: 'full',- })-- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()- expect(mocks.hostFullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'context-dependent-host' }),- ])- await watcher.stop()- })-- it('applies a pending CommunicationChannel lifecycle generation during Context recovery', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextCacheSynced = false- const pendingGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()-- await (watcher as any).recoverContextInventoryAndWatch()- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())-- expect((watcher as any).ccAppliedLifecycleGeneration).toBe(pendingGeneration)- await watcher.stop()- })-- it('starts McpServer and Context watches from their inventory resourceVersions', async () => {- const watcher = new McpServerWatcher()-- await (watcher as any).startMcpServerWatch('mcp-inventory-rv')- await (watcher as any).startContextWatch('context-inventory-rv')-- expect(mocks.watch).toHaveBeenNthCalledWith(- 1,- expect.stringContaining('/mcpservers'),- { resourceVersion: 'mcp-inventory-rv' },- expect.any(Function),- expect.any(Function)- )- expect(mocks.watch).toHaveBeenNthCalledWith(- 2,- expect.stringContaining('/contexts'),- { resourceVersion: 'context-inventory-rv' },- expect.any(Function),- expect.any(Function)- )- watcher.stop()- })-- it.each([- ['Context', 'startContextWatch', 'contextWatchGeneration'],- ['Host', 'startHostWatch', 'hostWatchGeneration'],- ] as const)(- 'assigns a fresh generation to every replacement %s watch',- async (_kind, method, generationField) => {- const watcher = new McpServerWatcher()- const before = (watcher as any)[generationField] as number-- const first = await (watcher as any)[method]('first-rv')- const second = await (watcher as any)[method]('second-rv')-- expect(first).toBe(before + 1)- expect(second).toBe(before + 2)- expect((watcher as any)[generationField]).toBe(before + 2)- await watcher.stop()- }- )-- it.each([- [- 'McpServer',- 'restartMcpServerWatch',- 'startMcpServerWatch',- 'mcpWatchGeneration',- 'mcpServerCacheSynced',- { resourceVersion: 'mcp-rv', servers: [] },- ],- [- 'Context',- 'restartContextWatch',- 'startContextWatch',- 'contextWatchGeneration',- 'contextCacheSynced',- { resourceVersion: 'context-rv', contexts: [] },- ],- ] as const)(- 'rejects a %s LIST snapshot that is not paired with a live watch request',- async (_kind, restartMethod, startMethod, generationField, cacheField, snapshot) => {- const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, startMethod).mockImplementation(async () => {- ;(watcher as any)[generationField] += 1- return (watcher as any)[generationField]- })-- await expect((watcher as any)[restartMethod](snapshot)).rejects.toThrow(- 'snapshot could not be paired with an active watch'- )- expect((watcher as any)[cacheField]).toBe(false)- await watcher.stop()- }- )-- it.each([- ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],- ['Context', 'contexts', 'contextCacheRecoveryTimer'],- ] as const)(- 'stays live and arms in-process recovery when the %s inventory LIST is unavailable',- async (kind, failingPlural, recoveryTimerField) => {- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural }: { plural: string }) => {- if (plural === failingPlural) throw new Error(`${plural} inventory unavailable`)- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- }- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()-- try {- await expect(watcher.start()).resolves.toBeUndefined()-- expect(errorSpy).toHaveBeenCalledWith(- `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,- expect.objectContaining({ message: `${failingPlural} inventory unavailable` })- )- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((watcher as any)[recoveryTimerField]).not.toBeNull()- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- } finally {- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- }- )-- it.each([- ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],- ['Context', 'contexts', 'contextCacheRecoveryTimer'],- ] as const)(- 'stays live and arms in-process recovery when the %s collection resourceVersion is missing',- async (kind, missingPlural, recoveryTimerField) => {- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural }: { plural: string }) => {- if (plural === missingPlural) return { metadata: {}, items: [] }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- }- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()-- try {- await expect(watcher.start()).resolves.toBeUndefined()-- expect(errorSpy).toHaveBeenCalledWith(- `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,- expect.objectContaining({ message: `${kind} snapshot missing resourceVersion` })- )- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((watcher as any)[recoveryTimerField]).not.toBeNull()- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- } finally {- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- }- )-- it('replays McpServer and Context changes from the inventory resourceVersion before convergence', async () => {- const staleServer = {- metadata: {- name: 'gap-server',- namespace: 'mcp-server',- uid: 'gap-server-uid',- generation: 1,- },- spec: {- contextRef: 'default',- image: 'clerum/gap-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const staleContext = {- metadata: { name: 'gap-context', namespace: 'mcp-server' },- spec: { contextId: 'gap-context', mcpServers: [] },- }- const currentContext = {- metadata: { name: 'gap-context', namespace: 'mcp-server' },- spec: { contextId: 'gap-context', mcpServers: ['current-server'] },- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') {- return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }- }- if (plural === 'contexts') {- return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }- }- return { items: [] }- })- mocks.watch.mockImplementation(async (path, options, callback) => {- if (path.endsWith('/mcpservers')) {- expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })- await callback('DELETED', staleServer)- }- if (path.endsWith('/contexts')) {- expect(options).toEqual({ resourceVersion: 'opaque/context:202' })- await callback('MODIFIED', currentContext)- }- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()-- await watcher.start()-- await vi.waitFor(() => {- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {- runEffect: expect.any(Function),- })- expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(- [expect.objectContaining({ spec: currentContext.spec })],- [],- expect.objectContaining({- serverInventoryComplete: true,- ensureDefaults: false,- contextInventoryAuthoritative: expect.any(Function),- serverInventoryAuthoritative: expect.any(Function),- contextDesiredRevision: expect.any(Function),- serverDesiredRevision: expect.any(Function),- })- )- })- expect(watcher.getAllServers()).toEqual([])- await watcher.stop()- })-- it('recovers a closed McpServer watch through a fresh authoritative snapshot', async () => {- vi.useFakeTimers()- const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []- const doneCallbacks: Array<(error: Error | null) => void> = []- const watchQueries: unknown[] = []- mocks.watch.mockImplementation(async (path, options, callback, done) => {- if (path.endsWith('/mcpservers')) {- callbacks.push(callback)- doneCallbacks.push(done)- watchQueries.push(options)- }- return { abort: vi.fn() }- })- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') {- return {- metadata: { resourceVersion: 'mcp-recovery-rv' },- items: [- {- metadata: {- name: 'recovered-server',- namespace: 'mcp-server',- uid: 'recovered-uid',- generation: 1,- },- spec: {- contextRef: 'default',- image: 'clerum/recovered:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- },- ],- }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).hostCacheSynced = true- await (watcher as any).restartMcpServerWatch({- resourceVersion: 'mcp-start-rv',- servers: [],- })- markNetworkPolicyRevocationAuthoritative(watcher)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)-- doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- await vi.advanceTimersByTimeAsync(5000)- await vi.waitFor(() => expect(watchQueries).toHaveLength(2))-- expect(watchQueries).toEqual([- { resourceVersion: 'mcp-start-rv' },- { resourceVersion: 'mcp-recovery-rv' },- ])- expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])- expect(callbacks).toHaveLength(2)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- await watcher.stop()- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- })-- it('withdraws readiness when the safety pass leaves its inventory uncertified', async () => {- // Losing a delete fence is the one doom cause that bumps no watch- // generation and moves no desired revision, so every equality below stays- // satisfied while an allow this pass classified as stale is still live.- // The reconciler already degrades its fence at the point of loss; readiness- // has to read it, or /ready keeps answering 200 over that allow until a- // retry lands (>=5s, and indefinitely against a persistent second writer —- // e.g. the two-replica overlap every rolling update opens, since the- // Deployment carries no leader election).- const watcher = newContextAuthoritativeWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)-- try {- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- } finally {- // vi.clearAllMocks() clears calls, not implementations, so an escaped- // mockReturnValue would silently re-gate every later test in this file.- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)- }- })-- it('restores readiness once a later safety pass certifies its inventory again', async () => {- // The withdrawal above must not be a one-way latch: a fence raised by a- // transient loss has to clear on the next completed pass, or the retry- // machinery would leave the controller permanently unready.- const watcher = newContextAuthoritativeWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- try {- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)-- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- } finally {- // An assertion failure above must not escape a false implementation into- // the rest of the file — vi.clearAllMocks() would not undo it.- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)- }- })-- it('ignores a late McpServer callback from a retired watch generation', async () => {- const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []- mocks.watch.mockImplementation(async (path, _options, callback) => {- if (path.endsWith('/mcpservers')) callbacks.push(callback)- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- await (watcher as any).restartMcpServerWatch({- resourceVersion: 'mcp-old-rv',- servers: [],- })- await (watcher as any).restartMcpServerWatch({- resourceVersion: 'mcp-new-rv',- servers: [- {- name: 'current-server',- namespace: 'mcp-server',- uid: 'current-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/current:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- },- ],- })-- await callbacks[0]('ADDED', {- metadata: {- name: 'stale-server',- namespace: 'mcp-server',- uid: 'stale-uid',- generation: 1,- },- spec: {- contextRef: 'default',- image: 'clerum/stale:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- })-- expect(watcher.getAllServers().map(server => server.name)).toEqual(['current-server'])- await watcher.stop()- })-- it('recovers a closed Context watch through a fresh authoritative snapshot', async () => {- vi.useFakeTimers()- const doneCallbacks: Array<(error: Error | null) => void> = []- const watchQueries: unknown[] = []- mocks.watch.mockImplementation(async (path, options, _callback, done) => {- if (path.endsWith('/contexts')) {- doneCallbacks.push(done)- watchQueries.push(options)- }- return { abort: vi.fn() }- })- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'contexts') {- return {- metadata: { resourceVersion: 'context-recovery-rv' },- items: [- {- metadata: { name: 'recovered-context', namespace: 'mcp-server' },- spec: { contextId: 'recovered-context', mcpServers: [] },- },- ],- }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- await (watcher as any).restartContextWatch({- resourceVersion: 'context-start-rv',- contexts: [],- })-- doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))- await vi.advanceTimersByTimeAsync(5000)- await vi.waitFor(() => expect(watchQueries).toHaveLength(2))-- expect(watchQueries).toEqual([- { resourceVersion: 'context-start-rv' },- { resourceVersion: 'context-recovery-rv' },- ])- expect((watcher as any).contexts.get('recovered-context')).toEqual(- expect.objectContaining({ name: 'recovered-context' })- )- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())- await watcher.stop()- })-- it('becomes ready after authoritative revocation while every additive convergence lane remains pending', async () => {- const initialServerFleet = deferred()- const initialNetworkPolicies = deferred()- const initialHostFleet = deferred()- const initialSfsFleet = deferred()- const initialGfsFleet = deferred()- let completeAuthoritativeRevocation: (() => void) | undefined- mocks.serverFullReconcile.mockImplementationOnce(() => initialServerFleet.promise)- mocks.netPolFullReconcile.mockImplementationOnce(- async (- _contexts: unknown,- _servers: unknown,- options?: { onAuthoritativeRevocationComplete?: () => void }- ) => {- completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete- await initialNetworkPolicies.promise- }- )- mocks.hostFullReconcile.mockImplementationOnce(() => initialHostFleet.promise)- mocks.sfsFullReconcile.mockImplementationOnce(() => initialSfsFleet.promise)- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'safe-bootstrap-rv' }, items: [] }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- const startMcpServerWatch = stubAuthoritativeInventoryWatch(watcher, 'McpServer')- const startContextWatch = stubAuthoritativeInventoryWatch(watcher, 'Context')- const startSharedFileSystemWatch = vi- .spyOn(watcher as any, 'startSharedFileSystemWatch')- .mockResolvedValue(undefined)- const startGlobalFileSystemWatch = vi- .spyOn(watcher as any, 'startGlobalFileSystemWatch')- .mockResolvedValue(undefined)- const gfsFullReconcile = vi- .spyOn((watcher as any).gfsReconciler, 'fullReconcile')- .mockImplementationOnce(() => initialGfsFleet.promise)-- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- const bootstrap = watcher.start()-- try {- await vi.waitFor(() => {- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()- expect(mocks.sfsFullReconcile).toHaveBeenCalledOnce()- expect(gfsFullReconcile).toHaveBeenCalledOnce()- })- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()-- await expect(bootstrap).resolves.toBeUndefined()- server.setReady(true)- const beforeRevocation = await requestReadyOverHttp(server)- expect(beforeRevocation.statusCode).toBe(503)- expect(completeAuthoritativeRevocation).toBeTypeOf('function')-- completeAuthoritativeRevocation?.()- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())- const ready = await requestReadyOverHttp(server)- expect(ready.statusCode).toBe(200)- expect(JSON.parse(ready.body)).toEqual({ status: 'ready', ready: true })- expect(startMcpServerWatch).toHaveBeenCalledOnce()- expect(startContextWatch).toHaveBeenCalledOnce()- expect(startSharedFileSystemWatch).toHaveBeenCalledOnce()- expect(startGlobalFileSystemWatch).toHaveBeenCalledOnce()- expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)-- // Runtime convergence starts at the safety-certification boundary, not- // after the NetworkPolicy additive fleet completes. This prevents the- // startup egress lane from racing the policy safety pass while retaining- // readiness decoupling from both additive fleets.- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()- } finally {- initialServerFleet.resolve(undefined)- initialNetworkPolicies.resolve(undefined)- initialHostFleet.resolve(undefined)- initialSfsFleet.resolve(undefined)- initialGfsFleet.resolve(undefined)- await bootstrap.catch(() => undefined)- await server.stop()- watcher.stop()- }- })-- it('keeps readiness after a current Context safety delta while an older additive pass is pending', async () => {- const olderAdditivePass = deferred()- const currentDeltaSafety = deferred<boolean>()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- let networkPolicyPasses = 0- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {- if (path.endsWith('/contexts')) contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {- networkPolicyPasses += 1- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined- options?.onAuthoritativeRevocationComplete?.()- if (networkPolicyPasses === 1) await olderAdditivePass.promise- })-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('context-delta-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('delta-context', {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- spec: { contextId: 'delta-context', mcpServers: [] },- })- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(- () => currentDeltaSafety.promise- )-- try {- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)-- const currentDelta = contextWatchCallback!('MODIFIED', {- metadata: {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- },- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },- })-- await vi.waitFor(() =>- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()- )- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- // The scoped revocation completed, which is what authorises the delta- // certificate; see the aborted-pass twin above for the negative side.- currentDeltaSafety.resolve(true)- await currentDelta- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()- } finally {- olderAdditivePass.resolve(undefined)- await initialPass- await server.stop()- await watcher.stop()- }- })-- it('does not certify a Context delta whose stale-allow revocation aborted mid-pass', async () => {- const olderAdditivePass = deferred()- const abortedDeltaSafety = deferred<boolean>()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- let networkPolicyPasses = 0- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {- if (path.endsWith('/contexts')) contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {- networkPolicyPasses += 1- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined- options?.onAuthoritativeRevocationComplete?.()- if (networkPolicyPasses === 1) await olderAdditivePass.promise- })-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('aborted-delta-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('delta-context', {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- spec: { contextId: 'delta-context', mcpServers: [] },- })- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>- // `reconcileContext` reports that its authority fence broke, so the scoped- // revocation never finished deleting this Context's stale allows.- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(- () => abortedDeltaSafety.promise- )-- try {- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)-- const abortedDelta = contextWatchCallback!('MODIFIED', {- metadata: {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- },- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },- })-- await vi.waitFor(() =>- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()- )- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- abortedDeltaSafety.resolve(false)- await abortedDelta-- // An incomplete revocation must leave readiness withheld until the- // authoritative full pass re-certifies. Certifying here would report- // Ready while a stale allow is still live.- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- } finally {- olderAdditivePass.resolve(undefined)- await initialPass- await server.stop()- await watcher.stop()- }- })-- it('does not certify a Context delta while the namespace-wide inventory is uncertified', async () => {- const olderAdditivePass = deferred()- const uncertifiedDelta = deferred<boolean>()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- let networkPolicyPasses = 0- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {- if (path.endsWith('/contexts')) contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {- networkPolicyPasses += 1- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined- options?.onAuthoritativeRevocationComplete?.()- if (networkPolicyPasses === 1) await olderAdditivePass.promise- })-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('uncertified-inventory-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('delta-context', {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- spec: { contextId: 'delta-context', mcpServers: [] },- })- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(- () => uncertifiedDelta.promise- )- try {- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())- // Baseline: the inventory is still certified here, so readiness is- // genuinely green. Degrading the fence before this point would assert a- // 200 that only the missing readiness gate could produce.- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)-- // The delta's own revocation completes, but the last authoritative pass left- // the namespace-wide inventory uncertified. A label-scoped delta cannot- // vouch for that namespace, so readiness must stay withheld.- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)-- const delta = contextWatchCallback!('MODIFIED', {- metadata: {- name: 'delta-context',- namespace: 'mcp-server',- uid: 'delta-context-uid',- },- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },- })-- await vi.waitFor(() =>- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()- )- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- uncertifiedDelta.resolve(true)- await delta-- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- } finally {- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)- olderAdditivePass.resolve(undefined)- await initialPass- await server.stop()- await watcher.stop()- }- })-- it('recovers when a Context delta loses the delete fence with no certificate to withhold', async () => {- // A MODIFIED that does not move the desired revision — a status or label- // write by another controller — still runs a scoped revocation, but builds- // no delta certificate. Nothing reads the returned boolean there, so a lost- // delete fence must stay loud and reach the convergence retry instead of- // leaving a stale allow live with no NetworkPolicy resync to recover it.- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string; generation?: number }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {- if (path.endsWith('/contexts')) contextWatchCallback = callback- return { abort: vi.fn() }- })-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('lost-fence-no-certificate-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('steady-context', {- name: 'steady-context',- namespace: 'mcp-server',- uid: 'steady-context-uid',- generation: 4,- spec: { contextId: 'steady-context', mcpServers: [] },- })- markNetworkPolicyRevocationAuthoritative(watcher)- const passesBefore = mocks.netPolFullReconcile.mock.calls.length- // With honorsLostFence false the real reconciler throws the 409 rather than- // reporting it, which is the whole point: the throw is what reaches the- // convergence retry below.- ;(watcher as any).netPolReconciler.reconcileContext.mockRejectedValueOnce(- Object.assign(new Error('the UID in the precondition does not match'), { code: 409 })- )-- try {- await contextWatchCallback!('MODIFIED', {- metadata: {- name: 'steady-context',- namespace: 'mcp-server',- uid: 'steady-context-uid',- generation: 4,- },- spec: { contextId: 'steady-context', mcpServers: [] },- })-- // This lane cannot act on the boolean, so it must not have opted into the- // reported outcome.- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledWith(- expect.objectContaining({ name: 'steady-context' }),- expect.objectContaining({ honorsLostFence: false })- )- await vi.waitFor(() =>- expect(mocks.netPolFullReconcile.mock.calls.length).toBeGreaterThan(passesBefore)- )- } finally {- await watcher.stop()- }- })-- it('does not issue a Context delta certificate before any authoritative safety certificate', async () => {- const pendingFullSafetyPass = deferred()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('uncertified-context-delta-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('uncertified-context', {- name: 'uncertified-context',- namespace: 'mcp-server',- uid: 'uncertified-context-uid',- spec: { contextId: 'uncertified-context', mcpServers: [] },- })- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)-- try {- await contextWatchCallback!('MODIFIED', {- metadata: {- name: 'uncertified-context',- namespace: 'mcp-server',- uid: 'uncertified-context-uid',- },- spec: { contextId: 'uncertified-context', mcpServers: ['delta-server'] },- })-- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- } finally {- pendingFullSafetyPass.resolve(undefined)- await watcher.stop()- await server.stop()- }- })-- it('does not let a Context delta certify an outstanding McpServer revision', async () => {- const pendingFullSafetyPass = deferred()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('server-revision-context-delta-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('unrelated-context', {- name: 'unrelated-context',- namespace: 'mcp-server',- uid: 'unrelated-context-uid',- spec: { contextId: 'unrelated-context', mcpServers: [] },- })- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).mcpServerDesiredRevision += 1- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)-- try {- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- await contextWatchCallback!('MODIFIED', {- metadata: {- name: 'unrelated-context',- namespace: 'mcp-server',- uid: 'unrelated-context-uid',- },- spec: { contextId: 'unrelated-context', mcpServers: ['delta-server'] },- })-- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- } finally {- pendingFullSafetyPass.resolve(undefined)- await watcher.stop()- await server.stop()- }- })-- it('does not let one Context delta certify another Context revision still in flight', async () => {- const pendingFullSafetyPass = deferred()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string; uid: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)-- const watcher = new McpServerWatcher()- await (watcher as any).startContextWatch('parallel-context-delta-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 1- ;(watcher as any).contexts.set('first-context', {- name: 'first-context',- namespace: 'mcp-server',- uid: 'first-context-uid',- spec: { contextId: 'first-context', mcpServers: [] },- })- ;(watcher as any).contexts.set('second-context', {- name: 'second-context',- namespace: 'mcp-server',- uid: 'second-context-uid',- spec: { contextId: 'second-context', mcpServers: [] },- })- markNetworkPolicyRevocationAuthoritative(watcher)- // This represents a prior Context change whose global safety sweep is- // pending. It invalidates the old certificate before the scoped delta.- ;(watcher as any).contextDesiredRevision += 1- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)-- try {- await contextWatchCallback!('MODIFIED', {- metadata: {- name: 'second-context',- namespace: 'mcp-server',- uid: 'second-context-uid',- },- spec: { contextId: 'second-context', mcpServers: ['delta-server'] },- })-- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)- } finally {- pendingFullSafetyPass.resolve(undefined)- await watcher.stop()- await server.stop()- }- })-- it.each([- ['Context', 'contextDesiredRevision'],- ['McpServer', 'mcpServerDesiredRevision'],- ] as const)(- 'does not restore readiness from a stale %s delta safety certificate',- async (_kind, revisionField) => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextWatchGeneration = 4- ;(watcher as any).mcpWatchGeneration = 7- markNetworkPolicyRevocationAuthoritative(watcher)- const certificate = {- contextGeneration: (watcher as any).contextWatchGeneration,- serverGeneration: (watcher as any).mcpWatchGeneration,- contextRevision: (watcher as any).contextDesiredRevision,- serverRevision: (watcher as any).mcpServerDesiredRevision,- }-- ;(watcher as any)[revisionField] += 1-- expect((watcher as any).recordNetworkPolicySafetyCertificate(certificate)).toBe(false)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- await watcher.stop()- }- )-- it.each([- ['SharedFileSystem', 'LIST', 'sharedfilesystems', 'startSharedFileSystemWatch'],- ['SharedFileSystem', 'WATCH', 'sharedfilesystems', 'startSharedFileSystemWatch'],- ['GlobalFileSystem', 'LIST', 'globalfilesystems', 'startGlobalFileSystemWatch'],- ['GlobalFileSystem', 'WATCH', 'globalfilesystems', 'startGlobalFileSystemWatch'],- ] as const)(- 'does not let a pending %s %s delay NetworkPolicy safety, bootstrap, or readiness certification',- async (_kind, boundary, plural, watchMethod) => {- const pendingBoundary = deferred<unknown>()- const pendingAdditivePolicies = deferred()- let completeAuthoritativeRevocation: (() => void) | undefined- mocks.listNamespacedCustomObject.mockImplementation(- async ({ plural: requestedPlural }: { plural: string }) => {- if (requestedPlural === 'communicationchannels') {- return { metadata: { resourceVersion: 'background-lane-rv' }, items: [] }- }- if (boundary === 'LIST' && requestedPlural === plural) {- return pendingBoundary.promise- }- return { items: [] }- }- )- mocks.netPolFullReconcile.mockImplementationOnce(- async (- _contexts: unknown,- _servers: unknown,- options?: { onAuthoritativeRevocationComplete?: () => void }- ) => {- completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete- await pendingAdditivePolicies.promise- }- )-- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- const sfsWatch = vi- .spyOn(watcher as any, 'startSharedFileSystemWatch')- .mockResolvedValue(undefined)- const gfsWatch = vi- .spyOn(watcher as any, 'startGlobalFileSystemWatch')- .mockResolvedValue(undefined)- const targetWatch = watchMethod === 'startSharedFileSystemWatch' ? sfsWatch : gfsWatch- if (boundary === 'WATCH') {- targetWatch.mockImplementationOnce(() => pendingBoundary.promise as Promise<void>)- }- const bootstrap = watcher.start()-- try {- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())- await expect(bootstrap).resolves.toBeUndefined()-- expect(completeAuthoritativeRevocation).toBeTypeOf('function')- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- if (boundary === 'LIST') {- expect(targetWatch).not.toHaveBeenCalled()- } else {- expect(targetWatch).toHaveBeenCalledOnce()- }-- completeAuthoritativeRevocation?.()- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))- } finally {- pendingBoundary.resolve({ items: [] })- pendingAdditivePolicies.resolve(undefined)- await flushMicrotasks()- await watcher.stop()- }- }- )-- it('starts every initial background convergence lane from the caches current after watches', async () => {- const staleServer = {- metadata: { name: 'stale-server', namespace: 'mcp-server' },- spec: {- contextRef: 'default',- image: 'clerum/stale-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const currentServer = {- name: 'current-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/current-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const staleContext = {- metadata: { name: 'stale-context', namespace: 'mcp-server' },- spec: { contextId: 'stale-context', mcpServers: [] },- }- const currentContext = {- name: 'current-context',- namespace: 'mcp-server',- spec: { contextId: 'current-context', mcpServers: [] },- }- const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }- const currentSfs = { name: 'current-sfs', namespace: 'mcp-host', spec: {} }- const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }- const currentGfs = { name: 'current-gfs', namespace: 'mcp-host', spec: {} }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [staleServer] }- if (plural === 'contexts') return { items: [staleContext] }- if (plural === 'sharedfilesystems') return { items: [staleSfs] }- if (plural === 'globalfilesystems') return { items: [staleGfs] }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'current-cache-rv' }, items: [] }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer', () => {- ;(watcher as any).servers.clear()- ;(watcher as any).servers.set(currentServer.name, currentServer)- })- stubAuthoritativeInventoryWatch(watcher, 'Context', () => {- ;(watcher as any).contexts.clear()- ;(watcher as any).contexts.set(currentContext.name, currentContext)- })- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => {- ;(watcher as any).sharedFileSystems.clear()- ;(watcher as any).sharedFileSystems.set(currentSfs.name, currentSfs)- })- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => {- ;(watcher as any).globalFileSystems.clear()- ;(watcher as any).globalFileSystems.set(currentGfs.name, currentGfs)- })- const gfsFullReconcile = vi.spyOn((watcher as any).gfsReconciler, 'fullReconcile')-- await watcher.start()-- await vi.waitFor(() => {- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([currentServer], {- runEffect: expect.any(Function),- })- expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(- [currentContext],- [currentServer],- expect.objectContaining({- serverInventoryComplete: true,- ensureDefaults: false,- contextInventoryAuthoritative: expect.any(Function),- serverInventoryAuthoritative: expect.any(Function),- })- )- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([currentSfs])- expect(gfsFullReconcile).toHaveBeenCalledWith([currentGfs])- })-- watcher.stop()- })-- it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {- const previousServer = {- name: 'status-server',- namespace: 'mcp-server',- uid: 'status-server-uid',- generation: 7,- annotations: { 'clerum.io/recipe': 'status-recipe' },- labels: { 'clerum.io/workload': 'status-server' },- spec: {- contextRef: 'default',- image: 'clerum/status-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const watcher = new McpServerWatcher()- ;(watcher as any).servers.set(previousServer.name, previousServer)- const changed = vi.fn()- watcher.onChange(changed)- const reconciler = (watcher as any).reconciler- const netPol = (watcher as any).netPolReconciler- const callback = (watcher as any).getMcpServerWatchCallback()- expect((watcher as any).mcpServerDesiredRevision).toBe(0)-- await callback('MODIFIED', {- metadata: {- name: previousServer.name,- namespace: previousServer.namespace,- uid: previousServer.uid,- generation: previousServer.generation,- annotations: previousServer.annotations,- labels: previousServer.labels,- },- spec: previousServer.spec,- status: { conditions: [{ type: 'Ready', status: 'True' }] },- })-- expect(watcher.getAllServers()[0]?.status).toEqual({- conditions: [{ type: 'Ready', status: 'True' }],- })- expect(changed).toHaveBeenCalledOnce()- expect(reconciler.reconcile).not.toHaveBeenCalled()- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()- expect((watcher as any).mcpServerDesiredRevision).toBe(0)-- await callback('MODIFIED', {- metadata: {- name: previousServer.name,- namespace: previousServer.namespace,- uid: previousServer.uid,- generation: 8,- annotations: previousServer.annotations,- labels: previousServer.labels,- },- spec: { ...previousServer.spec, image: 'clerum/status-server:next' },- })- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect((watcher as any).mcpServerDesiredRevision).toBe(1)- await watcher.stop()- })-- it('does not advance desired revision for the controller-owned network-ready handshake', async () => {- const previousServer = {- name: 'network-ready-server',- namespace: 'mcp-server',- uid: 'network-ready-server-uid',- generation: 7,- annotations: { 'clerum.io/pre-deploy': 'true' },- labels: { 'clerum.io/workload': 'network-ready-server' },- spec: {- contextRef: 'default',- image: 'clerum/network-ready-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const watcher = new McpServerWatcher()- ;(watcher as any).servers.set(previousServer.name, previousServer)- const callback = (watcher as any).getMcpServerWatchCallback()-- await callback('MODIFIED', {- metadata: {- name: previousServer.name,- namespace: previousServer.namespace,- uid: previousServer.uid,- generation: previousServer.generation,- annotations: {- ...previousServer.annotations,- 'clerum.io/network-ready': 'true',- },- labels: previousServer.labels,- },- spec: previousServer.spec,- })-- expect((watcher as any).mcpServerDesiredRevision).toBe(0)- expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()- await watcher.stop()- })-- it('advances McpServer desired revision for owner addition and deletion', async () => {- const serverObject = {- metadata: {- name: 'revision-server',- namespace: 'mcp-server',- uid: 'revision-server-uid',- generation: 1,- },- spec: {- contextRef: 'default',- image: 'clerum/revision-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const watcher = new McpServerWatcher()- const callback = (watcher as any).getMcpServerWatchCallback()-- expect((watcher as any).mcpServerDesiredRevision).toBe(0)- await callback('ADDED', serverObject)- expect((watcher as any).mcpServerDesiredRevision).toBe(1)- await callback('DELETED', serverObject)- expect((watcher as any).mcpServerDesiredRevision).toBe(2)-- await watcher.stop()- })-- it('advances Context desired revision only when the authoritative desired cache changes', async () => {- const watcher = new McpServerWatcher()- let contextWatchCallback:- | ((- type: string,- apiObj: {- metadata: { name: string; namespace: string }- spec: { contextId: string; mcpServers: string[] }- }- ) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- await (watcher as any).startContextWatch('context-desired-revision-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true-- const added = {- metadata: { name: 'revision-context', namespace: 'mcp-server' },- spec: { contextId: 'revision-context', mcpServers: [] as string[] },- }- expect((watcher as any).contextDesiredRevision).toBe(0)- await contextWatchCallback!('ADDED', added)- expect((watcher as any).contextDesiredRevision).toBe(1)-- await contextWatchCallback!('MODIFIED', added)- expect((watcher as any).contextDesiredRevision).toBe(1)-- const modified = {- ...added,- spec: { ...added.spec, mcpServers: ['revision-server'] },- }- await contextWatchCallback!('MODIFIED', modified)- expect((watcher as any).contextDesiredRevision).toBe(2)-- await contextWatchCallback!('DELETED', modified)- expect((watcher as any).contextDesiredRevision).toBe(3)-- await watcher.stop()- })-- it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {- const initialHosts = Array.from({ length: 50 }, (_, index) => ({- metadata: {- name: `initial-host-${index}`,- namespace: 'mcp-host',- uid: `initial-host-uid-${index}`,- generation: 1,- },- spec: {- host: `initial-host-${index}`,- contextRef: 'default',- secretRef: 'host-secret',- },- }))- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- return {- metadata: { resourceVersion: 'initial-host-rv' },- items: initialHosts,- }- }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }- }- return { items: [] }- })- const watcher = new McpServerWatcher()- const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')-- await watcher.start()- await vi.waitFor(() =>- expect(mocks.hostFullReconcile).toHaveBeenCalledWith(- initialHosts.map(host =>- expect.objectContaining({- name: host.metadata.name,- generation: host.metadata.generation,- })- )- )- )- await flushMicrotasks()-- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()- expect(urgentReconcile).not.toHaveBeenCalled()- await watcher.stop()- })-- it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {- const staleServer = {- name: 'stale-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/stale-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'stale.example', port: 443 }],- },- }- const currentServer = {- metadata: { name: 'current-server', namespace: 'mcp-server' },- spec: {- contextRef: 'default',- image: 'clerum/current-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const egressStarted = deferred()- const releaseEgress = deferred()- const reconciliationOrder: string[] = []- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).servers.set(staleServer.name, staleServer)- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {- await Promise.all(- servers.map((selected: { name: string }) =>- options.runEffect(selected.name, async () => {- reconciliationOrder.push('initial')- })- )- )- })- const reconciler = (watcher as any).reconciler- reconciler.reconcile.mockImplementationOnce(async () => {- reconciliationOrder.push('watch')- })-- const convergence = (watcher as any).runInitialMcpServerConvergence()- await egressStarted.promise- const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)-- await flushMicrotasks()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: 'current-server' }),- { isCurrent: expect.any(Function) }- )-- releaseEgress.resolve(undefined)- await Promise.all([convergence, watchConvergence])-- expect(reconciliationOrder).toEqual(['watch'])- expect(reconciler.reconcile).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'current-server' }),- { isCurrent: expect.any(Function) }- )- await watcher.stop()- })-- it('continues a same-server reconciliation lane after a rejected operation', async () => {- const watcher = new McpServerWatcher()- const server = { name: 'retry-safe-server', namespace: 'mcp-server' }- const order: string[] = []- const firstStarted = deferred()- const rejectFirst = deferred()-- const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {- order.push('rejected')- firstStarted.resolve(undefined)- await rejectFirst.promise- throw new Error('transient lane failure')- }) as Promise<void>- await firstStarted.promise- const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {- order.push('recovered')- }) as Promise<void>- rejectFirst.resolve(undefined)-- await expect(rejected).rejects.toThrow('transient lane failure')- await expect(recovered).resolves.toBeUndefined()- expect(order).toEqual(['rejected', 'recovered'])- expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)- await watcher.stop()- })-- it('revalidates env-Secret matches inside each server lane before reconciling', async () => {- const currentServer = {- name: 'current-secret-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/current-secret-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- envSecret: { name: 'shared-env' },- },- }- const staleServer = {- name: 'stale-secret-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/stale-secret-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- envSecret: { name: 'shared-env' },- },- }- const unrelatedServer = {- name: 'unrelated-secret-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/unrelated-secret-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- envSecret: { name: 'other-env' },- },- }- const blockerStarted = deferred()- const releaseBlocker = deferred()- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const reconciler = (watcher as any).reconciler- for (const server of [currentServer, staleServer, unrelatedServer]) {- ;(watcher as any).servers.set(server.name, server)- }-- const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- })- await blockerStarted.promise-- const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')- ;(watcher as any).servers.set(staleServer.name, {- ...staleServer,- spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },- })- await vi.waitFor(() =>- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: currentServer.name }),- { isCurrent: expect.any(Function) }- )- )- releaseBlocker.resolve(undefined)- await Promise.all([blocker, convergence])-- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- expect(- reconciler.reconcile.mock.calls.some(- ([server]: [McpServerCRD]) => server.name === staleServer.name- )- ).toBe(false)- expect(- reconciler.reconcile.mock.calls.some(- ([server]: [McpServerCRD]) => server.name === unrelatedServer.name- )- ).toBe(false)- await watcher.stop()- })-- it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {- const selected = {- name: 'authority-fenced-secret-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/authority-fenced-secret-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- envSecret: { name: 'shared-env' },- },- }- const watcher = new McpServerWatcher()- const reconciler = (watcher as any).reconciler- ;(watcher as any).servers.set(selected.name, selected)- ;(watcher as any).mcpServerCacheSynced = false-- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')-- expect(reconciler.reconcile).not.toHaveBeenCalled()-- markMcpServerInventoryAuthoritative(watcher)- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')-- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: selected.name }),- { isCurrent: expect.any(Function) }- )- await watcher.stop()- })-- it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {- const context = {- name: 'sfs-context',- namespace: 'mcp-server',- spec: {- contextId: 'sfs-context',- sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],- },- }- const sharedFileSystem = {- name: 'shared-data',- namespace: 'mcp-host',- spec: {},- }- const watcher = new McpServerWatcher()- const reconciler = (watcher as any).sharedFileSystemReconciler- ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)-- await (watcher as any).reconcileSharedFileSystemsReferencedByContext(- undefined,- context,- () => false- )-- expect(reconciler.reconcile).not.toHaveBeenCalled()-- await (watcher as any).reconcileSharedFileSystemsReferencedByContext(- undefined,- context,- () => true- )-- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: sharedFileSystem.name })- )- await watcher.stop()- })-- it('fences Context-triggered Host convergence to the current Context lease', async () => {- const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)- ;(watcher as any).hosts.set('context-host', {- name: 'context-host',- namespace: 'mcp-host',- spec: {- host: 'context-host',- contextRef: 'context-a',- secretRef: 'host-secret',- },- })- const hostReconciler = (watcher as any).hostReconciler-- await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)-- expect(hostReconciler.reconcile).not.toHaveBeenCalled()-- await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)-- expect(hostReconciler.reconcile).toHaveBeenCalledOnce()- expect(hostReconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: 'context-host' })- )- await watcher.stop()- })-- it('keeps env-Secret reconciliation a no-op after stop', async () => {- const selected = {- name: 'stopped-secret-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/stopped-secret-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- envSecret: { name: 'shared-env' },- },- }- const watcher = new McpServerWatcher()- const reconciler = (watcher as any).reconciler- ;(watcher as any).servers.set(selected.name, selected)-- await watcher.stop()- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')-- expect(reconciler.reconcile).not.toHaveBeenCalled()- })-- it('admits an initial server without egress while another initial server waits on DNS', async () => {- const dnsBlockedServer = {- name: 'dns-blocked-server',- namespace: 'mcp-server',- uid: 'dns-blocked-server-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/dns-blocked-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'blocked.example', port: 443 }],- },- }- const independentServer = {- name: 'independent-server',- namespace: 'mcp-server',- uid: 'independent-server-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/independent-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const egressStarted = deferred()- const releaseEgress = deferred()- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)- ;(watcher as any).servers.set(independentServer.name, independentServer)- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- mocks.serverFullReconcile.mockImplementationOnce(- async (- servers: Array<typeof dnsBlockedServer | typeof independentServer>,- options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }- ) => {- await Promise.all(- servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))- )- }- )-- const convergence = (watcher as any).runInitialMcpServerConvergence()- await egressStarted.promise- await flushMicrotasks()-- expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)- expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)-- releaseEgress.resolve(undefined)- await convergence-- expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)- await watcher.stop()- })-- it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {- const staleContext = {- name: 'stale-context',- namespace: 'mcp-server',- spec: { contextId: 'stale-context', mcpServers: [] },- }- const currentContext = {- metadata: { name: 'current-context', namespace: 'mcp-server' },- spec: { contextId: 'current-context', mcpServers: [] },- }- const fullReconcileStarted = deferred()- const releaseFullReconcile = deferred()- const reconciliationOrder: string[] = []- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).contexts.set(staleContext.name, staleContext)- mocks.netPolFullReconcile.mockImplementationOnce(async () => {- fullReconcileStarted.resolve(undefined)- await releaseFullReconcile.promise- reconciliationOrder.push('initial')- })- const netPol = (watcher as any).netPolReconciler- netPol.reconcileContext.mockImplementationOnce(async () => {- reconciliationOrder.push('watch')- })- let contextWatchCallback:- | ((type: string, apiObj: typeof currentContext) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- await (watcher as any).startContextWatch('context-queue-rv')-- const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)- await fullReconcileStarted.promise- const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)-- await flushMicrotasks()- expect(netPol.reconcileContext).toHaveBeenCalledWith(- expect.objectContaining({ name: 'current-context' }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )-- releaseFullReconcile.resolve(undefined)- await Promise.all([convergence, watchConvergence])-- expect(reconciliationOrder).toEqual(['watch', 'initial'])- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'current-context' }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )- await watcher.stop()- })-- it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {- const contextName = 'recreated-context-resource'- const contextId = 'recreated-context'- const recreatedContext = {- metadata: { name: contextName, namespace: 'mcp-server' },- spec: {- contextId,- description: 'recreated',- mcpServers: [],- },- }- const orphanCleanupStarted = deferred()- const releaseOrphanCleanup = deferred()- const order: string[] = []- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- mocks.netPolFullReconcile.mockImplementationOnce(- async (- _contexts: unknown[],- _servers: unknown[],- options: {- runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>- }- ) => {- await options.runContextEffect(contextId, async () => {- orphanCleanupStarted.resolve(undefined)- await releaseOrphanCleanup.promise- order.push('orphan-delete')- })- }- )- const netPol = (watcher as any).netPolReconciler- netPol.reconcileContext.mockImplementationOnce(async () => {- order.push('live-recreate')- })- let contextWatchCallback:- | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- await (watcher as any).startContextWatch('recreated-context-rv')-- const convergence = (watcher as any).runInitialNetworkPolicyConvergence()- await orphanCleanupStarted.promise- const recreation = contextWatchCallback!('ADDED', recreatedContext)- await flushMicrotasks()-- expect(netPol.reconcileContext).not.toHaveBeenCalled()-- releaseOrphanCleanup.resolve(undefined)- await Promise.all([convergence, recreation])-- expect(order).toEqual(['orphan-delete', 'live-recreate'])- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(- expect.objectContaining({- name: contextName,- spec: expect.objectContaining({ contextId, description: 'recreated' }),- }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )- await watcher.stop()- })-- it('retires old contextId policies before reconciling a mutable Context identity', async () => {- const oldContext = {- name: 'mutable-context-resource',- namespace: 'mcp-server',- spec: {- contextId: 'old-context-id',- mcpServers: ['redis-tools'],- },- }- const newContext = {- metadata: { name: oldContext.name, namespace: oldContext.namespace },- spec: {- contextId: 'new-context-id',- mcpServers: ['redis-tools'],- },- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'contexts') {- return {- metadata: { resourceVersion: 'mutable-context-rv' },- items: [newContext],- }- }- return { items: [] }- })- let contextWatchCallback:- | ((type: string, apiObj: typeof newContext) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- ;(watcher as any).contexts.set(oldContext.name, oldContext)- ;(watcher as any).contextCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const order: string[] = []- netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {- order.push(`delete:${contextId}`)- })- netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {- order.push(`reconcile:${context.spec.contextId}`)- })- await (watcher as any).startContextWatch('mutable-context-rv')-- await contextWatchCallback!('MODIFIED', newContext)-- expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])- expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(- 'old-context-id',- expect.any(Function)- )- await watcher.stop()- })-- it('retires queued Context policy work immediately when its watch ends', async () => {- const context = {- metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },- spec: { contextId: 'retired-context-id', mcpServers: [] },- }- const blockerStarted = deferred()- const releaseBlocker = deferred()- let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined- let contextWatchDone: ((error: Error | null) => void) | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {- contextWatchCallback = callback- contextWatchDone = done- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- await (watcher as any).startContextWatch('retired-context-rv')- const blocker = (watcher as any).enqueueContextReconciliation(- context.spec.contextId,- async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }- )- await blockerStarted.promise-- const event = contextWatchCallback!('MODIFIED', context)- contextWatchDone!(null)- releaseBlocker.resolve(undefined)- await Promise.all([blocker, event])-- expect(netPol.reconcileContext).not.toHaveBeenCalled()- await watcher.stop()- })-- it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {- const server = {- metadata: {- name: 'context-policy-server',- namespace: 'mcp-server',- uid: 'context-policy-server-uid',- generation: 1,- },- spec: {- contextRef: 'context-policy',- image: 'clerum/context-policy-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const oldContext = {- name: 'context-policy-resource',- namespace: 'mcp-server',- spec: {- contextId: 'context-policy',- description: 'old',- mcpServers: [server.metadata.name],- },- }- const newContext = {- metadata: { name: oldContext.name, namespace: oldContext.namespace },- spec: {- ...oldContext.spec,- description: 'new',- },- }- const oldStarted = deferred()- const releaseOld = deferred()- const order: string[] = []- let oldLease: (() => boolean) | undefined- let contextWatchCallback:- | ((type: string, apiObj: typeof newContext) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- ;(watcher as any).contexts.set(oldContext.name, oldContext)- const netPol = (watcher as any).netPolReconciler- netPol.reconcileContext.mockImplementation(- async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {- if (context.spec.description === 'old') {- oldLease = options.isCurrent- oldStarted.resolve(undefined)- await releaseOld.promise- order.push('old')- return- }- order.push('new')- }- )- await (watcher as any).startContextWatch('context-policy-rv')-- const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)- await oldStarted.promise- const contextEvent = contextWatchCallback!('MODIFIED', newContext)- await flushMicrotasks()-- expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)- expect(oldLease?.()).toBe(false)- releaseOld.resolve(undefined)- await Promise.all([serverEvent, contextEvent])-- expect(order).toEqual(['old', 'new'])- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(- expect.objectContaining({- spec: expect.objectContaining({ description: 'new' }),- }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )- await watcher.stop()- })-- it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {- const oldContext = {- metadata: { name: 'same-stream-context', namespace: 'mcp-server' },- spec: {- contextId: 'same-stream-context',- description: 'old',- mcpServers: ['old-server'],- },- }- const newContext = {- metadata: oldContext.metadata,- spec: {- ...oldContext.spec,- description: 'new',- mcpServers: ['new-server'],- },- }- const mutationStarted = deferred()- const releaseMutation = deferred()- let oldLease: (() => boolean) | undefined- let contextWatchCallback:- | ((type: string, apiObj: typeof oldContext) => Promise<void>)- | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- netPol.reconcileContext.mockImplementationOnce(- async (_context: unknown, options: { isCurrent: () => boolean }) => {- oldLease = options.isCurrent- mutationStarted.resolve()- await releaseMutation.promise- }- )- await (watcher as any).startContextWatch('same-stream-context-rv')- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true-- const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)- await mutationStarted.promise- expect(oldLease?.()).toBe(true)-- const newReconcile = contextWatchCallback!('MODIFIED', newContext)- expect(oldLease?.()).toBe(false)-- releaseMutation.resolve()- await Promise.all([oldReconcile, newReconcile])- await watcher.stop()- })-- it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {- const server = {- name: 'periodic-egress-server',- namespace: 'mcp-server',- uid: 'periodic-egress-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/periodic-egress:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'periodic.example', port: 443 }],- },- }- const context = {- metadata: { name: 'live-context', namespace: 'mcp-server' },- spec: { contextId: 'live-context', mcpServers: [] },- }- const resyncStarted = deferred()- const releaseResync = deferred()- let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {- contextWatchCallback = callback- return { abort: vi.fn() }- })- const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- ;(watcher as any).servers.set(server.name, server)- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- resyncStarted.resolve(undefined)- await releaseResync.promise- })- await (watcher as any).startContextWatch('live-context-rv')-- const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>- const overlappingResync = (- watcher as any- ).externalEgressCoordinator.runResync() as Promise<void>- expect(overlappingResync).toBe(firstResync)- await resyncStarted.promise-- const liveEvent = contextWatchCallback!('MODIFIED', context)- await flushMicrotasks()- expect(netPol.reconcileContext).toHaveBeenCalledWith(- expect.objectContaining({ name: 'live-context' }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )-- releaseResync.resolve(undefined)- await Promise.all([firstResync, liveEvent])- await watcher.stop()- randomSpy.mockRestore()- })-- it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {- const oldServer = {- name: 'same-stream-egress',- namespace: 'mcp-server',- uid: 'same-stream-egress-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/same-stream-egress:old',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'old.example', port: 443 }],- },- }- const newServer = {- metadata: {- name: oldServer.name,- namespace: oldServer.namespace,- uid: oldServer.uid,- generation: 2,- },- spec: {- ...oldServer.spec,- image: 'clerum/same-stream-egress:new',- egressBindings: [{ dns: 'new.example', port: 443 }],- },- }- const mutationStarted = deferred()- const releaseMutation = deferred()- let oldLease: (() => boolean) | undefined- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- ;(watcher as any).servers.set(oldServer.name, oldServer)- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementationOnce(- async (_server: unknown, options: { isCurrent: () => boolean }) => {- oldLease = options.isCurrent- mutationStarted.resolve()- await releaseMutation.promise- }- )- vi.spyOn(Math, 'random').mockReturnValue(0)-- const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>- await mutationStarted.promise- expect(oldLease?.()).toBe(true)-- const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)- expect(oldLease?.()).toBe(false)-- releaseMutation.resolve()- await Promise.all([periodic, replacement])- await watcher.stop()- })-- it('uses the latest cached McpServer after periodic external-egress jitter', async () => {- vi.useFakeTimers()- const staleServer = {- name: 'periodic-egress-server',- namespace: 'mcp-server',- uid: 'periodic-egress-uid',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/periodic-egress:old',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'old.example', port: 443 }],- },- }- const currentServer = {- metadata: {- name: staleServer.name,- namespace: staleServer.namespace,- uid: staleServer.uid,- generation: 2,- },- spec: {- ...staleServer.spec,- image: 'clerum/periodic-egress:new',- egressBindings: [{ dns: 'new.example', port: 443 }],- },- }- const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)- const watcher = new McpServerWatcher()- ;(watcher as any).servers.set(staleServer.name, staleServer)- const netPol = (watcher as any).netPolReconciler-- const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>- await flushMicrotasks()- await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)- await vi.advanceTimersByTimeAsync(2500)- await periodic-- expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(- expect.objectContaining({- name: staleServer.name,- generation: 2,- spec: expect.objectContaining({- egressBindings: [{ dns: 'new.example', port: 443 }],- }),- }),- { isCurrent: expect.any(Function) }- )- await watcher.stop()- randomSpy.mockRestore()- })-- it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {- const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }- const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }- const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }- const currentGfs = {- metadata: { name: 'stale-gfs', namespace: 'mcp-host' },- spec: {},- status: {- phase: 'Ready',- pvcName: 'gfs-drive',- serviceName: 'gfsc',- serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',- },- }- const sfsInitialStarted = deferred()- const releaseSfsInitial = deferred()- const gfsInitialStarted = deferred()- const releaseGfsInitial = deferred()- const sfsOrder: string[] = []- const gfsOrder: string[] = []- const watcher = new McpServerWatcher()- ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)- ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)-- mocks.sfsFullReconcile.mockImplementationOnce(async () => {- sfsInitialStarted.resolve(undefined)- await releaseSfsInitial.promise- sfsOrder.push('initial')- })- const sfsReconciler = (watcher as any).sharedFileSystemReconciler- sfsReconciler.reconcile.mockImplementationOnce(async () => {- sfsOrder.push('watch')- })- const gfsReconciler = (watcher as any).gfsReconciler- vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {- gfsInitialStarted.resolve(undefined)- await releaseGfsInitial.promise- gfsOrder.push('initial')- })- const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {- gfsOrder.push('watch')- })-- let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined- let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined- mocks.watch- .mockImplementationOnce(async (_path, _options, callback) => {- sfsWatchCallback = callback- return { abort: vi.fn() }- })- .mockImplementationOnce(async (_path, _options, callback) => {- gfsWatchCallback = callback- return { abort: vi.fn() }- })- await (watcher as any).startSharedFileSystemWatch()- await (watcher as any).startGlobalFileSystemWatch()-- const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()- const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()- await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])- const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)- const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)-- await flushMicrotasks()- expect(sfsReconciler.reconcile).not.toHaveBeenCalled()- expect(gfsReconcile).not.toHaveBeenCalled()-- releaseSfsInitial.resolve(undefined)- releaseGfsInitial.resolve(undefined)- await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])-- expect(sfsOrder).toEqual(['initial', 'watch'])- expect(gfsOrder).toEqual(['initial', 'watch'])- expect(gfsReconcile).toHaveBeenCalledWith(- expect.objectContaining({ status: currentGfs.status })- )- await watcher.stop()- })-- it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {- const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }- const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }- const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }- const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }- const sfsListStarted = deferred()- const releaseSfsList = deferred<unknown>()- const gfsListStarted = deferred()- const releaseGfsList = deferred<unknown>()- const watcher = new McpServerWatcher()- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'sharedfilesystems') {- sfsListStarted.resolve(undefined)- return releaseSfsList.promise- }- if (plural === 'globalfilesystems') {- gfsListStarted.resolve(undefined)- return releaseGfsList.promise- }- return { items: [] }- })- const gfsReconciler = (watcher as any).gfsReconciler- vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)-- let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined- let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined- mocks.watch- .mockImplementationOnce(async (_path, _options, callback) => {- sfsWatchCallback = callback- return { abort: vi.fn() }- })- .mockImplementationOnce(async (_path, _options, callback) => {- gfsWatchCallback = callback- return { abort: vi.fn() }- })- await (watcher as any).startSharedFileSystemWatch()- await (watcher as any).startGlobalFileSystemWatch()-- const sfsResync = (watcher as any).runSfsResync()- const gfsResync = (watcher as any).runGfsResync()- await Promise.all([sfsListStarted.promise, gfsListStarted.promise])- const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)- const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)- releaseSfsList.resolve({ items: [staleSfs] })- releaseGfsList.resolve({ items: [staleGfs] })- await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])-- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'current-sfs' }),- ])- expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'current-gfs' }),- ])- expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(- expect.objectContaining({ name: 'current-sfs' })- )- expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)- expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(- expect.objectContaining({ name: 'current-gfs' })- )- expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)- await watcher.stop()- })-- it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {- const listedSfs = {- metadata: { name: 'listed-sfs', namespace: 'mcp-host' },- spec: { capacity: '1Gi' },- }- const listedGfs = {- metadata: { name: 'listed-gfs', namespace: 'gfs' },- spec: { capacity: '10Gi' },- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'sharedfilesystems') return { items: [listedSfs] }- if (plural === 'globalfilesystems') return { items: [listedGfs] }- return { items: [] }- })- const watcher = new McpServerWatcher()- const gfsReconciler = (watcher as any).gfsReconciler- const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)-- await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])-- expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(- expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })- )- expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(- expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })- )- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),- ])- expect(gfsFullReconcile).toHaveBeenCalledWith([- expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),- ])- await watcher.stop()- })-- it('does not run queued fleet effects after the watcher stops', async () => {- const initialServer = {- name: 'initial-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/initial-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'initial.example', port: 443 }],- },- }- const queuedServer = {- metadata: { name: 'queued-server', namespace: 'mcp-server' },- spec: {- contextRef: 'default',- image: 'clerum/queued-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- const egressStarted = deferred()- const releaseEgress = deferred()- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).servers.set(initialServer.name, initialServer)- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- const reconciler = (watcher as any).reconciler- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {- await Promise.all(- servers.map((selected: { name: string }) =>- options.runEffect(selected.name, () => reconciler.reconcile(selected))- )- )- })-- const initial = (watcher as any).runInitialMcpServerConvergence()- await egressStarted.promise- const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)-- await watcher.stop()- releaseEgress.resolve(undefined)- await Promise.all([initial, queuedWatch])-- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).not.toHaveBeenCalled()- })-- it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {- const firstPassStarted = deferred()- const releaseFirstPass = deferred()- const snapshots: string[][] = []- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).servers.set('first-server', {- name: 'first-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/first-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- })- mocks.serverFullReconcile- .mockImplementationOnce(async (servers: Array<{ name: string }>) => {- snapshots.push(servers.map(server => server.name))- firstPassStarted.resolve(undefined)- await releaseFirstPass.promise- })- .mockImplementationOnce(async (servers: Array<{ name: string }>) => {- snapshots.push(servers.map(server => server.name))- })-- const first = (watcher as any).runInitialMcpServerConvergence()- await firstPassStarted.promise- ;(watcher as any).servers.set('second-server', {- name: 'second-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/second-server:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- })- const second = (watcher as any).runInitialMcpServerConvergence()- const third = (watcher as any).runInitialMcpServerConvergence()-- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)- releaseFirstPass.resolve(undefined)- await Promise.all([first, second, third])-- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)- expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])- await watcher.stop()- })-- it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {- const firstPassStarted = deferred()- const releaseFirstPass = deferred()- const snapshots: string[][] = []- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contexts.set('first-context', {- name: 'first-context',- namespace: 'mcp-server',- spec: { contextId: 'first-context', mcpServers: [] },- })- mocks.netPolFullReconcile- .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {- snapshots.push(contexts.map(context => context.name))- firstPassStarted.resolve(undefined)- await releaseFirstPass.promise- })- .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {- snapshots.push(contexts.map(context => context.name))- })-- const first = (watcher as any).runInitialNetworkPolicyConvergence()- await firstPassStarted.promise- ;(watcher as any).contexts.set('second-context', {- name: 'second-context',- namespace: 'mcp-server',- spec: { contextId: 'second-context', mcpServers: [] },- })- const second = (watcher as any).runInitialNetworkPolicyConvergence()- const third = (watcher as any).runInitialNetworkPolicyConvergence()-- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)- releaseFirstPass.resolve(undefined)- await Promise.all([first, second, third])-- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)- expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])- await watcher.stop()- })-- it('does not lose an initial convergence request queued in the settlement microtask', async () => {- const watcher = new McpServerWatcher()- let trailing: Promise<void> | undefined- const convergenceCore = vi- .fn()- .mockImplementationOnce(- () =>- ({- then: (resolve: () => void) => {- resolve()- queueMicrotask(() => {- trailing = (watcher as any).runInitialMcpServerConvergence()- })- },- }) as PromiseLike<void>- )- .mockResolvedValueOnce(undefined)- ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore-- const first = (watcher as any).runInitialMcpServerConvergence()- await first- await vi.waitFor(() => expect(trailing).toBeDefined())- await trailing-- expect(convergenceCore).toHaveBeenCalledTimes(2)- await watcher.stop()- })-- it('retains one keyed queue until every same-key waiter has drained', async () => {- const watcher = new McpServerWatcher()- const firstStarted = deferred()- const releaseFirst = deferred()- const effects: string[] = []- const queues = (watcher as any).contextReconciliationQueues as Map<- string,- { references: number }- >-- const first = (watcher as any).enqueueContextReconciliation('shared', async () => {- effects.push('first')- firstStarted.resolve(undefined)- await releaseFirst.promise- })- await firstStarted.promise- const second = (watcher as any).enqueueContextReconciliation('shared', async () => {- effects.push('second')- })-- expect(queues.get('shared')?.references).toBe(2)- expect(effects).toEqual(['first'])- releaseFirst.resolve(undefined)- await Promise.all([first, second])- expect(effects).toEqual(['first', 'second'])- expect(queues.size).toBe(0)- await watcher.stop()- })-- it('does not execute newly enqueued keyed work after stop', async () => {- const watcher = new McpServerWatcher()- const work = vi.fn()- ;(watcher as any).stopped = true-- await (watcher as any).enqueueContextReconciliation('stopped', work)-- expect(work).not.toHaveBeenCalled()- expect((watcher as any).contextReconciliationQueues.size).toBe(0)- })-- it('acquires multi-context identity keys in one deterministic sorted order', async () => {- const watcher = new McpServerWatcher()- const acquired: string[] = []- vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(- async (...args: unknown[]) => {- const [key, work] = args as [string, () => Promise<void>]- acquired.push(key)- await work()- }- )-- await (watcher as any).enqueueContextIdentityReconciliation(- ['z-context', 'a-context', 'm-context', 'z-context'],- async () => undefined- )-- expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])- await watcher.stop()- })-- it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {- const egressStarted = deferred()- const releaseEgress = deferred()- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpWatchGeneration = 7- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)- ;(watcher as any).servers.set('stale-server', {- name: 'stale-server',- namespace: 'mcp-server',- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/stale-server:v1',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'stale.example', port: 443 }],- },- })- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {- await Promise.all(- servers.map((selected: { name: string }) =>- options.runEffect(selected.name, () => reconciler.reconcile(selected))- )- )- })-- const stalePass = (watcher as any).runInitialMcpServerConvergence()- await egressStarted.promise- ;(watcher as any).mcpServerCacheSynced = false- ;(watcher as any).mcpWatchGeneration = 8- const trailing = (watcher as any).runInitialMcpServerConvergence()- releaseEgress.resolve(undefined)- await Promise.all([stalePass, trailing])-- expect(reconciler.reconcile).not.toHaveBeenCalled()- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)- ;(watcher as any).servers.set('stale-server', {- name: 'stale-server',- namespace: 'mcp-server',- generation: 2,- spec: {- contextRef: 'default',- image: 'clerum/stale-server:v2',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'current.example', port: 443 }],- },- })- ;(watcher as any).mcpServerCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- await (watcher as any).runInitialMcpServerConvergence()-- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({- generation: 2,- spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),- })- )- expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)- await watcher.stop()- })-- it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {- const firstPassStarted = deferred()- const releaseFirstPass = deferred()- const appliedContexts: string[] = []- const appliedServers: string[] = []- let pass = 0- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 11- ;(watcher as any).mcpWatchGeneration = 13- ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)- ;(watcher as any).contexts.set('stale-context', {- name: 'stale-context',- namespace: 'mcp-server',- spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },- })- ;(watcher as any).servers.set('stale-server', {- name: 'stale-server',- namespace: 'mcp-server',- spec: {- contextRef: 'stale-context',- image: 'clerum/stale-server:v1',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- })- mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {- pass += 1- if (pass === 1) {- firstPassStarted.resolve(undefined)- await releaseFirstPass.promise- }- await Promise.all(- contexts.map((context: { spec: { contextId: string } }) =>- options.runContextEffect(context.spec.contextId, async () => {- appliedContexts.push(context.spec.contextId)- })- )- )- await Promise.all(- servers.map((server: { name: string }) =>- options.runServerEffect(server.name, async () => {- appliedServers.push(server.name)- })- )- )- })-- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()- await firstPassStarted.promise- ;(watcher as any).contextCacheSynced = false- ;(watcher as any).mcpServerCacheSynced = false- ;(watcher as any).contextWatchGeneration = 12- ;(watcher as any).mcpWatchGeneration = 14- const trailing = (watcher as any).runInitialNetworkPolicyConvergence()- releaseFirstPass.resolve(undefined)- await Promise.all([stalePass, trailing])-- expect(appliedContexts).toEqual([])- expect(appliedServers).toEqual([])- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)- ;(watcher as any).contexts.clear()- ;(watcher as any).servers.clear()- ;(watcher as any).contexts.set('current-context', {- name: 'current-context',- namespace: 'mcp-server',- spec: { contextId: 'current-context', mcpServers: ['current-server'] },- })- ;(watcher as any).servers.set('current-server', {- name: 'current-server',- namespace: 'mcp-server',- spec: {- contextRef: 'current-context',- image: 'clerum/current-server:v2',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- })- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect(appliedContexts).toEqual(['current-context'])- expect(appliedServers).toEqual(['current-server'])- expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)- await watcher.stop()- })-- it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 21- ;(watcher as any).mcpWatchGeneration = 34- mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {- ;(watcher as any).contextCacheSynced = false- options.onAuthoritativeRevocationComplete()- })-- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)- expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)- await watcher.stop()- })-- it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 21- ;(watcher as any).mcpWatchGeneration = 34- const server = {- name: 'openai-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }- const retrySpy = vi- .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')- .mockImplementation(() => undefined)- let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined- mocks.netPolFullReconcile.mockImplementationOnce(- async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {- captured = options- }- )-- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')- captured!.onExternalEgressRevoked!(server)- expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)- await watcher.stop()- })-- it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 4- ;(watcher as any).mcpWatchGeneration = 7-- await (watcher as any).runInitialMcpServerConvergence()- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()-- await (watcher as any).runInitialNetworkPolicyConvergence()- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())-- await watcher.stop()- })-- it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {- // M1: with the revocation counters aligned, the ONLY thing that can null the- // certificate is the lost safety fence. Fence lost -> cert null -> the fleet- // lane must not run; fence restored -> it runs exactly once.- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 4- ;(watcher as any).mcpWatchGeneration = 7- markNetworkPolicyRevocationAuthoritative(watcher)- try {- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)- await (watcher as any).runInitialMcpServerConvergence()- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()-- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)- await (watcher as any).runInitialMcpServerConvergence()- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()- } finally {- // vi.clearAllMocks() clears calls, not implementations — restore explicitly- // or every later test would re-gate on a stale false.- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)- await watcher.stop()- }- })-- it('invalidates readiness when either desired policy revision advances', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 4- ;(watcher as any).mcpWatchGeneration = 7- markNetworkPolicyRevocationAuthoritative(watcher)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- ;(watcher as any).mcpServerDesiredRevision += 1- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)-- markNetworkPolicyRevocationAuthoritative(watcher)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- ;(watcher as any).contextDesiredRevision += 1- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)-- await watcher.stop()- })-- it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {- const previous = {- metadata: {- name: 'restrictive-change',- namespace: 'mcp-server',- uid: 'restrictive-change-uid',- generation: 1,- },- spec: {- contextRef: 'default',- image: 'clerum/restrictive-change:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],- },- }- const current = {- ...previous,- metadata: { ...previous.metadata, generation: 2 },- spec: {- ...previous.spec,- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],- },- }- const watcher = new McpServerWatcher()- ;(watcher as any).hostCacheSynced = true- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 4- ;(watcher as any).mcpWatchGeneration = 7- ;(watcher as any).servers.set(previous.metadata.name, {- name: previous.metadata.name,- namespace: previous.metadata.namespace,- uid: previous.metadata.uid,- generation: previous.metadata.generation,- spec: previous.spec,- })- markNetworkPolicyRevocationAuthoritative(watcher)- const safetyPass = vi- .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')- .mockResolvedValue(undefined)- ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(- new Error('replacement policy unavailable')- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})-- await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect(safetyPass).toHaveBeenCalled()- await watcher.stop()- errorSpy.mockRestore()- })-- it('fences runtime effects when either generation retires the safety certificate', async () => {- const selected = {- name: 'certificate-fenced-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'clerum/certificate-fenced:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],- },- }- const egressStarted = deferred()- const releaseEgress = deferred()- const runtimeEffect = vi.fn()- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 11- ;(watcher as any).mcpWatchGeneration = 13- markNetworkPolicyRevocationAuthoritative(watcher)- ;(watcher as any).servers.set(selected.name, selected)- ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {- await Promise.all(- servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))- )- })-- const convergence = (watcher as any).runInitialMcpServerConvergence()- await egressStarted.promise- ;(watcher as any).contextWatchGeneration = 12- releaseEgress.resolve(undefined)- await convergence-- expect(runtimeEffect).not.toHaveBeenCalled()- await watcher.stop()- })-- it.each([- ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],- ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],- ] as const)(- 'does not admit queued %s effects after the watcher stops',- async (_, enqueueMethod) => {- const firstStarted = deferred()- const releaseFirst = deferred()- const watcher = new McpServerWatcher()- let queuedEffectRan = false-- const first = (watcher as any)[enqueueMethod]('same-object', async () => {- firstStarted.resolve(undefined)- await releaseFirst.promise- }) as Promise<void>- await firstStarted.promise- const queued = (watcher as any)[enqueueMethod]('same-object', async () => {- queuedEffectRan = true- }) as Promise<void>-- await watcher.stop()- releaseFirst.resolve(undefined)- await Promise.all([first, queued])-- expect(queuedEffectRan).toBe(false)- }- )-- it.each([- ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],- ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],- ] as const)(- 'serializes the same %s identity without blocking an unrelated fleet member',- async (_, enqueueMethod) => {- const firstStarted = deferred()- const releaseFirst = deferred()- const secondStarted = deferred()- const unrelatedStarted = deferred()- const watcher = new McpServerWatcher()- let secondRan = false-- const first = (watcher as any)[enqueueMethod]('slow', async () => {- firstStarted.resolve(undefined)- await releaseFirst.promise- }) as Promise<void>- await firstStarted.promise- const second = (watcher as any)[enqueueMethod]('slow', async () => {- secondRan = true- secondStarted.resolve(undefined)- }) as Promise<void>- const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {- unrelatedStarted.resolve(undefined)- }) as Promise<void>-- await unrelatedStarted.promise- expect(secondRan).toBe(false)- releaseFirst.resolve(undefined)- await Promise.all([first, second, unrelated])- await expect(secondStarted.promise).resolves.toBeUndefined()- await watcher.stop()- }- )-- it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {- const firstStarted = deferred()- const releaseFirst = deferred()- const secondStarted = deferred()- const releaseSecond = deferred()- const thirdStarted = deferred()- let thirdRan = false- const watcher = new McpServerWatcher()-- const first = (watcher as any).enqueueContextReconciliation('shared', async () => {- firstStarted.resolve(undefined)- await releaseFirst.promise- })- await firstStarted.promise- const second = (watcher as any).enqueueContextReconciliation('shared', async () => {- secondStarted.resolve(undefined)- await releaseSecond.promise- })- const third = (watcher as any).enqueueContextReconciliation('shared', async () => {- thirdRan = true- thirdStarted.resolve(undefined)- })-- expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)- releaseFirst.resolve(undefined)- await secondStarted.promise- expect(thirdRan).toBe(false)- expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)- releaseSecond.resolve(undefined)- await Promise.all([first, second, third])- expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)- await watcher.stop()- })-- it('caps initial convergence retry delay after attempts exceed the schedule', async () => {- vi.useFakeTimers()- const watcher = new McpServerWatcher()- const convergence = vi- .spyOn(watcher as any, 'runInitialMcpServerConvergence')- .mockResolvedValue(undefined)- ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)- ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')- await vi.advanceTimersByTimeAsync(299_999)- expect(convergence).not.toHaveBeenCalled()- await vi.advanceTimersByTimeAsync(1)- expect(convergence).toHaveBeenCalledOnce()- await watcher.stop()- })-- it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = false- ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)-- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)- ;(watcher as any).mcpServerCacheSynced = true- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()- expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(- [],- [],- expect.objectContaining({- serverInventoryComplete: true,- ensureDefaults: false,- contextInventoryAuthoritative: expect.any(Function),- serverInventoryAuthoritative: expect.any(Function),- })- )- expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)- await watcher.stop()- })-- it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).contextCacheSynced = true- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextWatchGeneration = 17- ;(watcher as any).mcpWatchGeneration = 23- let options:- | {- contextInventoryAuthoritative: () => boolean- serverInventoryAuthoritative: () => boolean- }- | undefined- mocks.netPolFullReconcile.mockImplementationOnce(- async (- _contexts: unknown[],- _servers: unknown[],- received: {- contextInventoryAuthoritative: () => boolean- serverInventoryAuthoritative: () => boolean- }- ) => {- options = received- }- )-- await (watcher as any).runInitialNetworkPolicyConvergence()-- expect(options?.contextInventoryAuthoritative()).toBe(true)- expect(options?.serverInventoryAuthoritative()).toBe(true)-- // A recovered watch may already be authoritative again, but it is a- // different snapshot generation and cannot authorize cleanup by this pass.- ;(watcher as any).contextWatchGeneration = 18- ;(watcher as any).mcpWatchGeneration = 24-- expect(options?.contextInventoryAuthoritative()).toBe(false)- expect(options?.serverInventoryAuthoritative()).toBe(false)- await watcher.stop()- })-- it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {- vi.useFakeTimers()- const server = {- metadata: { name: 'initial-runtime', namespace: 'mcp-server' },- spec: {- contextRef: 'default',- image: 'clerum/initial-runtime:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- },- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [server] }- if (plural === 'contexts') return { items: [] }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }- }- return { items: [] }- })- mocks.serverFullReconcile- .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))- .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))- .mockResolvedValueOnce(undefined)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const retryMetricBefore = await readInitialConvergenceMetric(- 'clerum_hcc_initial_convergence_retries_total',- 'McpServer'- )- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)-- await watcher.start()- await vi.advanceTimersByTimeAsync(0)- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)-- await vi.advanceTimersByTimeAsync(5000)- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)-- await vi.advanceTimersByTimeAsync(14_999)- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)- await vi.advanceTimersByTimeAsync(1)- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)- expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)- expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)- expect(- await readInitialConvergenceMetric(- 'clerum_hcc_initial_convergence_retries_total',- 'McpServer'- )- ).toBe(retryMetricBefore + 2)- expect(- await readInitialConvergenceMetric(- 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',- 'McpServer'- )- ).toBeGreaterThan(0)-- await watcher.stop()- errorSpy.mockRestore()- })-- it('cancels a pending initial convergence retry when the watcher stops', async () => {- vi.useFakeTimers()- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- ;(watcher as any).contextCacheSynced = true- markNetworkPolicyRevocationAuthoritative(watcher)- mocks.serverFullReconcile.mockRejectedValueOnce(- new Error('initial runtime reconciliation unavailable')- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})-- await (watcher as any).runInitialMcpServerConvergence()-- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)- expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)-- await watcher.stop()- await vi.advanceTimersByTimeAsync(300_000)-- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)- expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)- expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)- errorSpy.mockRestore()- })-- it('retries failed initial NetworkPolicy convergence from the current caches', async () => {- vi.useFakeTimers()- const context = {- metadata: { name: 'initial-context', namespace: 'mcp-server' },- spec: { contextId: 'initial-context', mcpServers: [] },- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [] }- if (plural === 'contexts') return { items: [context] }- if (plural === 'communicationchannels') {- return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }- }- return { items: [] }- })- mocks.netPolFullReconcile- .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))- .mockImplementationOnce(- async (- _contexts: unknown,- _servers: unknown,- options?: { onAuthoritativeRevocationComplete?: () => void }- ) => {- options?.onAuthoritativeRevocationComplete?.()- }- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)-- await watcher.start()- await vi.advanceTimersByTimeAsync(0)- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)-- await vi.advanceTimersByTimeAsync(5000)- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(- [expect.objectContaining({ name: 'initial-context' })],- [],- expect.objectContaining({- serverInventoryComplete: true,- ensureDefaults: false,- contextInventoryAuthoritative: expect.any(Function),- serverInventoryAuthoritative: expect.any(Function),- })- )-- await watcher.stop()- errorSpy.mockRestore()- })-- it('does not report a post-certification additive failure as a readiness safety failure', async () => {- vi.useFakeTimers()- const additiveFailure = new AggregateError(- [new Error('Context policy API unavailable')],- 'One or more additive Context NetworkPolicy reconciliations failed'- )- mocks.netPolFullReconcile.mockImplementationOnce(- async (- _contexts: unknown,- _servers: unknown,- options?: { onAuthoritativeRevocationComplete?: () => void }- ) => {- options?.onAuthoritativeRevocationComplete?.()- throw additiveFailure- }- )- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)-- await watcher.start()- await vi.advanceTimersByTimeAsync(0)-- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- expect(errorSpy).toHaveBeenCalledWith(- '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',- additiveFailure- )- expect(errorSpy).not.toHaveBeenCalledWith(- '[K8s] Initial NetworkPolicy background reconciliation failed:',- additiveFailure- )- expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)-- await watcher.stop()- errorSpy.mockRestore()- })-- it.each([- ['McpServer', 'startMcpServerWatch'],- ['Context', 'startContextWatch'],- ] as const)(- 'keeps safe bootstrap authority-gated when the %s watch cannot be established',- async (_kind, method) => {- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const watcher = new McpServerWatcher()- if (method === 'startMcpServerWatch') {- vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))- } else {- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- }- if (method === 'startContextWatch') {- vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))- } else if (method !== 'startMcpServerWatch') {- stubAuthoritativeInventoryWatch(watcher, 'Context')- }- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)- const recoveryTimerField =- method === 'startMcpServerWatch'- ? 'mcpServerCacheRecoveryTimer'- : 'contextCacheRecoveryTimer'-- try {- await expect(watcher.start()).resolves.toBeUndefined()- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)- expect((watcher as any)[recoveryTimerField]).not.toBeNull()- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- } finally {- await watcher.stop()- errorSpy.mockRestore()- warnSpy.mockRestore()- }- }- )-- it.each([- ['SharedFileSystem', 'startSharedFileSystemWatch'],- ['GlobalFileSystem', 'startGlobalFileSystemWatch'],- ] as const)(- 'isolates, logs, and retries an initial %s background watch rejection',- async (kind, method) => {- vi.useFakeTimers()- const failure = new Error(`${method} unavailable`)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- const rejectedWatch = vi- .spyOn(watcher as any, method)- .mockRejectedValueOnce(failure)- .mockResolvedValueOnce(undefined)- const otherMethod =- method === 'startSharedFileSystemWatch'- ? 'startGlobalFileSystemWatch'- : 'startSharedFileSystemWatch'- vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)-- await expect(watcher.start()).resolves.toBeUndefined()- await vi.advanceTimersByTimeAsync(0)-- expect(errorSpy).toHaveBeenCalledWith(- `[K8s] ${kind} background watch failed to start:`,- failure- )- expect(rejectedWatch).toHaveBeenCalledOnce()- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()-- await vi.advanceTimersByTimeAsync(4_999)- expect(rejectedWatch).toHaveBeenCalledOnce()- await vi.advanceTimersByTimeAsync(1)- expect(rejectedWatch).toHaveBeenCalledTimes(2)-- await watcher.stop()- errorSpy.mockRestore()- }- )-- it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') return { items: [] }- return { items: [] }- })-- await expect(listAllHosts()).resolves.toEqual([])-- expect(mocks.hostListCallOptions).toHaveBeenCalledWith(- expect.objectContaining({- middleware: [expect.objectContaining({ pre: expect.any(Function) })],- middlewareMergeStrategy: 'append',- })- )- })-- it('preserves Host UID from the Kubernetes inventory boundary', async () => {- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- return {- items: [- {- metadata: {- name: 'uid-host',- namespace: 'mcp-host',- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',- generation: 4,- },- spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },- },- ],- }- }- return { items: [] }- })-- await expect(listAllHosts()).resolves.toEqual([- expect.objectContaining({- name: 'uid-host',- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',- generation: 4,- }),- ])- })-- it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {- const status = {- phase: 'Ready',- pvcName: 'gfs-drive',- serviceName: 'gfsc',- serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',- }- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'globalfilesystems') {- return {- items: [- {- metadata: { name: 'gfs', namespace: 'gfs' },- spec: {},- status,- },- ],- }- }- return { items: [] }- })-- await expect(listAllGlobalFileSystems()).resolves.toEqual([- expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),- ])- })-- it('reconciles startup external egress before runtime full reconciliation', async () => {- const eventLog: string[] = []- const server = {- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },- spec: {- contextRef: 'wf-research',- image: 'clerum/web-search:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],- },- }-- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [server] }- if (plural === 'contexts') return { items: [] }- if (plural === 'hosts') return { items: [] }- if (plural === 'sharedfilesystems') return { items: [] }- if (plural === 'communicationchannels')- return { metadata: { resourceVersion: '1' }, items: [] }- return { items: [] }- })- mocks.ensureDefaultPolicies.mockImplementation(async () => {- eventLog.push('defaults')- })- const watcher = new McpServerWatcher()- const reconciler = (watcher as any).reconciler- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {- await Promise.all(- servers.map((selected: { name: string }) =>- options.runEffect(selected.name, async () => {- eventLog.push('runtime')- await reconciler.reconcile(selected)- })- )- )- })- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockImplementation(async () => {- eventLog.push('egress')- })- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(- async () => undefined- )-- await watcher.start()-- await vi.waitFor(() => expect(eventLog).toContain('runtime'))- expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])- expect(mocks.serverFullReconcile).toHaveBeenCalledWith(- [expect.objectContaining({ name: 'web-search' })],- { runEffect: expect.any(Function) }- )-- watcher.stop()- })-- it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {- vi.useFakeTimers()- const server = {- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },- spec: {- contextRef: 'wf-research',- image: 'clerum/web-search:test',- transport: { type: 'streamableHttp' as const, port: 3000 },- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],- },- }-- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'mcpservers') return { items: [server] }- if (plural === 'contexts') return { items: [] }- if (plural === 'hosts') return { items: [] }- if (plural === 'sharedfilesystems') return { items: [] }- if (plural === 'communicationchannels')- return { metadata: { resourceVersion: '1' }, items: [] }- return { items: [] }- })-- const watcher = new McpServerWatcher()- const reconciler = (watcher as any).reconciler- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {- await Promise.all(- servers.map((selected: { name: string }) =>- options.runEffect(selected.name, () => reconciler.reconcile(selected))- )- )- })- const netPol = (watcher as any).netPolReconciler- const dnsFailure = new Error('dns resolution failed')- netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(- async () => undefined- )+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )await watcher.start()- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())- expect(mocks.serverFullReconcile).toHaveBeenCalledWith(- [expect.objectContaining({ name: 'web-search' })],- { runEffect: expect.any(Function) }- )+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)- expect(reconciler.reconcile).not.toHaveBeenCalled()- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)- expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)- expect(errorSpy).not.toHaveBeenCalledWith(- '[K8s] Initial NetworkPolicy background reconciliation failed:',- dnsFailure- )watcher.stop()errorSpy.mockRestore()@@ -4627,29 +448,6 @@ describe('McpServerWatcher startup', () => {})})-describe('getContext error semantics', () => {- beforeEach(() => {- vi.clearAllMocks()- })-- it('preserves not-found semantics only for an authoritative 404', async () => {- mocks.getNamespacedCustomObject.mockRejectedValueOnce(- Object.assign(new Error('context absent'), { code: 404 })- )-- await expect(getContext('missing-context')).resolves.toBeNull()- })-- it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {- const unavailable = Object.assign(new Error('apiserver unavailable'), {- response: { statusCode: 503 },- })- mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)-- await expect(getContext('production-context')).rejects.toBe(unavailable)- })-})-describe('McpServerWatcher external egress retries', () => {const serverObject = {metadata: { name: 'redis-tools', namespace: 'mcp-server' },@@ -4662,194 +460,41 @@ describe('McpServerWatcher external egress retries', () => {}beforeEach(() => {- vi.clearAllMocks()- vi.useFakeTimers()- })-- afterEach(() => {- vi.useRealTimers()- vi.restoreAllMocks()- })-- it('retries failed external egress reconciliation from an ADDED event', async () => {- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- const serverWithBindings = {- ...serverObject,- metadata: {- ...serverObject.metadata,- annotations: {- 'clerum.io/recipe-bindings': JSON.stringify([- { source: 'redis-tools', target: 'downstream' },- ]),- },- },- }- ;(watcher as any).contexts.set('context1', {- name: 'context1',- namespace: 'mcp-server',- spec: { contextId: 'context1', mcpServers: ['redis-tools'] },- })- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))- netPol.reconcileExternalEgress.mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('ADDED', serverWithBindings)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)- expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()- expect(netPol.reconcileContext).not.toHaveBeenCalled()-- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({ name: 'redis-tools' }),- { isCurrent: expect.any(Function) }- )- expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)- expect(netPol.reconcileContext).toHaveBeenCalledWith(- expect.objectContaining({- spec: expect.objectContaining({ contextId: 'context1' }),- }),- expect.objectContaining({ isCurrent: expect.any(Function) })- )-- watcher.stop()- })-- it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const bindingReconciler = (watcher as any).bindingReconciler- const bindingStarted = deferred()- const releaseBinding = deferred()- bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {- bindingStarted.resolve(undefined)- await releaseBinding.promise- })- const annotations = {- 'clerum.io/recipe-bindings': JSON.stringify([- { from: 'redis-tools', to: 'downstream', port: 443 },- ]),- }- const original = {- ...serverObject,- metadata: {- ...serverObject.metadata,- uid: 'redis-tools-uid',- generation: 1,- annotations,- },- }- const replacement = {- ...original,- metadata: { ...original.metadata, generation: 2 },- spec: { ...original.spec, image: 'redis-tools:v2' },- }- const watchCallback = (watcher as any).getMcpServerWatchCallback()-- const originalEffect = watchCallback('ADDED', original)- await bindingStarted.promise- const replacementEffect = watchCallback('MODIFIED', replacement)-- const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]- expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })- expect(originalOptions.isCurrent()).toBe(false)-- releaseBinding.resolve(undefined)- await Promise.all([originalEffect, replacementEffect])-- expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)- expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(- 'redis-tools',- expect.any(Array),- 'redis-tools',- 'redis-tools',- { isCurrent: expect.any(Function) }- )- await watcher.stop()- })-- it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const bindingReconciler = (watcher as any).bindingReconciler- const cleanupStarted = deferred()- const releaseCleanup = deferred()- bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {- cleanupStarted.resolve(undefined)- await releaseCleanup.promise- })- const deleted = {- ...serverObject,- metadata: {- ...serverObject.metadata,- uid: 'redis-tools-old-uid',- generation: 1,- },- }- const recreated = {- ...serverObject,- metadata: {- ...serverObject.metadata,- uid: 'redis-tools-new-uid',- generation: 1,- },- spec: { ...serverObject.spec, image: 'redis-tools:recreated' },- }- const watchCallback = (watcher as any).getMcpServerWatchCallback()-- const deleteEffect = watchCallback('DELETED', deleted)- await cleanupStarted.promise- const recreateEffect = watchCallback('ADDED', recreated)-- const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]- expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })- await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)-- releaseCleanup.resolve(undefined)- await Promise.all([deleteEffect, recreateEffect])+ vi.useFakeTimers()+ })- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)- await watcher.stop()+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()})- it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {+ it('retries failed external egress reconciliation from an ADDED event', async () => {const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconciler- const noEgressServer = {- ...serverObject,- spec: {- ...serverObject.spec,- egressBindings: undefined,- },- }- reconciler.reconcile- .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))- .mockResolvedValue(undefined)+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)vi.spyOn(console, 'error').mockImplementation(() => {})vi.spyOn(console, 'warn').mockImplementation(() => {})- await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)await vi.advanceTimersByTimeAsync(5000)- expect(reconciler.reconcile).toHaveBeenCalledTimes(2)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'redis-tools' })+ )+watcher.stop()})it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)@@ -4869,8 +514,7 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'redis-tools' }),- { isCurrent: expect.any(Function) }+ expect.objectContaining({ name: 'redis-tools' }))watcher.stop()@@ -4878,7 +522,6 @@ describe('McpServerWatcher external egress retries', () => {it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))@@ -4904,16 +547,14 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenLastCalledWith(- expect.objectContaining({ name: 'redis-tools' }),- { isCurrent: expect.any(Function) }+ expect.objectContaining({ name: 'redis-tools' }))watcher.stop()})- it('continues a pending runtime pipeline during periodic external egress resync', async () => {+ it('does not let periodic external egress resync cancel a pending runtime retry', async () => {const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilerconst reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)@@ -4922,7 +563,6 @@ describe('McpServerWatcher external egress retries', () => {vi.spyOn(console, 'error').mockImplementation(() => {})vi.spyOn(console, 'warn').mockImplementation(() => {})vi.spyOn(console, 'log').mockImplementation(() => {})- vi.spyOn(Math, 'random').mockReturnValue(0)const watchCallback = (watcher as any).getMcpServerWatchCallback()await watchCallback('ADDED', serverObject)@@ -4930,152 +570,18 @@ describe('McpServerWatcher external egress retries', () => {expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- await (watcher as any).externalEgressCoordinator.runResync()+ await (watcher as any).runExternalEgressResync()- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)- expect(reconciler.reconcile).toHaveBeenCalledTimes(2)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)await vi.advanceTimersByTimeAsync(5000)+expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)expect(reconciler.reconcile).toHaveBeenCalledTimes(2)watcher.stop()})- it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- netPol.reconcileExternalEgress- .mockRejectedValueOnce(new Error('initial outage'))- .mockRejectedValueOnce(new Error('outage after 5s'))- .mockRejectedValueOnce(new Error('outage after 15s'))- .mockRejectedValueOnce(new Error('outage after 30s'))- .mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)- await vi.advanceTimersByTimeAsync(5000)- await vi.advanceTimersByTimeAsync(15000)- await vi.advanceTimersByTimeAsync(30000)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)- expect(reconciler.reconcile).not.toHaveBeenCalled()-- await vi.advanceTimersByTimeAsync(30000)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)- watcher.stop()- })-- it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- netPol.cleanupExternalEgress- .mockRejectedValueOnce(new Error('initial cleanup outage'))- .mockRejectedValueOnce(new Error('cleanup outage after 5s'))- .mockRejectedValueOnce(new Error('cleanup outage after 15s'))- .mockRejectedValueOnce(new Error('cleanup outage after 30s'))- .mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)- await vi.advanceTimersByTimeAsync(5000)- await vi.advanceTimersByTimeAsync(15000)- await vi.advanceTimersByTimeAsync(30000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)- expect(reconciler.reconcileDelete).not.toHaveBeenCalled()- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()-- await vi.advanceTimersByTimeAsync(30000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {- deleteAllowed: expect.any(Function),- })- watcher.stop()- })-- it('bounds concurrent durable retries across McpServers', async () => {- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const reconciler = (watcher as any).reconciler- const release = deferred()- let active = 0- let maxActive = 0- reconciler.reconcile.mockImplementation(async () => {- active += 1- maxActive = Math.max(maxActive, active)- await release.promise- active -= 1- })- vi.spyOn(console, 'warn').mockImplementation(() => {})-- for (let index = 0; index < 11; index += 1) {- const server = {- name: `server-${index}`,- namespace: 'mcp-server',- spec: {- ...serverObject.spec,- egressBindings: undefined,- },- }- ;(watcher as any).servers.set(server.name, server)- ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)- }-- await vi.advanceTimersByTimeAsync(5000)- await flushMicrotasks()-- expect(maxActive).toBe(10)-- release.resolve(undefined)- await flushMicrotasks()- watcher.stop()- })-- it('replays a retry against the watcher generation current when the timer executes', async () => {- const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)- const selected = {- name: 'generation-fenced-server',- namespace: 'mcp-server',- spec: {- ...serverObject.spec,- egressBindings: undefined,- },- }- ;(watcher as any).servers.set(selected.name, selected)- ;(watcher as any).mcpWatchGeneration = 41- const replay = vi- .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')- .mockResolvedValue(undefined)- vi.spyOn(console, 'warn').mockImplementation(() => {})- ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)- ;(watcher as any).mcpWatchGeneration = 42- await vi.advanceTimersByTimeAsync(5000)-- expect(replay).toHaveBeenCalledWith(- 'MODIFIED',- expect.objectContaining({ name: selected.name }),- 42,- expect.objectContaining({- isCurrent: expect.any(Function),- complete: expect.any(Function),- })- )- watcher.stop()- })-it('reconciles external egress before runtime reconciliation on ADDED events', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler@@ -5102,340 +608,39 @@ describe('McpServerWatcher external egress retries', () => {const reconciler = (watcher as any).reconcilernetPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('ADDED', serverObject)-- expect(reconciler.reconcile).not.toHaveBeenCalled()-- watcher.stop()- })-- it('retries failed external egress cleanup from a DELETED event', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))- netPol.cleanupExternalEgress.mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await watchCallback('DELETED', serverObject)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)-- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)- expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(- 'redis-tools',- 'mcp-server',- undefined,- expect.any(Function)- )- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {- deleteAllowed: expect.any(Function),- })-- watcher.stop()- })-- it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {- response: { statusCode: 503 },- })- mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()-- expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)- expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()- expect(reconciler.reconcileDelete).not.toHaveBeenCalled()- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()-- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {- deleteAllowed: expect.any(Function),- })- expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)-- watcher.stop()- })-- it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- reconciler.reconcileDelete.mockImplementationOnce(async () => {- ;(watcher as any).mcpServerCacheSynced = false- })- vi.spyOn(console, 'warn').mockImplementation(() => {})-- const watchCallback = (watcher as any).getMcpServerWatchCallback()- await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()-- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)- expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()- ;(watcher as any).mcpServerCacheSynced = true- await vi.advanceTimersByTimeAsync(14_999)- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)- await vi.advanceTimersByTimeAsync(1)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)- expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {- deleteAllowed: expect.any(Function),- })-- watcher.stop()- })-- it('retires queued McpServer work immediately when its watch ends', async () => {- const blockerStarted = deferred()- const releaseBlocker = deferred()- let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined- let watchDone: ((error: Error | null) => void) | undefined- mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {- watchCallback = callback- watchDone = done- return { abort: vi.fn() }- })- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- await (watcher as any).startMcpServerWatch('mcp-watch-rv')- const blocker = (watcher as any).enqueueMcpServerReconciliation(- { name: 'redis-tools', namespace: 'mcp-server' },- async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }- )- await blockerStarted.promise-- const event = watchCallback!('ADDED', serverObject)- watchDone!(null)- releaseBlocker.resolve(undefined)- await Promise.all([blocker, event])-- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()- await watcher.stop()- })-- it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {- const deletedServer = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-old-uid',- generation: 1,- },- spec: serverObject.spec,- }- const recreatedServer = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-new-uid',- generation: 1,- },- spec: { ...serverObject.spec, image: 'redis-tools:recreated' },- }- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))- netPol.reconcileExternalEgress- .mockRejectedValueOnce(new Error('recreated policy unavailable'))- .mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})- const watchCallback = (watcher as any).getMcpServerWatchCallback()-- await watchCallback('DELETED', deletedServer)- await watchCallback('ADDED', recreatedServer)- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({- uid: 'redis-tools-new-uid',- generation: 1,- spec: expect.objectContaining({ image: 'redis-tools:recreated' }),- }),- { isCurrent: expect.any(Function) }- )-- watcher.stop()- })-- it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {- const server = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-uid',- generation: 3,- },- spec: serverObject.spec,- }- const watcher = new McpServerWatcher()- ;(watcher as any).mcpServerCacheSynced = true- const netPol = (watcher as any).netPolReconciler- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))- netPol.cleanupExternalEgress- .mockRejectedValueOnce(new Error('cleanup unavailable'))- .mockResolvedValue(undefined)- vi.spyOn(console, 'error').mockImplementation(() => {})- vi.spyOn(console, 'warn').mockImplementation(() => {})- const watchCallback = (watcher as any).getMcpServerWatchCallback()-- await watchCallback('ADDED', server)- await watchCallback('DELETED', server)- await vi.advanceTimersByTimeAsync(5000)-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)-- watcher.stop()- })-- it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {- const egressStarted = deferred()- const releaseEgress = deferred()- const server = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-uid',- generation: 1,- annotations: {- 'clerum.io/recipe-bindings': JSON.stringify([- { source: 'redis-tools', target: 'downstream' },- ]),- },- },- spec: serverObject.spec,- }- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- const bindingReconciler = (watcher as any).bindingReconciler- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- const watchCallback = (watcher as any).getMcpServerWatchCallback()-- const reconcile = watchCallback('ADDED', server)- await egressStarted.promise- await watcher.stop()- releaseEgress.resolve(undefined)- await reconcile-- expect(reconciler.reconcile).not.toHaveBeenCalled()- expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()- })-- it('does not run an older runtime revision after newer desired state arrives during egress', async () => {- const egressStarted = deferred()- const releaseEgress = deferred()- const oldServer = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-uid',- generation: 1,- },- spec: { ...serverObject.spec, image: 'redis-tools:old' },- }- const newServer = {- metadata: {- name: 'redis-tools',- namespace: 'mcp-server',- uid: 'redis-tools-uid',- generation: 2,- },- spec: { ...serverObject.spec, image: 'redis-tools:new' },- }- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- const reconciler = (watcher as any).reconciler- netPol.reconcileExternalEgress- .mockImplementationOnce(async () => {- egressStarted.resolve(undefined)- await releaseEgress.promise- })- .mockResolvedValue(undefined)- const watchCallback = (watcher as any).getMcpServerWatchCallback()+ vi.spyOn(console, 'warn').mockImplementation(() => {})- const oldReconcile = watchCallback('ADDED', oldServer)- await egressStarted.promise- const newReconcile = watchCallback('MODIFIED', newServer)- releaseEgress.resolve(undefined)- await Promise.all([oldReconcile, newReconcile])+ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)- expect(reconciler.reconcile).toHaveBeenCalledOnce()- expect(reconciler.reconcile).toHaveBeenCalledWith(- expect.objectContaining({- generation: 2,- spec: expect.objectContaining({ image: 'redis-tools:new' }),- }),- { isCurrent: expect.any(Function) }- )+ expect(reconciler.reconcile).not.toHaveBeenCalled()watcher.stop()})- it('does not resume periodic external egress work from jitter after stop', async () => {+ it('retries failed external egress cleanup from a DELETED event', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler- ;(watcher as any).servers.set('redis-tools', {- name: 'redis-tools',- namespace: 'mcp-server',- spec: serverObject.spec,- })- vi.spyOn(Math, 'random').mockReturnValue(0.5)+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})- const resync = (watcher as any).externalEgressCoordinator.runResync()- await flushMicrotasks()- await watcher.stop()- await vi.advanceTimersByTimeAsync(2500)- await resync+ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('DELETED', serverObject)- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith('redis-tools', 'mcp-server')++ watcher.stop()})it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {const watcher = new McpServerWatcher()- markMcpServerInventoryAuthoritative(watcher)const netPol = (watcher as any).netPolReconcilernetPol.reconcileExternalEgress.mockResolvedValue(undefined)vi.spyOn(Math, 'random').mockReturnValue(0)@@ -5445,7 +650,7 @@ describe('McpServerWatcher external egress retries', () => {spec: serverObject.spec,})- await (watcher as any).externalEgressCoordinator.runResync()+ await (watcher as any).runExternalEgressResync()expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)@@ -5455,92 +660,55 @@ describe('McpServerWatcher external egress retries', () => {it('does not run parallel external egress reconciles for the same McpServer', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler- const mutationStarted = deferred()- const releaseMutation = deferred()const server = {name: 'redis-tools',namespace: 'mcp-server',spec: serverObject.spec,};(watcher as any).servers.set('redis-tools', server)- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- mutationStarted.resolve(undefined)- await releaseMutation.promise- })+ ;(watcher as any).externalEgressInFlight.set(+ 'mcp-server/redis-tools',+ new Promise(() => undefined)+ )vi.spyOn(Math, 'random').mockReturnValue(0)- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await mutationStarted.promise- await (watcher as any).externalEgressCoordinator.runResync()+ await (watcher as any).runExternalEgressResync()- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()- releaseMutation.resolve(undefined)- await activewatcher.stop()})it('waits for already in-flight external egress before reconciling the current server', async () => {const watcher = new McpServerWatcher()const netPol = (watcher as any).netPolReconciler- const mutationStarted = deferred()- const releaseMutation = deferred()const server = {name: 'redis-tools',namespace: 'mcp-server',spec: serverObject.spec,}- ;(watcher as any).servers.set(server.name, server)- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- mutationStarted.resolve(undefined)- await releaseMutation.promise+ let releaseExisting!: () => void+ const existing = new Promise<void>(resolve => {+ releaseExisting = resolve})+ ;(watcher as any).externalEgressInFlight.set('mcp-server/redis-tools', existing)const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await mutationStarted.promise- const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await flushMicrotasks()+ const run = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await Promise.resolve()- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()expect(warn).toHaveBeenCalledWith('[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight')- releaseMutation.resolve(undefined)- await Promise.all([active, waiting])- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ releaseExisting()+ await expect(run).resolves.toBeUndefined()+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)warn.mockRestore()watcher.stop()})-- it('does not start another external egress mutation after stop releases an in-flight wait', async () => {- const watcher = new McpServerWatcher()- const netPol = (watcher as any).netPolReconciler- const mutationStarted = deferred()- const releaseMutation = deferred()- const server = {- name: 'redis-tools',- namespace: 'mcp-server',- spec: serverObject.spec,- }- ;(watcher as any).servers.set(server.name, server)- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {- mutationStarted.resolve(undefined)- await releaseMutation.promise- })-- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await mutationStarted.promise- const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)- await flushMicrotasks()- await watcher.stop()- releaseMutation.resolve(undefined)- await Promise.all([active, waiting])-- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()- })})describe('McpServerWatcher.countCommunicationChannels', () => {@@ -5611,139 +779,9 @@ describe('McpServerWatcher wires CC counter into HostReconciler', () => {})})-describe('McpServerWatcher Host mutation dependency wiring', () => {- it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {- const watcher = new McpServerWatcher()- const reflect = (- watcher.getHostReconciler() as unknown as {- _reflectHostOutcomeFn:- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)- | null- }- )._reflectHostOutcomeFn- if (!reflect) {- throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')- }- const entry = {- name: 'h2-host',- namespace: 'mcp-host',- uid: 'uid-1',- spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },- } as unknown as HostCRD- ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)- const apply = vi.fn()- reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid- expect(apply).toHaveBeenCalledTimes(1)- expect(apply).toHaveBeenCalledWith(entry)- apply.mockClear()- reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip- expect(apply).not.toHaveBeenCalled()- reflect('h2-host', undefined, apply) // fail-closed on missing uid- expect(apply).not.toHaveBeenCalled()- reflect('absent-host', 'uid-1', apply) // absent entry → skip- expect(apply).not.toHaveBeenCalled()- })-- it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {- const watcher = new McpServerWatcher()- const host = {- name: 'alpha-host',- namespace: 'mcp-host',- spec: { contextRef: 'alpha-context' },- } as HostCRD- const context = {- name: 'alpha-context',- namespace: 'mcp-server',- spec: {- contextId: 'alpha-context',- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],- },- }- const documentsV1 = {- name: 'documents',- namespace: 'mcp-host',- spec: { storageClassName: 'fast-v1' },- }- const alphaChannelV1 = {- name: 'telegram-alpha',- namespace: 'channels',- spec: { hostRef: 'alpha-host' },- }- ;(watcher as any).contexts.set(context.name, context)- ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)- ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)- ;(watcher as any).ccCacheSynced = true- ;(watcher as any).ccWatchGeneration = 7- const sfsReconciler = (watcher as any).sharedFileSystemReconciler- sfsReconciler.isMountable.mockReturnValue(true)- const hostReconciler = watcher.getHostReconciler() as any- const resolveDependencies = hostReconciler._hostMutationDependenciesFn-- const original = resolveDependencies(host)- expect(original).toEqual([- { name: context.name, namespace: context.namespace, spec: context.spec },- true,- 7,- { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },- true,- {- name: alphaChannelV1.name,- namespace: alphaChannelV1.namespace,- spec: alphaChannelV1.spec,- },- ])- ;(watcher as any).sharedFileSystems.set('unrelated', {- name: 'unrelated',- namespace: 'mcp-host',- spec: {},- })- ;(watcher as any).communicationChannels.set('telegram-beta', {- name: 'telegram-beta',- namespace: 'channels',- spec: { hostRef: 'beta-host' },- })- const afterUnrelatedChanges = resolveDependencies(host)- expect(afterUnrelatedChanges).toEqual(original)-- const documentsV2 = {- ...documentsV1,- spec: { storageClassName: 'fast-v2' },- }- ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)- const afterSfsRevision = resolveDependencies(host)- expect(afterSfsRevision[3]).toEqual({- name: documentsV2.name,- namespace: documentsV2.namespace,- spec: documentsV2.spec,- })- expect(afterSfsRevision[3]).not.toEqual(original[3])-- sfsReconciler.isMountable.mockReturnValue(false)- const afterMountabilityChange = resolveDependencies(host)- expect(afterMountabilityChange[4]).toBe(false)-- const alphaChannelV2 = {- ...alphaChannelV1,- spec: {- ...alphaChannelV1.spec,- credentialsSecretRef: { name: 'telegram-alpha-v2' },- },- }- ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)- const afterChannelRevision = resolveDependencies(host)- expect(afterChannelRevision.at(-1)).toEqual({- name: alphaChannelV2.name,- namespace: alphaChannelV2.namespace,- spec: alphaChannelV2.spec,- })- expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))- })-})-describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('re-reconciles the Host whose name matches the CC hostRef', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>hosts.set('marketing', {name: 'marketing',@@ -5762,7 +800,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('is a no-op when the hostRef does not match any cached Host', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const reconcileSpy = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -5774,7 +811,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>hosts.set('marketing', {name: 'marketing',@@ -5792,7 +828,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher);(watcher as any).hosts.set('marketing', {name: 'marketing',namespace: 'mcp-host',@@ -5861,8 +896,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {})const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')@@ -5901,8 +936,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {const watcher = new McpServerWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)const ccWatchSpy = vi@@ -5970,7 +1005,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('partial-channel', {name: 'partial-channel',@@ -6010,7 +1045,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {if (plural === 'hosts') return { items: [host] }return { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('watch-observed-channel', {name: 'watch-observed-channel',@@ -6051,7 +1086,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockImplementationOnce(() => firstFleetPass.promise).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const failClosedPass = (watcher as any).requestHostFleetReconcile(@@ -6127,7 +1162,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {})const fleetFailure = new Error('fleet did not converge')mocks.hostReconcileHosts.mockRejectedValueOnce(fleetFailure).mockResolvedValueOnce(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -6169,7 +1204,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {if (hostListAttempts === 1) throw inventoryFailurereturn { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -6179,12 +1214,14 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {lifecycleGeneration)expect((watcher as any).ccAppliedLifecycleGeneration).not.toBe(lifecycleGeneration)+ expect((watcher as any).ccFleetRetryTimer).not.toBeNull()expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()await vi.advanceTimersByTimeAsync(5000)await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())expect(hostListAttempts).toBe(2)expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)+ expect((watcher as any).ccFleetRetryTimer).toBeNull()} finally {errorSpy.mockRestore()watcher.stop()@@ -6200,7 +1237,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([], [cleanupFailure]))- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -6212,6 +1249,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {)expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)+ expect((watcher as any).ccFleetRetryTimer).toBeNull()await vi.advanceTimersByTimeAsync(300000)expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()} finally {@@ -6232,7 +1270,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([hostFailure], [cleanupFailure])).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher();(watcher as any).ccCacheSynced = trueconst errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -6249,6 +1287,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {await vi.advanceTimersByTimeAsync(5000)await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)+ expect((watcher as any).ccFleetRetryTimer).toBeNull()await (watcher as any).performHostResync()expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2)@@ -6273,7 +1312,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockRejectedValueOnce(new Error('lifecycle pass failed')).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition();(watcher as any).ccCacheSynced = true@@ -6318,7 +1357,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.hostReconcileHosts.mockRejectedValue(new Error('persistent fleet failure'))- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()@@ -6364,13 +1403,15 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {mocks.hostReconcileHosts.mockRejectedValueOnce(new Error('old generation failed')).mockResolvedValueOnce(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})try {const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()await (watcher as any).requestHostFleetReconcile('fail-closed', staleGeneration)+ expect((watcher as any).ccFleetRetryTimer).not.toBeNull()const currentGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()+ expect((watcher as any).ccFleetRetryTimer).toBeNull()await (watcher as any).requestHostFleetReconcile('recovered', currentGeneration)expect(mocks.hostReconcileHosts).toHaveBeenCalledTimes(2)@@ -6398,7 +1439,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {}return { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('previous-channel', {name: 'previous-channel',@@ -6439,7 +1480,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -6497,7 +1538,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {.mockImplementationOnce(() => firstHostConvergence.promise).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const firstRecovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>try {@@ -6555,7 +1596,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {}).mockResolvedValue({ abort: vi.fn() })- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()try {await expect((watcher as any).recoverCommunicationChannelCache()).resolves.toBe(false)@@ -6588,7 +1629,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { items: [] }})mocks.watch.mockImplementationOnce(() => pendingWatch.promise)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const recovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>try {@@ -6633,7 +1674,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -6695,9 +1736,9 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {maxActivePasses = Math.max(maxActivePasses, activePasses)activePasses -= 1})- const watcher = newContextAuthoritativeWatcher()- stubAuthoritativeInventoryWatch(watcher, 'McpServer')- stubAuthoritativeInventoryWatch(watcher, 'Context')+ const watcher = new McpServerWatcher()+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(@@ -6717,11 +1758,8 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()expect(maxActivePasses).toBe(1)- // Startup now resolves after safe bootstrap; the serialized fleet tail- // keeps running independently and must still drain the queued lifecycle pass.- await startinitialPass.resolve(undefined)- await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())+ await startexpect(mocks.hostFullReconcile).toHaveBeenCalledOnce()expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce()@@ -6748,7 +1786,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return { abort }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher();(watcher as any).ccCacheSynced = trueawait (watcher as any).startCommunicationChannelWatch('504')@@ -6782,7 +1820,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {return requests[callbacks.length - 1]})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const scheduleSpy = vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(() => {})@@ -6833,7 +1871,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {const abort = vi.fn()const watchRequest = { abort }mocks.watch.mockReset().mockResolvedValue(watchRequest)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()await (watcher as any).startHostWatch('stream-rv')await vi.advanceTimersByTimeAsync(30_001)@@ -6868,11 +1906,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => activePass).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher();(watcher as any).ccCacheSynced = true- ;(watcher as any).hostFleetScheduler.markLifecycleApplied(- (watcher as any).ccLifecycleGeneration- )+ ;(watcher as any).ccAppliedLifecycleGeneration = (watcher as any).ccLifecycleGenerationconst first = (watcher as any).runHostResync() as Promise<void>await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))@@ -6891,53 +1927,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)})- it('queues a trailing full Host pass when Context recovery lands during an active pass', async () => {- vi.clearAllMocks()- const activePass = deferred()- mocks.hostFullReconcile- .mockReset()- .mockImplementationOnce(() => activePass.promise)- .mockResolvedValue(undefined)- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'contexts') {- return {- metadata: { resourceVersion: 'recovered-context-rv' },- items: [- {- metadata: { name: 'recovered-context', namespace: 'mcp-server' },- spec: {- contextId: 'recovered-context',- mcpServers: [],- sharedFileSystems: [{ name: 'recovered-sfs' }],- },- },- ],- }- }- return { items: [] }- })- mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })- const watcher = newContextAuthoritativeWatcher()- ;(watcher as any).hostCacheSynced = true-- const first = (watcher as any).requestHostFleetReconcile(- 'before Context recovery'- ) as Promise<void>- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))-- await expect((watcher as any).recoverContextInventoryAndWatch()).resolves.toBe(true)- expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)-- activePass.resolve(undefined)- await first- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2))-- await watcher.stop()- })-it('does not invent another fleet transition during a failed periodic cache recovery', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher();(watcher as any).ccCacheSynced = falseconst recover = vi.spyOn(watcher as any, 'recoverCommunicationChannelCache')@@ -6953,33 +1945,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {watcher.stop()})- it('coalesces the newest Host input revision into the serialized trailing pass', async () => {- vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()- const activePass = deferred()- const requests: Array<{ inputRevision: number }> = []- vi.spyOn(watcher as any, 'performHostFleetReconcile').mockImplementation(- async (...args: unknown[]) => {- const request = args[0] as { inputRevision: number }- requests.push({ ...request })- if (requests.length === 1) await activePass.promise- }- )- ;(watcher as any).hostFleetScheduler.bumpInputRevision()- const first = (watcher as any).requestHostFleetReconcile('active') as Promise<void>- ;(watcher as any).hostFleetScheduler.bumpInputRevision()- const second = (watcher as any).requestHostFleetReconcile('pending-v2') as Promise<void>- ;(watcher as any).hostFleetScheduler.bumpInputRevision()- const third = (watcher as any).requestHostFleetReconcile('pending-v3') as Promise<void>-- activePass.resolve(undefined)- await Promise.all([first, second, third])-- expect(requests).toHaveLength(2)- expect(requests[1].inputRevision).toBe(3)- watcher.stop()- })-it('collapses overlapping fleet requests into one serialized trailing pass', async () => {vi.clearAllMocks()const fleetV1 = [@@ -7023,7 +1988,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {maxActivePasses = Math.max(maxActivePasses, activePasses)activePasses -= 1})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const first = (watcher as any).requestHostFleetReconcile('first',@@ -7076,7 +2041,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {it('uses fresh lifecycle inventory without replacing newer Host watch-cache entries', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const initialHost = {name: 'stateless-primary',namespace: 'mcp-host',@@ -7141,7 +2106,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()const lifecycle = (watcher as any).requestHostFleetReconcile('CommunicationChannel interruption',@@ -7186,7 +2151,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()const lifecycle = (watcher as any).requestHostFleetReconcile('CommunicationChannel recovery',@@ -7231,7 +2196,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {.mockReset().mockImplementationOnce(() => lifecyclePass).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const active = (watcher as any).requestHostFleetReconcile('CommunicationChannel interruption',@@ -7278,7 +2243,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const full = (watcher as any).requestHostFleetReconcile('initial Host reconciliation',@@ -7330,7 +2295,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { items: [] }})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')@@ -7415,14 +2380,13 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { items: [] }})mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient event failure')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)await hostWatchCallback('MODIFIED', {metadata: { name: 'retry-before-list', namespace: 'mcp-host', generation: 1 },@@ -7431,9 +2395,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {expect(reconcile).toHaveBeenCalledTimes(1)expect((watcher as any).hostWatchRetryTimers.size).toBe(1)- // The active Host watch is then lost. A full request must recover a fresh- // snapshot and retire the failed event's pending retry.- ;(watcher as any).hostCacheSynced = falseawait (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')expect((watcher as any).hostWatchRetryTimers.size).toBe(0)@@ -7486,7 +2447,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {await fullReconcile.promiseorder.push('full:end')})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const hostReconciler = watcher.getHostReconciler()const reconcile = vi.spyOn(hostReconciler, 'reconcile').mockImplementation(async host => {order.push(`watch:${host.name}:${host.generation}`)@@ -7539,7 +2500,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})mocks.listNamespacedCustomObject.mockImplementation(async () => ({ items: [] }))- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const hostReconciler = watcher.getHostReconciler()// The reconcile()/reconcileDelete() spies stand in for the per-Host chain;// the actual serializeByHost non-overlap guarantee is proven in@@ -7590,7 +2551,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const order: string[] = []const timeout = new HostK8sRequestTimeoutError('AppsV1Api.readNamespacedDeployment', 30_000)const reconcile = vi@@ -7601,7 +2562,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {})await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)const first = hostWatchCallback('MODIFIED', {metadata: { name: 'host-a', namespace: 'mcp-host', generation: 1 },@@ -7637,7 +2597,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {if (plural === 'hosts') return { items: [] }return { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -7686,7 +2646,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {// wins). Block that full pass so it stays running until shutdown unblocks// the promoted trailing waiter — the semantic this test protects.mocks.hostFullReconcile.mockReset().mockImplementation(() => trailingPass)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const activeRequest = (watcher as any).requestHostFleetReconcile('startup',@@ -7732,7 +2692,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {resolveActivePass = resolve})mocks.hostReconcileHosts.mockReset().mockImplementationOnce(() => activePass)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const activeRequest = (watcher as any).requestHostFleetReconcile('active',@@ -7771,7 +2731,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {return inventory})mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()const staleRequest = (watcher as any).requestHostFleetReconcile('stale transition',@@ -7825,7 +2785,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {if (plural === 'hosts') return hostInventoryreturn { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const run = (watcher as any).requestHostFleetReconcile('shutdown race') as Promise<void>await vi.waitFor(() => {@@ -7850,7 +2810,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {it('releases the periodic single-flight guard after a rejected pass', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const failure = new Error('transient resync failure')const performSpy = vi.spyOn(watcher as any, 'performHostResync')@@ -7883,189 +2843,12 @@ describe('McpServerWatcher Host watch generation', () => {vi.useRealTimers()})- it('folds a newer lifecycle generation into an in-flight Host inventory recovery', async () => {- const hostListStarted = deferred()- const hostList = deferred<{- metadata: { resourceVersion: string }- items: unknown[]- }>()- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- hostListStarted.resolve(undefined)- return hostList.promise- }- return { items: [] }- })- const watcher = newContextAuthoritativeWatcher()- const fleetRequest = vi- .spyOn(watcher as any, 'requestHostFleetReconcile')- .mockResolvedValue(undefined)-- const recovery = (watcher as any).recoverHostInventoryAndWatch(- 'Host watch recovery convergence',- 1,- 'watch-recovery'- ) as Promise<unknown>- await hostListStarted.promise- const joined = (watcher as any).recoverHostInventoryAndWatch(- 'initial Host reconciliation',- 2,- 'cold-start'- ) as Promise<unknown>-- expect(joined).toBe(recovery)- hostList.resolve({ metadata: { resourceVersion: 'joined-host-rv' }, items: [] })- await Promise.all([recovery, joined])-- expect(fleetRequest).toHaveBeenCalledOnce()- expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')- await watcher.stop()- })-- it('adopts a pending cold-start intent when a direct Host recovery starts first', async () => {- vi.useFakeTimers()- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- return { metadata: { resourceVersion: 'adopted-host-rv' }, items: [] }- }- return { items: [] }- })- const watcher = newContextAuthoritativeWatcher()- const fleetRequest = vi- .spyOn(watcher as any, 'requestHostFleetReconcile')- .mockResolvedValue(undefined)-- ;(watcher as any).scheduleHostCacheRecovery({- convergenceReason: 'initial Host reconciliation',- ccLifecycleGeneration: 2,- cause: 'cold-start',- })- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()-- await (watcher as any).recoverHostInventoryAndWatch(- 'Host watch recovery convergence',- 1,- 'watch-recovery'- )-- expect(mocks.listNamespacedCustomObject).toHaveBeenCalledTimes(1)- expect(fleetRequest).toHaveBeenCalledOnce()- expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()- await watcher.stop()- })-- it('folds a recovery intent scheduled while Host LIST is in flight into the successful pass', async () => {- vi.useFakeTimers()- const hostListStarted = deferred()- const hostList = deferred<{- metadata: { resourceVersion: string }- items: unknown[]- }>()- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'hosts') {- hostListStarted.resolve(undefined)- return hostList.promise- }- return { items: [] }- })- const watcher = newContextAuthoritativeWatcher()- const fleetRequest = vi- .spyOn(watcher as any, 'requestHostFleetReconcile')- .mockResolvedValue(undefined)-- const recovery = (watcher as any).recoverHostInventoryAndWatch(- 'Host watch recovery convergence',- 1,- 'watch-recovery'- ) as Promise<unknown>- await hostListStarted.promise- ;(watcher as any).scheduleHostCacheRecovery({- convergenceReason: 'CommunicationChannel event Host convergence',- ccLifecycleGeneration: 3,- cause: 'cold-start',- })- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()- hostList.resolve({ metadata: { resourceVersion: 'in-flight-intent-rv' }, items: [] })- await recovery-- expect(fleetRequest).toHaveBeenCalledOnce()- expect(fleetRequest).toHaveBeenCalledWith(- 'CommunicationChannel event Host convergence',- 3,- 'full'- )- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()- await watcher.stop()- })-- it('preserves an in-flight recovery intent across failure and the scheduled retry', async () => {- vi.useFakeTimers()- const firstHostListStarted = deferred()- const firstHostList = deferred<{- metadata: { resourceVersion: string }- items: unknown[]- }>()- let hostListCalls = 0- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural !== 'hosts') return { items: [] }- hostListCalls += 1- if (hostListCalls === 1) {- firstHostListStarted.resolve(undefined)- return firstHostList.promise- }- return { metadata: { resourceVersion: 'retry-preserved-rv' }, items: [] }- })- const watcher = newContextAuthoritativeWatcher()- const fleetRequest = vi- .spyOn(watcher as any, 'requestHostFleetReconcile')- .mockResolvedValue(undefined)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})-- const recovery = (watcher as any).recoverHostInventoryAndWatch(- 'Host watch recovery convergence',- 1,- 'watch-recovery'- ) as Promise<unknown>- await firstHostListStarted.promise- ;(watcher as any).scheduleHostCacheRecovery({- convergenceReason: 'CommunicationChannel event Host convergence',- ccLifecycleGeneration: 3,- cause: 'cold-start',- })- firstHostList.reject(new Error('transient Host LIST failure'))- await expect(recovery).rejects.toThrow('transient Host LIST failure')-- expect((watcher as any).hostCacheRecoveryIntent).toEqual({- convergenceReason: 'CommunicationChannel event Host convergence',- ccLifecycleGeneration: 3,- cause: 'cold-start',- })- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()-- await vi.advanceTimersByTimeAsync(5_000)- await vi.waitFor(() => expect(fleetRequest).toHaveBeenCalledOnce())-- expect(hostListCalls).toBe(2)- expect(fleetRequest).toHaveBeenCalledWith(- 'CommunicationChannel event Host convergence',- 3,- 'full'- )- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()- errorSpy.mockRestore()- await watcher.stop()- })-it('rejects a Host snapshot that cannot anchor a continuing watch', async () => {mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {if (plural === 'hosts') return { metadata: {}, items: [] }return { items: [] }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -8101,7 +2884,7 @@ describe('McpServerWatcher Host watch generation', () => {if (watchQueries.length === 1) throw new Error('watch connection failed')return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -8145,7 +2928,7 @@ describe('McpServerWatcher Host watch generation', () => {if (path.endsWith('/hosts')) return lateRequest.promisereturn { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const requested = (watcher as any).requestHostFleetReconcile('Periodic resync',@@ -8154,8 +2937,7 @@ describe('McpServerWatcher Host watch generation', () => {) as Promise<void>await vi.waitFor(() => expect(mocks.watch).toHaveBeenCalledOnce())- const internal = (watcher as any).hostFleetScheduler.hostFleetReconcileInFlight- .promise as Promise<void>+ const internal = (watcher as any).hostFleetReconcileInFlight.promise as Promise<void>expect((watcher as any).hosts.has('snapshot-host')).toBe(true)watcher.stop()@@ -8188,7 +2970,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()await (watcher as any).startHostWatch('initial-rv');(watcher as any).hostCacheSynced = true@@ -8228,7 +3010,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient event failure'))@@ -8305,7 +3087,7 @@ describe('McpServerWatcher Host watch generation', () => {mocks.hostFullReconcile.mockImplementationOnce(() => firstFullReconcile.promise).mockResolvedValue(undefined)- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const full = (watcher as any).requestHostFleetReconcile('Periodic resync',undefined,@@ -8371,7 +3153,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()await (watcher as any).startHostWatch('test-host-rv')await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')@@ -8412,7 +3194,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -8466,7 +3248,7 @@ describe('McpServerWatcher Host watch generation', () => {],}})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -8488,8 +3270,7 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()- markHostInventoryAuthoritative(watcher)+ const watcher = new McpServerWatcher()const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -8520,8 +3301,7 @@ describe('McpServerWatcher Host watch generation', () => {// short-circuits on snapshotNames.has(name) and never dispatches at all.it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()- markHostInventoryAuthoritative(watcher)+ const watcher = new McpServerWatcher()const reconcileDelete = vi.spyOn(watcher.getHostReconciler(), 'reconcileDelete').mockResolvedValue(undefined)@@ -8555,8 +3335,7 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()- markHostInventoryAuthoritative(watcher)+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -8592,7 +3371,7 @@ describe('McpServerWatcher Host watch generation', () => {})it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher();(watcher as any).hosts.set('x', {name: 'x',namespace: 'mcp-host',@@ -8621,8 +3400,7 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()- markHostInventoryAuthoritative(watcher)+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -8664,7 +3442,7 @@ describe('McpServerWatcher Host watch generation', () => {it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {vi.clearAllMocks()- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockResolvedValue(undefined)@@ -8712,14 +3490,13 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('transient')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)await hostWatchCallback('MODIFIED', {metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },@@ -8755,7 +3532,7 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const unrelatedHost = {name: 'unrelated-host',namespace: 'mcp-host',@@ -8770,7 +3547,6 @@ describe('McpServerWatcher Host watch generation', () => {await (watcher as any).startHostWatch('test-host-rv')const watchGeneration = (watcher as any).hostWatchGenerationif (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },@@ -8806,13 +3582,12 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('persistent failure'))await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },@@ -8842,14 +3617,13 @@ describe('McpServerWatcher Host watch generation', () => {return { abort: vi.fn() }})- const watcher = newContextAuthoritativeWatcher()+ const watcher = new McpServerWatcher()const reconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValueOnce(new Error('stale event failed')).mockResolvedValue(undefined)await (watcher as any).startHostWatch('test-host-rv')if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- markHostInventoryAuthoritative(watcher)await hostWatchCallback('ADDED', {metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },@@ -8871,7 +3645,6 @@ describe('McpServerWatcher Host watch generation', () => {describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('caches ADDED events and triggers Host reconcile', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',@@ -8895,7 +3668,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('removes from cache on DELETED and triggers Host reconcile', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-marketing', {name: 'cc-marketing',@@ -8924,7 +3696,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-1', {name: 'cc-1',@@ -8989,7 +3760,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',@@ -9012,7 +3782,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('preserves credentialsSecretRef from the watch event into the cache', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {@@ -9043,7 +3812,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-1', {name: 'cc-1',@@ -9078,7 +3846,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const cache = (watcher as any).communicationChannels as Map<string, any>cache.set('cc-marketing', {name: 'cc-marketing',@@ -9107,7 +3874,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revit('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {const watcher = new McpServerWatcher()- markHostInventoryAuthoritative(watcher)const hosts = (watcher as any).hosts as Map<string, any>hosts.set('marketing', {name: 'marketing',diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 1dc96adcf..7fe7ceaec 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -14,19 +14,9 @@ import {} from './administrativeOutcomeReporter'import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'import { config } from './config'-import {- ExternalEgressConvergenceCoordinator,- type ExternalEgressRetryHandle,- type ExternalEgressWatchEventType,-} from './externalEgressConvergenceCoordinator'import { gfsDefaultFactoryConfig } from './gfsConfig'import { GfsReconciler } from './gfsReconciler'import { ControlApiGfsSeedClient } from './gfsSeedClient'-import {- type HostFleetReconcileMode,- type HostFleetReconcileRequest,- HostFleetScheduler,-} from './hostFleetScheduler'import {HostFleetReconcileError,type HostReconcileSource,@@ -40,19 +30,8 @@ import {import { K8sGfsApi } from './k8s/gfsK8sApi'import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'-import {- confirmAuthoritativeMcpServerAbsence,- isMcpServerStatusOnlyUpdate,- sameMcpServerDesiredRevision,-} from './mcpServerSafety'-import {- hostDeleteCleanupTotal,- hostFleetRequestsTotal,- hostWatchRecoverySeconds,- initialConvergenceLastSuccessTimestampSeconds,- initialConvergenceRetriesTotal,-} from './metrics'-import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'+import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'+import { NetworkPolicyReconciler } from './networkPolicyReconciler'import { McpServerReconciler } from './reconciler'import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'import {@@ -69,7 +48,6 @@ import {SharedFileSystemCRD,SharedFileSystemSpec,} from './types'-import { getErrorCode } from './utils'// Only initialize K8s client if not in dev modelet customObjectsApi: k8s.CustomObjectsApi | null = null@@ -105,13 +83,30 @@ const PLURAL_HOSTS = 'hosts'const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'-// Before readiness was decoupled, a failed initial McpServer or NetworkPolicy-// sweep made provider.start() fail and Kubernetes restarted HCC. These retries-// retain that convergence guarantee now that the sweeps run in the background.-const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]-const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000+const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000]+const EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY = 10+const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000+/**+ * External-egress resync delay (ms), H2 (issue #299). The configured interval is+ * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a+ * rotating low-TTL host is sampled every rotation. The floor is the hard lower+ * bound to avoid a hot-loop. Pure and unit-testable.+ */+export function externalEgressResyncDelayMs(+ intervalSec: number,+ floorSec: number,+ minObservedTtlMs = Infinity+): number {+ const floorMs = floorSec * 1000+ const configuredMs = intervalSec * 1000+ const ttlAwareMs = Number.isFinite(minObservedTtlMs)+ ? Math.min(configuredMs, minObservedTtlMs / 2)+ : configuredMs+ return Math.max(ttlAwareMs, floorMs)+}const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000+const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]const HOST_CACHE_RECOVERY_RETRY_MS = 5000const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]// Wake-pending Hosts get immediate per-Host admission after watch recovery@@ -123,92 +118,57 @@ type CommunicationChannelSnapshot = {resourceVersion?: string}-type McpServerSnapshot = {- servers: McpServerCRD[]- resourceVersion?: string-}--type ContextSnapshot = {- contexts: ContextCRD[]- resourceVersion?: string-}-type HostSnapshot = {hosts: HostCRD[]resourceVersion?: string}-type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'+type HostFleetReconcileMode = 'full' | 'lifecycle'type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'-type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'-type NetworkPolicySafetyCertificate = {- contextGeneration: number- serverGeneration: number- contextRevision: number- serverRevision: number+type HostFleetReconcileRequest = {+ reason: string+ mode: HostFleetReconcileMode+ ccLifecycleGeneration?: number}-type ActiveInitialConvergenceRun = {- trailingRequested: boolean+type PendingHostFleetReconcile = HostFleetReconcileRequest & {promise: Promise<void>+ resolve: () => void}-type HostInventoryRecoveryRequest = {- convergenceReason: string- ccLifecycleGeneration?: number- cause: HostInventoryRecoveryCause-}--type ActiveHostInventoryRecovery = {- request: HostInventoryRecoveryRequest- promise: Promise<HostCRD[]>-}+type ActiveHostFleetReconcile = HostFleetReconcileRequest & { promise: Promise<void> }-function mergeHostInventoryRecoveryRequest(- target: HostInventoryRecoveryRequest,- requested: HostInventoryRecoveryRequest-): void {- // Cold start is the stronger admission mode: its snapshot is consumed only- // by the bounded full pass, whereas watch recovery also dispatches urgent- // per-Host work. Preserve that mode when either joined caller requires it.- if (requested.cause === 'cold-start' || target.cause !== 'cold-start') {- target.convergenceReason = requested.convergenceReason- }- if (target.cause === 'cold-start' || requested.cause === 'cold-start') {- target.cause = 'cold-start'- }- if (requested.ccLifecycleGeneration !== undefined) {- target.ccLifecycleGeneration =- target.ccLifecycleGeneration === undefined- ? requested.ccLifecycleGeneration- : Math.max(target.ccLifecycleGeneration, requested.ccLifecycleGeneration)- }+function hostFleetRequestCovers(+ active: HostFleetReconcileRequest,+ requested: HostFleetReconcileRequest+): boolean {+ const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'+ if (!modeCovered) return false+ if (requested.ccLifecycleGeneration === undefined) return true+ return active.ccLifecycleGeneration === requested.ccLifecycleGeneration}-/**- * Serializes a reconciliation domain without turning a rejected operation into- * a permanent queue failure. Kubernetes Watch invokes async callbacks without- * awaiting the preceding callback, while a full reconciliation can delete- * orphaned resources owned by the same domain. A single ordered effect stream- * therefore makes a later watch event the final writer after an older sweep.- */-class SerializedReconciliationQueue {- private tail: Promise<void> = Promise.resolve()-- enqueue(work: () => Promise<void>): Promise<void> {- const result = this.tail.then(work)- this.tail = result.then(- () => undefined,- () => undefined- )- return result+function mergeHostFleetRequests(+ pending: HostFleetReconcileRequest,+ requested: HostFleetReconcileRequest+): HostFleetReconcileRequest {+ const pendingGeneration = pending.ccLifecycleGeneration+ const requestedGeneration = requested.ccLifecycleGeneration+ return {+ reason: requested.reason,+ mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',+ ccLifecycleGeneration:+ pendingGeneration === undefined+ ? requestedGeneration+ : requestedGeneration === undefined+ ? pendingGeneration+ : Math.max(pendingGeneration, requestedGeneration),}}-type KeyedReconciliationQueue = {- queue: SerializedReconciliationQueue- references: number+function delay(ms: number): Promise<void> {+ return new Promise(resolve => setTimeout(resolve, ms))}type McpServerWatchObject = {@@ -245,7 +205,7 @@ export interface McpServerProvider {/*** List all McpServer CRDs in the namespace.*/-async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {+export async function listAllMcpServers(): Promise<McpServerCRD[]> {if (!customObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}@@ -261,7 +221,6 @@ async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {})const list = response as {- metadata?: { resourceVersion?: string }items: Array<{metadata: {name: string@@ -288,21 +247,17 @@ async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {}))console.log(`[K8s] Found ${servers.length} McpServer(s)`)- return { servers, resourceVersion: list.metadata?.resourceVersion }+ return servers} catch (error) {console.error('[K8s] Failed to list McpServers:', error)throw error}}-export async function listAllMcpServers(): Promise<McpServerCRD[]> {- return (await listMcpServerSnapshot()).servers-}-/*** List all Context CRDs in the namespace.*/-async function listContextSnapshot(): Promise<ContextSnapshot> {+export async function listAllContexts(): Promise<ContextCRD[]> {if (!customObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}@@ -318,14 +273,8 @@ async function listContextSnapshot(): Promise<ContextSnapshot> {})const list = response as {- metadata?: { resourceVersion?: string }items: Array<{- metadata: {- name: string- namespace?: string- uid?: string- generation?: number- }+ metadata: { name: string; namespace?: string }spec: ContextSpec}>}@@ -333,23 +282,17 @@ async function listContextSnapshot(): Promise<ContextSnapshot> {const contexts = list.items.map(item => ({name: item.metadata.name,namespace: item.metadata.namespace || config.namespace,- uid: item.metadata.uid,- generation: item.metadata.generation,spec: item.spec,}))console.log(`[K8s] Found ${contexts.length} Context(s)`)- return { contexts, resourceVersion: list.metadata?.resourceVersion }+ return contexts} catch (error) {console.error('[K8s] Failed to list Contexts:', error)throw error}}-export async function listAllContexts(): Promise<ContextCRD[]> {- return (await listContextSnapshot()).contexts-}-/*** List all Host CRDs in host namespace.*/@@ -456,16 +399,12 @@ export async function getAuthToken(* SharedFileSystems are allowed to live in, per CRD validation).*/export async function listAllSharedFileSystems(): Promise<SharedFileSystemCRD[]> {- if (!hostCustomObjectsApi) {+ if (!customObjectsApi) {throw new Error('K8s client not initialized - are you in dev mode?')}try {console.log(`[K8s] Listing all SharedFileSystems in namespace ${config.hostNamespace}`)- // Deadline-bearing client, like listHostSnapshot: the cold-start Host fleet- // pass waits on this inventory, so an apiserver that never answers would- // otherwise strand the fleet behind an already-certified readiness — and- // leave the SharedFileSystem watch, which starts after this await, dead.- const response = await hostCustomObjectsApi.listNamespacedCustomObject({+ const response = await customObjectsApi.listNamespacedCustomObject({group: GROUP,version: VERSION,namespace: config.hostNamespace,@@ -507,17 +446,12 @@ export async function listAllGlobalFileSystems(): Promise<GlobalFileSystemCRD[]>plural: PLURAL_GLOBALFILESYSTEMS,})const list = response as {- items: Array<{- metadata: { name: string; namespace?: string }- spec: GlobalFileSystemSpec- status?: GlobalFileSystemCRD['status']- }>+ items: Array<{ metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }>}return list.items.map(item => ({name: item.metadata.name,namespace: item.metadata.namespace || namespace,spec: item.spec,- status: item.status,}))} catch (error) {console.error('[K8s] Failed to list GlobalFileSystems:', error)@@ -618,7 +552,7 @@ export async function getContext(contextId: string): Promise<ContextCRD | null>return null}console.error(`[K8s] Failed to read Context CRD:`, error)- throw error+ return null}}@@ -638,26 +572,7 @@ export class McpServerWatcher implements McpServerProvider {private hosts: Map<string, HostCRD> = new Map()private contexts: Map<string, ContextCRD> = new Map()private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()- private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()- private mcpWatchGeneration = 0- private contextWatchGeneration = 0- private mcpServerDesiredRevision = 0- private contextDesiredRevision = 0- // Readiness covers authoritative revocation, not additive fleet completion.- // Generations bind the completed safety sweep to the exact LIST -> WATCH- // pair that supplied its absence decisions. A recovered watch invalidates- // this marker until the new inventory's orphan-allow sweep completes.- private networkPolicyRevocationContextGeneration = 0- private networkPolicyRevocationServerGeneration = 0- private networkPolicyRevocationContextRevision = 0- private networkPolicyRevocationServerRevision = 0- private mcpServerCacheSynced = false- private contextCacheSynced = false- private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null- private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null- private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null- private contextCacheRecoveryInFlight: Promise<boolean> | null = null// A watch callback can still settle after its stream reports completion.// Host reconciles change the mcp-host pod template, so stale callbacks must// never replay an older Host spec after a replacement watch is active.@@ -670,18 +585,11 @@ export class McpServerWatcher implements McpServerProvider {private readonly hostWatchRetryAttempts = new Map<string, number>()private hostCacheSynced = falseprivate hostCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null- private sfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null- private gfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null// Host watch LIST-to-WATCH recovery is a dedicated, deduplicated operation// (§10.2). Concurrent recovery signals reuse the in-flight promise so exactly// one LIST + WATCH is installed. Independent per-Host events no longer share a// process-wide convergence tail — they enter the per-Host serializer directly.- private hostRecoveryInFlight: ActiveHostInventoryRecovery | null = null- private hostCacheRecoveryIntent: HostInventoryRecoveryRequest | null = null- // Only the cold-start Host fleet pass waits for the initial SFS inventory.- // The inventory itself remains asynchronous with respect to provider.start- // and NetworkPolicy readiness certification.- private initialHostFleetSfsInventory: Promise<boolean> | null = null+ private hostRecoveryInFlight: Promise<HostCRD[]> | null = null// B2: tracks whether the CC snapshot is paired with a continuing watch.// Set to true only while a complete snapshot is paired with a live watch;// used by HostReconciler to make fail-closed lifecycle decisions.@@ -689,16 +597,16 @@ export class McpServerWatcher implements McpServerProvider {private ccWatchGeneration = 0private ccCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = nullprivate ccCacheRecoveryInFlight: Promise<boolean> | null = null- private readonly hostFleetScheduler = new HostFleetScheduler({- perform: request => this.performHostFleetReconcile(request),- recordRequest: result => hostFleetRequestsTotal.inc({ result }),+ private ccLifecycleGeneration = 0+ private ccAppliedLifecycleGeneration = -1+ private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null+ private ccFleetRetryAttempt = 0+ private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null+ private hostFleetReconcilePending: PendingHostFleetReconcile | null = null+ private resolveHostFleetShutdown: () => void = () => {}+ private readonly hostFleetShutdown = new Promise<void>(resolve => {+ this.resolveHostFleetShutdown = resolve})- private get ccLifecycleGeneration(): number {- return this.hostFleetScheduler.currentLifecycleGeneration- }- private get ccAppliedLifecycleGeneration(): number {- return this.hostFleetScheduler.appliedLifecycleGeneration- }private hostResyncInFlight: Promise<void> | null = nullprivate changeCallback?: () => voidprivate stopped = false@@ -710,46 +618,15 @@ export class McpServerWatcher implements McpServerProvider {private gfsReconciler: GfsReconcilerprivate readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporterprivate readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter- private readonly externalEgressCoordinator = new ExternalEgressConvergenceCoordinator({- listServers: () => [...this.servers.values()],- getCurrentServer: name => this.servers.get(name),- inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,- sameDesiredRevision: sameMcpServerDesiredRevision,- enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),- mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),- replay: (type, server, retry) =>- this.reconcileMcpServerWatchEvent(type, server, this.mcpWatchGeneration, retry),- externalEgressRefreshMinTtlMs: () => this.netPolReconciler.externalEgressRefreshMinTtlMs,- })- private readonly initialConvergenceRetryTimers = new Map<- InitialConvergenceLane,- ReturnType<typeof setTimeout>- >()- private readonly initialConvergenceRetryAttempts = new Map<InitialConvergenceLane, number>()- private readonly initialConvergenceRuns = new Map<- InitialConvergenceLane,- ActiveInitialConvergenceRun- >()- // Every resource effect serializes by resource identity, so one slow fleet- // member cannot block unrelated live events while same-resource updates- // still retain last-writer ordering.- private readonly mcpServerReconciliationQueues = new Map<string, KeyedReconciliationQueue>()- private readonly contextReconciliationQueues = new Map<string, KeyedReconciliationQueue>()- private readonly sharedFileSystemReconciliationQueues = new Map<- string,- KeyedReconciliationQueue- >()- private readonly globalFileSystemReconciliationQueues = new Map<- string,- KeyedReconciliationQueue- >()- private sharedFileSystemCacheRevision = 0- private globalFileSystemCacheRevision = 0+ private readonly externalEgressRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()+ private readonly externalEgressRetryAttempts = new Map<string, number>()+ private readonly externalEgressInFlight = new Map<string, Promise<void>>()// Periodic resync timer: K8s watches can drop events on long disconnects;// running fullReconcile every N minutes guarantees mcp-host-runtime-token Secret// rotation eventually catches up even after a missed MODIFIED event.// Disabled when interval <= 0 (tests).private resyncTimer: ReturnType<typeof setInterval> | null = null+ private externalEgressResyncTimer: ReturnType<typeof setTimeout> | null = null// Periodic SharedFileSystem resync (#592): the SFS watch fires only on SFS CRD// changes, not on PVC binding / wfc pod readiness, so a SharedFileSystem that// reported Initializing/Degraded needs a periodic re-reconcile to converge to a@@ -767,11 +644,6 @@ export class McpServerWatcher implements McpServerProvider {}this.watch = new k8s.Watch(kc)this.reconciler = new McpServerReconciler(kc)- this.reconciler.setInventoryAuthority(() => ({- known: this.mcpServerCacheSynced,- generation: this.mcpWatchGeneration,- }))- this.reconciler.setResolveCurrentServer(name => this.servers.get(name))const infrastructureTelemetryReporter = config.controlApiBaseUrl? createInfrastructureTelemetryReporter(config.governedTracingEnabled, {baseUrl: config.controlApiBaseUrl,@@ -821,14 +693,6 @@ export class McpServerWatcher implements McpServerProvider {// entries a delete/newer event has since removed), and orphan cleanup// compares candidates with the CURRENT inventory rather than a pass snapshot.this.hostReconciler.setResolveCurrentHost(name => this.hosts.get(name))- // H2: reflect committed lifecycle outcomes onto the CURRENT cache entry —- // but never onto a same-name recreation. uid === undefined fails closed- // (production watch objects always carry uid).- this.hostReconciler.setReflectHostOutcome((name, uid, apply) => {- const cached = this.hosts.get(name)- if (!cached || uid === undefined || cached.uid !== uid) return- apply(cached)- })// Orphan cleanup requires known watch authority and a stable watch// generation; expose both so cleanup fail-closes while authority is unknown// or the watch has been retired mid-pass.@@ -836,40 +700,6 @@ export class McpServerWatcher implements McpServerProvider {known: this.hostCacheSynced,generation: this.hostWatchGeneration,}))- this.hostReconciler.setHostMutationAuthority(() => ({- known: this.hostCacheSynced && this.contextCacheSynced,- hostGeneration: this.hostWatchGeneration,- contextGeneration: this.contextWatchGeneration,- }))- this.hostReconciler.setResolveHostMutationDependencies(host => {- const context = this.contexts.get(host.spec.contextRef)- // Lease desired semantics, not cache-object identity or observed status.- // Kubernetes watch events replace objects and status-only changes may- // legitimately advance resourceVersion without changing Host inputs.- const desiredRevision = <T extends { name: string; namespace: string; spec: unknown }>(- resource: T | undefined- ) =>- resource- ? { name: resource.name, namespace: resource.namespace, spec: resource.spec }- : undefined- const dependencies: unknown[] = [- desiredRevision(context),- this.ccCacheSynced,- this.ccWatchGeneration,- ]- for (const ref of context?.spec.sharedFileSystems ?? []) {- const sfs = this.sharedFileSystems.get(ref.name)- dependencies.push(- desiredRevision(sfs),- sfs ? this.sharedFileSystemReconciler.isMountable(sfs) : false- )- }- const channels = this.findCommunicationChannelsByHostRef(host.name).sort((a, b) =>- a.name.localeCompare(b.name)- )- dependencies.push(...channels.map(channel => desiredRevision(channel)))- return dependencies- })// Per-CC credentials Secret migration: HostReconciler computes the// channel-reader credentials-revision annotation by hashing the Secret(s)// referenced by all CCs of a host. Wire both finders so@@ -898,487 +728,65 @@ export class McpServerWatcher implements McpServerProvider {return this.ccCacheSynced}- /**- * Readiness authority covers the inventories that back HCC discovery,- * Context policy decisions, and Host API/runtime ownership. Communication- * Channel loss is intentionally excluded: that subsystem already fails safe- * by preserving replicas and disabling stateless suspension until recovery.- * SFS/GFS retain their established eventual-resync contract.- */- isReadinessInventoryAuthoritative(): boolean {- return (- !this.stopped &&- this.mcpServerCacheSynced &&- this.contextCacheSynced &&- this.hostCacheSynced &&- // The generation/revision equalities below prove no newer desired state- // is outstanding; they cannot prove the last pass actually revoked what- // it set out to revoke. Losing a delete fence bumps no generation and- // moves no revision, so it is invisible to every other term here while- // leaving live an allow the pass classified as stale. The reconciler- // degrades this fence at the point of loss; readiness reads it so the- // gate re-closes for the same reason it opened.- this.netPolReconciler.hasCertifiedSafetyInventory() &&- this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&- this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&- this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&- this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision- )- }-- /**- * Host-managed runtime mutation requires a LIST snapshot paired with its- * continuing WATCH. This narrower predicate is also used by non-HTTP effect- * producers while the broader readiness predicate additionally covers- * McpServer and Context discovery authority.- */- isHostInventoryAuthoritative(): boolean {- return !this.stopped && this.hostCacheSynced- }-- /**- * Host runtime state is derived from both the Host and Context inventories:- * Context owns the SharedFileSystem mount set rendered into each Host pod.- * Mutation must therefore stop when either LIST -> WATCH pair is unavailable.- */- private isHostEffectInventoryAuthoritative(): boolean {- return this.isHostInventoryAuthoritative() && this.contextCacheSynced- }-- /**- * Fail closed at every Host-effect admission boundary while recovery owns- * inventory authority. Preserve the reason/generation in the retry intent so- * the successful LIST -> WATCH recovery's full pass applies the deferred- * effect from current state instead of acting on the stale cache.- */- private admitHostDependentEffects(- convergenceReason: string,- ccLifecycleGeneration?: number- ): boolean {- if (this.isHostEffectInventoryAuthoritative()) return true- if (!this.stopped && !this.hostCacheSynced) {- this.scheduleHostCacheRecovery({- convergenceReason,- ccLifecycleGeneration,- cause: 'watch-recovery',- })- }- return false- }-- private retireMcpServerWatch(expectedGeneration?: number): boolean {- if (expectedGeneration !== undefined && expectedGeneration !== this.mcpWatchGeneration) {- return false- }- this.mcpServerCacheSynced = false- this.mcpWatchGeneration += 1- this.mcpWatchRequest?.abort()- this.mcpWatchRequest = null- return true- }-- private retireContextWatch(expectedGeneration?: number): boolean {- if (expectedGeneration !== undefined && expectedGeneration !== this.contextWatchGeneration) {- return false- }- this.contextCacheSynced = false- this.contextWatchGeneration += 1- this.ctxWatchRequest?.abort()- this.ctxWatchRequest = null- return true- }-- private async enqueueKeyedReconciliation(- queues: Map<string, KeyedReconciliationQueue>,- key: string,- work: () => Promise<void>- ): Promise<void> {- let entry = queues.get(key)- if (!entry) {- entry = { queue: new SerializedReconciliationQueue(), references: 0 }- queues.set(key, entry)- }- entry.references++- try {- await entry.queue.enqueue(async () => {- if (this.stopped) return- await work()- })- } finally {- entry.references--- if (entry.references === 0 && queues.get(key) === entry) queues.delete(key)- }- }-- private enqueueMcpServerReconciliation(- server: Pick<McpServerCRD, 'name' | 'namespace'>,- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedReconciliation(- this.mcpServerReconciliationQueues,- `${server.namespace}/${server.name}`,- work- )- }-- private enqueueContextReconciliation(- contextId: string,- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedReconciliation(this.contextReconciliationQueues, contextId, work)- }-- private enqueueContextIdentityReconciliation(- contextIds: string[],- work: () => Promise<void>- ): Promise<void> {- const keys = [...new Set(contextIds)].sort()- const enter = (index: number): Promise<void> => {- const key = keys[index]- if (!key) return work()- return this.enqueueContextReconciliation(key, () => enter(index + 1))- }- return enter(0)- }-- private hasMcpServerInventoryAuthority(watchGeneration: number): boolean {- return this.mcpServerCacheSynced && this.mcpWatchGeneration === watchGeneration- }-- private hasContextInventoryAuthority(watchGeneration: number): boolean {- return this.contextCacheSynced && this.contextWatchGeneration === watchGeneration- }-- private async mcpServerAbsentForDelete(- name: string,- namespace: string,- watchGeneration: number- ): Promise<boolean> {- if (!customObjectsApi) return false- return confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => this.hasMcpServerInventoryAuthority(watchGeneration),- resolveCurrent: () => this.servers.get(name),- readCurrent: () =>- customObjectsApi.getNamespacedCustomObject({- group: GROUP,- version: VERSION,- namespace,- plural: PLURAL_MCPSERVERS,- name,- }),- })- }-- private async authorizeMcpServerDeleteOrRetry(- server: McpServerCRD,- watchGeneration: number- ): Promise<boolean> {- try {- const authorized = await this.mcpServerAbsentForDelete(- server.name,- server.namespace,- watchGeneration- )- if (!authorized && !this.hasMcpServerInventoryAuthority(watchGeneration)) {- console.warn(- `[K8s] McpServer inventory authority changed while deleting ${server.name}; ` +- 'full cleanup deferred until retry'- )- this.scheduleExternalEgressRetry('DELETED', server)- this.changeCallback?.()- }- return authorized- } catch (error) {- console.error(- `[K8s] Authoritative absence check failed for deleted McpServer ${server.name}; ` +- 'cleanup blocked until retry:',- error- )- this.scheduleExternalEgressRetry('DELETED', server)- this.changeCallback?.()- return false- }- }-- private async contextAbsentForDelete(- name: string,- namespace: string,- watchGeneration: number- ): Promise<boolean> {- if (- !this.contextCacheSynced ||- this.contextWatchGeneration !== watchGeneration ||- this.contexts.has(name) ||- !customObjectsApi- ) {- return false- }- try {- await customObjectsApi.getNamespacedCustomObject({- group: GROUP,- version: VERSION,- namespace,- plural: PLURAL_CONTEXTS,- name,- })- return false- } catch (error: unknown) {- if (getErrorCode(error) !== 404) throw error- }- return (- this.contextCacheSynced &&- this.contextWatchGeneration === watchGeneration &&- !this.contexts.has(name)- )- }-- private async contextIdAbsentForDelete(- contextId: string,- watchGeneration: number- ): Promise<boolean> {- const inventoryAuthoritative = () =>- this.contextCacheSynced && this.contextWatchGeneration === watchGeneration- const resolveCurrent = () =>- [...this.contexts.values()].find(context => context.spec.contextId === contextId)- if (!customObjectsApi || !inventoryAuthoritative() || resolveCurrent()) return false-- const response = (await customObjectsApi.listNamespacedCustomObject({- group: GROUP,- version: VERSION,- namespace: config.namespace,- plural: PLURAL_CONTEXTS,- })) as { items?: Array<{ spec?: { contextId?: string } }> }- if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false-- return inventoryAuthoritative() && !resolveCurrent()- }-- private enqueueSharedFileSystemReconciliation(- name: string,- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedReconciliation(this.sharedFileSystemReconciliationQueues, name, work)- }-- private enqueueGlobalFileSystemReconciliation(- name: string,- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedReconciliation(this.globalFileSystemReconciliationQueues, name, work)- }-- /**- * Fleet passes acquire every selected object lane in stable order. This- * preserves their snapshot-wide ordering without making two unrelated live- * watch events wait behind one another.- */- private enqueueKeyedFleetReconciliation(- queues: Map<string, KeyedReconciliationQueue>,- keys: string[],- work: () => Promise<void>- ): Promise<void> {- const orderedKeys = [...new Set(keys)].sort()- const enter = (index: number): Promise<void> => {- const key = orderedKeys[index]- if (!key) return work()- return this.enqueueKeyedReconciliation(queues, key, () => enter(index + 1))- }- return enter(0)- }-- private enqueueSharedFileSystemFleetReconciliation(- names: string[],- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedFleetReconciliation(- this.sharedFileSystemReconciliationQueues,- names,- work- )- }-- private enqueueGlobalFileSystemFleetReconciliation(- names: string[],- work: () => Promise<void>- ): Promise<void> {- return this.enqueueKeyedFleetReconciliation(- this.globalFileSystemReconciliationQueues,- names,- work- )- }-- private requireInventoryResourceVersion(- kind: 'McpServer' | 'Context',- resourceVersion: string | undefined- ): string {- if (!resourceVersion) {- throw new Error(`${kind} snapshot missing resourceVersion`)+ private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {+ this.communicationChannels.clear()+ for (const channel of snapshot.channels) {+ this.communicationChannels.set(channel.name, channel)}- return resourceVersion}- private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {- this.servers.clear()- for (const server of snapshot.servers) {- this.servers.set(server.name, server)+ private beginCommunicationChannelLifecycleTransition(): number {+ this.ccLifecycleGeneration += 1+ this.ccFleetRetryAttempt = 0+ if (this.ccFleetRetryTimer) {+ clearTimeout(this.ccFleetRetryTimer)+ this.ccFleetRetryTimer = null}+ return this.ccLifecycleGeneration}- private installContextSnapshot(snapshot: ContextSnapshot): void {- this.contexts.clear()- for (const context of snapshot.contexts) {- this.contexts.set(context.name, context)+ private markCommunicationChannelLifecycleApplied(generation: number): void {+ if (generation !== this.ccLifecycleGeneration) return+ this.ccAppliedLifecycleGeneration = generation+ this.ccFleetRetryAttempt = 0+ if (this.ccFleetRetryTimer) {+ clearTimeout(this.ccFleetRetryTimer)+ this.ccFleetRetryTimer = null}}- private async restartMcpServerWatch(snapshot: McpServerSnapshot): Promise<number> {- const resourceVersion = this.requireInventoryResourceVersion(- 'McpServer',- snapshot.resourceVersion- )- this.mcpServerCacheSynced = false- this.installMcpServerSnapshot(snapshot)- const watchGeneration = await this.startMcpServerWatch(resourceVersion)+ private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {+ const generation = request.ccLifecycleGeneration+ if (generation === undefined) returnif (this.stopped ||- watchGeneration !== this.mcpWatchGeneration ||- this.mcpWatchRequest === null+ generation !== this.ccLifecycleGeneration ||+ generation === this.ccAppliedLifecycleGeneration ||+ this.ccFleetRetryTimer) {- throw new Error('McpServer snapshot could not be paired with an active watch')+ return}- this.mcpServerCacheSynced = true- return watchGeneration- }-- private async restartContextWatch(snapshot: ContextSnapshot): Promise<number> {- const resourceVersion = this.requireInventoryResourceVersion(- 'Context',- snapshot.resourceVersion+ const delayIndex = Math.min(+ this.ccFleetRetryAttempt,+ COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1)- this.contextCacheSynced = false- this.installContextSnapshot(snapshot)- const watchGeneration = await this.startContextWatch(resourceVersion)- if (- this.stopped ||- watchGeneration !== this.contextWatchGeneration ||- this.ctxWatchRequest === null- ) {- throw new Error('Context snapshot could not be paired with an active watch')- }- this.contextCacheSynced = true- return watchGeneration- }-- private recoverMcpServerInventoryAndWatch(): Promise<boolean> {- if (this.stopped) return Promise.resolve(false)- if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight-- const recovery = (async () => {- try {- const snapshot = await listMcpServerSnapshot()- if (this.stopped) return false- await this.restartMcpServerWatch(snapshot)- if (this.stopped) return false- this.changeCallback?.()- if (this.contextCacheSynced) {- void this.runInitialNetworkPolicyConvergence()- }- return true- } catch (error) {- this.mcpServerCacheSynced = false- console.error('[K8s] McpServer cache recovery failed:', error)- return false- }- })()-- this.mcpServerCacheRecoveryInFlight = recovery- void recovery.finally(() => {- if (this.mcpServerCacheRecoveryInFlight === recovery) {- this.mcpServerCacheRecoveryInFlight = null- }- })- return recovery- }-- private recoverContextInventoryAndWatch(): Promise<boolean> {- if (this.stopped) return Promise.resolve(false)- if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight-- const recovery = (async () => {- try {- const snapshot = await listContextSnapshot()- if (this.stopped) return false- await this.restartContextWatch(snapshot)- if (this.stopped) return false- this.hostFleetScheduler.bumpInputRevision()- void this.runInitialNetworkPolicyConvergence()- void this.runInitialSharedFileSystemConvergence()- if (this.hostCacheSynced) {- const pendingCcLifecycleGeneration =- this.ccLifecycleGeneration > this.ccAppliedLifecycleGeneration- ? this.ccLifecycleGeneration- : undefined- void this.requestHostFleetReconcile(- 'Context cache recovery',- pendingCcLifecycleGeneration,- 'full'- )- }- return true- } catch (error) {- this.contextCacheSynced = false- console.error('[K8s] Context cache recovery failed:', error)- return false- }- })()-- this.contextCacheRecoveryInFlight = recovery- void recovery.finally(() => {- if (this.contextCacheRecoveryInFlight === recovery) {- this.contextCacheRecoveryInFlight = null+ const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]+ this.ccFleetRetryAttempt += 1+ this.ccFleetRetryTimer = setTimeout(() => {+ this.ccFleetRetryTimer = null+ if (+ this.stopped ||+ generation !== this.ccLifecycleGeneration ||+ generation === this.ccAppliedLifecycleGeneration+ ) {+ return}- })- return recovery- }-- private scheduleMcpServerCacheRecovery(): void {- if (this.stopped || this.mcpServerCacheRecoveryTimer) return- this.mcpServerCacheRecoveryTimer = setTimeout(() => {- this.mcpServerCacheRecoveryTimer = null- void this.recoverMcpServerInventoryAndWatch().then(recovered => {- if (!recovered) this.scheduleMcpServerCacheRecovery()- })- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)- }-- private scheduleContextCacheRecovery(): void {- if (this.stopped || this.contextCacheRecoveryTimer) return- this.contextCacheRecoveryTimer = setTimeout(() => {- this.contextCacheRecoveryTimer = null- void this.recoverContextInventoryAndWatch().then(recovered => {- if (!recovered) this.scheduleContextCacheRecovery()- })- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)- }-- private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {- this.communicationChannels.clear()- for (const channel of snapshot.channels) {- this.communicationChannels.set(channel.name, channel)- }- }-- private beginCommunicationChannelLifecycleTransition(): number {- return this.hostFleetScheduler.beginLifecycleTransition()- }-- private markCommunicationChannelLifecycleApplied(generation: number): void {- this.hostFleetScheduler.markLifecycleApplied(generation)- }-- private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {- this.hostFleetScheduler.scheduleLifecycleRetry(request)+ void this.requestHostFleetReconcile(+ 'CommunicationChannel lifecycle convergence retry',+ generation,+ request.mode+ )+ }, retryDelay)}/**@@ -1476,45 +884,19 @@ export class McpServerWatcher implements McpServerProvider {* Hosts through their per-Host chains (step 7) and requests — without awaiting* — a coalesced background full pass (step 8).*/- private recoverHostInventoryAndWatch(- convergenceReason = 'Host watch recovery convergence',- ccLifecycleGeneration?: number,- cause: HostInventoryRecoveryCause = 'watch-recovery'- ): Promise<HostCRD[]> {+ private recoverHostInventoryAndWatch(): Promise<HostCRD[]> {if (this.stopped) return Promise.resolve([])- const requested = { convergenceReason, ccLifecycleGeneration, cause }- // A scheduled retry and an immediate caller are two entrances to the same- // monotonic recovery intent. Adopt the pending request before choosing or- // joining the active operation so a direct recovery cannot erase a stronger- // cold-start cause or newer lifecycle generation when it cancels the timer.- const request = this.hostCacheRecoveryIntent- ? { ...this.hostCacheRecoveryIntent }- : { ...requested }- if (this.hostCacheRecoveryIntent) {- mergeHostInventoryRecoveryRequest(request, requested)- this.hostCacheRecoveryIntent = null- if (this.hostCacheRecoveryTimer) {- clearTimeout(this.hostCacheRecoveryTimer)- this.hostCacheRecoveryTimer = null- }- }- if (this.hostRecoveryInFlight) {- mergeHostInventoryRecoveryRequest(this.hostRecoveryInFlight.request, request)- return this.hostRecoveryInFlight.promise- }- const recovery = this.performHostInventoryRecovery(request)- const active = { request, promise: recovery }- this.hostRecoveryInFlight = active+ if (this.hostRecoveryInFlight) return this.hostRecoveryInFlight+ const recovery = this.performHostInventoryRecovery()+ this.hostRecoveryInFlight = recoveryconst clear = (): void => {- if (this.hostRecoveryInFlight === active) this.hostRecoveryInFlight = null+ if (this.hostRecoveryInFlight === recovery) this.hostRecoveryInFlight = null}void recovery.then(clear, clear)return recovery}- private async performHostInventoryRecovery(- request: HostInventoryRecoveryRequest- ): Promise<HostCRD[]> {+ private async performHostInventoryRecovery(): Promise<HostCRD[]> {const startedAt = Date.now()// Capture the pre-recovery inventory so step 7 can distinguish genuinely// new/changed Hosts from ones the fleet pass will converge anyway.@@ -1559,58 +941,24 @@ export class McpServerWatcher implements McpServerProvider {{ phase: 'total', outcome: 'success' },(Date.now() - startedAt) / 1000)- // Effect producers can discover newer work while LIST/WATCH recovery is- // already in flight. Fold that scheduled intent into this successful- // operation before cancelling its timer so no lifecycle generation,- // convergence reason, or stronger cold-start cause is discarded.- if (this.hostCacheRecoveryIntent) {- mergeHostInventoryRecoveryRequest(request, this.hostCacheRecoveryIntent)- }if (this.hostCacheRecoveryTimer) {clearTimeout(this.hostCacheRecoveryTimer)this.hostCacheRecoveryTimer = null}- this.hostCacheRecoveryIntent = null- if (request.cause === 'watch-recovery') {- this.enqueueRecoveredUrgentHosts(- snapshot.hosts,- previousNames,- previousGenerations,- previousUids- )- // Addendum 4 (#827): a Host present before recovery and absent from the- // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate- // per-Host delete cleanup so a DELETE lost to watch retirement is never- // silently dropped.- this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)- }+ this.enqueueRecoveredUrgentHosts(+ snapshot.hosts,+ previousNames,+ previousGenerations,+ previousUids+ )+ // Addendum 4 (#827): a Host present before recovery and absent from the+ // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate+ // per-Host delete cleanup so a DELETE lost to watch retirement is never+ // silently dropped.+ this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)// Request a coalesced background full pass, but do NOT await it before- // declaring watch recovery complete. The cold-start pass alone waits for- // the concurrent SFS LIST: otherwise it can write every Host template- // without its declared mounts, then immediately roll the fleet again- // when SFS discovery finishes. NetworkPolicy readiness and start() stay- // independent from that finite inventory request.- const requestFleetPass = () => {- if (this.stopped) return- void this.requestHostFleetReconcile(- request.convergenceReason,- request.ccLifecycleGeneration,- 'full'- )- }- const initialSfsInventory =- request.cause === 'cold-start' ? this.initialHostFleetSfsInventory : null- if (initialSfsInventory) {- // Waiting for the inventory is deliberate: a Host fleet pass without it- // writes mount-less templates and immediately rerolls the whole fleet.- // The wait cannot strand the fleet, because the LIST underneath runs on- // the deadline-bearing client: an apiserver that never answers aborts- // the request, `startSharedFileSystemInventoryAndWatch` logs and- // resolves false, and this settles. One path, one pass.- void initialSfsInventory.then(() => requestFleetPass())- } else {- requestFleetPass()- }+ // declaring watch recovery complete.+ void this.requestHostFleetReconcile('Host watch recovery convergence', undefined, 'full')return snapshot.hosts} catch (error) {// Keep authority unknown, record the failure, and use the existing@@ -1622,7 +970,7 @@ export class McpServerWatcher implements McpServerProvider {)if (!this.stopped) {console.error('[K8s] Host watch recovery failed:', error)- this.scheduleHostCacheRecovery(request)+ this.scheduleHostCacheRecovery()}throw error}@@ -1662,7 +1010,6 @@ export class McpServerWatcher implements McpServerProvider {private async dispatchUrgentHostReconcile(name: string): Promise<void> {if (this.stopped) return- if (!this.admitHostDependentEffects('Recovered urgent Host convergence')) returnconst host = this.hosts.get(name)if (!host) returntry {@@ -1694,7 +1041,6 @@ export class McpServerWatcher implements McpServerProvider {private async dispatchRecoveredHostDelete(name: string): Promise<void> {if (this.stopped) return- if (!this.admitHostDependentEffects('Recovered Host deletion convergence')) return// The fresh authoritative LIST already confirmed absence and the cache no// longer holds this Host, so cleanup is authorized. Route it through the// per-Host serializer (reconcileDelete → serializeByHost) so an older@@ -1726,30 +1072,13 @@ export class McpServerWatcher implements McpServerProvider {}}- private scheduleHostCacheRecovery(- requested: HostInventoryRecoveryRequest = {- convergenceReason: 'Host watch recovery convergence',- cause: 'watch-recovery',- }- ): void {- if (this.stopped) return- if (this.hostCacheRecoveryIntent) {- mergeHostInventoryRecoveryRequest(this.hostCacheRecoveryIntent, requested)- } else {- this.hostCacheRecoveryIntent = { ...requested }- }- if (this.hostCacheRecoveryTimer) return+ private scheduleHostCacheRecovery(): void {+ if (this.stopped || this.hostCacheRecoveryTimer) returnthis.hostCacheRecoveryTimer = setTimeout(() => {this.hostCacheRecoveryTimer = null- const request = this.hostCacheRecoveryIntent ?? requested- this.hostCacheRecoveryIntent = null// Watch recovery is an independent operation, no longer coupled to a full// fleet pass. Recovery itself requests the background convergence pass.- void this.recoverHostInventoryAndWatch(- request.convergenceReason,- request.ccLifecycleGeneration,- request.cause- ).catch(() => undefined)+ void this.recoverHostInventoryAndWatch().catch(() => undefined)}, HOST_CACHE_RECOVERY_RETRY_MS)}@@ -1794,7 +1123,6 @@ export class McpServerWatcher implements McpServerProvider {source: HostReconcileSource = 'urgent'): Promise<void> {if (this.stopped) return- if (!this.admitHostDependentEffects('Host watch event convergence')) returnif (!this.isCurrentHostWatchEvent(type, host, eventRevision)) {// #827: a DELETE that is no longer current because the cache now holds a// (recreated) same-name Host is a superseded delete — record it and do@@ -1889,7 +1217,6 @@ export class McpServerWatcher implements McpServerProvider {await this.recoverHostInventoryAndWatch()if (this.stopped) return}- if (!this.isHostEffectInventoryAuthoritative()) return// Reconcile the CURRENT cache. Fleet workers dispatch Host keys and// resolve the freshest cached spec at execution (see collectHostReconcile// Failures); only fullReconcile runs the authority-gated orphan cleanup.@@ -1952,7 +1279,78 @@ export class McpServerWatcher implements McpServerProvider {ccLifecycleGeneration?: number,mode?: HostFleetReconcileMode): Promise<void> {- return this.hostFleetScheduler.request(reason, ccLifecycleGeneration, mode)+ if (this.stopped) return Promise.resolve()+ const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')+ if (+ ccLifecycleGeneration !== undefined &&+ (ccLifecycleGeneration !== this.ccLifecycleGeneration ||+ (requestMode === 'lifecycle' &&+ ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))+ ) {+ return Promise.resolve()+ }++ const request: HostFleetReconcileRequest = {+ reason,+ mode: requestMode,+ ccLifecycleGeneration,+ }+ const active = this.hostFleetReconcileInFlight+ if (!active) {+ hostFleetRequestsTotal.inc({ result: 'started' })+ return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))+ }+ if (hostFleetRequestCovers(active, request)) {+ hostFleetRequestsTotal.inc({ result: 'coalesced' })+ return this.waitForHostFleetOrShutdown(active.promise)+ }++ if (!this.hostFleetReconcilePending) {+ let resolve!: () => void+ const promise = new Promise<void>(resolvePromise => {+ resolve = resolvePromise+ })+ this.hostFleetReconcilePending = {+ ...request,+ promise,+ resolve,+ }+ } else {+ const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)+ this.hostFleetReconcilePending.reason = merged.reason+ this.hostFleetReconcilePending.mode = merged.mode+ this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration+ }+ // The request did not start its own pass; it is queued behind the active one.+ hostFleetRequestsTotal.inc({ result: 'coalesced' })+ return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)+ }++ private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {+ return Promise.race([coverage, this.hostFleetShutdown])+ }++ private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {+ const promise = this.performHostFleetReconcile(request)+ const active: ActiveHostFleetReconcile = { ...request, promise }+ this.hostFleetReconcileInFlight = active++ const settle = () => {+ if (this.hostFleetReconcileInFlight !== active) return+ this.hostFleetReconcileInFlight = null+ const pending = this.hostFleetReconcilePending+ this.hostFleetReconcilePending = null+ if (!pending) return+ if (this.stopped) {+ pending.resolve()+ return+ }+ hostFleetRequestsTotal.inc({ result: 'trailing' })+ const trailing = this.startHostFleetReconcile(pending)+ void trailing.then(pending.resolve, pending.resolve)+ }+ void promise.then(settle, settle)+ return promise}private contextsReferencingSfs(sfsName: string): Array<{ namespace: string; name: string }> {@@ -2012,35 +1410,14 @@ export class McpServerWatcher implements McpServerProvider {return out}- private reconcileSharedFileSystemsReferencedByContext(- prevContext: ContextCRD | undefined,- nextContext: ContextCRD | undefined,- isCurrent: () => boolean = () => true- ): Promise<void> {- const names = [- ...(prevContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),- ...(nextContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),- ]- return this.enqueueSharedFileSystemFleetReconciliation(names, async () => {- if (!isCurrent()) return- await this.reconcileSharedFileSystemsReferencedByContextCore(- prevContext,- nextContext,- isCurrent- )- })- }-- private async reconcileSharedFileSystemsReferencedByContextCore(+ private async reconcileSharedFileSystemsReferencedByContext(prevContext: ContextCRD | undefined,- nextContext: ContextCRD | undefined,- isCurrent: () => boolean+ nextContext: ContextCRD | undefined): Promise<void> {const names = new Set<string>()for (const r of prevContext?.spec.sharedFileSystems ?? []) names.add(r.name)for (const r of nextContext?.spec.sharedFileSystems ?? []) names.add(r.name)for (const name of names) {- if (!isCurrent()) returnconst sfs = this.sharedFileSystems.get(name)if (!sfs) continuetry {@@ -2057,9 +1434,6 @@ export class McpServerWatcher implements McpServerProvider {* Deployment template picks up (or drops) the volume + volumeMount.*/private async reconcileHostsReferencingSfs(sfsName: string): Promise<void> {- if (!this.admitHostDependentEffects(`SharedFileSystem "${sfsName}" Host convergence`)) {- return- }const affectedContexts = new Set<string>()for (const ctx of this.contexts.values()) {if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {@@ -2077,24 +1451,6 @@ export class McpServerWatcher implements McpServerProvider {}}- /**- * Reconcile Hosts affected by a Context change only while the Host cache is- * authoritative. A recovery full pass consumes the current Context cache, so- * deferral loses no effect and never acts on an unpaired Host LIST.- */- private async reconcileHostsReferencingContext(- contextName: string,- isCurrent: () => boolean = () => true- ): Promise<void> {- if (!isCurrent()) return- if (!this.admitHostDependentEffects(`Context "${contextName}" Host convergence`)) return- for (const host of this.hosts.values()) {- if (!isCurrent()) return- if (host.spec.contextRef !== contextName) continue- await this.hostReconciler.reconcile(host)- }- }-/*** Re-reconcile the Host whose name matches the given hostRef. Called by the* CommunicationChannel watch handler so per-Host channel-reader Deployment@@ -2107,9 +1463,6 @@ export class McpServerWatcher implements McpServerProvider {* its caller schedules bounded fleet convergence.*/private async reconcileHostsReferencingCC(hostRef: string): Promise<boolean> {- if (!this.admitHostDependentEffects(`CommunicationChannel Host "${hostRef}" convergence`)) {- return false- }const host = this.hosts.get(hostRef)if (!host) return truetry {@@ -2201,28 +1554,9 @@ export class McpServerWatcher implements McpServerProvider {* tracker resolves the heartbeat's hostRef through this).*/getHost(name: string): HostCRD | undefined {- if (!this.isHostInventoryAuthoritative()) return undefinedreturn this.hosts.get(name)}- /**- * Route channel credentials-Secret events through the same Host authority- * boundary as watch, Context, SFS, and heartbeat effects.- */- async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {- const convergenceReason = `CommunicationChannel Secret "${secretNamespace}/${secretName}" Host convergence`- if (!this.ccCacheSynced) {- console.warn(- `[K8s] Deferring ${convergenceReason}; CommunicationChannel cache is not authoritative`- )- return- }- if (!this.admitHostDependentEffects(convergenceReason)) {- return- }- await this.hostReconciler.reconcileChannelReaderRevision(secretName, secretNamespace)- }-/*** Trigger a reconcile for any cached McpServer whose `envSecret.name`* matches the given Secret. Used by SecretInformer to react to Secret@@ -2230,45 +1564,18 @@ export class McpServerWatcher implements McpServerProvider {* event.*/async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {- const inventoryGeneration = this.mcpWatchGeneration- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return- const selected = [...this.servers.values()].filter(- server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName- )- await Promise.all(- selected.map(server =>- this.enqueueMcpServerReconciliation(server, async () => {- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return- const current = this.servers.get(server.name)- if (- !current ||- current.namespace !== secretNamespace ||- current.spec.envSecret?.name !== secretName- ) {- return- }- try {- console.log(- `[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`- )- await this.reconciler.reconcile(current, {- isCurrent: () => {- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false- const latest = this.servers.get(current.name)- return (- latest !== undefined &&- latest.namespace === secretNamespace &&- latest.spec.envSecret?.name === secretName &&- sameMcpServerDesiredRevision(current, latest)- )- },- })- } catch (err) {- console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)- }- })- )- )+ for (const server of this.servers.values()) {+ if (server.namespace !== secretNamespace) continue+ if (server.spec.envSecret?.name !== secretName) continue+ try {+ console.log(+ `[K8s] Re-reconciling McpServer "${server.name}" after Secret "${secretName}" change`+ )+ await this.reconciler.reconcile(server)+ } catch (err) {+ console.error(`[K8s] Secret-triggered reconcile failed for "${server.name}":`, err)+ }+ }}/**@@ -2296,7 +1603,7 @@ export class McpServerWatcher implements McpServerProvider {transport,auth: server.spec.auth,enabled: server.spec.enabled !== false,- status: this.reconciler.getStatus(server),+ status: this.reconciler.getStatus(server.name),}}@@ -2351,55 +1658,61 @@ export class McpServerWatcher implements McpServerProvider {* Start watching for changes and run initial reconciliation.*/async start(): Promise<void> {- // L0/L1 isolation and removal of legacy permissive policies are mandatory- // bootstrap barriers. They are a fixed-size safety operation, independent- // of fleet size, and must succeed before HCC can watch or reconcile runtimes.- await this.netPolReconciler.ensureDefaultPolicies()-- // Discovery and policy caches become authoritative only when each complete- // LIST snapshot is paired with a watch continuing from its opaque collection- // resourceVersion. Full fleet convergence happens later in the background.- // A transient LIST/WATCH failure on either inventory must not kill the- // retry that restart*Watch already armed (it sets *CacheSynced=false before- // it can throw). Mirror the Host lane below: log, keep the flag false, and- // schedule in-process recovery. The HTTP server stays live, dynamic- // readiness stays 503, and recovery promotes readiness only after a fresh- // LIST is paired with its continuing WATCH.+ let initialServers: McpServerCRD[] = []+ let serverInventoryComplete = falsetry {- const initialServerSnapshot = await listMcpServerSnapshot()- await this.restartMcpServerWatch(initialServerSnapshot)+ initialServers = await listAllMcpServers()+ serverInventoryComplete = true+ for (const server of initialServers) {+ this.servers.set(server.name, server)+ }} catch (error) {- this.mcpServerCacheSynced = falseconsole.error(- '[K8s] Initial McpServer inventory is unavailable; HCC remains unready while in-process recovery continues:',+ '[K8s] Skipping initial McpServer full reconciliation because server discovery failed:',error)}- if (!this.mcpServerCacheSynced) {- this.scheduleMcpServerCacheRecovery()++ // Ensure baseline network policies exist even if initial Context discovery fails.+ try {+ await this.netPolReconciler.ensureDefaultPolicies()+ } catch (error) {+ console.error('[K8s] Failed to ensure default NetworkPolicies before startup:', error)+ }++ let initialExternalEgressFailures = new Set<string>()+ if (serverInventoryComplete) {+ initialExternalEgressFailures =+ await this.reconcileInitialExternalEgressBeforeRuntime(initialServers)+ }++ // Full McpServer reconciliation (Deployments + Services). Servers with+ // egressBindings are reconciled only after their external egress policies+ // have converged, matching the ADDED/MODIFIED watch-event contract.+ if (serverInventoryComplete) {+ const runtimeServers = initialServers.filter(+ server => !initialExternalEgressFailures.has(this.externalEgressRetryKey(server))+ )+ console.log('[K8s] Running initial full reconciliation...')+ await this.reconciler.fullReconcile(runtimeServers)}try {- const initialContextSnapshot = await listContextSnapshot()- await this.restartContextWatch(initialContextSnapshot)+ const initialContexts = await listAllContexts()+ for (const context of initialContexts) {+ this.contexts.set(context.name, context)+ }+ console.log('[K8s] Running initial NetworkPolicy reconciliation...')+ await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {+ serverInventoryComplete,+ ensureDefaults: false,+ })} catch (error) {- this.contextCacheSynced = falseconsole.error(- '[K8s] Initial Context inventory is unavailable; HCC remains unready while in-process recovery continues:',+ '[K8s] Skipping initial NetworkPolicy reconciliation because context discovery failed:',error)}- if (!this.contextCacheSynced) {- this.scheduleContextCacheRecovery()- }-- // Host templates resolve Context SharedFileSystem references from this- // cache. Inventory is therefore a fixed startup boundary, not part of the- // unbounded SFS reconciliation fleet: letting the first Host pass run- // without it writes mount-less templates and immediately rolls the entire- // Host fleet a second time once discovery catches up.- const initialSharedFileSystemInventory = this.startSharedFileSystemInventoryAndWatch()- this.initialHostFleetSfsInventory = initialSharedFileSystemInventory// ── CommunicationChannel snapshot + watch — MUST complete before Host fullReconcile ──// A stateless Host may suspend only after this controller has a complete@@ -2448,30 +1761,54 @@ export class McpServerWatcher implements McpServerProvider {}console.log(- `[K8s] Starting initial Host background convergence... (ccCacheSynced=${this.ccCacheSynced})`+ `[K8s] Running initial Host reconciliation... (ccCacheSynced=${this.ccCacheSynced})`)const initialLifecycleGeneration =this.ccLifecycleGeneration === 0? this.beginCommunicationChannelLifecycleTransition(): this.ccLifecycleGeneration+ await this.requestHostFleetReconcile(+ 'initial Host reconciliation',+ initialLifecycleGeneration,+ 'full'+ )++ // ── SharedFileSystem initial load + reconciliation ──try {- await this.recoverHostInventoryAndWatch(- 'initial Host reconciliation',- initialLifecycleGeneration,- 'cold-start'+ const initialSfses = await listAllSharedFileSystems()+ for (const sfs of initialSfses) {+ this.sharedFileSystems.set(sfs.name, sfs)+ }+ console.log('[K8s] Running initial SharedFileSystem reconciliation...')+ await this.sharedFileSystemReconciler.fullReconcile(initialSfses)+ } catch (error) {+ console.error(+ '[K8s] Skipping initial SharedFileSystem reconciliation because discovery failed:',+ error)+ }++ // ── GlobalFileSystem (gfs) initial load + reconciliation ──+ // gfs is a cluster singleton (DISTINCT from SharedFileSystem). A failed+ // discovery (e.g. the CRD not installed yet) is skipped, not fatal — the+ // next HCC restart / resync retries. Live updates land via the watch+ // (follow-up); the deploy applies the singleton before HCC reconciles it.+ try {+ const initialGfses = await listAllGlobalFileSystems()+ console.log('[K8s] Running initial GlobalFileSystem reconciliation...')+ await this.gfsReconciler.fullReconcile(initialGfses)} catch (error) {- // Host inventory is an essential readiness authority, but a transient- // LIST/WATCH failure must not kill the retry that recovery just armed.- // Continue startup with hostCacheSynced=false: the HTTP server stays live,- // dynamic readiness remains 503, and recovery promotes readiness only- // after a fresh LIST is paired with its continuing WATCH.- console.warn(- '[K8s] Initial Host inventory is unavailable; HCC remains unready while in-process recovery continues:',+ console.error(+ '[K8s] Skipping initial GlobalFileSystem reconciliation because discovery failed:',error)}+ await this.startMcpServerWatch()+ await this.startContextWatch()+ await this.startSharedFileSystemWatch()+ await this.startGlobalFileSystemWatch()+const resyncSec = config.hostResyncIntervalSecif (resyncSec > 0) {this.resyncTimer = setInterval(() => {@@ -2509,375 +1846,35 @@ export class McpServerWatcher implements McpServerProvider {}const externalEgressResyncSec = config.externalEgressResyncIntervalSec- // #205 delegates external-egress periodic resync to the convergence- // coordinator. Its runResyncCore drives reconcileExternalEgress, so the- // #299 sliding-window accumulation still converges on DNS rotation between- // McpServer events. The coordinator owns cadence with a self-rescheduling- // timer that advances to <= observed TTL/2 (H2, issue #299), bounded by the- // configured floor.- this.externalEgressCoordinator.startPeriodicResync(- externalEgressResyncSec,- config.externalEgressRefreshFloorSec- )-- // Full convergence is observable and retains each lane's existing retry or- // periodic-resync contract, but it no longer extends provider.start().- // NetworkPolicy safety starts first and launches McpServer convergence from- // its certification callback, before additive policy convergence completes.- // This prevents the startup egress refresh from racing or undoing the safety- // pass while keeping readiness independent from both additive fleets.- void this.runInitialNetworkPolicyConvergence()- void initialSharedFileSystemInventory.then(inventoryComplete => {- if (inventoryComplete && !this.stopped) {- void this.runInitialSharedFileSystemConvergence()- }- })- void this.startGlobalFileSystemBackgroundLane()- }-- private async startSharedFileSystemInventoryAndWatch(): Promise<boolean> {- let inventoryComplete = false- try {- const initialSfses = await listAllSharedFileSystems()- inventoryComplete = true- for (const sfs of initialSfses) this.sharedFileSystems.set(sfs.name, sfs)- } catch (error) {- console.error(- '[K8s] Skipping initial SharedFileSystem background convergence because discovery failed:',- error- )- }- // The cold-start Host fleet pass gates on this promise, so it must settle on- // the inventory alone. `k8s.Watch` carries no transport deadline — the- // client builds `AbortSignal.any([controller, timeout])` and then overwrites- // it with the bare controller — so awaiting the watch start here would- // re-strand the fleet behind an already-certified readiness, which is the- // exact failure bounding the LIST was meant to close.- void this.startSharedFileSystemWatch().catch(error => {- console.error('[K8s] SharedFileSystem background watch failed to start:', error)- this.scheduleSharedFileSystemWatchRestart(5000)- })- if (inventoryComplete && !this.stopped) {- return true- }- return false- }-- private async startGlobalFileSystemBackgroundLane(): Promise<void> {- let inventoryComplete = false- try {- const initialGfses = await listAllGlobalFileSystems()- inventoryComplete = true- for (const gfs of initialGfses) this.globalFileSystems.set(gfs.name, gfs)- } catch (error) {- console.error(- '[K8s] Skipping initial GlobalFileSystem background convergence because discovery failed:',- error- )- }- try {- await this.startGlobalFileSystemWatch()- } catch (error) {- console.error('[K8s] GlobalFileSystem background watch failed to start:', error)- this.scheduleGlobalFileSystemWatchRestart(5000)- }- if (inventoryComplete && !this.stopped) {- void this.runInitialGlobalFileSystemConvergence()- }- }-- private runInitialMcpServerConvergence(): Promise<void> {- return this.runInitialConvergence('McpServer')- }-- private currentNetworkPolicySafetyCertificate(): NetworkPolicySafetyCertificate | null {- if (- this.stopped ||- !this.contextCacheSynced ||- !this.mcpServerCacheSynced ||- !this.netPolReconciler.hasCertifiedSafetyInventory() ||- this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||- this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||- this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||- this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision- ) {- return null- }- return {- contextGeneration: this.contextWatchGeneration,- serverGeneration: this.mcpWatchGeneration,- contextRevision: this.contextDesiredRevision,- serverRevision: this.mcpServerDesiredRevision,- }- }-- private recordNetworkPolicySafetyCertificate(- certificate: NetworkPolicySafetyCertificate- ): boolean {- if (- this.stopped ||- !this.contextCacheSynced ||- !this.mcpServerCacheSynced ||- this.contextWatchGeneration !== certificate.contextGeneration ||- this.mcpWatchGeneration !== certificate.serverGeneration ||- this.contextDesiredRevision !== certificate.contextRevision ||- this.mcpServerDesiredRevision !== certificate.serverRevision- ) {- return false- }- this.networkPolicyRevocationContextGeneration = certificate.contextGeneration- this.networkPolicyRevocationServerGeneration = certificate.serverGeneration- this.networkPolicyRevocationContextRevision = certificate.contextRevision- this.networkPolicyRevocationServerRevision = certificate.serverRevision- return true- }-- private isNetworkPolicySafetyCertificateCurrent(- certificate: NetworkPolicySafetyCertificate- ): boolean {- const current = this.currentNetworkPolicySafetyCertificate()- return (- current !== null &&- current.contextGeneration === certificate.contextGeneration &&- current.serverGeneration === certificate.serverGeneration &&- current.contextRevision === certificate.contextRevision &&- current.serverRevision === certificate.serverRevision- )- }-- private async runInitialMcpServerConvergenceCore(): Promise<void> {- const safetyCertificate = this.currentNetworkPolicySafetyCertificate()- if (safetyCertificate === null) return- const inventoryGeneration = this.mcpWatchGeneration- const inventoryAuthoritative = () =>- this.hasMcpServerInventoryAuthority(inventoryGeneration) &&- this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)- try {- const initialServers = [...this.servers.values()]- const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(- initialServers,- inventoryAuthoritative- )- if (this.stopped) return- console.log('[K8s] Running initial McpServer background reconciliation...')- await this.reconciler.fullReconcile(initialServers, {- runEffect: async (serverName, work) => {- const selected = this.servers.get(serverName)- const laneOwner = selected ?? {- name: serverName,- namespace: config.namespace,- }- if (!(await initialExternalEgressGates.waitFor(laneOwner))) return- if (!inventoryAuthoritative()) return- await this.enqueueMcpServerReconciliation(laneOwner, async () => {- if (!inventoryAuthoritative()) return- await work()- })- },- })- if (!inventoryAuthoritative()) return- initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)- this.clearInitialConvergenceRetry('McpServer')- } catch (error) {- console.error('[K8s] Initial McpServer background reconciliation failed:', error)- this.scheduleInitialConvergenceRetry('McpServer')- }- }-- private runInitialNetworkPolicyConvergence(): Promise<void> {- return this.runInitialConvergence('NetworkPolicy')- }-- private runInitialConvergence(lane: InitialConvergenceLane): Promise<void> {- const active = this.initialConvergenceRuns.get(lane)- if (active) {- active.trailingRequested = true- return active.promise- }-- const run: ActiveInitialConvergenceRun = {- trailingRequested: false,- promise: Promise.resolve(),- }- this.initialConvergenceRuns.set(lane, run)- run.promise = Promise.resolve()- .then(async () => {- do {- run.trailingRequested = false- if (lane === 'McpServer') {- await this.runInitialMcpServerConvergenceCore()- } else {- await this.runInitialNetworkPolicyConvergenceCore()- }- } while (!this.stopped && run.trailingRequested)- // Retire the active slot synchronously before this promise settles.- // A request arriving in the following settlement microtask must start- // a new run instead of attaching to a pass whose loop already exited.- if (this.initialConvergenceRuns.get(lane) === run) {- this.initialConvergenceRuns.delete(lane)- }- })- .finally(() => {- // Errors bypass the normal retirement above.- if (this.initialConvergenceRuns.get(lane) === run) {- this.initialConvergenceRuns.delete(lane)- }- })- return run.promise- }-- private async runInitialNetworkPolicyConvergenceCore(): Promise<void> {- // A NetworkPolicy full pass combines authoritative Context/McpServer allow- // revocation with additive Context policy effects. External egress- // creation and DNS refresh have one separate owner: the startup/resync- // coordinator. Running safety from a partial inventory would be neither a- // full convergence nor safe evidence of success. Recovery of either- // missing LIST -> WATCH pair schedules a fresh current-cache pass.- if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return- let authoritativeRevocationCompleted = false- try {- const initialContexts = [...this.contexts.values()]- const initialServers = [...this.servers.values()]- const contextInventoryGeneration = this.contextWatchGeneration- const serverInventoryGeneration = this.mcpWatchGeneration- const safetyCertificate: NetworkPolicySafetyCertificate = {- contextGeneration: contextInventoryGeneration,- serverGeneration: serverInventoryGeneration,- contextRevision: this.contextDesiredRevision,- serverRevision: this.mcpServerDesiredRevision,- }- const serverInventoryComplete = this.mcpServerCacheSynced- const contextInventoryAuthoritative = () =>- this.hasContextInventoryAuthority(contextInventoryGeneration)- const serverInventoryAuthoritative = () =>- this.hasMcpServerInventoryAuthority(serverInventoryGeneration)- console.log('[K8s] Running initial NetworkPolicy background reconciliation...')- await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {- serverInventoryComplete,- ensureDefaults: false,- contextInventoryAuthoritative,- serverInventoryAuthoritative,- runContextEffect: (contextId, work) =>- this.enqueueContextReconciliation(contextId, async () => {- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return- await work()- }),- runServerEffect: (serverName, work) => {- const selected = this.servers.get(serverName)- return this.enqueueMcpServerReconciliation(- selected ?? { name: serverName, namespace: config.namespace },- async () => {- if (!serverInventoryAuthoritative()) return- await work()- }- )- },- resolveCurrentContext: name => this.contexts.get(name),- resolveCurrentContextById: contextId =>- [...this.contexts.values()].find(context => context.spec.contextId === contextId),- resolveCurrentServer: name => this.servers.get(name),- contextDesiredRevision: () => this.contextDesiredRevision,- serverDesiredRevision: () => this.mcpServerDesiredRevision,- // B3 piece-2: queue an additive recreation for every server whose- // external-egress allow the authoritative pass revoked — fired at the- // deletion, so an abort/throw cannot strand a revoked-but-desired allow.- // The coordinator maps against the current cache (absent → DELETED, no- // recreate) and rebuilds from the current spec with fresh DNS.- onExternalEgressRevoked: server =>- this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),- onAuthoritativeRevocationComplete: () => {- if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return- authoritativeRevocationCompleted = true- // Start runtime convergence at the safety boundary, before this full- // pass enters additive Context work. Startup egress gates may now- // preserve only policies certified by this exact authoritative- // inventory generation.- void this.runInitialMcpServerConvergence()- },- })- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return- initialConvergenceLastSuccessTimestampSeconds.set(- { lane: 'NetworkPolicy' },- Date.now() / 1000+ if (externalEgressResyncSec > 0) {+ // Self-rescheduling (not a fixed setInterval) so the delay can advance to+ // <= observed TTL/2 (H2, issue #299) and rescheduling AFTER each pass+ // completes guarantees no overlapping resyncs.+ const scheduleNext = (): void => {+ if (this.stopped) return+ const delayMs = externalEgressResyncDelayMs(+ externalEgressResyncSec,+ config.externalEgressRefreshFloorSec,+ this.netPolReconciler.externalEgressRefreshMinTtlMs+ )+ this.externalEgressResyncTimer = setTimeout(() => {+ void this.runExternalEgressResync().finally(() => scheduleNext())+ }, delayMs)+ this.externalEgressResyncTimer.unref?.()+ }+ // Idempotency guard (audit F5): never arm a second self-rescheduling chain+ // if one is already live (start() runs once today, but a future restart+ // path must not double the cadence).+ if (!this.externalEgressResyncTimer) scheduleNext()+ console.log(+ `[K8s] External egress periodic DNS resync enabled (every ${externalEgressResyncSec}s, advancing to <= TTL/2)`)- this.clearInitialConvergenceRetry('NetworkPolicy')- } catch (error) {- console.error(- authoritativeRevocationCompleted- ? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'- : '[K8s] Initial NetworkPolicy background reconciliation failed:',- error+ } else {+ console.warn(+ '[K8s] External egress periodic DNS resync DISABLED (HCC_EXTERNAL_EGRESS_RESYNC_SEC=0); ' ++ 'exact-host FQDN egress will NOT converge on DNS rotation between McpServer events — ' ++ 'accumulated IPs can go stale and block workloads (issue #299). Enable it unless you have a specific reason.')- this.scheduleInitialConvergenceRetry('NetworkPolicy')- }- }-- private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {- const timer = this.initialConvergenceRetryTimers.get(lane)- if (timer) {- clearTimeout(timer)- this.initialConvergenceRetryTimers.delete(lane)- }- this.initialConvergenceRetryAttempts.delete(lane)- }-- private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {- if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return-- const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1- const delayMs =- INITIAL_CONVERGENCE_RETRY_DELAYS_MS[- Math.min(attempt - 1, INITIAL_CONVERGENCE_RETRY_DELAYS_MS.length - 1)- ]- this.initialConvergenceRetryAttempts.set(lane, attempt)- initialConvergenceRetriesTotal.inc({ lane })- console.warn(- `[K8s] Scheduling initial ${lane} background convergence retry ${attempt} in ${delayMs}ms`- )-- const timer = setTimeout(() => {- this.initialConvergenceRetryTimers.delete(lane)- if (this.stopped) return- if (lane === 'McpServer') {- void this.runInitialMcpServerConvergence()- return- }- void this.runInitialNetworkPolicyConvergence()- }, delayMs)- this.initialConvergenceRetryTimers.set(lane, timer)- }-- private runInitialSharedFileSystemConvergence(): Promise<void> {- const names = [...this.sharedFileSystems.keys()]- return this.enqueueSharedFileSystemFleetReconciliation(names, () =>- this.runInitialSharedFileSystemConvergenceCore()- )- }-- private async runInitialSharedFileSystemConvergenceCore(): Promise<void> {- try {- const initialSfses = [...this.sharedFileSystems.values()]- console.log('[K8s] Running initial SharedFileSystem background reconciliation...')- await this.sharedFileSystemReconciler.fullReconcile(initialSfses)- } catch (error) {- console.error('[K8s] Initial SharedFileSystem background reconciliation failed:', error)- }- }-- private runInitialGlobalFileSystemConvergence(): Promise<void> {- const names = [...this.globalFileSystems.keys()]- return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>- this.runInitialGlobalFileSystemConvergenceCore()- )- }-- private async runInitialGlobalFileSystemConvergenceCore(): Promise<void> {- try {- const initialGfses = [...this.globalFileSystems.values()]- console.log('[K8s] Running initial GlobalFileSystem background reconciliation...')- await this.gfsReconciler.fullReconcile(initialGfses)- } catch (error) {- console.error('[K8s] Initial GlobalFileSystem background reconciliation failed:', error)}}@@ -2905,58 +1902,42 @@ export class McpServerWatcher implements McpServerProvider {} else if (type === 'DELETED') {this.sharedFileSystems.delete(sfs.name)}- this.sharedFileSystemCacheRevision += 1- await this.enqueueSharedFileSystemReconciliation(sfs.name, async () => {- try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.sharedFileSystemReconciler.reconcile(sfs)- } else if (type === 'DELETED') {- await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)- }- } catch (error) {- console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.sharedFileSystemReconciler.reconcile(sfs)+ } else if (type === 'DELETED') {+ await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)}+ } catch (error) {+ console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)+ }- // Re-reconcile any Host whose Context references this SharedFileSystem.- // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on- // the next mcp-host pod template diff so the pod stops failing to mount.- try {- await this.reconcileHostsReferencingSfs(sfs.name)- } catch (error) {- console.error(- `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,- error- )- }- })+ // Re-reconcile any Host whose Context references this SharedFileSystem.+ // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on+ // the next mcp-host pod template diff so the pod stops failing to mount.+ try {+ await this.reconcileHostsReferencingSfs(sfs.name)+ } catch (error) {+ console.error(+ `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,+ error+ )+ }}const doneCallback = (err: Error | null) => {if (this.stopped) return- this.sfsWatchRequest = nullif (err) {console.error('[K8s] SharedFileSystem watch error:', err)}console.log('[K8s] SharedFileSystem watch ended, restarting...')- this.scheduleSharedFileSystemWatchRestart(err ? 5000 : 1000)+ setTimeout(() => this.startSharedFileSystemWatch(), err ? 5000 : 1000)}this.sfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}- private scheduleSharedFileSystemWatchRestart(delayMs: number): void {- if (this.stopped || this.sfsWatchRestartTimer) return- this.sfsWatchRestartTimer = setTimeout(() => {- this.sfsWatchRestartTimer = null- if (this.stopped) return- void this.startSharedFileSystemWatch().catch(error => {- console.error('[K8s] SharedFileSystem background watch restart failed:', error)- this.scheduleSharedFileSystemWatchRestart(5000)- })- }, delayMs)- }-/*** Watch the GlobalFileSystem singleton (gfs). ADDED/MODIFIED → reconcile the* drive stack; DELETED → tear it down (respecting retainOnDelete). DISTINCT@@ -2970,63 +1951,37 @@ export class McpServerWatcher implements McpServerProvider {const watchCallback = async (type: string,- apiObj: {- metadata: { name: string; namespace?: string }- spec: GlobalFileSystemSpec- status?: GlobalFileSystemCRD['status']- }+ apiObj: { metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }) => {const gfs: GlobalFileSystemCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || namespace,spec: apiObj.spec,- status: apiObj.status,}console.log(`[K8s] GlobalFileSystem watch event: ${type} for ${gfs.name}`)- if (type === 'ADDED' || type === 'MODIFIED') {- this.globalFileSystems.set(gfs.name, gfs)- } else if (type === 'DELETED') {- this.globalFileSystems.delete(gfs.name)- }- this.globalFileSystemCacheRevision += 1- await this.enqueueGlobalFileSystemReconciliation(gfs.name, async () => {- try {- if (type === 'ADDED' || type === 'MODIFIED') {- await this.gfsReconciler.reconcile(gfs)- } else if (type === 'DELETED') {- await this.gfsReconciler.reconcileDelete(gfs)- }- } catch (error) {- console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.gfsReconciler.reconcile(gfs)+ } else if (type === 'DELETED') {+ await this.gfsReconciler.reconcileDelete(gfs)}- })+ } catch (error) {+ console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)+ }}const doneCallback = (err: Error | null) => {if (this.stopped) return- this.gfsWatchRequest = nullif (err) {console.error('[K8s] GlobalFileSystem watch error:', err)}console.log('[K8s] GlobalFileSystem watch ended, restarting...')- this.scheduleGlobalFileSystemWatchRestart(err ? 5000 : 1000)+ setTimeout(() => this.startGlobalFileSystemWatch(), err ? 5000 : 1000)}this.gfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}- private scheduleGlobalFileSystemWatchRestart(delayMs: number): void {- if (this.stopped || this.gfsWatchRestartTimer) return- this.gfsWatchRestartTimer = setTimeout(() => {- this.gfsWatchRestartTimer = null- if (this.stopped) return- void this.startGlobalFileSystemWatch().catch(error => {- console.error('[K8s] GlobalFileSystem background watch restart failed:', error)- this.scheduleGlobalFileSystemWatchRestart(5000)- })- }, delayMs)- }-/*** Extracted so unit tests can invoke the handler without spinning up a real* K8s watch. Production calls this from `startCommunicationChannelWatch` and@@ -3081,14 +2036,6 @@ export class McpServerWatcher implements McpServerProvider {if (previousHostRef && previousHostRef !== cc.spec.hostRef) {affectedHostRefs.add(previousHostRef)}- if (!this.isHostInventoryAuthoritative()) {- const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()- this.admitHostDependentEffects(- 'CommunicationChannel event Host convergence',- lifecycleGeneration- )- return- }let needsFleetRetry = falsefor (const hostRef of affectedHostRefs) {if (!(await this.reconcileHostsReferencingCC(hostRef))) {@@ -3250,41 +2197,8 @@ export class McpServerWatcher implements McpServerProvider {private async runGfsResync(): Promise<void> {if (this.stopped) returntry {- const cacheRevisionAtListStart = this.globalFileSystemCacheRevisionconst gfses = await listAllGlobalFileSystems()- const names = [- ...this.globalFileSystems.keys(),- ...gfses.map(globalFileSystem => globalFileSystem.name),- ]- return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>- this.runGfsResyncCore(gfses, cacheRevisionAtListStart)- )- } catch (err) {- console.error(- `[K8s] GlobalFileSystem periodic resync failed: ${- err instanceof Error ? err.message : String(err)- }`- )- }- }-- private async runGfsResyncCore(- listedGfses?: GlobalFileSystemCRD[],- listedAtCacheRevision?: number- ): Promise<void> {- if (this.stopped) return- try {- const cacheRevisionAtListStart = listedAtCacheRevision ?? this.globalFileSystemCacheRevision- const gfses = listedGfses ?? (await listAllGlobalFileSystems())- let reconcileInventory: GlobalFileSystemCRD[]- if (cacheRevisionAtListStart === this.globalFileSystemCacheRevision) {- this.globalFileSystems.clear()- for (const gfs of gfses) this.globalFileSystems.set(gfs.name, gfs)- reconcileInventory = gfses- } else {- reconcileInventory = [...this.globalFileSystems.values()]- }- await this.gfsReconciler.fullReconcile(reconcileInventory)+ await this.gfsReconciler.fullReconcile(gfses)} catch (err) {console.error(`[K8s] GlobalFileSystem periodic resync failed: ${@@ -3297,36 +2211,9 @@ export class McpServerWatcher implements McpServerProvider {private async runSfsResync(): Promise<void> {if (this.stopped) returntry {- const cacheRevisionAtListStart = this.sharedFileSystemCacheRevisionconst sfses = await listAllSharedFileSystems()- const names = [- ...this.sharedFileSystems.keys(),- ...sfses.map(sharedFileSystem => sharedFileSystem.name),- ]- return this.enqueueSharedFileSystemFleetReconciliation(names, () =>- this.runSfsResyncCore(sfses, cacheRevisionAtListStart)- )- } catch (error) {- console.error('[K8s] Periodic SharedFileSystem resync failed:', error)- }- }-- private async runSfsResyncCore(- listedSfses?: SharedFileSystemCRD[],- listedAtCacheRevision?: number- ): Promise<void> {- if (this.stopped) return- try {- const cacheRevisionAtListStart = listedAtCacheRevision ?? this.sharedFileSystemCacheRevision- const sfses = listedSfses ?? (await listAllSharedFileSystems())- let reconcileInventory: SharedFileSystemCRD[]- if (cacheRevisionAtListStart === this.sharedFileSystemCacheRevision) {- this.sharedFileSystems.clear()- for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)- reconcileInventory = sfses- } else {- reconcileInventory = [...this.sharedFileSystems.values()]- }+ this.sharedFileSystems.clear()+ for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)// Capture each SFS's mountability (PVC Bound) BEFORE reconciling so we can// detect a flip. resolveContextMounts() injects the RO mount + podAffinity@@ -3335,11 +2222,11 @@ export class McpServerWatcher implements McpServerProvider {// on phase==='Ready': a transient wfc readiness dip while the PVC stays// Bound must NOT re-roll consumers — see resolveContextMounts.)const wasMountable = new Map<string, boolean>()- for (const sfs of reconcileInventory) {+ for (const sfs of sfses) {wasMountable.set(sfs.name, this.sharedFileSystemReconciler.isMountable(sfs))}- await this.sharedFileSystemReconciler.fullReconcile(reconcileInventory)+ await this.sharedFileSystemReconciler.fullReconcile(sfses)// #592 gap fix: heal the consuming mcp-host Deployments on a mountability// flip — inject the mount + affinity once the PVC binds, or drop it if the@@ -3349,7 +2236,7 @@ export class McpServerWatcher implements McpServerProvider {// an already-Bound SFS picked up on HCC restart still re-injects its mount// within one resync interval instead of waiting for the (much longer) Host// resync.- for (const sfs of reconcileInventory) {+ for (const sfs of sfses) {const nowMountable = this.sharedFileSystemReconciler.isMountable(sfs)if (nowMountable !== (wasMountable.get(sfs.name) ?? false)) {await this.reconcileHostsReferencingSfs(sfs.name)@@ -3360,14 +2247,86 @@ export class McpServerWatcher implements McpServerProvider {}}+ private async runExternalEgressResync(): Promise<void> {+ if (this.stopped) return+ const servers = [...this.servers.values()]+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)+ .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))++ if (servers.length === 0) return++ console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)+ let index = 0+ const workers = Array.from(+ { length: Math.min(EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY, servers.length) },+ async () => {+ while (!this.stopped) {+ const server = servers[index++]+ if (!server) return+ const key = this.externalEgressRetryKey(server)+ if (this.externalEgressInFlight.has(key)) {+ console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)+ continue+ }+ if (this.externalEgressRetryTimers.has(key)) {+ console.log(`[K8s] Skipping external egress resync for ${key}; retry already scheduled`)+ continue+ }+ const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)+ if (jitterMs > 0) await delay(jitterMs)+ try {+ await this.runExternalEgressOnce('MODIFIED', server)+ } catch (error) {+ console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)+ this.scheduleExternalEgressRetry('MODIFIED', server)+ }+ }+ }+ )+ await Promise.all(workers)+ }++ private async reconcileInitialExternalEgressBeforeRuntime(+ servers: McpServerCRD[]+ ): Promise<Set<string>> {+ const failures = new Set<string>()+ const serversWithExternalEgress = servers+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)+ .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))++ if (serversWithExternalEgress.length === 0) return failures++ console.log(+ `[K8s] Reconciling external egress before startup runtime reconciliation for ` ++ `${serversWithExternalEgress.length} McpServer(s)`+ )++ for (const server of serversWithExternalEgress) {+ const key = this.externalEgressRetryKey(server)+ try {+ await this.runExternalEgressOnce('MODIFIED', server)+ } catch (error) {+ failures.add(key)+ console.error(+ `[K8s] Initial external egress reconciliation failed for ${key}; ` ++ 'runtime reconciliation will stay blocked until retry succeeds:',+ error+ )+ this.scheduleExternalEgressRetry('MODIFIED', server)+ }+ }++ return failures+ }+/*** Start watching McpServer CRDs.*/- private getMcpServerWatchCallback(- watchGeneration = this.mcpWatchGeneration- ): (type: string, apiObj: McpServerWatchObject) => Promise<void> {+ private getMcpServerWatchCallback(): (+ type: string,+ apiObj: McpServerWatchObject+ ) => Promise<void> {return async (type, apiObj) => {- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) returnconst server: McpServerCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.namespace,@@ -3381,324 +2340,227 @@ export class McpServerWatcher implements McpServerProvider {console.log(`[K8s] McpServer watch event: ${type} for ${server.name}`)- const previous = this.servers.get(server.name)-// Update cache- let desiredStateChanged = falseif (type === 'ADDED' || type === 'MODIFIED') {- desiredStateChanged =- previous === undefined || !sameMcpServerDesiredRevision(previous, server)this.servers.set(server.name, server)} else if (type === 'DELETED') {- if (!previous?.uid || !server.uid || previous.uid === server.uid) {- desiredStateChanged = this.servers.delete(server.name)- }- }- if (desiredStateChanged) {- this.mcpServerDesiredRevision += 1- void this.runInitialNetworkPolicyConvergence()+ this.servers.delete(server.name)}- // HCC writes McpServer status during reconciliation. Those writes emit- // MODIFIED events but do not change the desired runtime or policy state.- // Suppress that self-induced work while still publishing the fresh status- // through discovery. Any UID/spec/annotation/label change remains live.- if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {- this.changeCallback?.()- return+ // External egress is part of the workload's pre-start contract. Reconcile+ // it before HCC creates or updates any managed runtime Deployment so a+ // stdio MCP with egressBindings cannot start before ExternalEgressReady.+ if (type === 'ADDED' || type === 'MODIFIED') {+ try {+ await this.runExternalEgressOnce(type, server)+ } catch (error) {+ console.error(+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,+ error+ )+ this.scheduleExternalEgressRetry(type, server)+ this.changeCallback?.()+ return+ }}- await this.enqueueMcpServerReconciliation(server, () =>- this.reconcileMcpServerWatchEvent(- type as ExternalEgressWatchEventType,- server,- watchGeneration- )- )- }- }-- private async reconcileMcpServerWatchEvent(- type: ExternalEgressWatchEventType,- server: McpServerCRD,- watchGeneration: number,- retry?: ExternalEgressRetryHandle- ): Promise<void> {- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return- if (retry && !retry.isCurrent()) return- if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {- this.scheduleExternalEgressRetry(type, server)- return- }- const current = this.servers.get(server.name)- if (type === 'DELETED') {- // A same-name recreation may already be accepted while this callback- // waits for its per-server effect. The newer identity owns the key.- if (current) return- } else {- if (!current || current.namespace !== server.namespace) return- server = current- }- const deleteAllowed = () =>- this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)- const contextInventoryGeneration = this.contextWatchGeneration- const contextPolicyInventoryIsCurrent = (): boolean =>- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&- this.hasContextInventoryAuthority(contextInventoryGeneration)-- // External egress is part of the workload's pre-start contract. Reconcile- // it before HCC creates or updates any managed runtime Deployment so a- // stdio MCP with egressBindings cannot start before ExternalEgressReady.- let retryCompletion: ExternalEgressRetryHandle | undefined- if (type === 'ADDED' || type === 'MODIFIED') {- try {- retryCompletion = await this.runExternalEgressOnce(type, server, {- retry,- isCurrent: () =>- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&- (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),- })- } catch (error) {- console.error(- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,- error- )- this.scheduleExternalEgressRetry(type, server)- this.changeCallback?.()- return- }- } else {- if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) return+ // Trigger deployment reconciliationtry {- retryCompletion = await this.runExternalEgressOnce(type, server, {- deleteAllowed,- retry,- isCurrent: () =>- !this.stopped &&- watchGeneration === this.mcpWatchGeneration &&- (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),- })+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.reconciler.reconcile(server)+ } else if (type === 'DELETED') {+ await this.reconciler.reconcileDelete(server.name, server.namespace)+ }} catch (error) {- console.error(- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,- error- )- this.scheduleExternalEgressRetry(type, server)- this.changeCallback?.()- return- }- }-- if (type === 'ADDED' || type === 'MODIFIED') {- if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {- this.scheduleExternalEgressRetry(type, server)- return- }- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- server = this.servers.get(server.name)!- } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {- return- }-- // Trigger deployment reconciliation.- try {- if (type === 'ADDED' || type === 'MODIFIED') {- const currentServer = server- await this.reconciler.reconcile(currentServer, {- isCurrent: () => this.mcpServerWatchEffectIsCurrent(currentServer, watchGeneration),- })- } else if (type === 'DELETED') {- await this.reconciler.reconcileDelete(server.name, server.namespace)+ console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)+ if (+ (type === 'ADDED' || type === 'MODIFIED') &&+ (server.spec.egressBindings?.length ?? 0) > 0+ ) {+ this.scheduleExternalEgressRetry(type, server)+ }}- } catch (error) {- console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)- this.scheduleExternalEgressRetry(type, server)- this.changeCallback?.()- return- }- if (type === 'ADDED' || type === 'MODIFIED') {- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- server = this.servers.get(server.name)!- } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {- return- }-- // Trigger binding policy reconciliation (L3 ingress/egress).- try {- if (type === 'ADDED' || type === 'MODIFIED') {- const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']- if (bindingsJson) {- const bindings: BindingDef[] = JSON.parse(bindingsJson)- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name- const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name- const bindingServer = server- await this.bindingReconciler.reconcileBindings(- recipeName,- bindings,- mcpWorkloadName,- bindingServer.name,- {- isCurrent: () => this.mcpServerWatchEffectIsCurrent(bindingServer, watchGeneration),+ // Trigger binding policy reconciliation (L3 ingress/egress)+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']+ if (bindingsJson) {+ const bindings: BindingDef[] = JSON.parse(bindingsJson)+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name+ const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name+ await this.bindingReconciler.reconcileBindings(+ recipeName,+ bindings,+ mcpWorkloadName,+ server.name+ )+ }+ // Re-reconcile any cached Context that references this server.+ // Fixes race condition where Context MODIFIED arrives before+ // the McpServer ADDED event populates the cache.+ for (const ctx of this.contexts.values()) {+ if (ctx.spec.mcpServers?.includes(server.name)) {+ console.log(+ `[K8s] Re-reconciling context "${ctx.name}" after McpServer "${server.name}" cached`+ )+ await this.netPolReconciler.reconcileContext(ctx)}- )- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- server = this.servers.get(server.name)!+ }+ } else if (type === 'DELETED') {+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name+ await this.bindingReconciler.cleanupBindings(recipeName)+ await this.runExternalEgressOnce(type, server)}- // Re-reconcile cached Contexts after the server cache is populated.- for (const selectedContext of this.contexts.values()) {- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- if (!selectedContext.spec.mcpServers?.includes(server.name)) continue- const selectedContextId = selectedContext.spec.contextId- console.log(- `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`- )- await this.enqueueContextReconciliation(selectedContextId, async () => {- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return- const currentContext = this.contexts.get(selectedContext.name)- if (- currentContext?.spec.contextId !== selectedContextId ||- !currentContext.spec.mcpServers?.includes(server.name)- ) {- return- }- await this.netPolReconciler.reconcileContext(currentContext, {- isCurrent: () =>- contextPolicyInventoryIsCurrent() &&- this.contexts.get(currentContext.name) === currentContext,- })- })+ } catch (error) {+ console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)+ if (type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED') {+ this.scheduleExternalEgressRetry(type, server)}- } else if (type === 'DELETED') {- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name- await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })}- } catch (error) {- console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)- this.scheduleExternalEgressRetry(type, server)++ // Notify listenersthis.changeCallback?.()- return}-- retryCompletion?.complete()- this.changeCallback?.()}- private async startMcpServerWatch(resourceVersion: string): Promise<number> {- this.requireInventoryResourceVersion('McpServer', resourceVersion)+ private async startMcpServerWatch(): Promise<void> {const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_MCPSERVERS}`console.log(`[K8s] Starting McpServer watch`)- const watchGeneration = ++this.mcpWatchGeneration- if (this.mcpWatchRequest) {- this.mcpWatchRequest.abort()- this.mcpWatchRequest = null- }- let watchEnded = false- const watchCallback = this.getMcpServerWatchCallback(watchGeneration)+ const watchCallback = this.getMcpServerWatchCallback()const doneCallback = (err: Error | null) => {- if (watchEnded) return- watchEnded = true- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return- if (!this.retireMcpServerWatch(watchGeneration)) return+ if (this.stopped) returnif (err) {console.error('[K8s] McpServer watch error:', err)}- console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')- this.scheduleMcpServerCacheRecovery()+ console.log('[K8s] McpServer watch ended, restarting...')+ setTimeout(() => this.startMcpServerWatch(), err ? 5000 : 1000)}- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)- if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {- request.abort()- return watchGeneration- }- this.mcpWatchRequest = request- return watchGeneration+ this.mcpWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}- private mcpServerWatchEffectIsCurrent(expected: McpServerCRD, watchGeneration: number): boolean {- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return false- const current = this.servers.get(expected.name)- return current !== undefined && sameMcpServerDesiredRevision(expected, current)+ private externalEgressRetryKey(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {+ return `${server.namespace}/${server.name}`}- private runExternalEgressOnce(- type: ExternalEgressWatchEventType,+ private async runExternalEgressOnce(+ type: string,server: McpServerCRD,- options: {- deleteAllowed?: () => Promise<boolean>- retry?: ExternalEgressRetryHandle- isCurrent?: () => boolean- } = {}- ): Promise<ExternalEgressRetryHandle | undefined> {- return this.externalEgressCoordinator.reconcile(type, server, options)- }+ options: { clearRetryOnSuccess?: boolean } = {}+ ): Promise<void> {+ const clearRetryOnSuccess = options.clearRetryOnSuccess ?? true+ const key = this.externalEgressRetryKey(server)+ const existing = this.externalEgressInFlight.get(key)+ if (existing) {+ console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)+ await existing+ }- private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {- this.externalEgressCoordinator.scheduleRetry(type, server)+ const run = (async () => {+ if (type === 'DELETED') {+ await this.netPolReconciler.cleanupExternalEgress(server.name, server.namespace)+ } else {+ await this.netPolReconciler.reconcileExternalEgress(server)+ }+ })()+ this.externalEgressInFlight.set(key, run)+ try {+ await run+ if (clearRetryOnSuccess) {+ this.clearExternalEgressRetry(server)+ }+ } finally {+ if (this.externalEgressInFlight.get(key) === run) {+ this.externalEgressInFlight.delete(key)+ }+ }}- private async performExternalEgressMutation(- type: ExternalEgressWatchEventType,- server: McpServerCRD,- options: {- deleteAllowed?: () => Promise<boolean>- isCurrent: () => boolean+ private clearExternalEgressRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {+ const key = this.externalEgressRetryKey(server)+ const timer = this.externalEgressRetryTimers.get(key)+ if (timer) {+ clearTimeout(timer)+ this.externalEgressRetryTimers.delete(key)}- ): Promise<void> {- if (type === 'DELETED') {- const deleteAllowed = async (): Promise<boolean> => {- if (!options.isCurrent()) return false- if (options.deleteAllowed && !(await options.deleteAllowed())) return false- return options.isCurrent()- }- await this.netPolReconciler.cleanupExternalEgress(- server.name,- server.namespace,- undefined,- deleteAllowed+ this.externalEgressRetryAttempts.delete(key)+ }++ private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {+ if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return+ if (this.stopped) return++ const key = this.externalEgressRetryKey(server)+ if (this.externalEgressRetryTimers.has(key)) return++ const attempt = (this.externalEgressRetryAttempts.get(key) ?? 0) + 1+ const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]+ if (!delayMs) {+ console.error(+ `[K8s] External egress retry exhausted for McpServer "${server.name}" in namespace "${server.namespace}"`)return}- await this.netPolReconciler.reconcileExternalEgress(server, {- isCurrent: options.isCurrent,- })++ this.externalEgressRetryAttempts.set(key, attempt)+ console.warn(+ `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length} ` ++ `for McpServer "${server.name}" in ${delayMs}ms`+ )++ const timer = setTimeout(() => {+ this.externalEgressRetryTimers.delete(key)+ void this.retryExternalEgress(type, server)+ }, delayMs)+ this.externalEgressRetryTimers.set(key, timer)+ }++ private async retryExternalEgress(type: string, server: McpServerCRD): Promise<void> {+ if (this.stopped) return+ try {+ if (type === 'DELETED') {+ await this.runExternalEgressOnce(type, server)+ } else {+ const current = this.servers.get(server.name)+ if (!current || current.namespace !== server.namespace) {+ this.clearExternalEgressRetry(server)+ return+ }+ await this.runExternalEgressOnce(type, current, { clearRetryOnSuccess: false })+ try {+ await this.reconciler.reconcile(current)+ this.clearExternalEgressRetry(current)+ } catch (error) {+ console.error(+ `[K8s] Runtime reconciliation after external egress retry failed for ${server.name}:`,+ error+ )+ this.scheduleExternalEgressRetry(type, current)+ }+ }+ } catch (error) {+ console.error(`[K8s] External egress retry failed for ${server.name}:`, error)+ this.scheduleExternalEgressRetry(type, server)+ }}/*** Start watching Context CRDs for NetworkPolicy reconciliation.*/- private async startContextWatch(resourceVersion: string): Promise<number> {- this.requireInventoryResourceVersion('Context', resourceVersion)+ private async startContextWatch(): Promise<void> {const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_CONTEXTS}`console.log(`[K8s] Starting Context watch`)- const watchGeneration = ++this.contextWatchGeneration- if (this.ctxWatchRequest) {- this.ctxWatchRequest.abort()- this.ctxWatchRequest = null- }- let watchEnded = falseconst watchCallback = async (type: string,- apiObj: {- metadata: {- name: string- namespace?: string- uid?: string- generation?: number- }- spec: ContextSpec- }+ apiObj: { metadata: { name: string; namespace?: string }; spec: ContextSpec }) => {- if (this.stopped || watchGeneration !== this.contextWatchGeneration) returnconst context: ContextCRD = {name: apiObj.metadata.name,namespace: apiObj.metadata.namespace || config.namespace,- uid: apiObj.metadata.uid,- generation: apiObj.metadata.generation,spec: apiObj.spec,}@@ -3707,168 +2569,58 @@ export class McpServerWatcher implements McpServerProvider {// Capture the previous spec before mutating the cache so we can// re-reconcile any SFS that *was* referenced but is no longer.const previous = this.contexts.get(context.name)- // A scoped Context delta may replace only a prior, fully authoritative- // safety certificate. Capturing it before this event changes desired- // state proves there was no outstanding Context or McpServer revocation- // work that the scoped reconciliation could accidentally certify.- const previousSafetyCertificate = this.currentNetworkPolicySafetyCertificate()// Cache the context for cross-resource re-reconciliation- let desiredStateChanged = false- let deltaSafetyCertificate: NetworkPolicySafetyCertificate | undefinedif (type === 'ADDED' || type === 'MODIFIED') {- desiredStateChanged =- previous === undefined || !sameContextDesiredRevision(previous, context)this.contexts.set(context.name, context)} else if (type === 'DELETED') {- desiredStateChanged = this.contexts.delete(context.name)- }- if (desiredStateChanged) {- this.contextDesiredRevision += 1- // A same-identity Context MODIFIED event can complete its own stale- // allow revocation below. Once that exact delta is current, it is safe- // to restore readiness without waiting for an older pass's additive- // fleet. Deletes and identity changes remain fail-closed until the- // authoritative full pass certifies their broader cleanup.- if (- type === 'MODIFIED' &&- previous?.uid === context.uid &&- previous?.spec.contextId === context.spec.contextId &&- previousSafetyCertificate !== null- ) {- deltaSafetyCertificate = {- contextGeneration: previousSafetyCertificate.contextGeneration,- serverGeneration: previousSafetyCertificate.serverGeneration,- contextRevision: this.contextDesiredRevision,- serverRevision: previousSafetyCertificate.serverRevision,- }- }- void this.runInitialNetworkPolicyConvergence()+ this.contexts.delete(context.name)}// Re-reconcile every SFS this Context referenced before or after the// change so SharedFileSystem.status.mountedByContexts stays in sync// without waiting for an SFS-level event.const nextForSfs = type === 'DELETED' ? undefined : context- const contextInventoryAuthoritative = () => this.hasContextInventoryAuthority(watchGeneration)- const serverInventoryGeneration = this.mcpWatchGeneration- void this.reconcileSharedFileSystemsReferencedByContext(- previous,- nextForSfs,- contextInventoryAuthoritative- )-- const previousContextId = previous?.spec.contextId- const effectContextIds =- previousContextId && previousContextId !== context.spec.contextId- ? [previousContextId, context.spec.contextId]- : [context.spec.contextId]- await this.enqueueContextIdentityReconciliation(effectContextIds, async () => {- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return+ void this.reconcileSharedFileSystemsReferencedByContext(previous, nextForSfs)- try {- if (previousContextId && previousContextId !== context.spec.contextId) {- const oldIdDeleteAllowed = () =>- this.contextIdAbsentForDelete(previousContextId, watchGeneration)- if (await oldIdDeleteAllowed()) {- await this.netPolReconciler.reconcileDeleteContext(- previousContextId,- oldIdDeleteAllowed- )- }- }-- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return- const current = this.contexts.get(context.name)- if (type === 'DELETED') {- if (current) return- } else if (current?.spec.contextId !== context.spec.contextId) {- return- }- if (type === 'ADDED' || type === 'MODIFIED') {- const selectedContext = current!- // A scoped delta may only replace the safety certificate when its- // own stale-allow revocation ran to completion — `reconcileContext`- // aborts mid-pass whenever its authority fence breaks — and when- // the last authoritative pass certified its namespace-wide- // inventory, which a label-scoped delta cannot vouch for.- // `deltaSafetyCertificate` is only ever built for MODIFIED, so this- // is the one branch where it can be recorded.- const scopedRevocationCompleted = await this.netPolReconciler.reconcileContext(- selectedContext,- {- isCurrent: () =>- contextInventoryAuthoritative() &&- this.hasMcpServerInventoryAuthority(serverInventoryGeneration) &&- this.contexts.get(selectedContext.name) === selectedContext,- // Only opt in when there is a certificate to withhold. Without- // one the result below is discarded, so a reported lost fence- // would vanish: no throw to reach the retry, and the recovery- // in the desired-state branch does not run for an event that- // did not move the revision.- honorsLostFence: deltaSafetyCertificate !== undefined,- }- )- if (deltaSafetyCertificate && scopedRevocationCompleted) {- if (this.netPolReconciler.hasCertifiedSafetyInventory()) {- this.recordNetworkPolicySafetyCertificate(deltaSafetyCertificate)- } else {- console.warn(- `[K8s] Not certifying the scoped delta for context "${context.spec.contextId}": the last authoritative safety pass ended without certifying its namespace-wide inventory`- )- }- }- } else if (type === 'DELETED') {- const deleteAllowed = () =>- this.contextAbsentForDelete(context.name, context.namespace, watchGeneration)- if (!(await deleteAllowed())) return- await this.netPolReconciler.reconcileDeleteContext(- context.spec.contextId,- deleteAllowed- )- }- } catch (error) {- console.error(- `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,- error- )- void this.runInitialNetworkPolicyConvergence()- return+ try {+ if (type === 'ADDED' || type === 'MODIFIED') {+ await this.netPolReconciler.reconcileContext(context)+ } else if (type === 'DELETED') {+ await this.netPolReconciler.reconcileDeleteContext(context.spec.contextId)}+ } catch (error) {+ console.error(+ `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,+ error+ )+ }- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return- // Re-reconcile any Host that points at this Context so that changes to- // spec.sharedFileSystems[] propagate into the mcp-host pod template.- try {- await this.reconcileHostsReferencingContext(context.name, contextInventoryAuthoritative)- } catch (error) {- console.error(- `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,- error- )+ // Re-reconcile any Host that points at this Context so that changes to+ // spec.sharedFileSystems[] propagate into the mcp-host pod template.+ try {+ for (const host of this.hosts.values()) {+ if (host.spec.contextRef !== context.name) continue+ await this.hostReconciler.reconcile(host)}- })+ } catch (error) {+ console.error(+ `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,+ error+ )+ }}const doneCallback = (err: Error | null) => {- if (watchEnded) return- watchEnded = true- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return- if (!this.retireContextWatch(watchGeneration)) return+ if (this.stopped) returnif (err) {console.error('[K8s] Context watch error:', err)}- console.log('[K8s] Context watch ended; recovering authoritative inventory...')- this.scheduleContextCacheRecovery()+ console.log('[K8s] Context watch ended, restarting...')+ setTimeout(() => this.startContextWatch(), err ? 5000 : 1000)}- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)- if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {- request.abort()- return watchGeneration- }- this.ctxWatchRequest = request- return watchGeneration+ this.ctxWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)}/**@@ -3970,39 +2722,25 @@ export class McpServerWatcher implements McpServerProvider {*/async stop(): Promise<void> {this.stopped = true- this.externalEgressCoordinator.stop()- this.retireMcpServerWatch()- this.retireContextWatch()this.ccCacheSynced = falsethis.hostCacheSynced = falsethis.hostWatchGeneration += 1this.ccWatchGeneration += 1- this.hostFleetScheduler.stop()+ this.resolveHostFleetShutdown()+ this.hostFleetReconcilePending?.resolve()+ this.hostFleetReconcilePending = nullif (this.ccCacheRecoveryTimer) {clearTimeout(this.ccCacheRecoveryTimer)this.ccCacheRecoveryTimer = null}+ if (this.ccFleetRetryTimer) {+ clearTimeout(this.ccFleetRetryTimer)+ this.ccFleetRetryTimer = null+ }if (this.hostCacheRecoveryTimer) {clearTimeout(this.hostCacheRecoveryTimer)this.hostCacheRecoveryTimer = null}- if (this.sfsWatchRestartTimer) {- clearTimeout(this.sfsWatchRestartTimer)- this.sfsWatchRestartTimer = null- }- if (this.gfsWatchRestartTimer) {- clearTimeout(this.gfsWatchRestartTimer)- this.gfsWatchRestartTimer = null- }- this.hostCacheRecoveryIntent = null- if (this.mcpServerCacheRecoveryTimer) {- clearTimeout(this.mcpServerCacheRecoveryTimer)- this.mcpServerCacheRecoveryTimer = null- }- if (this.contextCacheRecoveryTimer) {- clearTimeout(this.contextCacheRecoveryTimer)- this.contextCacheRecoveryTimer = null- }this.clearAllHostWatchRetries()if (this.resyncTimer) {clearInterval(this.resyncTimer)@@ -4016,11 +2754,16 @@ export class McpServerWatcher implements McpServerProvider {clearInterval(this.gfsResyncTimer)this.gfsResyncTimer = null}- for (const timer of this.initialConvergenceRetryTimers.values()) {+ if (this.externalEgressResyncTimer) {+ clearTimeout(this.externalEgressResyncTimer)+ this.externalEgressResyncTimer = null+ }+ for (const timer of this.externalEgressRetryTimers.values()) {clearTimeout(timer)}- this.initialConvergenceRetryTimers.clear()- this.initialConvergenceRetryAttempts.clear()+ this.externalEgressRetryTimers.clear()+ this.externalEgressRetryAttempts.clear()+ this.externalEgressInFlight.clear()if (this.mcpWatchRequest) {console.log('[K8s] Stopping McpServer watch')this.mcpWatchRequest.abort()diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.tsindex 565c537d3..0e1502aa5 100644--- a/host-context-controller/src/main.ts+++ b/host-context-controller/src/main.ts@@ -18,7 +18,6 @@ import {createMcpServerProvider,getKubeConfig,} from './k8sClient'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'import { ContextMapperServer } from './server'import { StatelessLifecycleTracker } from './statelessLifecycleTracker'import {@@ -114,16 +113,6 @@ async function main(): Promise<void> {const hasDesktopFn = hostReconciler? (hostRef: string) => hostReconciler.hasDesktop(hostRef): undefined- // Dev (no watcher) gets an explicit always-authoritative gate: no inventory- // to certify there, and the server's fail-closed default would otherwise pin- // /ready at 503 forever (R3-B1). See resolveProviderAuthoritativeFn.- const providerAuthoritativeFn = resolveProviderAuthoritativeFn(watcher)- // Desktop status gates on Host inventory authority alone (not full readiness).- // Dev (no watcher) mirrors the provider gate above: there is no Host inventory- // to certify, so authority is unconditional (permissive) — otherwise the- // server's fail-closed default would pin /api/v1/desktop/* at 503 forever in- // dev (R2-M1). See resolveHostAuthoritativeFn.- const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)// Stateless heartbeat consumption — mcp-host pods authenticate their// heartbeats toward control-api's /mcp-host facade (control-api is the@@ -156,23 +145,16 @@ async function main(): Promise<void> {`target=${config.controlApiBaseUrl})`)}- server = new ContextMapperServer(- provider,- config.port,- hostReconciler,- hasDesktopFn,- providerAuthoritativeFn,- hostAuthoritativeFn- )+ server = new ContextMapperServer(provider, config.port, hostReconciler, hasDesktopFn)await server.start()// One-shot legacy sweep: delete the static `clerum-channel-reader`// Deployment if it still exists in the channels namespace. MUST run- // BEFORE provider.start(): provider startup establishes the Host inventory- // watch and schedules its initial fleet convergence, which creates per-Host- // channel-reader Deployments in the background. If the static is still alive- // when that work begins, both pods compete on the same bot's getUpdates and- // Telegram 409s one of them. Sweeping first guarantees zero overlap.+ // BEFORE provider.start(): provider.start() invokes fullReconcile on+ // the initial Host list, which creates per-Host channel-reader Deployments+ // that immediately try to long-poll Telegram. If the static is still+ // alive at that moment, both pods compete on the same bot's getUpdates+ // and Telegram 409s one of them. Sweeping first guarantees zero overlap.// Idempotent; 404 (already gone) is the steady state. See issue #273// for the empirical reproduction.if (!config.devMode && hostReconciler) {@@ -217,11 +199,12 @@ async function main(): Promise<void> {// reconcileChannelReaderRevision is self-contained (does not throw), so// the .catch here is defensive — it catches any unexpected runtime error// in the SecretInformer's own callback path.- if (!config.devMode && watcher) {+ if (!config.devMode && hostReconciler) {const kc = getKubeConfig()if (kc) {+ const reconciler = hostReconcilerchannelSecretInformer = new SecretInformer(kc, config.channelsNamespace, evt => {- watcher.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {+ reconciler.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {console.error('[Main] reconcileChannelReaderRevision failed:', err)})})diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.tsdeleted file mode 100644index 19f75caf3..000000000--- a/host-context-controller/src/mcpServerSafety.test.ts+++ /dev/null@@ -1,267 +0,0 @@-import { describe, expect, it, vi } from 'vitest'-import {- confirmAuthoritativeMcpServerAbsence,- isMcpServerStatusOnlyUpdate,- sameMcpServerDesiredRevision,-} from './mcpServerSafety'-import type { McpServerCRD } from './types'--function makeServer(overrides: Partial<McpServerCRD> = {}): McpServerCRD {- return {- name: 'web-search',- namespace: 'mcp-server',- uid: 'web-search-uid',- generation: 7,- annotations: { 'clerum.io/recipe': 'research' },- labels: { 'clerum.io/workload': 'web-search' },- spec: {- contextRef: 'default',- image: 'clerum/web-search:test',- transport: { type: 'streamableHttp', port: 3000 },- },- ...overrides,- }-}--describe('sameMcpServerDesiredRevision', () => {- it('matches equal desired state while ignoring status and one-sided identity metadata', () => {- const expected = makeServer({- status: { conditions: [{ type: 'Ready', status: 'False' }] },- })- const current = makeServer({- uid: undefined,- generation: undefined,- status: { conditions: [{ type: 'Ready', status: 'True' }] },- })-- expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)- expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)- })-- it('rejects a different resource key', () => {- const expected = makeServer()-- expect(sameMcpServerDesiredRevision(expected, makeServer({ name: 'doc-generator' }))).toBe(- false- )- expect(sameMcpServerDesiredRevision(expected, makeServer({ namespace: 'other' }))).toBe(false)- })-- it('rejects differing UID or generation only when both sides provide the field', () => {- const expected = makeServer()-- expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: 'replacement-uid' }))).toBe(- false- )- expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: 8 }))).toBe(false)- expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: undefined }))).toBe(true)- expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: undefined }))).toBe(true)- })-- it('compares desired object fields by semantic content, not Kubernetes map key order', () => {- const expected = makeServer({- annotations: { first: '1', second: '2' },- labels: { first: '1', second: '2' },- })-- expect(- sameMcpServerDesiredRevision(- expected,- makeServer({- spec: {- ...expected.spec,- image: 'clerum/web-search:next',- },- labels: expected.labels,- })- )- ).toBe(false)- expect(- sameMcpServerDesiredRevision(- expected,- makeServer({- annotations: { second: '2', first: '1' },- labels: expected.labels,- })- )- ).toBe(true)- expect(- sameMcpServerDesiredRevision(- expected,- makeServer({- annotations: { 'clerum.io/recipe': 'other' },- labels: expected.labels,- })- )- ).toBe(false)- expect(- sameMcpServerDesiredRevision(- expected,- makeServer({- labels: { second: '2', first: '1' },- annotations: expected.annotations,- })- )- ).toBe(true)- })-- it('ignores the controller-owned network-ready handshake annotation', () => {- const expected = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })- const current = makeServer({- annotations: {- 'clerum.io/pre-deploy': 'true',- 'clerum.io/network-ready': 'true',- },- })-- expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)- expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)- })-})--describe('isMcpServerStatusOnlyUpdate', () => {- it('accepts only an exact identity and generation with unchanged desired state', () => {- const previous = makeServer({- status: { conditions: [{ type: 'Ready', status: 'False' }] },- })- const current = makeServer({- status: { conditions: [{ type: 'Ready', status: 'True' }] },- })-- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)- })-- it('does not suppress events when UID or generation equality is only one-sided', () => {- const previous = makeServer()-- expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ uid: undefined }))).toBe(false)- expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ generation: undefined }))).toBe(false)- })-- it('does not suppress a desired-state change', () => {- const previous = makeServer()- const current = makeServer({- labels: { 'clerum.io/workload': 'replacement' },- })-- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(false)- })-- it('treats the controller-owned network-ready handshake as a status-only update', () => {- const previous = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })- const current = makeServer({- annotations: {- 'clerum.io/pre-deploy': 'true',- 'clerum.io/network-ready': 'true',- },- })-- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)- })-})--describe('confirmAuthoritativeMcpServerAbsence', () => {- it('stops before the API read when inventory is not authoritative', async () => {- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)- const readCurrent = vi.fn()-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => false,- resolveCurrent,- readCurrent,- })- ).resolves.toBe(false)- expect(resolveCurrent).not.toHaveBeenCalled()- expect(readCurrent).not.toHaveBeenCalled()- })-- it('stops before the API read when the cache already contains the current server', async () => {- const readCurrent = vi.fn()-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => true,- resolveCurrent: () => makeServer(),- readCurrent,- })- ).resolves.toBe(false)- expect(readCurrent).not.toHaveBeenCalled()- })-- it('rejects absence when the API read returns a current object', async () => {- const readCurrent = vi.fn().mockResolvedValue({- metadata: { name: 'web-search', namespace: 'mcp-server' },- })-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => true,- resolveCurrent: () => undefined,- readCurrent,- })- ).resolves.toBe(false)- expect(readCurrent).toHaveBeenCalledOnce()- })-- it('propagates a non-404 API read failure', async () => {- const apiError = Object.assign(new Error('unavailable'), { code: 503 })-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => true,- resolveCurrent: () => undefined,- readCurrent: vi.fn().mockRejectedValue(apiError),- })- ).rejects.toBe(apiError)- })-- it('rejects absence when inventory authority is lost after a 404', async () => {- const inventoryAuthoritative = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative,- resolveCurrent,- readCurrent: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),- })- ).resolves.toBe(false)- expect(resolveCurrent).toHaveBeenCalledOnce()- })-- it('rejects absence when the server is recreated in cache after a 404', async () => {- const resolveCurrent = vi- .fn<() => McpServerCRD | undefined>()- .mockReturnValueOnce(undefined)- .mockReturnValueOnce(makeServer())-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => true,- resolveCurrent,- readCurrent: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),- })- ).resolves.toBe(false)- expect(resolveCurrent).toHaveBeenCalledTimes(2)- })-- it('confirms absence after a 404 when authority and cache absence remain stable', async () => {- const inventoryAuthoritative = vi.fn(() => true)- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)-- await expect(- confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative,- resolveCurrent,- readCurrent: vi.fn().mockRejectedValue({ response: { statusCode: 404 } }),- })- ).resolves.toBe(true)- expect(inventoryAuthoritative).toHaveBeenCalledTimes(2)- expect(resolveCurrent).toHaveBeenCalledTimes(2)- })-})diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.tsdeleted file mode 100644index 7f8d8438d..000000000--- a/host-context-controller/src/mcpServerSafety.ts+++ /dev/null@@ -1,91 +0,0 @@-import type { McpServerCRD } from './types'-import { getErrorCode } from './utils'--const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'--/**- * This acknowledgement is written by HCC after it has applied a server's- * NetworkPolicies so WRC can admit its workload. It is not user intent and it- * cannot change any NetworkPolicy decision. Treating that write as desired- * state makes HCC invalidate its own authoritative revocation pass under a- * large startup fleet, coupling /ready to additive runtime convergence again.- */-function desiredAnnotations(annotations: Record<string, string> | undefined): Record<string, string> {- if (!annotations?.[NETWORK_READY_ANNOTATION]) return annotations ?? {}- const { [NETWORK_READY_ANNOTATION]: _networkReady, ...remaining } = annotations- return remaining-}--/**- * Kubernetes represents objects such as labels and annotations as maps, whose- * wire key order is not meaningful or stable across LIST and WATCH payloads.- * Preserve array order (it is semantic) while sorting object keys so a- * status-only event cannot masquerade as a desired-state revision.- */-function stableJson(value: unknown): string {- if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`- if (value && typeof value === 'object') {- const record = value as Record<string, unknown>- return `{${Object.keys(record)- .sort()- .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)- .join(',')}}`- }- return JSON.stringify(value) ?? 'undefined'-}--export function sameMcpServerDesiredRevision(- expected: McpServerCRD,- current: McpServerCRD-): boolean {- if (expected.name !== current.name || expected.namespace !== current.namespace) return false- if (expected.uid && current.uid && expected.uid !== current.uid) return false- if (- expected.generation !== undefined &&- current.generation !== undefined &&- expected.generation !== current.generation- ) {- return false- }-- return (- stableJson(expected.spec) === stableJson(current.spec) &&- stableJson(desiredAnnotations(expected.annotations)) ===- stableJson(desiredAnnotations(current.annotations)) &&- stableJson(expected.labels ?? {}) === stableJson(current.labels ?? {})- )-}--export function isMcpServerStatusOnlyUpdate(- previous: McpServerCRD,- current: McpServerCRD-): boolean {- return (- previous.uid === current.uid &&- previous.generation === current.generation &&- sameMcpServerDesiredRevision(previous, current)- )-}--export interface AuthoritativeMcpServerAbsenceCheck {- inventoryAuthoritative: () => boolean- resolveCurrent: () => McpServerCRD | undefined- readCurrent: () => Promise<unknown>-}--export async function confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative,- resolveCurrent,- readCurrent,-}: AuthoritativeMcpServerAbsenceCheck): Promise<boolean> {- if (!inventoryAuthoritative() || resolveCurrent()) return false-- try {- await readCurrent()- return false- } catch (error: unknown) {- if (getErrorCode(error) !== 404) throw error- }-- return inventoryAuthoritative() && !resolveCurrent()-}diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 59465c3b3..75b8cc5f0 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -79,30 +79,6 @@ export const networkPoliciesTotal = gauge({labelNames: ['layer'] as const,})-export const initialConvergenceRetriesTotal = counter({- name: 'clerum_hcc_initial_convergence_retries_total',- help: 'Initial background convergence retries scheduled after a failed fleet pass.',- labelNames: ['lane'] as const,-})--export const initialConvergenceLastSuccessTimestampSeconds = gauge({- name: 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',- help: 'Unix timestamp of the last successful initial background convergence pass.',- labelNames: ['lane'] as const,-})--export const networkPolicySafetyPassDurationSeconds = histogram({- name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',- help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',- labelNames: ['outcome'] as const,-})--export const networkPolicySafetyPassPoliciesTotal = counter({- name: 'clerum_hcc_networkpolicy_safety_pass_policies_total',- help: 'NetworkPolicies listed and revoked by authoritative HCC safety passes.',- labelNames: ['operation'] as const,-})-export const contextReconciliationsTotal = counter({name: 'clerum_hcc_context_reconciliations_total',help: 'Total Context CRD reconciliations',@@ -225,13 +201,3 @@ export const hostDeleteCleanupTotal = counter({help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',labelNames: ['outcome'] as const,})--// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding-// whose DNS never resolves is retried forever at the capped (maximum) backoff;-// without this signal the still-denied binding converges nowhere yet stays-// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server-// label — so cardinality stays flat regardless of fleet size.-export const externalEgressRetriesAtCap = gauge({- name: 'clerum_hcc_external_egress_retries_at_cap',- help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',-})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 047ccd88e..8f3d3f3e1 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -4,37 +4,9 @@ import type { RecordWithTtl } from 'node:dns'import * as dns from 'node:dns/promises'import { readFileSync } from 'node:fs'import { join } from 'node:path'-import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'-import {- networkPolicySafetyPassDurationSeconds,- networkPolicySafetyPassPoliciesTotal,-} from './metrics'-import {- NetworkPolicyReconciler,- PUBLIC_EGRESS_EXCEPT_CIDRS,- sameContextDesiredRevision,-} from './networkPolicyReconciler'+import { NetworkPolicyReconciler, PUBLIC_EGRESS_EXCEPT_CIDRS } from './networkPolicyReconciler'import { ContextCRD, McpServerCRD } from './types'-function deferred<T = void>(): {- promise: Promise<T>- resolve: (value: T | PromiseLike<T>) => void-} {- let resolve!: (value: T | PromiseLike<T>) => void- const promise = new Promise<T>(res => {- resolve = res- })- return { promise, resolve }-}--vi.mock('./mcpServerSafety', async () => {- const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')- return {- ...actual,- confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),- }-})-// resolve4({ ttl: true }) returns RecordWithTtl[]; `rec` builds that shape and// `resolve4Mock` sidesteps the string[]/RecordWithTtl[] overload inference.const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))@@ -50,7 +22,6 @@ vi.mock('./config', () => ({port: 8081,runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],k8sApiCidrs: [],- externalEgressDnsResolveTimeoutMs: 5_000,// #299 sliding-window knobs read by reconcileExternalEgress.externalEgressOverlapSec: 300,externalEgressMaxEntries: 128,@@ -68,23 +39,10 @@ vi.mock('node:dns/promises', () => ({]),}))-vi.mock('./metrics', () => ({- networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },- networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },-}))-function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {- if (- name === 'allow-desktop-egress-rpc-proxy' ||- name === 'allow-rpc-proxy-to-managed-mcp-servers'- ) {- throw Object.assign(new Error('not found'), { code: 404 })- }- return { metadata: { resourceVersion: '1' } }- }),+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -93,17 +51,7 @@ function makeMockNetworkingApi() {function makeMockCustomApi() {return {- getNamespacedCustomObject: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),- listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),- getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({- metadata: {- generation: 7,- resourceVersion: 'status-rv-default',- },- status: {},- }),+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({ status: {} }),patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),}}@@ -122,48 +70,7 @@ function makeReconciler(})const cache = serverCache ?? new Map()const reconciler = new NetworkPolicyReconciler(kc, cache)- const listedPolicies = new Map<string, k8s.V1NetworkPolicy>()- const networkingApi = {- ...mockApi,- listNamespacedNetworkPolicy: async (args: { namespace: string }) => {- const response = await mockApi.listNamespacedNetworkPolicy(args)- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {- const name = policy.metadata?.name ?? `unnamed-${index}`- const materialized = {- ...policy,- metadata: {- ...policy.metadata,- name,- namespace: policy.metadata?.namespace ?? args.namespace,- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,- resourceVersion: policy.metadata?.resourceVersion ?? '1',- },- }- listedPolicies.set(`${args.namespace}/${name}`, materialized)- return materialized- })- return { ...response, items }- },- readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {- const response = await mockApi.readNamespacedNetworkPolicy(args)- const listed = listedPolicies.get(`${args.namespace}/${args.name}`)- return {- ...listed,- ...response,- metadata: {- ...listed?.metadata,- ...response.metadata,- name: response.metadata?.name ?? listed?.metadata?.name ?? args.name,- namespace: response.metadata?.namespace ?? listed?.metadata?.namespace ?? args.namespace,- uid:- response.metadata?.uid ?? listed?.metadata?.uid ?? `${args.namespace}:${args.name}:uid`,- resourceVersion:- response.metadata?.resourceVersion ?? listed?.metadata?.resourceVersion ?? '1',- },- }- },- }- ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = mockApi;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApireturn reconciler}@@ -174,7 +81,6 @@ describe('NetworkPolicyReconciler', () => {let reconciler: NetworkPolicyReconcilerbeforeEach(() => {- vi.useRealTimers()mockApi = makeMockNetworkingApi()mockCustomApi = makeMockCustomApi()reconciler = makeReconciler(mockApi, undefined, mockCustomApi)@@ -182,102 +88,6 @@ describe('NetworkPolicyReconciler', () => {resolve4Mock.mockResolvedValue(rec('1.2.3.4', '5.6.7.8'))})- it('compares Context desired revisions canonically and fences identity changes', () => {- const expected: ContextCRD = {- name: 'default',- namespace: 'mcp-server',- uid: 'context-uid',- generation: 7,- spec: {- contextId: 'default',- mcpServers: ['alpha'],- sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace' }],- },- }- const reordered: ContextCRD = {- namespace: 'mcp-server',- name: 'default',- generation: 7,- uid: 'context-uid',- spec: {- sharedFileSystems: [{ mountPath: '/workspace', name: 'workspace' }],- mcpServers: ['alpha'],- contextId: 'default',- },- }-- expect(sameContextDesiredRevision(expected, reordered)).toBe(true)- expect(sameContextDesiredRevision(expected, { ...reordered, uid: 'replacement-uid' })).toBe(- false- )- expect(sameContextDesiredRevision(expected, { ...reordered, generation: 8 })).toBe(false)- })-- it('records the authoritative safety-pass inventory, completed revocations, and duration', async () => {- mockApi.listNamespacedNetworkPolicy- .mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-context',- labels: { 'clerum.io/context': 'missing-context' },- },- },- ],- })- .mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-host-egress',- labels: { 'clerum.io/context': 'missing-context' },- },- },- ],- })- .mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-rpc-egress',- labels: { 'clerum.io/context': 'missing-context' },- },- },- ],- })- .mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-external-egress',- labels: { 'clerum.io/mcpserver': 'missing-server' },- },- },- ],- })- const complete = vi.fn()-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- onAuthoritativeRevocationComplete: complete,- })-- expect(complete).toHaveBeenCalledOnce()- expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(- { operation: 'listed' },- 4- )- expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(- { operation: 'revoked' },- 4- )- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(- { outcome: 'completed' },- expect.any(Number)- )- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)- })-describe('public egress blocklist parity', () => {it('keeps the HCC and WRC IPv4 exclusion lists in sync', () => {const wrcFactory = readFileSync(@@ -340,50 +150,6 @@ describe('NetworkPolicyReconciler', () => {// ─── L1: Infrastructure Policies ───────────────────────────────────describe('L1 — ensureInfrastructurePolicies', () => {- it.each([- ['deny-all-mcp-server', 'default-deny'],- ['allow-dns-egress-mcp-server', 'infrastructure'],- ['allow-hcc-api-egress-mcp-server', 'infrastructure'],- ['allow-k8s-api-egress-mcp-server', 'infrastructure'],- ])(- 'refuses to adopt a foreign policy that collides with %s',- async (targetName, policyType) => {- mockApi.createNamespacedNetworkPolicy.mockImplementation(- async ({ body }: { body: k8s.V1NetworkPolicy }) => {- if (body.metadata?.name === targetName) {- throw Object.assign(new Error('already exists'), { code: 409 })- }- return {}- }- )- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {- if (- name === 'allow-desktop-egress-rpc-proxy' ||- name === 'allow-rpc-proxy-to-managed-mcp-servers'- ) {- throw Object.assign(new Error('not found'), { code: 404 })- }- return {- metadata: {- name,- namespace,- uid: 'foreign-baseline-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': policyType,- },- },- }- })-- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ name: targetName })- )- }- )-it('creates DNS egress policies in every runtime namespace', async () => {await reconciler.ensureDefaultPolicies()@@ -542,204 +308,34 @@ describe('NetworkPolicyReconciler', () => {})it('removes legacy static policies now replaced by generated scoped policies', async () => {- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({- metadata: {- name,- namespace,- uid: `${name}-uid`,- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- },- },- }))-await reconciler.ensureDefaultPolicies()expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({namespace: 'rpc-proxy',name: 'allow-desktop-egress-rpc-proxy',- body: {- preconditions: {- uid: 'allow-desktop-egress-rpc-proxy-uid',- resourceVersion: '7',- },- },})expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({namespace: 'mcp-server',name: 'allow-rpc-proxy-to-managed-mcp-servers',- body: {- preconditions: {- uid: 'allow-rpc-proxy-to-managed-mcp-servers-uid',- resourceVersion: '7',- },- },})})- it.each([- ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],- ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],- ])('refuses to delete a foreign legacy policy %s/%s', async (targetNamespace, targetName) => {- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {- if (name !== targetName) {- throw Object.assign(new Error('not found'), { code: 404 })- }- return {- metadata: {- name,- namespace,- uid: 'foreign-legacy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- },- },+ it('fails closed when legacy static policy cleanup fails', async () => {+ const cleanupError = Object.assign(new Error('rbac denied'), { code: 403 })+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === 'allow-rpc-proxy-to-managed-mcp-servers') {+ throw cleanupError}+ return {}})- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ namespace: targetNamespace, name: targetName })- )+ await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(cleanupError)})-- it.each([- ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],- ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],- ])(- 'propagates an identity conflict deleting legacy policy %s/%s',- async (targetNamespace, targetName) => {- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {- if (name !== targetName) {- throw Object.assign(new Error('not found'), { code: 404 })- }- return {- metadata: {- name,- namespace,- uid: 'legacy-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- },- },- }- })- const conflict = Object.assign(new Error('identity changed'), { code: 409 })- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {- if (name === targetName && namespace === targetNamespace) throw conflict- return {}- })-- await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(conflict)- }- )})// ─── L1: Allow API ─────────────────────────────────────────────────describe('L1 — ensureAllowContextMapperApi', () => {- it('updates the existing HCC-owned allow-api policy without requiring Context owner labels', async () => {- mockApi.createNamespacedNetworkPolicy.mockImplementation(- async ({ body }: { body: k8s.V1NetworkPolicy }) => {- if (body.metadata?.name === 'allow-host-context-controller-api') {- throw Object.assign(new Error('already exists'), { code: 409 })- }- return {}- }- )- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({- metadata: {- name,- namespace,- uid: 'existing-default-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'allow-api',- },- },- }))-- await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'allow-host-context-controller-api',- namespace: 'mcp-server',- })- )- })-- it('refuses to adopt a foreign policy that collides with the allow-api policy name', async () => {- mockApi.createNamespacedNetworkPolicy.mockImplementation(- async ({ body }: { body: k8s.V1NetworkPolicy }) => {- if (body.metadata?.name === 'allow-host-context-controller-api') {- throw Object.assign(new Error('already exists'), { code: 409 })- }- return {}- }- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({- metadata: {- name: 'allow-host-context-controller-api',- namespace: 'mcp-server',- uid: 'foreign-default-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'allow-api',- },- },- })-- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('revalidates allow-api ownership after a replace conflict', async () => {- mockApi.createNamespacedNetworkPolicy.mockImplementation(- async ({ body }: { body: k8s.V1NetworkPolicy }) => {- if (body.metadata?.name === 'allow-host-context-controller-api') {- throw Object.assign(new Error('already exists'), { code: 409 })- }- return {}- }- )- mockApi.readNamespacedNetworkPolicy- .mockResolvedValueOnce({- metadata: {- name: 'allow-host-context-controller-api',- namespace: 'mcp-server',- uid: 'owned-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'allow-api',- },- },- })- .mockResolvedValueOnce({- metadata: {- name: 'allow-host-context-controller-api',- namespace: 'mcp-server',- uid: 'foreign-uid',- resourceVersion: '8',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'allow-api',- },- },- })- mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('conflict'), { code: 409 })- )-- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- })-it('creates allow-api ingress policy using _from (K8s client convention)', async () => {await reconciler.ensureDefaultPolicies()@@ -759,82 +355,6 @@ describe('NetworkPolicyReconciler', () => {// ─── L2: Context-Based Policies ────────────────────────────────────describe('L2 — reconcileContext', () => {- it('refuses to adopt a foreign policy that collides with a live Context policy name', async () => {- const server: McpServerCRD = {- name: 'mongo',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'mongo:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]))- const context: ContextCRD = {- name: 'dev',- namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: [server.name] },- }- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: 'ctx-dev-mongo',- namespace: 'mcp-server',- uid: 'foreign-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'dev',- 'clerum.io/mcpserver': 'mongo',- },- },- })-- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('refuses to replace an HCC Context policy owned by a different Context/server pair', async () => {- const server: McpServerCRD = {- name: 'b-c',- namespace: 'mcp-server',- spec: {- contextRef: 'a',- image: 'mongo:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]))- const context: ContextCRD = {- name: 'a',- namespace: 'mcp-server',- spec: { contextId: 'a', mcpServers: ['b-c'] },- }- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: 'ctx-a-b-c',- namespace: 'mcp-server',- uid: 'other-hcc-owner-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'a-b',- 'clerum.io/mcpserver': 'c',- },- },- })-- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-it('creates context-allow ingress policy per server in mcp-server namespace', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {@@ -854,7 +374,7 @@ describe('NetworkPolicyReconciler', () => {spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context, { isCurrent: () => true })+ await rec.reconcileContext(context)// First call: ingress in mcp-server namespaceconst ingressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {@@ -896,352 +416,106 @@ describe('NetworkPolicyReconciler', () => {)})- it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {- const oldServer: McpServerCRD = {+ it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {+ const cache = new Map<string, McpServerCRD>()+ cache.set('mongo', {name: 'mongo',namespace: 'mcp-server',spec: {contextRef: 'dev',image: 'mongo:latest',- transport: { type: 'streamableHttp', port: 3000 },+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },},- }- const cache = new Map([[oldServer.name, oldServer]])+ })+const rec = makeReconciler(mockApi, cache)const context: ContextCRD = {name: 'dev',namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: [oldServer.name] },+ spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context, { isCurrent: () => true })- const livePolicies = new Map(- mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body- return [`${policy.metadata?.namespace}/${policy.metadata?.name}`, policy]- })- )+ await rec.reconcileContext(context)- vi.clearAllMocks()- cache.set(oldServer.name, {- ...oldServer,- generation: 2,- spec: {- ...oldServer.spec,- transport: { type: 'streamableHttp', port: 4000 },- },- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => ({- items: [...livePolicies.values()].filter(- policy => policy.metadata?.namespace === namespace- ),- })- )- mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {- livePolicies.set(`${namespace}/${body.metadata?.name}`, body)- return {}- })- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, name, body }) => {- livePolicies.set(`${namespace}/${name}`, body)- return {}+ // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {+ namespace: string+ body: {+ metadata: { name: string; namespace: string; labels: Record<string, string> }+ spec: {+ podSelector: { matchLabels: Record<string, string> }+ policyTypes: string[]+ egress: Array<{+ to: Array<{+ namespaceSelector: { matchLabels: Record<string, string> }+ podSelector: { matchLabels: Record<string, string> }+ }>+ ports: Array<{ port: number }>+ }>+ }}- )- // The delete mock MUST remove from livePolicies, or the port-4000- // assertion below survives a spurious deletion and the test passes with- // the bug reinjected (R3-H7).- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {- livePolicies.delete(`${namespace}/${name}`)- return {}+ }+ expect(egressCall.namespace).toBe('mcp-host')+ expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')+ expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')+ expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'dev',})-- await rec.reconcileContext(context, { isCurrent: () => true })-- // Filter the real calls by name+namespace: the production delete always- // carries `preconditions`, so `not.toHaveBeenCalledWith({name,namespace})`- // never matched any call and was vacuously true (R3-H7).- const deletedCtxDevMongo = mockApi.deleteNamespacedNetworkPolicy.mock.calls.some(- call =>- (call[0] as { name?: string; namespace?: string })?.name === 'ctx-dev-mongo' &&- (call[0] as { name?: string; namespace?: string })?.namespace === 'mcp-server'+ // Egress targets mcp-server namespace + specific server pod+ const egressRule = egressCall.body.spec.egress[0]+ expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(+ 'mcp-server')- expect(deletedCtxDevMongo).toBe(false)- const ingress = livePolicies.get('mcp-server/ctx-dev-mongo')- expect(ingress?.spec?.ingress?.[0]?.ports).toEqual([expect.objectContaining({ port: 4000 })])+ expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')+ expect(egressRule.ports[0].port).toBe(3000)})- it('stops remaining policy mutations after its combined authority lease is retired', async () => {+ it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',namespace: 'mcp-server',spec: {- contextRef: 'dev',+ contextRef: 'ctx-name',image: 'mongo:latest',- transport: { type: 'streamableHttp', port: 3000 },+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },},})- const firstCreateStarted = deferred()- const releaseFirstCreate = deferred()- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- firstCreateStarted.resolve(undefined)- await releaseFirstCreate.promise- return {}- })+const rec = makeReconciler(mockApi, cache)- let contextGeneration = 4- let serverGeneration = 8- const capturedContextGeneration = contextGeneration- const capturedServerGeneration = serverGenerationconst context: ContextCRD = {- name: 'dev',+ name: 'ctx-name',namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['mongo'] },+ spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },}- const reconcile = rec.reconcileContext(context, {- isCurrent: () =>- contextGeneration === capturedContextGeneration &&- serverGeneration === capturedServerGeneration,- })- await firstCreateStarted.promise-- contextGeneration = 6- serverGeneration = 10- releaseFirstCreate.resolve(undefined)- await reconcile+ await rec.reconcileContext(context)- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {+ body: {+ metadata: { labels: Record<string, string> }+ spec: { podSelector: { matchLabels: Record<string, string> } }+ }+ }+ expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/context': 'ctx-name',+ })})- it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {+ it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',namespace: 'mcp-server',- uid: 'mongo-uid',- generation: 1,spec: {contextRef: 'dev',- image: 'mongo:old',- transport: { type: 'streamableHttp', port: 3000 },- },- })- const firstCreateStarted = deferred()- const releaseFirstCreate = deferred()- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- firstCreateStarted.resolve(undefined)- await releaseFirstCreate.promise- return {}- })- const rec = makeReconciler(mockApi, cache)- const context: ContextCRD = {- name: 'dev',- namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['mongo'] },- }-- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })- await firstCreateStarted.promise- cache.set('mongo', {- name: 'mongo',- namespace: 'mcp-server',- uid: 'mongo-uid',- generation: 2,- spec: {- contextRef: 'dev',- image: 'mongo:new',- transport: { type: 'streamableHttp', port: 4000 },- },- })- releaseFirstCreate.resolve(undefined)- await reconcile-- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('continues a Context policy pass across a status-only McpServer replacement', async () => {- const cache = new Map<string, McpServerCRD>()- const selected: McpServerCRD = {- name: 'mongo',- namespace: 'mcp-server',- uid: 'mongo-uid',- generation: 1,- spec: {- contextRef: 'dev',- image: 'mongo:stable',- transport: { type: 'streamableHttp', port: 3000 },- },- status: { conditions: [{ type: 'Ready', status: 'False' }] },- }- cache.set(selected.name, selected)- const firstCreateStarted = deferred()- const releaseFirstCreate = deferred()- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- firstCreateStarted.resolve(undefined)- await releaseFirstCreate.promise- return {}- })- const rec = makeReconciler(mockApi, cache)- const context: ContextCRD = {- name: 'dev',- namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['mongo'] },- }-- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })- await firstCreateStarted.promise- cache.set(selected.name, {- ...selected,- status: { conditions: [{ type: 'Ready', status: 'True' }] },- })- releaseFirstCreate.resolve(undefined)- await reconcile-- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)- })-- it('does not adopt an McpServer that appeared after Context selection', async () => {- const cache = new Map<string, McpServerCRD>()- const inventoryListed = deferred()- const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()- mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {- inventoryListed.resolve(undefined)- return releaseInventory.promise- })- const rec = makeReconciler(mockApi, cache)- const context: ContextCRD = {- name: 'dev',- namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['late-server'] },- }-- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })- await inventoryListed.promise- cache.set('late-server', {- name: 'late-server',- namespace: 'mcp-server',- uid: 'late-server-uid',- generation: 1,- spec: {- contextRef: 'dev',- image: 'late-server:new',- transport: { type: 'streamableHttp', port: 3000 },- },- })- releaseInventory.resolve({ items: [] })- await reconcile-- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {- const cache = new Map<string, McpServerCRD>()- cache.set('mongo', {- name: 'mongo',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },- },- })-- const rec = makeReconciler(mockApi, cache)- const context: ContextCRD = {- name: 'dev',- namespace: 'mcp-server',- spec: { contextId: 'dev', mcpServers: ['mongo'] },- }-- await rec.reconcileContext(context, { isCurrent: () => true })-- // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {- namespace: string- body: {- metadata: { name: string; namespace: string; labels: Record<string, string> }- spec: {- podSelector: { matchLabels: Record<string, string> }- policyTypes: string[]- egress: Array<{- to: Array<{- namespaceSelector: { matchLabels: Record<string, string> }- podSelector: { matchLabels: Record<string, string> }- }>- ports: Array<{ port: number }>- }>- }- }- }- expect(egressCall.namespace).toBe('mcp-host')- expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')- expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')- expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'dev',- })- // Egress targets mcp-server namespace + specific server pod- const egressRule = egressCall.body.spec.egress[0]- expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(- 'mcp-server'- )- expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')- expect(egressRule.ports[0].port).toBe(3000)- })-- it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {- const cache = new Map<string, McpServerCRD>()- cache.set('mongo', {- name: 'mongo',- namespace: 'mcp-server',- spec: {- contextRef: 'ctx-name',- image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },- },- })-- const rec = makeReconciler(mockApi, cache)- const context: ContextCRD = {- name: 'ctx-name',- namespace: 'mcp-server',- spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },- }-- await rec.reconcileContext(context, { isCurrent: () => true })-- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {- body: {- metadata: { labels: Record<string, string> }- spec: { podSelector: { matchLabels: Record<string, string> } }- }- }- expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'ctx-name',- })- })-- it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {- const cache = new Map<string, McpServerCRD>()- cache.set('mongo', {- name: 'mongo',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'mongo:latest',- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },+ image: 'mongo:latest',+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },},})@@ -1252,7 +526,7 @@ describe('NetworkPolicyReconciler', () => {spec: { contextId: 'dev', mcpServers: ['mongo'] },}- await rec.reconcileContext(context, { isCurrent: () => true })+ await rec.reconcileContext(context)const rpcProxyEgressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[2][0] as {namespace: string@@ -1292,118 +566,6 @@ describe('NetworkPolicyReconciler', () => {// ─── L3: External Egress ───────────────────────────────────────────describe('L3 — reconcileExternalEgress', () => {- it('removes a deterministic safety policy created after its desired-state fence is lost', async () => {- const server: McpServerCRD = {- name: 'race-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'race:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],- },- }- const createStarted = deferred()- const releaseCreate = deferred()- let current = true- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- createStarted.resolve(undefined)- await releaseCreate.promise- return {- metadata: {- name: 'ext-egress-race-server-104-18-0-0-16-443',- namespace: server.namespace,- uid: 'created-race-policy',- resourceVersion: '7',- },- }- })-- const safety = (reconciler as any).reconcileExternalEgressSafety(- server,- [{ metadata: { name: 'ext-egress-race-server-stale' } }],- () => current- ) as Promise<boolean>-- await createStarted.promise- current = false- releaseCreate.resolve(undefined)-- expect(await safety).toBe(false)- const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: created.metadata.name,- namespace: server.namespace,- body: { preconditions: { uid: 'created-race-policy', resourceVersion: '7' } },- })- )- })-- it('refuses to adopt a foreign policy that collides with an external-egress name', async () => {- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],- },- }- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: 'ext-egress-openai-mcp-104-18-0-0-16-443',- namespace: 'mcp-server',- uid: 'foreign-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': server.name,- },- },- })-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('refuses to replace an HCC external-egress policy owned by a different server', async () => {- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],- },- }- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: 'ext-egress-openai-mcp-104-18-0-0-16-443',- namespace: 'mcp-server',- uid: 'other-hcc-owner-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': 'different-server',- },- },- })-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-it('creates egress policy for CIDR binding', async () => {const server: McpServerCRD = {name: 'openai-mcp',@@ -1416,7 +578,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body@@ -1439,7 +601,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body@@ -1477,69 +639,35 @@ describe('NetworkPolicyReconciler', () => {)})- it('does not mutate policy or status after its McpServer authority lease is retired', async () => {- const dnsStarted = deferred()- const releaseDns = deferred<string[]>()- vi.mocked(dns.resolve4).mockImplementationOnce(async () => {- dnsStarted.resolve(undefined)- return releaseDns.promise- })- let inventoryGeneration = 7- const capturedGeneration = inventoryGeneration+ it('fails closed when ExternalEgressReady status cannot be written', async () => {+ const statusError = new Error('status update denied')+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)const server: McpServerCRD = {- name: 'authority-race-mcp',+ name: 'openai-mcp',namespace: 'mcp-server',- uid: 'authority-race-uid',- generation: 3,+ generation: 7,spec: {contextRef: 'dev',- image: 'authority-race:test',- transport: { type: 'streamableHttp', port: 3000 },+ image: 'openai:latest',+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },egressBindings: [{ dns: 'api.openai.com', port: 443 }],},}- const reconcile = reconciler.reconcileExternalEgress(server, {- isCurrent: () => inventoryGeneration === capturedGeneration,- })- await dnsStarted.promise-- // Retire and recover to a different generation. The recovered cache is- // authoritative again, but never for this stale pass.- inventoryGeneration = 9- releaseDns.resolve(['1.2.3.4'])- await reconcile+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(+ 'status update denied'+ )- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()})- it('identity-fences the status patch and preserves unrelated conditions', async () => {- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- metadata: {- uid: 'openai-mcp-uid',- generation: 7,- resourceVersion: 'status-rv-17',- },- status: {- conditions: [- {- type: 'RuntimeReady',- status: 'True',- reason: 'DeploymentAvailable',- message: 'Runtime deployment is available',- lastTransitionTime: '2026-07-29T00:00:00.000Z',- observedGeneration: 7,- },- ],- },- })+ it('fails closed when ExternalEgressReady status cannot be read', async () => {+ const statusError = new Error('status read denied')+ mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)const server: McpServerCRD = {name: 'openai-mcp',namespace: 'mcp-server',- uid: 'openai-mcp-uid',generation: 7,spec: {contextRef: 'dev',@@ -1549,194 +677,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })-- const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body- expect(patch.slice(0, 3)).toEqual([- { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },- { op: 'test', path: '/metadata/generation', value: 7 },- { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },- ])- expect(patch).toEqual(- expect.arrayContaining([- expect.objectContaining({- path: '/status/conditions',- value: expect.arrayContaining([- expect.objectContaining({- type: 'RuntimeReady',- reason: 'DeploymentAvailable',- }),- expect.objectContaining({- type: 'ExternalEgressReady',- status: 'True',- observedGeneration: 7,- }),- ]),- }),- ])- )- })-- it('fences every identity field read from status when synthetic input omits them', async () => {- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- metadata: {- uid: 'openai-mcp-uid',- generation: 7,- resourceVersion: 'status-rv-17',- },- status: {},- })- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })-- const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body- expect(patch.slice(0, 3)).toEqual([- { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },- { op: 'test', path: '/metadata/generation', value: 7 },- { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },- ])- })-- it.each([- {- name: 'UID',- currentMetadata: {- uid: 'replacement-uid',- generation: 7,- resourceVersion: 'status-rv-18',- },- },- {- name: 'generation',- currentMetadata: {- uid: 'openai-mcp-uid',- generation: 8,- resourceVersion: 'status-rv-18',- },- },- ])(- 'rejects a stale $name before patching external egress status',- async ({ currentMetadata }) => {- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- metadata: currentMetadata,- status: {},- })- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- uid: 'openai-mcp-uid',- generation: 7,- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/stale McpServer/i)-- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- }- )-- it('fails closed when a same-generation status writer wins the resourceVersion race', async () => {- const conflict = Object.assign(new Error('JSON patch test failed'), { code: 409 })- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- metadata: {- uid: 'openai-mcp-uid',- generation: 7,- resourceVersion: 'status-rv-17',- },- status: {- conditions: [- {- type: 'RuntimeReady',- status: 'True',- reason: 'DeploymentAvailable',- lastTransitionTime: '2026-07-29T00:00:00.000Z',- },- ],- },- })- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict)- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- uid: 'openai-mcp-uid',- generation: 7,- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toBe(conflict)-- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(- expect.objectContaining({- body: expect.arrayContaining([- {- op: 'test',- path: '/metadata/resourceVersion',- value: 'status-rv-17',- },- ]),- })- )- })-- it('fails closed when ExternalEgressReady status cannot be written', async () => {- const statusError = new Error('status update denied')- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- generation: 7,- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(- 'status update denied'- )-- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()- })-- it('fails closed when ExternalEgressReady status cannot be read', async () => {- const statusError = new Error('status read denied')- mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)- const server: McpServerCRD = {- name: 'openai-mcp',- namespace: 'mcp-server',- generation: 7,- spec: {- contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('status read denied')+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('status read denied')expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()@@ -1754,7 +695,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(dns.resolve4).not.toHaveBeenCalled()expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)@@ -1796,7 +737,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/public-web external egress bindings/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -1814,7 +755,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/public-web external egress bindings/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -1852,7 +793,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})@@ -1898,7 +839,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})@@ -1923,7 +864,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].bodyconst resolvedPatch = patch.find(@@ -1948,7 +889,7 @@ describe('NetworkPolicyReconciler', () => {}// First reconcile bootstraps and writes the external-egress policy.- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -1975,7 +916,7 @@ describe('NetworkPolicyReconciler', () => {// Second resync: same DNS answers → same IP set → must NOT rewrite the// policy (a timestamp-only refresh is a no-op). This is the churn the audit// caught: before the fix, RESOLVED_AT + renewed expiry forced a write here.- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -1991,7 +932,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -2023,7 +964,7 @@ describe('NetworkPolicyReconciler', () => {// Same DNS → set unchanged (changed=false) but window aging → renewalDue=true// → the policy MUST be re-persisted (deleting `|| egressRenewalDue` breaks this,// reintroducing the M1 overlap-grace loss).- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const wrote =mockApi.createNamespacedNetworkPolicy.mock.calls.length +mockApi.replaceNamespacedNetworkPolicy.mock.calls.length@@ -2041,7 +982,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -2094,7 +1035,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)// The reconcile must rewrite the policy WITHOUT the private IP.const call =mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??@@ -2119,7 +1060,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ dns: 'api.example.com', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)// The min observed TTL (ms) must reflect the 15s answer — deleting the// Math.min update leaves it Infinity and the resync degrades to the fixed// interval (the #299 under-sampling failure mode).@@ -2139,7 +1080,7 @@ describe('NetworkPolicyReconciler', () => {}// Round 1: DNS serves A only.resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.3', ttl: 15 }])- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const written = (mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }).body@@ -2152,7 +1093,7 @@ describe('NetworkPolicyReconciler', () => {// Round 2: DNS rotates to B. The written policy must be the UNION {A, B} —// this only holds if previousAnnotations rehydration is actually wired in.resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.4', ttl: 15 }])- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const call =mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??mockApi.createNamespacedNetworkPolicy.mock.calls[0]?.[0]@@ -2174,7 +1115,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -2185,7 +1126,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -2205,7 +1146,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -2222,7 +1163,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -2238,7 +1179,7 @@ describe('NetworkPolicyReconciler', () => {egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {body: k8s.V1NetworkPolicy}@@ -2257,7 +1198,7 @@ describe('NetworkPolicyReconciler', () => {mockApi.createNamespacedNetworkPolicy.mockClear()mockApi.replaceNamespacedNetworkPolicy.mockClear()- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)// The reconciler must rewrite the policy back to the declared CIDR.const wrote =mockApi.createNamespacedNetworkPolicy.mock.calls.length +@@ -2278,7 +1219,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/External egress reconciliation failed/)}@@ -2299,7 +1240,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/resolved disallowed address/)@@ -2334,7 +1275,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/private, internal, metadata/)@@ -2355,7 +1296,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/resolved disallowed address/)@@ -2381,15 +1322,13 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-openai-mcp-old-example-com-443',- namespace: 'mcp-server',- })- )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'ext-egress-openai-mcp-old-example-com-443',+ namespace: 'mcp-server',+ })})it('fails closed when an orphaned external egress policy cannot be deleted', async () => {@@ -2412,7 +1351,7 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('delete denied')+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('delete denied')})it('deletes stale policies and fails when DNS resolution for that binding fails', async () => {@@ -2435,21 +1374,17 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/dns timeout/)+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/dns timeout/)expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-openai-mcp-api.openai.com-443',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-openai-mcp-old-example-com-443',- namespace: 'mcp-server',- })- )+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'ext-egress-openai-mcp-api.openai.com-443',+ namespace: 'mcp-server',+ })+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'ext-egress-openai-mcp-old-example-com-443',+ namespace: 'mcp-server',+ })expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(expect.objectContaining({body: expect.arrayContaining([@@ -2480,36 +1415,13 @@ describe('NetworkPolicyReconciler', () => {},}- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/must declare dns or cidr/)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()})- it('bounds DNS resolution so a stuck resolver cannot block reconciliation indefinitely', async () => {- vi.useFakeTimers()- vi.mocked(dns.resolve4).mockImplementationOnce(() => new Promise(() => undefined))- const server: McpServerCRD = {- name: 'stuck-dns-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'stuck-dns:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],- },- }-- const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })- const rejection = expect(reconcile).rejects.toThrow(/DNS resolution timed out after 5000ms/)- await vi.advanceTimersByTimeAsync(5_000)- await rejection-- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-it('skips if no egressBindings', async () => {const server: McpServerCRD = {name: 'basic-mcp',@@ -2521,7 +1433,7 @@ describe('NetworkPolicyReconciler', () => {},}- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await reconciler.reconcileExternalEgress(server)expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -2542,14 +1454,6 @@ describe('NetworkPolicyReconciler', () => {await expect((reconciler as any).listAllContextPolicies()).rejects.toThrow('api list denied')})- it('propagates all-context mcp-host egress policy list failures', async () => {- mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))-- await expect((reconciler as any).listAllContextEgressPolicies()).rejects.toThrow(- 'api list denied'- )- })-it('propagates all external-egress policy list failures', async () => {mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))@@ -2559,2646 +1463,98 @@ describe('NetworkPolicyReconciler', () => {})})- describe('delete-context TOCTOU re-check', () => {- it('stops mid-inventory when the context is recreated between two deletes', async () => {- // deleteAllowed is a live GET, so its answer can change while the loop- // runs. Hoisting the check out of the loop would satisfy every existing- // test and still delete the recreated context's own policies.- mockApi.listNamespacedNetworkPolicy.mockResolvedValueOnce({- items: [- { metadata: { name: 'ctx-allow-first' } },- { metadata: { name: 'ctx-allow-second' } },- ],- })- const deleteAllowed = vi- .fn<() => Promise<boolean>>()- .mockResolvedValueOnce(true)- .mockResolvedValue(false)-- await reconciler.reconcileDeleteContext('dev', deleteAllowed)-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: 'ctx-allow-first' })- )- expect(deleteAllowed).toHaveBeenCalledTimes(2)- })-- it('re-checks before every mcp-host egress delete', async () => {- mockApi.listNamespacedNetworkPolicy- .mockResolvedValueOnce({ items: [] }) // context-allow- .mockResolvedValueOnce({- items: [- { metadata: { name: 'host-egress-first' } },- { metadata: { name: 'host-egress-second' } },- ],- })- const deleteAllowed = vi- .fn<() => Promise<boolean>>()- .mockResolvedValueOnce(true)- .mockResolvedValue(false)-- await reconciler.reconcileDeleteContext('dev', deleteAllowed)-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: 'host-egress-first' })- )- })-- it('re-checks before every rpc-proxy egress delete', async () => {- mockApi.listNamespacedNetworkPolicy- .mockResolvedValueOnce({ items: [] }) // context-allow- .mockResolvedValueOnce({ items: [] }) // mcp-host egress- .mockResolvedValueOnce({- items: [- { metadata: { name: 'rpc-egress-first' } },- { metadata: { name: 'rpc-egress-second' } },- ],- })- const deleteAllowed = vi- .fn<() => Promise<boolean>>()- .mockResolvedValueOnce(true)- .mockResolvedValue(false)-- await reconciler.reconcileDeleteContext('dev', deleteAllowed)-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: 'rpc-egress-first' })- )- })- })-- describe('L3 — DNS-derived retention by binding identity (B3)', () => {- // Live DNS policy built through the reconciler's OWN builder so its identity- // is exact-by-construction; each test drifts exactly one dimension.- function makeDnsFixture() {+ describe('startup fullReconcile', () => {+ it('reconciles external egress policies for existing servers on startup', async () => {const server: McpServerCRD = {- name: 'openai-mcp',+ name: 'airtable-mcp',namespace: 'mcp-server',spec: {contextRef: 'dev',- image: 'openai:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ image: 'airtable:latest',+ transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },+ egressBindings: [{ dns: 'api.airtable.com', port: 443 }],},}- const binding = server.spec.egressBindings![0]- const name = (reconciler as any).externalEgressPolicyName(server.name, binding) as string- const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [- '1.2.3.4/32',- ]) as k8s.V1NetworkPolicy- live.metadata!.uid = 'live-dns-uid'- live.metadata!.resourceVersion = '11'- return { server, binding, name, live }- }-- async function runSafety(server: McpServerCRD, live: k8s.V1NetworkPolicy): Promise<boolean> {- return (reconciler as any).reconcileExternalEgressSafety(- server,- [live],- () => true- ) as Promise<boolean>- }-- function expectRevoked(name: string) {- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name,- namespace: 'mcp-server',- body: { preconditions: { uid: 'live-dns-uid', resourceVersion: '11' } },- })- )- }- it('(a) retains an unchanged DNS-derived allow — no revocation', async () => {- const { server, live } = makeDnsFixture()- await expect(runSafety(server, live)).resolves.toBe(true)- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })+ await reconciler.fullReconcile([], [server])- it('(b) revokes on spec-level port drift under the same policy name', async () => {- const { server, name, live } = makeDnsFixture()- live.spec!.egress![0].ports = [{ port: 8443, protocol: 'TCP' }]- await expect(runSafety(server, live)).resolves.toBe(true)- expectRevoked(name)+ const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(+ (call: unknown[]) => {+ const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }+ return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'+ }+ )+ expect(externalPolicies).toHaveLength(1)+ const body = (+ externalPolicies[0][0] as {+ body: {+ metadata: { name: string }+ spec: { egress: Array<{ ports: Array<{ port: number }> }> }+ }+ }+ ).body+ expect(body.metadata.name).toContain('ext-egress-airtable-mcp')+ expect(body.spec.egress[0].ports[0].port).toBe(443)})- it('(c) revokes on protocol drift TCP→UDP', async () => {- const { server, name, live } = makeDnsFixture()- live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]- await expect(runSafety(server, live)).resolves.toBe(true)- expectRevoked(name)- })+ it('deletes orphaned external egress policies for servers no longer present', async () => {+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('context-allow')) return { items: [] }+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-old-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'old-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ }+ )- it('(d) revokes a foreign-owned policy even when the spec is identical', async () => {- const { server, name, live } = makeDnsFixture()- live.metadata!.labels!['clerum.io/managed-by'] = 'foreign-controller'- await expect(runSafety(server, live)).resolves.toBe(true)- expectRevoked(name)- })+ await reconciler.fullReconcile([], [], { serverInventoryComplete: true })- it('(f) revokes a degenerate zero-cidr "allow" (deny in disguise)', async () => {- const { server, name, live } = makeDnsFixture()- live.spec!.egress = []- await expect(runSafety(server, live)).resolves.toBe(true)- expectRevoked(name)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'ext-egress-old-server-api-example-com-443',+ namespace: 'mcp-server',+ })})- it('(e) queues recreation AT the revocation even when the pass then aborts and throws', async () => {- const { server, binding, name } = makeDnsFixture()- // Identity-drifted (protocol UDP) → this DNS allow is revoked (deleted).- const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [- '1.2.3.4/32',- ]) as k8s.V1NetworkPolicy- live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]- live.metadata!.uid = 'live-dns-uid'- live.metadata!.resourceVersion = '11'-+ it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: !labelSelector && namespace === 'mcp-server' ? [live] : [],- })+ async ({ labelSelector }: { labelSelector?: string }) => {+ if (labelSelector?.includes('context-allow')) return { items: [] }+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }+ if (labelSelector?.includes('external-egress')) {+ return {+ items: [+ {+ metadata: {+ name: 'ext-egress-old-server-api-example-com-443',+ labels: { 'clerum.io/mcpserver': 'old-server' },+ },+ },+ ],+ }+ }+ return { items: [] }+ })- let resolved: McpServerCRD | undefined = server- const spy = vi.fn()- // Flip authority false AT the deletion, so the pass aborts right after the- // revocation hook fires — reproducing the abort/throw window B3 must survive.- mockApi.deleteNamespacedNetworkPolicy.mockImplementationOnce(async () => {- resolved = undefined- return {}+ await reconciler.fullReconcile([], [], { serverInventoryComplete: false })++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({+ name: 'ext-egress-old-server-api-example-com-443',+ namespace: 'mcp-server',})-- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)-- await expect(- rec.fullReconcile([], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentServer: () => resolved,- onExternalEgressRevoked: spy,- })- ).rejects.toThrow(/inventory changed during authoritative revocation/)-- expect(spy).toHaveBeenCalledTimes(1)- expect(spy).toHaveBeenCalledWith(- expect.objectContaining({ name: server.name, namespace: server.namespace })- )- })- })-- describe('startup fullReconcile', () => {- it('discovers and revokes orphaned policies with one missing ownership marker in every safety lane', async () => {- const partiallyLabelledByNamespace: Record<string, k8s.V1NetworkPolicy[]> = {- 'mcp-server': [- {- metadata: {- name: 'ctx-orphan-context-old-server',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'orphan-context',- 'clerum.io/mcpserver': 'old-server',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- },- {- metadata: {- name: 'ext-egress-orphan-server-old-example-com-443',- labels: {- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': 'orphan-server',- 'clerum.io/egress-class': 'exact-host',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- 'mcp-host': [- {- metadata: {- name: 'ctx-orphan-context-old-server-egress',- labels: {- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'orphan-context',- 'clerum.io/mcpserver': 'old-server',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- 'rpc-proxy': [- {- metadata: {- name: 'rpc-egress-orphan-context-old-server',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/context': 'orphan-context',- 'clerum.io/mcpserver': 'old-server',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: labelSelector ? [] : (partiallyLabelledByNamespace[namespace ?? ''] ?? []),- })- )- const onAuthoritativeRevocationComplete = vi.fn()-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server-egress',- namespace: 'mcp-host',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'rpc-egress-orphan-context-old-server',- namespace: 'rpc-proxy',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-orphan-server-old-example-com-443',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it('repairs one-marker live widened policies in every lane before certification', async () => {- const server: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'live-context',- image: 'live:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '8.8.8.8/32', port: 443 }],- },- }- const context: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [server.name] },- }- const cache = new Map([[server.name, server]])- const rec = makeReconciler(mockApi, cache, mockCustomApi)-- await rec.reconcileContext(context, { isCurrent: () => true })- await rec.reconcileExternalEgress(server, { isCurrent: () => true })- const generated = mockApi.createNamespacedNetworkPolicy.mock.calls.map(- call => (call[0] as { body: k8s.V1NetworkPolicy }).body- )- const widened = generated.map((policy, index) => ({- ...policy,- metadata: {- ...policy.metadata,- labels: Object.fromEntries(- Object.entries(policy.metadata?.labels ?? {}).filter(- ([label]) =>- label !== (index % 2 === 0 ? 'clerum.io/managed-by' : 'clerum.io/policy-type')- )- ),- },- spec: {- ...policy.spec,- policyTypes: [...(policy.spec?.policyTypes ?? []), 'Ingress'],- },- }))-- vi.clearAllMocks()- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: labelSelector- ? []- : widened.filter(policy => policy.metadata?.namespace === namespace),- })- )- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('already exists'), { code: 409 })- )- const ordering: string[] = []- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {- ordering.push(`replace:${name}`)- return {}- })- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))-- await rec.fullReconcile([context], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentContext: () => context,- resolveCurrentServer: () => server,- onAuthoritativeRevocationComplete,- })-- const certificationIndex = ordering.indexOf('certify')- expect(certificationIndex).toBeGreaterThanOrEqual(4)- for (const expectedName of [- 'ctx-live-context-live-server',- 'ctx-live-context-live-server-egress',- 'rpc-egress-live-context-live-server',- 'ext-egress-live-server-8-8-8-8-32-443',- ]) {- expect(ordering.indexOf(`replace:${expectedName}`)).toBeGreaterThanOrEqual(0)- expect(ordering.indexOf(`replace:${expectedName}`)).toBeLessThan(certificationIndex)- }- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it.each([- [- 'context ingress',- 'mcp-server',- {- metadata: { name: 'ctx-reserved-context-server', labels: {} },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- },- ],- [- 'host egress',- 'mcp-host',- {- metadata: { name: 'ctx-reserved-context-server-egress', labels: {} },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- [- 'rpc egress',- 'rpc-proxy',- {- metadata: { name: 'rpc-egress-reserved-context-server', labels: {} },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- [- 'external egress',- 'mcp-server',- {- metadata: { name: 'ext-egress-reserved-server-example-443', labels: {} },- spec: { podSelector: {}, policyTypes: ['Egress'] },- },- ],- ])(- 'fails closed without takeover for a both-markers-missing reserved %s name',- async (_lane, namespace, candidate) => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({- namespace: listedNamespace,- labelSelector,- }: {- namespace?: string- labelSelector?: string- }) => ({- items: !labelSelector && listedNamespace === namespace ? [candidate] : [],- })- )- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- }- )-- it('fails closed for an explicit foreign ownership conflict on an HCC-shaped policy', async () => {- const candidate: k8s.V1NetworkPolicy = {- metadata: {- name: 'ext-egress-live-server-api-example-com-443',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': 'live-server',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],- })- )- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('treats a non-reserved external policy missing its manager marker as ambiguous', async () => {- const candidate: k8s.V1NetworkPolicy = {- metadata: {- name: 'custom-egress-policy',- labels: {- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': 'live-server',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],- })- )- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)-- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('lets the broad snapshot shadow a selected-owned object that became unrelated at the same UID', async () => {- const selected: k8s.V1NetworkPolicy = {- metadata: {- name: 'legacy-custom-policy',- uid: 'stable-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'orphan-context',- 'clerum.io/mcpserver': 'old-server',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- const broadRelabelled: k8s.V1NetworkPolicy = {- ...selected,- metadata: {- ...selected.metadata,- resourceVersion: '8',- labels: { 'tenant.example/managed-by': 'tenant-controller' },- },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace !== 'mcp-server') return { items: [] }- if (labelSelector?.includes('policy-type=context-allow')) return { items: [selected] }- if (!labelSelector) return { items: [broadRelabelled] }- return { items: [] }- }- )- const onAuthoritativeRevocationComplete = vi.fn()-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it('leaves unrelated namespace policies untouched and still certifies', async () => {- const unrelated: k8s.V1NetworkPolicy = {- metadata: {- name: 'tenant-owned-policy',- labels: {- 'app.kubernetes.io/managed-by': 'tenant-operator',- 'tenant.example/purpose': 'custom-egress',- },- },- spec: { podSelector: {}, policyTypes: ['Egress'] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: !labelSelector && namespace === 'mcp-server' ? [unrelated] : [],- })- )- const onAuthoritativeRevocationComplete = vi.fn()-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('fails closed when the broad safety inventory LIST fails', async () => {- mockApi.listNamespacedNetworkPolicy- .mockResolvedValueOnce({ items: [] })- .mockResolvedValueOnce({ items: [] })- .mockResolvedValueOnce({ items: [] })- .mockResolvedValueOnce({ items: [] })- .mockRejectedValueOnce(new Error('broad inventory denied'))- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow('broad inventory denied')-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('withholds certification and mutation when Context authority is lost during the broad LIST', async () => {- let contextAuthoritative = true- const candidate: k8s.V1NetworkPolicy = {- metadata: {- name: 'ctx-orphan-context-old-server',- labels: {- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'orphan-context',- 'clerum.io/mcpserver': 'old-server',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (!labelSelector && namespace === 'mcp-server') {- contextAuthoritative = false- return { items: [candidate] }- }- return { items: [] }- }- )- const onAuthoritativeRevocationComplete = vi.fn()-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => contextAuthoritative,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('does not reconcile a queued Context when its current contextId changed', async () => {- const queuedContext: ContextCRD = {- name: 'mutable-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'captured-context', mcpServers: [] },- }- const currentContext: ContextCRD = {- ...queuedContext,- spec: { contextId: 'replacement-context', mcpServers: [] },- }- const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)- const effectKeys: string[] = []-- await reconciler.fullReconcile([queuedContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => false,- resolveCurrentContext: () => currentContext,- runContextEffect: async (contextId, work) => {- effectKeys.push(contextId)- await work()- },- })-- expect(effectKeys).toEqual(['captured-context', 'captured-context'])- expect(reconcileContext).not.toHaveBeenCalled()- })-- it('retires a full-pass Context lease when the selected object is replaced', async () => {- const selected: ContextCRD = {- name: 'full-pass-context',- namespace: 'mcp-server',- spec: { contextId: 'full-pass-context', description: 'old', mcpServers: [] },- }- const replacement: ContextCRD = {- ...selected,- spec: { ...selected.spec, description: 'new' },- }- let current = selected- let selectedLease: (() => boolean) | undefined- const mutationStarted = deferred()- const releaseMutation = deferred()- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async (_context, options) => {- selectedLease = options?.isCurrent- mutationStarted.resolve(undefined)- await releaseMutation.promise- return true- })-- const fullPass = reconciler.fullReconcile([selected], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentContext: () => current,- })- await mutationStarted.promise- expect(selectedLease?.()).toBe(true)-- current = replacement- expect(selectedLease?.()).toBe(false)- releaseMutation.resolve(undefined)- await fullPass- })-- it('deletes orphaned Context policies in all three namespaces through the canonical contextId effect key', async () => {- const effectKeys: string[] = []- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-orphan-context-old-server-egress',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'rpc-egress-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- runContextEffect: async (contextId, work) => {- effectKeys.push(contextId)- await work()- },- })-- expect(effectKeys).toEqual(['orphan-context', 'orphan-context', 'orphan-context'])- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server-egress',- namespace: 'mcp-host',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'rpc-egress-orphan-context-old-server',- namespace: 'rpc-proxy',- })- )- })-- it('routes desired and orphan Context policy lanes through canonical contextId effect keys', async () => {- const desiredContext: ContextCRD = {- name: 'desired-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'desired-context', mcpServers: [] },- }- const effectKeys: string[] = []- vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'rpc-egress-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([desiredContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- runContextEffect: async (contextId, work) => {- effectKeys.push(contextId)- await work()- },- })-- expect(effectKeys).toEqual([- 'orphan-context',- 'orphan-context',- 'desired-context',- 'desired-context',- ])- })-- it('revokes every orphan allow lane before a slow additive Context effect', async () => {- const desiredContext: ContextCRD = {- name: 'desired-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'desired-context', mcpServers: [] },- }- const additiveStarted = deferred()- const releaseAdditive = deferred()- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {- additiveStarted.resolve(undefined)- await releaseAdditive.promise- return true- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-orphan-context-old-server-egress',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'rpc-egress-orphan-context-old-server',- labels: { 'clerum.io/context': 'orphan-context' },- },- },- ],- }- }- if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-orphan-server-old.example',- labels: { 'clerum.io/mcpserver': 'orphan-server' },- },- },- ],- }- }- return { items: [] }- }- )-- const fullPass = reconciler.fullReconcile([desiredContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- })-- await additiveStarted.promise- try {- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-orphan-context-old-server-egress',- namespace: 'mcp-host',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'rpc-egress-orphan-context-old-server',- namespace: 'rpc-proxy',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-orphan-server-old.example',- namespace: 'mcp-server',- })- )- } finally {- releaseAdditive.resolve(undefined)- await fullPass- }- })-- it('revokes every stale allow lane for a live Context before signaling authoritative revocation', async () => {- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [] },- }- const additiveStarted = deferred()- const releaseAdditive = deferred()- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {- additiveStarted.resolve(undefined)- await releaseAdditive.promise- return true- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-live-context-removed-server',- labels: { 'clerum.io/context': 'live-context' },- },- },- ],- }- }- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-live-context-removed-server-egress',- labels: { 'clerum.io/context': 'live-context' },- },- },- ],- }- }- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'rpc-egress-live-context-removed-server',- labels: { 'clerum.io/context': 'live-context' },- },- },- ],- }- }- return { items: [] }- }- )- const onAuthoritativeRevocationComplete = vi.fn()-- const fullPass = reconciler.fullReconcile([liveContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- await additiveStarted.promise- try {- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-live-context-removed-server',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ctx-live-context-removed-server-egress',- namespace: 'mcp-host',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'rpc-egress-live-context-removed-server',- namespace: 'rpc-proxy',- })- )- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- } finally {- releaseAdditive.resolve(undefined)- await fullPass- }- })-- it('revokes stale external egress for a live McpServer before readiness certification', async () => {- const liveServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [],- },- }- const ordering: string[] = []- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-live-server-old.example-443',- labels: { 'clerum.io/mcpserver': 'live-server' },- },- },- ],- }- }- return { items: [] }- }- )- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('delete')- return {}- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- ordering.push('certify')- })-- await reconciler.fullReconcile([], [liveServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-live-server-old.example-443',- namespace: 'mcp-server',- })- )- expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it('does not let one post-certification Context failure starve later additive owners', async () => {- const contexts: ContextCRD[] = ['aaa-poisoned', 'bbb-healthy'].map(contextId => ({- name: `${contextId}-resource`,- namespace: 'mcp-server',- spec: { contextId, mcpServers: [] },- }))- const ordering: string[] = []- const additiveFailure = new Error('poisoned additive Context')- vi.spyOn(reconciler, 'reconcileContext').mockImplementation(async context => {- ordering.push(`context:${context.spec.contextId}`)- if (context.spec.contextId === 'aaa-poisoned') throw additiveFailure- return true- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- ordering.push('certify')- })- const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)-- const error = await reconciler- .fullReconcile(contexts, [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- .then(- () => undefined,- failure => failure- )-- expect(error).toBeInstanceOf(AggregateError)- expect((error as AggregateError).errors).toEqual([additiveFailure])- expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])- expect(errorLog).toHaveBeenCalledWith(- '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',- additiveFailure- )- })-- it('replaces same-name Context allow policies in every lane before certification', async () => {- const oldServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- transport: { type: 'streamableHttp', port: 4000 },- },- }- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },- }- const serverCache = new Map([[oldServer.name, oldServer]])- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)-- await rec.reconcileContext(liveContext, { isCurrent: () => true })- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(- call => (call[0] as { body: k8s.V1NetworkPolicy }).body- )- expect(oldPolicies).toHaveLength(3)-- vi.clearAllMocks()- serverCache.set(currentServer.name, currentServer)- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) return { items: [] }- return {- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),- }- }- )- vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)- vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)- const ordering: string[] = []- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => {- ordering.push(`replace:${namespace}`)- return {}- }- )- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))-- await rec.fullReconcile([liveContext], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(ordering).toEqual([- 'replace:mcp-server',- 'replace:mcp-host',- 'replace:rpc-proxy',- 'certify',- ])- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it.each([- ['context-ingress', 'mcp-server'],- ['context-host-egress', 'mcp-host'],- ['context-rpc-egress', 'rpc-proxy'],- ])(- 'refuses a same-name %s safety replacement owned by another server',- async (_lane, targetNamespace) => {- const oldServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- transport: { type: 'streamableHttp', port: 4000 },- },- }- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },- }- const serverCache = new Map([[oldServer.name, oldServer]])- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)-- await rec.reconcileContext(liveContext, { isCurrent: () => true })- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body- if (policy.metadata?.namespace !== targetNamespace) return policy- return {- ...policy,- metadata: {- ...policy.metadata,- labels: {- ...policy.metadata?.labels,- 'clerum.io/mcpserver': 'different-server',- },- },- }- })-- vi.clearAllMocks()- serverCache.set(currentServer.name, currentServer)- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) return { items: [] }- return {- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),- }- }- )- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- rec.fullReconcile([liveContext], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/ownership/i)-- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ namespace: targetNamespace })- )- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- }- )-- it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {- const oldServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- transport: { type: 'streamableHttp', port: 4000 },- },- }- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },- }- const serverCache = new Map([[oldServer.name, oldServer]])- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)-- await rec.reconcileContext(liveContext, { isCurrent: () => true })- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(- call => (call[0] as { body: k8s.V1NetworkPolicy }).body- )-- vi.clearAllMocks()- serverCache.set(currentServer.name, currentServer)- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) return { items: [] }- return {- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),- }- }- )- const replacementFailure = new Error('replace unavailable')- mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(replacementFailure)- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- rec.fullReconcile([liveContext], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toBe(replacementFailure)-- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(- { outcome: 'failed' },- expect.any(Number)- )- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)- })-- it('retains authoritative same-name policies when only API object key order differs', async () => {- const server: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [server.name] },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)- const reverseObjectKeys = (value: unknown): unknown => {- if (Array.isArray(value)) return value.map(reverseObjectKeys)- if (value === null || typeof value !== 'object') return value- return Object.fromEntries(- Object.entries(value as Record<string, unknown>)- .reverse()- .map(([key, entry]) => [key, reverseObjectKeys(entry)])- )- }-- await rec.reconcileContext(liveContext, { isCurrent: () => true })- const existingPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body- return { ...policy, spec: reverseObjectKeys(policy.spec) as k8s.V1NetworkPolicySpec }- })- vi.clearAllMocks()- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) return { items: [] }- return {- items: existingPolicies.filter(policy => policy.metadata?.namespace === namespace),- }- }- )- vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)- vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)-- await rec.fullReconcile([liveContext], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- })-- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('revokes same-name DNS egress before certifying and leaves refresh to its coordinator', async () => {- const oldServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)-- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')- expect(oldPolicy).toBeDefined()-- vi.clearAllMocks()- const ordering: string[] = []- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: [oldPolicy!] } : { items: [] }- )- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('replace')- return {}- })- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('delete')- return {}- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- ordering.push('certify')- })-- await rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])- expect(ordering).not.toContain('replace')- expect(dns.resolve4).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it('revokes a divergent same-name DNS policy when its safe replacement cannot resolve', async () => {- const oldServer: McpServerCRD = {- name: 'live-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'live:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)-- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')- expect(oldPolicy).toBeDefined()-- vi.clearAllMocks()- const ordering: string[] = []- vi.mocked(dns.resolve4).mockImplementation(async () => {- ordering.push('dns')- throw new Error('dns unavailable')- })- let currentPolicies = [oldPolicy!]- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }- )- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('delete')- currentPolicies = []- return {}- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- ordering.push('certify')- })-- await expect(- rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).resolves.toBeUndefined()-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: oldPolicy!.metadata!.name!,- namespace: oldServer.namespace,- })- )- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(dns.resolve4).not.toHaveBeenCalled()- expect(ordering).toEqual(['delete', 'certify'])- })-- it('revokes a renamed DNS binding before certification and performs no DNS lookup on the safety path', async () => {- const oldServer: McpServerCRD = {- name: 'renamed-dns-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'dns:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'old.example.com', port: 443 }],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [{ dns: 'new.example.com', port: 443 }],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!-- vi.clearAllMocks()- const ordering: string[] = []- let currentPolicies = [oldPolicy]- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }- )- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {- ordering.push('delete')- currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)- return {}- })- vi.mocked(dns.resolve4).mockImplementation(async () => {- ordering.push('dns')- return ['1.2.3.4']- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- expect(dns.resolve4).not.toHaveBeenCalled()- ordering.push('certify')- })-- await rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(ordering).toEqual(['delete', 'certify'])- expect(dns.resolve4).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: oldPolicy.metadata!.name!,- namespace: oldServer.namespace,- body: {- preconditions: {- uid: `${oldServer.namespace}:${oldPolicy.metadata!.name!}:uid`,- resourceVersion: '1',- },- },- })- })-- it('replaces deterministic drift and revokes renamed DNS before certification without resolving DNS', async () => {- const oldServer: McpServerCRD = {- name: 'mixed-drift-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'mixed:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [- { cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' },- { dns: 'old.example.com', port: 443, protocol: 'TCP' },- ],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [- { cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' },- { dns: 'new.example.com', port: 443, protocol: 'TCP' },- ],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .filter(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')-- vi.clearAllMocks()- const ordering: string[] = []- let currentPolicies = oldPolicies- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }- )- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('replace')- return {}- })- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {- ordering.push('delete')- currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)- return {}- })- vi.mocked(dns.resolve4).mockImplementation(async () => {- ordering.push('dns')- return ['1.2.3.4']- })- const onAuthoritativeRevocationComplete = vi.fn(() => {- expect(dns.resolve4).not.toHaveBeenCalled()- ordering.push('certify')- })-- await rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(ordering).toEqual(['replace', 'delete', 'certify'])- expect(dns.resolve4).not.toHaveBeenCalled()- })-- it('aborts certification when an identity-drifted safety delete reports a recreate conflict', async () => {- const server: McpServerCRD = {- name: 'recreated-dns-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'dns:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443 }],- },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)- await rec.reconcileExternalEgress(server, { isCurrent: () => true })- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!- // B3: identity-drift the live policy (protocol TCP→UDP) so the safety pass- // still revokes it. A same-intent (identity-stable) DNS allow is now- // RETAINED (see the retention pins in the B3 describe block); the revoke /- // abort machinery this test guards fires only for a drifted policy.- existing.spec!.egress = existing.spec!.egress!.map(rule => ({- ...rule,- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),- }))-- vi.clearAllMocks()- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }- )- const conflict = Object.assign(new Error('delete precondition conflict'), { code: 409 })- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(conflict)- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- rec.fullReconcile([], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toBe(conflict)-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: existing.metadata!.name!,- namespace: server.namespace,- body: {- preconditions: {- uid: `${server.namespace}:${existing.metadata!.name!}:uid`,- resourceVersion: '1',- },- },- })- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it.each([- [- 'foreign relabel',- {- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': 'fresh-read-server',- },- },- ],- ['UID drift', { uid: 'replacement-uid' }],- ['resourceVersion drift', { resourceVersion: '2' }],- ])(- 'aborts a deterministic safety replace after fresh-read %s',- async (_case, metadataDrift) => {- const oldServer: McpServerCRD = {- name: 'fresh-read-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'fresh:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!-- vi.clearAllMocks()- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({- metadata: {- ...metadataDrift,- },- })- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/ownership/)-- expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: existing.metadata!.name!,- namespace: oldServer.namespace,- })- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- }- )-- it('uses the fresh identity to complete a deterministic safety replacement before certification', async () => {- const oldServer: McpServerCRD = {- name: 'fresh-success-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'fresh:test',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],- },- }- const currentServer: McpServerCRD = {- ...oldServer,- spec: {- ...oldServer.spec,- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!-- vi.clearAllMocks()- const ordering: string[] = []- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }- )- mockApi.readNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('read')- return { metadata: { resourceVersion: '1' } }- })- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {- ordering.push('replace')- return {}- })- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))-- await rec.fullReconcile([], [currentServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(ordering.slice(0, 3)).toEqual(['read', 'replace', 'certify'])- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: existing.metadata!.name!,- namespace: oldServer.namespace,- body: expect.objectContaining({- metadata: expect.objectContaining({- name: existing.metadata!.name!,- namespace: oldServer.namespace,- resourceVersion: '1',- }),- spec: expect.objectContaining({- egress: expect.arrayContaining([- expect.objectContaining({- ports: [expect.objectContaining({ protocol: 'UDP' })],- }),- ]),- }),- }),- })- })-- it('revokes an identity-drifted DNS policy before readiness and defers its refresh', async () => {- const server: McpServerCRD = {- name: 'stable-dns-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'stable:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)- await rec.reconcileExternalEgress(server, { isCurrent: () => true })- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')- expect(existing).toBeDefined()- // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a- // same-intent DNS allow is now retained. The revoke-before-readiness +- // defer-refresh machinery this test guards still fires on the drifted one.- existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({- ...rule,- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),- }))-- vi.clearAllMocks()- let currentPolicies = [existing!]- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }- )- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {- currentPolicies = []- return {}- })- const onAuthoritativeRevocationComplete = vi.fn()-- await rec.fullReconcile([], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: existing!.metadata!.name!,- namespace: server.namespace,- })- )- expect(dns.resolve4).not.toHaveBeenCalled()- })-- it('does not certify safety when an identity-drifted DNS policy cannot be revoked', async () => {- const server: McpServerCRD = {- name: 'undeletable-dns-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'stable:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],- },- }- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)- await rec.reconcileExternalEgress(server, { isCurrent: () => true })- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')- expect(existing).toBeDefined()- // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a- // same-intent DNS allow is now retained. The no-certify-when-revoke-fails- // machinery this test guards still fires on the drifted one.- existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({- ...rule,- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),- }))-- vi.clearAllMocks()- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) =>- labelSelector?.includes('external-egress') ? { items: [existing!] } : { items: [] }- )- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('delete denied'))- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- rec.fullReconcile([], [server], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow('delete denied')-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: existing!.metadata!.name!,- namespace: server.namespace,- })- )- })-- it('does not certify revocation when an earlier Context changes while a later owner is paused', async () => {- const contexts: ContextCRD[] = ['first', 'later'].map(contextId => ({- name: `${contextId}-resource`,- namespace: 'mcp-server',- spec: { contextId, mcpServers: [] },- }))- const currentContexts = new Map(contexts.map(context => [context.name, context]))- let contextDesiredRevision = 10- let laterPaused = false- const laterStarted = deferred()- const releaseLater = deferred()- const onAuthoritativeRevocationComplete = vi.fn()-- const fullPass = reconciler.fullReconcile(contexts, [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentContext: name => currentContexts.get(name),- runContextEffect: async (contextId, work) => {- if (contextId === 'later' && !laterPaused) {- laterPaused = true- laterStarted.resolve(undefined)- await releaseLater.promise- }- await work()- },- contextDesiredRevision: () => contextDesiredRevision,- onAuthoritativeRevocationComplete,- })-- await laterStarted.promise- currentContexts.set('first-resource', {- ...contexts[0],- spec: { ...contexts[0].spec, mcpServers: ['replacement'] },- })- contextDesiredRevision += 1- releaseLater.resolve(undefined)-- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('does not certify revocation when a new owner is added after the global inventory LISTs', async () => {- const anchorContext: ContextCRD = {- name: 'anchor-resource',- namespace: 'mcp-server',- spec: { contextId: 'anchor', mcpServers: [] },- }- const currentContexts = new Map([[anchorContext.name, anchorContext]])- let contextDesiredRevision = 21- const safetyEffectStarted = deferred()- const releaseSafetyEffect = deferred()- let safetyEffectPaused = false- const onAuthoritativeRevocationComplete = vi.fn()-- const fullPass = reconciler.fullReconcile([anchorContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentContext: name => currentContexts.get(name),- runContextEffect: async (_contextId, work) => {- if (!safetyEffectPaused) {- safetyEffectPaused = true- safetyEffectStarted.resolve(undefined)- await releaseSafetyEffect.promise- }- await work()- },- contextDesiredRevision: () => contextDesiredRevision,- onAuthoritativeRevocationComplete,- })-- await safetyEffectStarted.promise- currentContexts.set('late-resource', {- name: 'late-resource',- namespace: 'mcp-server',- spec: { contextId: 'late', mcpServers: [] },- })- contextDesiredRevision += 1- releaseSafetyEffect.resolve(undefined)-- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('does not certify revocation when a new McpServer owner is added after inventory LISTs', async () => {- const anchorServer: McpServerCRD = {- name: 'anchor-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'anchor:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const currentServers = new Map([[anchorServer.name, anchorServer]])- let serverDesiredRevision = 31- const safetyEffectStarted = deferred()- const releaseSafetyEffect = deferred()- let safetyEffectPaused = false- const onAuthoritativeRevocationComplete = vi.fn()-- const fullPass = reconciler.fullReconcile([], [anchorServer], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- resolveCurrentServer: name => currentServers.get(name),- runServerEffect: async (_serverName, work) => {- if (!safetyEffectPaused) {- safetyEffectPaused = true- safetyEffectStarted.resolve(undefined)- await releaseSafetyEffect.promise- }- await work()- },- serverDesiredRevision: () => serverDesiredRevision,- onAuthoritativeRevocationComplete,- })-- await safetyEffectStarted.promise- currentServers.set('late-server', {- name: 'late-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'late:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- })- serverDesiredRevision += 1- releaseSafetyEffect.resolve(undefined)-- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('does not certify revocation when desired inventory changes while default policies are ensured', async () => {- const context: ContextCRD = {- name: 'default-window-context',- namespace: 'mcp-server',- spec: { contextId: 'default-window', mcpServers: [] },- }- const server: McpServerCRD = {- name: 'default-window-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default-window',- image: 'default-window:latest',- transport: { type: 'streamableHttp', port: 3000 },- },- }- let contextDesiredRevision = 41- let serverDesiredRevision = 51- const defaultsStarted = deferred()- const releaseDefaults = deferred()- const onAuthoritativeRevocationComplete = vi.fn()- vi.spyOn(reconciler, 'ensureDefaultPolicies').mockImplementation(async () => {- defaultsStarted.resolve(undefined)- await releaseDefaults.promise- })-- const fullPass = reconciler.fullReconcile([context], [server], {- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- contextDesiredRevision: () => contextDesiredRevision,- serverDesiredRevision: () => serverDesiredRevision,- onAuthoritativeRevocationComplete,- })-- await defaultsStarted.promise- contextDesiredRevision += 1- serverDesiredRevision += 1- releaseDefaults.resolve(undefined)-- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('does not signal authoritative revocation when a live-owner delete fails', async () => {- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-live-context-removed-server',- labels: { 'clerum.io/context': 'live-context' },- },- },- ],- }- }- return { items: [] }- }- )- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(- new Error('policy delete unavailable')- )- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([liveContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow('policy delete unavailable')- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('does not delete or signal revocation after live Context authority is lost', async () => {- const liveContext: ContextCRD = {- name: 'live-context-resource',- namespace: 'mcp-server',- spec: { contextId: 'live-context', mcpServers: [] },- }- let authoritative = true- const inventoryListed = deferred()- const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()- mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {- inventoryListed.resolve(undefined)- return releaseInventory.promise- })- const onAuthoritativeRevocationComplete = vi.fn()-- const fullPass = reconciler.fullReconcile([liveContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => authoritative,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- await inventoryListed.promise- authoritative = false- releaseInventory.resolve({- items: [- {- metadata: {- name: 'ctx-live-context-removed-server',- labels: { 'clerum.io/context': 'live-context' },- },- },- ],- })-- await fullPass- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('uses selected lane LISTs plus one broad LIST per bounded namespace before additive convergence', async () => {- const contexts: ContextCRD[] = ['alpha', 'beta', 'gamma'].map(contextId => ({- name: `${contextId}-resource`,- namespace: 'mcp-server',- spec: { contextId, mcpServers: [] },- }))- const servers: McpServerCRD[] = ['one', 'two'].map(name => ({- name,- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: `${name}:latest`,- transport: { type: 'streamableHttp', port: 3000 },- },- }))- const additiveStarted = deferred()- const releaseAdditive = deferred()- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {- additiveStarted.resolve(undefined)- await releaseAdditive.promise- return true- })- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })-- const fullPass = reconciler.fullReconcile(contexts, servers, {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- })-- await additiveStarted.promise- try {- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'mcp-server',- labelSelector: expect.stringContaining('policy-type=context-allow'),- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'mcp-host',- labelSelector: expect.stringContaining('policy-type=context-allow'),- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'rpc-proxy',- labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'mcp-server',- labelSelector: expect.stringContaining('policy-type=external-egress'),- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'mcp-server',- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'mcp-host',- })- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'rpc-proxy',- })- } finally {- releaseAdditive.resolve(undefined)- await fullPass- }- })-- it('leaves startup external egress creation to its dedicated coordinator', async () => {- const server: McpServerCRD = {- name: 'airtable-mcp',- namespace: 'mcp-server',- spec: {- contextRef: 'dev',- image: 'airtable:latest',- transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },- egressBindings: [{ dns: 'api.airtable.com', port: 443 }],- },- }-- await reconciler.fullReconcile([], [server])-- const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(- (call: unknown[]) => {- const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }- return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'- }- )- expect(externalPolicies).toHaveLength(0)- expect(dns.resolve4).not.toHaveBeenCalled()- })-- it('deletes orphaned external egress policies for servers no longer present', async () => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('context-allow')) return { items: [] }- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-old-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'old-server' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], { serverInventoryComplete: true })-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'ext-egress-old-server-api-example-com-443',- namespace: 'mcp-server',- })- )- })-- it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }- if (labelSelector?.includes('policy-type=context-allow')) {- return {- items: [- {- metadata: {- name:- namespace === 'mcp-host'- ? 'malformed-host-egress'- : 'malformed-context-ingress',- labels: managedLabels,- },- },- ],- }- }- if (labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'malformed-rpc-proxy-egress',- labels: managedLabels,- },- },- ],- }- }- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'malformed-external-egress',- labels: managedLabels,- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], {- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- serverInventoryComplete: true,- })-- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'malformed-context-ingress',- namespace: 'mcp-server',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'malformed-host-egress',- namespace: 'mcp-host',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'malformed-rpc-proxy-egress',- namespace: 'rpc-proxy',- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- name: 'malformed-external-egress',- namespace: 'mcp-server',- })- )- })-- it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('context-allow')) return { items: [] }- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-old-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'old-server' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], { serverInventoryComplete: false })-- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({- name: 'ext-egress-old-server-api-example-com-443',- namespace: 'mcp-server',- })- })-- it('finishes desired Context reconciliation but stops Context and rpc-proxy orphan cleanup when authority is lost after inventory listing', async () => {- const desiredContext: ContextCRD = {- name: 'current',- namespace: 'mcp-server',- spec: { contextId: 'current', mcpServers: [] },- }- const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)- const contextInventoryAuthoritative = vi- .fn()- .mockReturnValueOnce(true)- .mockReturnValueOnce(false)- .mockReturnValue(true)-- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('policy-type=context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-deleted-context-old-server',- labels: { 'clerum.io/context': 'deleted-context' },- },- },- ],- }- }- if (labelSelector?.includes('external-egress')) return { items: [] }- return { items: [] }- }- )-- await reconciler.fullReconcile([desiredContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative,- })-- expect(reconcileContext).toHaveBeenCalledWith(desiredContext, {- isCurrent: expect.any(Function),- })- expect(contextInventoryAuthoritative).toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({- name: 'ctx-deleted-context-old-server',- namespace: 'mcp-server',- })- expect(mockApi.listNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({- namespace: 'rpc-proxy',- labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),- })- )- })-- it('stops external-egress orphan cleanup when authority is lost after inventory listing', async () => {- const desiredServer: McpServerCRD = {- name: 'current-server',- namespace: 'mcp-server',- spec: {- contextRef: 'current',- image: 'current:latest',- transport: { type: 'streamableHttp', url: 'http://current:3000', port: 3000 },- },- }- const serverInventoryAuthoritative = vi- .fn()- .mockReturnValueOnce(true)- .mockReturnValueOnce(false)- .mockReturnValue(true)-- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-deleted-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'deleted-server' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [desiredServer], {- ensureDefaults: false,- serverInventoryComplete: true,- serverInventoryAuthoritative,- })-- expect(serverInventoryAuthoritative).toHaveBeenCalled()- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({- name: 'ext-egress-deleted-server-api-example-com-443',- namespace: 'mcp-server',- })- })-- it('does not delete any Context policy lane when a same-contextId CRD exists before its watch event arrives', async () => {- mockCustomApi.listNamespacedCustomObject.mockResolvedValue({- items: [{ metadata: { name: 'recreated-context' }, spec: { contextId: 'recreated' } }],- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-recreated-server',- labels: { 'clerum.io/context': 'recreated' },- },- },- ],- }- }- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {- return {- items: [- {- metadata: {- name: 'ctx-recreated-server-egress',- labels: { 'clerum.io/context': 'recreated' },- },- },- ],- }- }- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {- return {- items: [- {- metadata: {- name: 'rpc-egress-recreated-server',- labels: { 'clerum.io/context': 'recreated' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- })-- expect(mockCustomApi.listNamespacedCustomObject).toHaveBeenCalled()- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({- namespace: 'mcp-host',- labelSelector: expect.stringContaining('policy-type=context-allow'),- })- )- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('does not delete external-egress policy when a same-name McpServer exists before ADDED arrives', async () => {- mockCustomApi.getNamespacedCustomObject.mockResolvedValue({- metadata: { name: 'recreated-server', uid: 'new-uid' },- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-recreated-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'recreated-server' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- serverInventoryComplete: true,- serverInventoryAuthoritative: () => true,- })-- expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledWith(- expect.objectContaining({ plural: 'mcpservers', name: 'recreated-server' })- )- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('delegates startup McpServer orphan safety to the canonical absence helper', async () => {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ labelSelector }: { labelSelector?: string }) => {- if (labelSelector?.includes('external-egress')) {- return {- items: [- {- metadata: {- name: 'ext-egress-orphan-server-api-example-com-443',- labels: { 'clerum.io/mcpserver': 'orphan-server' },- },- },- ],- }- }- return { items: [] }- }- )-- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- serverInventoryComplete: true,- serverInventoryAuthoritative: () => true,- })-- expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledTimes(1)- expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledWith({- inventoryAuthoritative: expect.any(Function),- resolveCurrent: expect.any(Function),- readCurrent: expect.any(Function),- })- })- })+ })+ })describe('L3 — cleanupExternalEgress', () => {it('deletes all external egress policies for a server', async () => {@@ -5208,16 +1564,7 @@ describe('NetworkPolicyReconciler', () => {await reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({- name: 'ext-egress-openai-mcp-api-443',- namespace: 'mcp-server',- body: {- preconditions: {- uid: 'mcp-server:ext-egress-openai-mcp-api-443:uid',- resourceVersion: '1',- },- },- })+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)})it('fails closed when stale external egress policy deletion fails', async () => {@@ -5243,467 +1590,5 @@ describe('NetworkPolicyReconciler', () => {reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')).resolves.toBeUndefined()})-- it('does not report a revocation when the policy was already absent', async () => {- const error = Object.assign(new Error('not found'), { code: 404 })- const onDeleted = vi.fn()- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({- items: [{ metadata: { name: 'ext-egress-openai-mcp-old-api-443' } }],- })- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(error)-- await reconciler.cleanupExternalEgress(- 'openai-mcp',- 'mcp-server',- undefined,- undefined,- onDeleted- )-- expect(onDeleted).not.toHaveBeenCalled()- })- })-- describe('scoped delta certification vs. a stuck safety inventory', () => {- const ambiguousReservedPolicy: k8s.V1NetworkPolicy = {- metadata: { name: 'ctx-reserved-context-server', labels: {} },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- const deltaContext: ContextCRD = {- name: 'default',- namespace: 'mcp-server',- spec: { contextId: 'default', mcpServers: [] },- }-- function stickSafetyInventory(): void {- // Only the namespace-wide safety inventory sees the foreign policy. The- // label-scoped LISTs the delta lane uses stay clean, which is exactly why- // the delta cannot vouch for the namespace.- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items: !labelSelector && namespace === 'mcp-server' ? [ambiguousReservedPolicy] : [],- })- )- }-- it('refuses to certify a scoped delta while the safety pass is stuck on its inventory', async () => {- stickSafetyInventory()- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()-- // The delta's own revocation still completes — its label-scoped LISTs are- // clean. What it may not do is vouch for the namespace-wide inventory,- // which the readiness lane checks separately.- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- })-- it('certifies a scoped delta again once a later safety pass completes', async () => {- stickSafetyInventory()- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: vi.fn(),- })- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)-- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- const onAuthoritativeRevocationComplete = vi.fn()- await reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })-- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)- })-- it('does not certify the safety inventory until an authoritative pass has run', async () => {- // Fail-closed default: before any authoritative safety pass certifies, the- // namespace-wide inventory is NOT certified, even though the scoped delta's- // own label-scoped revocation completes (returns true). A delta cannot- // vouch for the namespace-wide inventory it never enumerated.- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- })- })-- describe('lost delete fence during stale-allow revocation', () => {- const staleContext: ContextCRD = {- name: 'default',- namespace: 'mcp-server',- spec: { contextId: 'default', mcpServers: [] },- }- const stalePolicy: k8s.V1NetworkPolicy = {- metadata: {- name: 'ctx-default-alpha',- namespace: 'mcp-server',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'default',- 'clerum.io/mcpserver': 'alpha',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- const lostFence = Object.assign(new Error('the UID in the precondition does not match'), {- code: 409,- })-- function listOnlyTheStalePolicy(): void {- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => ({- items: namespace === 'mcp-server' ? [stalePolicy] : [],- })- )- }-- it('refuses to certify a scoped delta instead of throwing the lost fence', async () => {- listOnlyTheStalePolicy()- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)-- // 409 is what Kubernetes returns when the uid/resourceVersion- // preconditions no longer match: the object live under that name is not- // the one this pass classified. Nothing was revoked, so the delta must- // decline to certify — and it must not blow up the caller, which would- // only force a spurious full re-convergence.- // The scoped delta lane reads this boolean and withholds the certificate,- // so it is the one caller allowed to take the lost fence as an outcome- // rather than a throw.- expect(await reconciler.reconcileContext(staleContext, { honorsLostFence: true, isCurrent: () => true })).toBe(false)- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: 'ctx-default-alpha', namespace: 'mcp-server' })- )- })-- it('withdraws certification when an additive revocation loses the fence after certifying', async () => {- // The additive phase re-runs reconcileContext, which carries a second- // revocation lane over a fresh LIST. Losing the fence there happens after- // the pass already certified, and the outcome recorder is first-wins — so- // the duration sample stays 'completed', which is right, but the safety- // fence must still degrade. It is an assertion about live allows, and it- // may only ever move toward unsafe.- const additiveContext: ContextCRD = {- name: 'additive',- namespace: 'mcp-server',- spec: { contextId: 'additive', mcpServers: [] },- }- const additiveStalePolicy: k8s.V1NetworkPolicy = {- metadata: {- name: 'ctx-additive-alpha',- namespace: 'mcp-server',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'additive',- 'clerum.io/mcpserver': 'alpha',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- // Self-synchronising: the stale policy only becomes visible once the pass- // has already certified, so the safety phase is clean by construction and- // only the additive re-LIST finds work.- let certified = false- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => ({- items: certified && namespace === 'mcp-server' ? [additiveStalePolicy] : [],- })- )- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(lostFence)- const onAuthoritativeRevocationComplete = vi.fn(() => {- certified = true- })-- await expect(- reconciler.fullReconcile([additiveContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow()-- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- })-- it('marks the inventory uncertified the moment the authoritative pass loses the fence', async () => {- // The doom must be visible mid-pass, not only after the unwind. A lost- // fence is the one doom cause that does not bump a watch generation, so- // it is the only one where the whole certification machinery stays green- // while the pass is already condemned. A scoped delta interleaving in- // that window would certify readiness over a stale allow.- //- // Two contexts, each with its own stale policy: the first loses the fence- // and the second's delete gives an observation point while the pass is- // condemned but still unwinding.- const secondContext: ContextCRD = {- name: 'second',- namespace: 'mcp-server',- spec: { contextId: 'second', mcpServers: [] },- }- const secondStalePolicy: k8s.V1NetworkPolicy = {- metadata: {- name: 'ctx-second-beta',- namespace: 'mcp-server',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'context-allow',- 'clerum.io/context': 'second',- 'clerum.io/mcpserver': 'beta',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },- }- mockApi.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => ({- items: namespace === 'mcp-server' ? [stalePolicy, secondStalePolicy] : [],- })- )- let certifiedDuringUnwind: boolean | undefined- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {- if (- certifiedDuringUnwind === undefined &&- mockApi.deleteNamespacedNetworkPolicy.mock.calls.length > 1- ) {- certifiedDuringUnwind = reconciler.hasCertifiedSafetyInventory()- return {}- }- throw lostFence- })-- await expect(- reconciler.fullReconcile([staleContext, secondContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: vi.fn(),- })- ).rejects.toThrow()-- expect(certifiedDuringUnwind).toBe(false)- })-- it('fails the additive phase when it loses the delete fence', async () => {- listOnlyTheStalePolicy()- // Safety phase revokes cleanly and certifies; the additive phase then- // races another writer and loses the fence. The additive caller discards- // reconcileContext's boolean, so swallowing the 409 there would record- // the pass as a success, disarm the convergence retry, and leave a stale- // allow live with no NetworkPolicy resync to recover it.- mockApi.deleteNamespacedNetworkPolicy- .mockResolvedValueOnce({})- .mockRejectedValueOnce(lostFence)- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([staleContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow()-- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- })-- it('refuses to certify the safety pass instead of throwing the lost fence', async () => {- listOnlyTheStalePolicy()- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)- const onAuthoritativeRevocationComplete = vi.fn()-- await expect(- reconciler.fullReconcile([staleContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/Desired NetworkPolicy inventory changed/)-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- })-- it('still fails loud when the delete fails for any other reason', async () => {- listOnlyTheStalePolicy()- const forbidden = Object.assign(new Error('forbidden'), { code: 403 })- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)-- await expect(reconciler.reconcileContext(staleContext, { isCurrent: () => true })).rejects.toBe(forbidden)- })-- it('still certifies when the stale policy was already gone', async () => {- listOnlyTheStalePolicy()- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('not found'), { code: 404 })- )-- expect(await reconciler.reconcileContext(staleContext, { isCurrent: () => true })).toBe(true)- })- })-- describe('same-name conflict creating a safety external-egress policy', () => {- const server: McpServerCRD = {- name: 'renamed-server',- namespace: 'mcp-server',- spec: {- contextRef: 'default',- image: 'renamed:latest',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],- },- }- const desiredName = 'ext-egress-renamed-server-104-18-0-0-16-443'- const stalePredecessor: k8s.V1NetworkPolicy = {- metadata: {- name: 'ext-egress-renamed-server-stale',- namespace: 'mcp-server',- uid: 'stale-uid',- resourceVersion: '3',- },- }-- function reconcileExternalEgressSafety(): Promise<boolean> {- return (- reconciler as unknown as {- reconcileExternalEgressSafety: (- server: McpServerCRD,- existingPolicies: k8s.V1NetworkPolicy[],- isCurrent: () => boolean- ) => Promise<boolean>- }- ).reconcileExternalEgressSafety(server, [stalePredecessor], () => true)- }-- it('converges an HCC-owned same-name policy instead of aborting the pass', async () => {- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: desiredName,- namespace: server.namespace,- uid: 'own-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': server.name,- },- },- })-- // This is the only creation route that bypasses applyNetworkPolicy. A- // same-name object HCC already owns is an idempotent re-run, not a- // reason to fail the pass and hold readiness down.- expect(await reconcileExternalEgressSafety()).toBe(true)- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: desiredName, namespace: server.namespace })- )- })-- it('still fails closed when the same-name policy is foreign', async () => {- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(- Object.assign(new Error('already exists'), { code: 409 })- )- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- name: desiredName,- namespace: server.namespace,- uid: 'foreign-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'external-egress',- 'clerum.io/mcpserver': server.name,- },- },- })-- await expect(reconcileExternalEgressSafety()).rejects.toThrow(/ownership/i)- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })-- it('still fails loud when the create fails for any other reason', async () => {- const forbidden = Object.assign(new Error('forbidden'), { code: 403 })- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)-- await expect(reconcileExternalEgressSafety()).rejects.toBe(forbidden)- })- })-- describe('safety-pass duration on the uncertified return path', () => {- it('samples a failed outcome when the pass returns without certifying', async () => {- const onAuthoritativeRevocationComplete = vi.fn()-- // Losing inventory authority ends the pass without an exception and- // without certifying. Operators alert on "the pass never completed", so- // this path has to produce a `failed` sample — absence is not a signal.- await expect(- reconciler.fullReconcile([], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => false,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).resolves.toBeUndefined()-- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(- { outcome: 'failed' },- expect.any(Number)- )- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)- })-- it('keeps the completed sample when a certified pass fails an additive effect afterwards', async () => {- const context: ContextCRD = {- name: 'default',- namespace: 'mcp-server',- spec: { contextId: 'default', mcpServers: [] },- }- const additiveFailure = new Error('additive list unavailable')- let certified = false- const onAuthoritativeRevocationComplete = vi.fn(() => {- certified = true- })- mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {- if (certified) throw additiveFailure- return { items: [] }- })-- await expect(- reconciler.fullReconcile([context], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete,- })- ).rejects.toThrow(/additive Context NetworkPolicy reconciliations failed/)-- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(- { outcome: 'completed' },- expect.any(Number)- )- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)- })})})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 70eab2a1a..8778887cd 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -27,14 +27,6 @@ import {POLICY_TYPE_LABEL,} from './constants'import { accumulateHostExactHostEgress } from './externalEgressAccumulator'-import {- confirmAuthoritativeMcpServerAbsence,- sameMcpServerDesiredRevision,-} from './mcpServerSafety'-import {- networkPolicySafetyPassDurationSeconds,- networkPolicySafetyPassPoliciesTotal,-} from './metrics'import {ContextCRD,EgressBinding,@@ -43,181 +35,21 @@ import {McpServerCrdStatus,McpServerResolvedEgressIP,} from './types'-import { applyNetworkPolicy, getErrorCode, replaceWithConflictRetry } from './utils'+import { applyNetworkPolicy, getErrorCode } from './utils'type JsonPatchOperation = {- op: 'add' | 'replace' | 'remove' | 'test'+ op: 'add' | 'replace' | 'remove'path: stringvalue?: unknown}-type NetworkPolicyMutationOptions = {- // Required (not optional): reconcileContext and reconcileExternalEgress are the- // two functions that CREATE/RETAIN context-allow (L2) and external-egress ALLOW- // (L3) policies, so the authority fence must be supplied at compile time. A- // fail-open `?? (() => true)` default here would silently readmit the exact- // fail-open class hardened out of bindingPolicyReconciler (required isCurrent).- isCurrent: () => boolean- onRevoked?: () => void- /**- * Set only by a caller that reads the returned boolean and withholds- * readiness on false. It downgrades a lost delete fence from a throw to a- * reported outcome; a caller that discards the boolean must not set it, or a- * stale allow survives while the pass is recorded as a success.- */- honorsLostFence?: boolean-}--export function sameContextDesiredRevision(expected: ContextCRD, current: ContextCRD): boolean {- return (- expected.name === current.name &&- expected.namespace === current.namespace &&- expected.uid === current.uid &&- expected.generation === current.generation &&- JSON.stringify(canonicalizeNetworkPolicyValue(expected.spec)) ===- JSON.stringify(canonicalizeNetworkPolicyValue(current.spec))- )-}--function canonicalizeNetworkPolicyValue(value: unknown): unknown {- if (Array.isArray(value)) return value.map(canonicalizeNetworkPolicyValue)- if (value === null || typeof value !== 'object') return value- return Object.fromEntries(- Object.entries(value as Record<string, unknown>)- .filter(([, entry]) => entry !== undefined)- .sort(([left], [right]) => left.localeCompare(right))- .map(([key, entry]) => [key, canonicalizeNetworkPolicyValue(entry)])- )-}--function sameNetworkPolicySpec(- existing: k8s.V1NetworkPolicy,- desired: k8s.V1NetworkPolicy-): boolean {- return (- JSON.stringify(canonicalizeNetworkPolicyValue(existing.spec)) ===- JSON.stringify(canonicalizeNetworkPolicyValue(desired.spec))- )-}--function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {- const labels = policy.metadata?.labels ?? {}- return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[POLICY_TYPE_LABEL] === policyType-}-const GROUP = 'clerum.io'const VERSION = 'v1alpha1'const PLURAL_MCPSERVERS = 'mcpservers'-const PLURAL_CONTEXTS = 'contexts'const CONTEXT_LABEL = 'clerum.io/context'const EGRESS_CLASS_LABEL = 'clerum.io/egress-class'const RPC_PROXY_EGRESS_POLICY_TYPE = 'rpc-proxy-egress'const RPC_PROXY_APP_LABEL = 'rpc-proxy'--type SafetyInventoryLane =- | 'context-ingress'- | 'context-host-egress'- | 'context-rpc-egress'- | 'external-egress'--type SafetyInventoryClassification = 'owned' | 'repairable' | 'ambiguous' | 'unrelated'--function policyHasOwnerShape(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {- const labels = policy.metadata?.labels ?? {}- const hasServerOwner = Boolean(labels[MCPSERVER_LABEL])- if (lane === 'external-egress') return hasServerOwner- return Boolean(labels[CONTEXT_LABEL]) && hasServerOwner-}--function policyHasReservedName(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {- const name = policy.metadata?.name ?? ''- switch (lane) {- case 'context-ingress':- return name.startsWith('ctx-')- case 'context-host-egress':- return name.startsWith('ctx-') && name.endsWith('-egress')- case 'context-rpc-egress':- return name.startsWith('rpc-egress-')- case 'external-egress':- return name.startsWith('ext-egress-')- }-}--function policyNameBelongsToDifferentLane(- policy: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane-): boolean {- const name = policy.metadata?.name ?? ''- if (lane === 'context-ingress') return name.startsWith('ext-egress-')- if (lane === 'external-egress') return name.startsWith('ctx-')- return false-}--function expectedPolicyType(lane: SafetyInventoryLane): string {- return lane === 'context-rpc-egress'- ? RPC_PROXY_EGRESS_POLICY_TYPE- : lane === 'external-egress'- ? EXTERNAL_EGRESS_POLICY_TYPE- : 'context-allow'-}--/**- * Classify broad namespace inventory without taking ownership of unrelated- * policies. A single missing HCC marker is recoverable only when owner labels- * corroborate the lane. Reserved HCC names and conflicting ownership markers- * are never adopted implicitly.- */-function classifySafetyInventoryPolicy(- policy: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane-): SafetyInventoryClassification {- const labels = policy.metadata?.labels ?? {}- const managedBy = labels[MANAGED_BY_LABEL]- const policyType = labels[POLICY_TYPE_LABEL]- const expectedType = expectedPolicyType(lane)- const managedMatches = managedBy === MANAGED_BY_VALUE- const typeMatches = policyType === expectedType- const managedMissing = managedBy === undefined- const typeMissing = policyType === undefined- const reservedName = policyHasReservedName(policy, lane)- const ownerShape = policyHasOwnerShape(policy, lane)-- if (managedMatches && typeMatches) return 'owned'-- const oneMarkerMissing = (managedMatches && typeMissing) || (managedMissing && typeMatches)- if (managedMissing && typeMatches && ownerShape) {- return reservedName ? 'repairable' : 'ambiguous'- }- if (managedMatches && typeMissing && ownerShape) {- if (reservedName) return 'repairable'- if (policyNameBelongsToDifferentLane(policy, lane)) return 'unrelated'- return 'ambiguous'- }-- const knownOtherHccType =- policyType !== undefined &&- ['context-allow', RPC_PROXY_EGRESS_POLICY_TYPE, EXTERNAL_EGRESS_POLICY_TYPE].includes(- policyType- ) &&- !typeMatches- const hasRawConflictingMarker =- (managedBy !== undefined && !managedMatches) || (policyType !== undefined && !typeMatches)- const hasConflictingMarker =- (managedBy !== undefined && !managedMatches) ||- (policyType !== undefined && !typeMatches && !knownOtherHccType)- if (- reservedName &&- ((managedMissing && typeMissing) || hasRawConflictingMarker || oneMarkerMissing)- ) {- return 'ambiguous'- }- if (ownerShape && hasConflictingMarker) {- return 'ambiguous'- }-- return 'unrelated'-}-export const PUBLIC_EGRESS_EXCEPT_CIDRS = ['0.0.0.0/8','10.0.0.0/8',@@ -366,45 +198,6 @@ function k8sApiEgressPodSelector(namespace: string): k8s.V1LabelSelector {return { matchLabels: { 'clerum.io/k8s-api-egress': 'true' } }}-export interface NetworkPolicyFullReconcileOptions {- serverInventoryComplete?: boolean- ensureDefaults?: boolean- /**- * Dynamic authority checks are intentionally callbacks: watch authority can- * be lost after the startup snapshot but before an orphan delete begins.- * Omitted callbacks preserve the historical authoritative-by-default API.- */- contextInventoryAuthoritative?: () => boolean- serverInventoryAuthoritative?: () => boolean- runContextEffect?: (contextId: string, work: () => Promise<void>) => Promise<void>- runServerEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>- resolveCurrentContext?: (name: string) => ContextCRD | undefined- resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined- resolveCurrentServer?: (name: string) => McpServerCRD | undefined- /**- * Monotonic desired-state revisions fence ordinary watch events that arrive- * after an owner has completed its safety turn. Watch generations remain the- * separate authority fence for LIST -> WATCH recovery.- */- contextDesiredRevision?: () => number- serverDesiredRevision?: () => number- /**- * Called only after every authoritative orphan-allow revocation lane has- * completed. Additive Context/McpServer convergence intentionally happens- * afterwards and must not delay this safety boundary.- */- onAuthoritativeRevocationComplete?: () => void- /**- * Fired at the moment each external-egress allow owned by an McpServer is- * revoked (or replaced) during the authoritative safety pass — before any- * isCurrent abort or inventory-changed throw can unwind the pass. The owner- * uses it to queue an additive recreation for the affected server; queuing in- * a pass epilogue instead would reproduce exactly the abort gap this hook- * closes.- */- onExternalEgressRevoked?: (server: McpServerCRD) => void-}-export class NetworkPolicyReconciler {private networkingApi: k8s.NetworkingV1Apiprivate customApi: k8s.CustomObjectsApi@@ -412,22 +205,6 @@ export class NetworkPolicyReconciler {/** Reference to the MCP server cache so we can look up ports. */private serverCache: Map<string, McpServerCRD>- /**- * Set whenever an authoritative safety pass ends without certifying, cleared- * as soon as one certifies again. Only that pass enumerates the bounded- * namespaces and classifies every object it finds; the scoped delta lane sees- * nothing but its own label selectors. A delta therefore cannot vouch for a- * namespace whose classified inventory is still stuck on an unresolved- * policy, and must not certify readiness on its behalf.- *- * Initialised fail-closed (true): hasCertifiedSafetyInventory() reports false- * until the first authoritative pass actually certifies, so certification- * cannot be implied before any pass has run. This does not depend on the- * readiness gate's generation-equality check for safety — it removes that- * coupling rather than relying on it.- */- private safetyPassLeftUncertified = true-// H2 (issue #299): smallest DNS TTL (ms) observed across external-egress// resolutions. The resync loop advances to <= this/2 so a rotating low-TTL host// is sampled every rotation. Ratchets down and never recovers until restart —@@ -526,7 +303,7 @@ export class NetworkPolicyReconciler {},}- await this.applyFixedPolicy(name, ns, policy, 'default-deny')+ await applyNetworkPolicy(this.networkingApi, name, ns, policy, '[NetPol]')}}@@ -600,7 +377,7 @@ export class NetworkPolicyReconciler {egress,},}- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')}/** Allow egress to host-context-controller API gateway in control-plane namespace. */@@ -637,7 +414,7 @@ export class NetworkPolicyReconciler {],},}- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')}/**@@ -682,7 +459,7 @@ export class NetworkPolicyReconciler {],},}- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')}/**@@ -725,26 +502,46 @@ export class NetworkPolicyReconciler {},}- await this.applyFixedPolicy(name, config.namespace, policy, 'allow-api')+ await this.applyPolicy(name, policy)}// ─── Context-Based Policies ──────────────────────────────────────────- private buildContextAllowPolicies(- context: ContextCRD,- server: McpServerCRD- ): {- ingress: k8s.V1NetworkPolicy- hostEgress: k8s.V1NetworkPolicy- rpcProxyEgress: k8s.V1NetworkPolicy- } {+ /**+ * Reconcile NetworkPolicies for a Context CRD.+ * Creates one policy per allowed MCP server.+ */+ async reconcileContext(context: ContextCRD): Promise<void> {const contextId = context.spec.contextId- const serverName = server.name- const name = this.policyName(contextId, serverName)- const port = server.spec.transport.port || 3000+ const allowedServers = context.spec.mcpServers || []++ console.log(+ `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`+ )- return {- ingress: {+ // Build desired policy names+ const desiredNames = new Set(+ allowedServers.map(serverName => this.policyName(contextId, serverName))+ )++ // Get existing policies for this context+ const existingPolicies = await this.listPoliciesForContext(contextId)++ // Create or update policies for each allowed server+ for (const serverName of allowedServers) {+ const server = this.serverCache.get(serverName)+ if (!server) {+ console.warn(+ `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`+ )+ continue+ }++ const name = this.policyName(contextId, serverName)+ const port = server.spec.transport.port || 3000++ // L2 ingress: allow scoped mcp-host/rpc-proxy access to this MCP server.+ const ingressPolicy: k8s.V1NetworkPolicy = {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {@@ -793,6 +590,7 @@ export class NetworkPolicyReconciler {},},{+ // MCP Proxy ingress: allow proxy to reach MCP serversnamespaceSelector: {matchLabels: {'kubernetes.io/metadata.name': config.namespace,@@ -809,12 +607,18 @@ export class NetworkPolicyReconciler {},],},- },- hostEgress: {+ }++ await this.applyPolicy(name, ingressPolicy)++ // L2 egress counterpart: allow traffic FROM mcp-host pods TO this MCP server+ // Without this, L0 egress deny-all blocks agents from reaching MCP servers.+ const egressName = `${name}-egress`+ const egressPolicy: k8s.V1NetworkPolicy = {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {- name: `${name}-egress`,+ name: egressName,namespace: config.hostNamespace,labels: {[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,@@ -851,322 +655,78 @@ export class NetworkPolicyReconciler {},],},- },- rpcProxyEgress: this.buildRpcProxyEgressPolicy(contextId, serverName, port),- }- }-- /**- * Reconcile NetworkPolicies for a Context CRD.- * Creates one policy per allowed MCP server.- *- * Returns whether this Context's stale-allow revocation ran to completion.- * Callers that certify readiness from a scoped delta MUST honour it: the pass- * aborts as soon as its authority fence breaks, and an aborted pass leaves- * stale allows live.- */- async reconcileContext(- context: ContextCRD,- options: NetworkPolicyMutationOptions- ): Promise<boolean> {- const callerIsCurrent = options.isCurrent- if (!callerIsCurrent()) return false- const contextId = context.spec.contextId- const allowedServers = context.spec.mcpServers || []- const selectedServers = new Map(- allowedServers.map(serverName => [serverName, this.serverCache.get(serverName)])- )- const isCurrent = (): boolean =>- callerIsCurrent() &&- allowedServers.every(serverName => {- const selected = selectedServers.get(serverName)- const current = this.serverCache.get(serverName)- if (!selected || !current) return selected === current- return sameMcpServerDesiredRevision(selected, current)- })- if (!isCurrent()) return false-- console.log(- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`- )-- // Create or update policies for each allowed server.- for (const serverName of allowedServers) {- if (!isCurrent()) return false- const server = this.serverCache.get(serverName)- if (!server) {- console.warn(- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`- )- continue}- const desired = this.buildContextAllowPolicies(context, server)- const ingressName = desired.ingress.metadata!.name!- await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)- if (!isCurrent()) return false-- const hostEgressName = desired.hostEgress.metadata!.name!- await this.applyOwnedPolicy(- hostEgressName,+ await applyNetworkPolicy(+ this.networkingApi,+ egressName,config.hostNamespace,- desired.hostEgress,- 'context-host-egress',- isCurrent- )- if (!isCurrent()) return false-- const rpcProxyEgressName = desired.rpcProxyEgress.metadata!.name!- await this.applyOwnedPolicy(- rpcProxyEgressName,- config.rpcProxyNamespace,- desired.rpcProxyEgress,- 'context-rpc-egress',- isCurrent+ egressPolicy,+ '[NetPol]')- }-- // Re-LIST every lane after writes so a same-name policy that was just- // replaced is compared in its new form and cannot delete itself.- return await this.revokeStaleContextAllows(- context,- isCurrent,- undefined,- undefined,- options.honorsLostFence === true- )- }- /**- * Whether the last authoritative safety pass certified its namespace-wide- * inventory. Only that pass enumerates the bounded namespaces and classifies- * every object it finds; a scoped delta sees nothing but its own label- * selectors, so it cannot vouch for a namespace whose classified inventory is- * still stuck. The readiness lane consults this before letting a delta- * replace the safety certificate.- */- hasCertifiedSafetyInventory(): boolean {- return !this.safetyPassLeftUncertified- }-- private async revokeStaleContextAllows(- context: ContextCRD,- isCurrent: () => boolean,- listedPolicies?: {- context?: k8s.V1NetworkPolicy[]- hostEgress?: k8s.V1NetworkPolicy[]- rpcProxyEgress?: k8s.V1NetworkPolicy[]- },- onRevoked?: () => void,- callerHonorsLostFence = false- ): Promise<boolean> {- if (!isCurrent()) return false- const safetySnapshotProvided = listedPolicies !== undefined- // A delete whose uid/resourceVersion preconditions were rejected revoked- // nothing. Reporting it instead of throwing is only safe for a caller that- // reads the returned boolean: the authoritative snapshot lane aborts its- // pass on it, and the scoped delta lane declines to certify. Every other- // caller discards the boolean, so for them a swallowed 409 would leave a- // stale allow live while the pass is recorded as a success — they keep the- // loud default and let the failure reach the retry policy.- const lostFenceOutcome =- safetySnapshotProvided || callerHonorsLostFence ? ('report' as const) : ('throw' as const)- let lostDeleteFence = false- /**- * Revoke one orphaned policy in any lane. Returns false only when the- * caller must abort the pass outright, which is what the authoritative- * safety-snapshot lane does with a lost delete fence; the scoped lane- * records it and keeps going so the rest of its work still lands.- */- const revokeOrphanedPolicy = async (- namespace: string,- existing: k8s.V1NetworkPolicy- ): Promise<boolean> => {- const deleted = await this.deleteSafetyPolicySnapshot(- namespace,- existing,- isCurrent,- undefined,- lostFenceOutcome- )- if (deleted) return true- if (safetySnapshotProvided) {- // The authoritative pass is condemned from here: it will unwind and- // throw. Losing the fence is the only doom cause that does not bump a- // watch generation, so without this the certification machinery stays- // green for the whole unwind and a concurrent scoped delta can certify- // readiness over the allow this pass failed to revoke. Record it the- // moment it is known rather than when the pass finally reports.- this.safetyPassLeftUncertified = true- return false- }- lostDeleteFence = true- return true- }- const contextId = context.spec.contextId- const allowedServers = context.spec.mcpServers || []- const desiredIngress = new Map<string, k8s.V1NetworkPolicy>()- const desiredHostEgress = new Map<string, k8s.V1NetworkPolicy>()- const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()- for (const serverName of allowedServers) {- const server = this.serverCache.get(serverName)- if (!server) continue- const desired = this.buildContextAllowPolicies(context, server)- desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)- desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)- desiredRpcProxyEgress.set(desired.rpcProxyEgress.metadata!.name!, desired.rpcProxyEgress)+ // L2 rpc-proxy egress: allow rpc-proxy pods to reach this MCP server+ await this.ensureRpcProxyEgress(contextId, serverName, port)}- const existingPolicies =- listedPolicies?.context ?? (await this.listPoliciesForContext(contextId))- if (!isCurrent()) return false+ // Delete policies for servers no longer in the contextfor (const existing of existingPolicies) {- if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- const desired = desiredIngress.get(existingName)- if (!desired) {+ if (!desiredNames.has(existingName)) {console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)- if (!(await revokeOrphanedPolicy(config.namespace, existing))) return false- onRevoked?.()- } else if (- !sameNetworkPolicySpec(existing, desired) ||- !hasExpectedPolicyOwnership(existing, 'context-allow')- ) {- console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)- if (safetySnapshotProvided) {- if (- !(await this.replaceSafetyPolicySnapshot(- config.namespace,- existing,- desired,- 'context-ingress',- isCurrent- ))- ) {- return false- }- } else {- await this.replaceSafetyPolicySnapshot(- config.namespace,- existing,- desired,- 'context-ingress',- isCurrent- )- }- onRevoked?.()+ await this.deletePolicy(existingName)}}- const existingEgressPolicies =- listedPolicies?.hostEgress ?? (await this.listEgressPoliciesForContext(contextId))- if (!isCurrent()) return false+ // Delete orphaned L2 egress counterparts in mcp-host namespace+ const existingEgressPolicies = await this.listEgressPoliciesForContext(contextId)+ const desiredEgressNames = new Set(+ allowedServers.map(serverName => `${this.policyName(contextId, serverName)}-egress`)+ )for (const existing of existingEgressPolicies) {- if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- const desired = desiredHostEgress.get(existingName)- if (!desired) {+ if (!desiredEgressNames.has(existingName)) {console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)- if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return false- onRevoked?.()- } else if (- !sameNetworkPolicySpec(existing, desired) ||- !hasExpectedPolicyOwnership(existing, 'context-allow')- ) {- console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)- if (safetySnapshotProvided) {- if (- !(await this.replaceSafetyPolicySnapshot(- config.hostNamespace,- existing,- desired,- 'context-host-egress',- isCurrent- ))- ) {- return false- }- } else {- await this.replaceSafetyPolicySnapshot(- config.hostNamespace,- existing,- desired,- 'context-host-egress',- isCurrent- )- }- onRevoked?.()+ await this.deleteEgressPolicy(existingName)}}- const existingRpcProxyPolicies =- listedPolicies?.rpcProxyEgress ?? (await this.listRpcProxyEgressPoliciesForContext(contextId))- if (!isCurrent()) return false+ // Delete orphaned rpc-proxy egress counterparts. Keeping these after a+ // server is removed from a Context would preserve an old in-cluster route.+ const existingRpcProxyPolicies = await this.listRpcProxyEgressPoliciesForContext(contextId)+ const desiredRpcProxyNames = new Set(+ allowedServers.map(serverName => `rpc-egress-${contextId}-${serverName}`)+ )for (const existing of existingRpcProxyPolicies) {- if (!isCurrent()) return falseconst existingName = existing.metadata?.name || ''- const desired = desiredRpcProxyEgress.get(existingName)- if (!desired) {+ if (!desiredRpcProxyNames.has(existingName)) {console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)- if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return false- onRevoked?.()- } else if (- !sameNetworkPolicySpec(existing, desired) ||- !hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)- ) {- console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)- if (safetySnapshotProvided) {- if (- !(await this.replaceSafetyPolicySnapshot(- config.rpcProxyNamespace,- existing,- desired,- 'context-rpc-egress',- isCurrent- ))- ) {- return false- }- } else {- await this.replaceSafetyPolicySnapshot(- config.rpcProxyNamespace,- existing,- desired,- 'context-rpc-egress',- isCurrent- )- }- onRevoked?.()+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, existingName)}}-- return isCurrent() && !lostDeleteFence}/*** Delete all NetworkPolicies for a deleted Context.*/- async reconcileDeleteContext(- contextId: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ async reconcileDeleteContext(contextId: string): Promise<void> {console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)const policies = await this.listPoliciesForContext(contextId)for (const policy of policies) {- if (deleteAllowed && !(await deleteAllowed())) return- await this.deleteSafetyPolicySnapshot(config.namespace, policy)+ const name = policy.metadata?.name || ''+ await this.deletePolicy(name)}// Also delete L2 egress counterparts in mcp-host namespaceconst egressPolicies = await this.listEgressPoliciesForContext(contextId)for (const policy of egressPolicies) {- if (deleteAllowed && !(await deleteAllowed())) return- await this.deleteSafetyPolicySnapshot(config.hostNamespace, policy)+ const name = policy.metadata?.name || ''+ await this.deleteEgressPolicy(name)}// Also delete L2 rpc-proxy egress policies- await this.deleteRpcProxyPoliciesForContext(contextId, deleteAllowed)+ await this.deleteRpcProxyPoliciesForContext(contextId)}/**@@ -1175,742 +735,80 @@ export class NetworkPolicyReconciler {async fullReconcile(contexts: ContextCRD[],servers: McpServerCRD[] = [],- options: NetworkPolicyFullReconcileOptions = {}- ): Promise<void> {- const onAuthoritativeRevocationComplete = options.onAuthoritativeRevocationComplete- if (!onAuthoritativeRevocationComplete) {- await this.fullReconcileInternal(contexts, servers, options)- return- }-- const safetyPassStartedAt = performance.now()- let safetyPassOutcomeRecorded = false- /**- * Settles both observable results of this pass: the duration sample and the- * cross-lane fence the scoped delta reads. First outcome wins for both — a- * pass that certified and only then failed an additive effect still revoked- * every stale allow its inventory implied.- */- const recordSafetyPassOutcome = (outcome: 'completed' | 'failed'): void => {- // The fence is an assertion about live allows, so it only ever moves- // toward unsafe. The additive phase re-runs reconcileContext, which- // carries a second revocation lane over a fresh LIST; losing the fence- // there happens after this pass certified, and first-outcome-wins would- // otherwise leave the gate green over an allow that was not revoked.- if (outcome === 'failed') this.safetyPassLeftUncertified = true- // The duration sample describes the pass's primary outcome, so it stays- // first-wins: a certified pass that later fails an additive effect did- // complete its authoritative revocation.- if (safetyPassOutcomeRecorded) return- safetyPassOutcomeRecorded = true- this.safetyPassLeftUncertified = outcome === 'failed'- networkPolicySafetyPassDurationSeconds.observe(- { outcome },- (performance.now() - safetyPassStartedAt) / 1_000- )- }-- try {- await this.fullReconcileInternal(contexts, servers, {- ...options,- onAuthoritativeRevocationComplete: () => {- onAuthoritativeRevocationComplete()- recordSafetyPassOutcome('completed')- },- })- // The pass returned without certifying: no exception, no certificate.- // Operators alert on this exact condition, so it must produce a sample- // rather than the absence of one. Stated as a condition rather than- // leaning on first-outcome-wins to swallow it, because the fence below is- // deliberately not first-wins.- if (!safetyPassOutcomeRecorded) recordSafetyPassOutcome('failed')- } catch (error) {- recordSafetyPassOutcome('failed')- throw error- }- }-- private async fullReconcileInternal(- contexts: ContextCRD[],- servers: McpServerCRD[] = [],- options: NetworkPolicyFullReconcileOptions = {}+ options: { serverInventoryComplete?: boolean; ensureDefaults?: boolean } = {}): Promise<void> {console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)- // This is intentionally scoped to callers that use the authoritative- // revocation callback. Ordinary additive reconciles must not be presented- // as readiness safety passes.- const measureSafetyPass = options.onAuthoritativeRevocationComplete !== undefined- let safetyPassListedPolicies = 0- let safetyPassRevokedPolicies = 0- const recordSafetyPassRevocation = (): void => {- if (measureSafetyPass) safetyPassRevokedPolicies += 1- }-- // The revision fence starts at invocation, before any asynchronous- // default-policy setup can interleave a watch update with this snapshot.- const selectedContextDesiredRevision = options.contextDesiredRevision?.()- const selectedServerDesiredRevision = options.serverDesiredRevision?.()- const runContextEffect =- options.runContextEffect ?? ((_contextId: string, work: () => Promise<void>) => work())- const runServerEffect =- options.runServerEffect ?? ((_serverName: string, work: () => Promise<void>) => work())- const contextInventoryIsCurrent = (): boolean =>- (options.contextInventoryAuthoritative?.() ?? true) &&- (options.serverInventoryAuthoritative?.() ?? true)-if (options.ensureDefaults !== false) {await this.ensureDefaultPolicies()}- // Authority is monotonic within this pass. Once a watch loses authority,- // this inventory snapshot cannot safely become authoritative again even if- // the callback flips back before the next delete.- let contextAuthorityLost = false- let contextAuthorityWarningLogged = false- const contextCleanupAuthoritative = (): boolean => {- if (- !contextAuthorityLost &&- options.contextInventoryAuthoritative &&- !options.contextInventoryAuthoritative()- ) {- contextAuthorityLost = true- }- if (contextAuthorityLost && !contextAuthorityWarningLogged) {- contextAuthorityWarningLogged = true- console.warn(- '[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost'- )- }- return !contextAuthorityLost+ // Reconcile each context+ for (const context of contexts) {+ await this.reconcileContext(context)}- let serverAuthorityLost = false- let serverAuthorityWarningLogged = false- const serverCleanupAuthoritative = (): boolean => {- if (- !serverAuthorityLost &&- options.serverInventoryAuthoritative &&- !options.serverInventoryAuthoritative()- ) {- serverAuthorityLost = true- }- if (serverAuthorityLost && !serverAuthorityWarningLogged) {- serverAuthorityWarningLogged = true- console.warn(- '[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost'- )- }- return !serverAuthorityLost+ // Reconcile external egress for all existing servers on startup so a+ // controller restart converges pre-existing McpServer CRDs too.+ for (const server of servers) {+ await this.reconcileExternalEgress(server)}+ // Delete policies for contexts that no longer existconst desiredContextIds = new Set(contexts.map(c => c.spec.contextId))- const cleanupOrphanedContextPolicies = async (- policies: k8s.V1NetworkPolicy[],- deletePolicy: (policy: k8s.V1NetworkPolicy) => Promise<boolean>,- describePolicy: (name: string, contextId: string) => string- ): Promise<void> => {- if (!contextCleanupAuthoritative()) return- for (const policy of policies) {- const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]- if (!contextId) {- if (!contextCleanupAuthoritative()) return- const name = policy.metadata?.name- if (!name) {- throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')- }- console.warn(- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`- )- if (await deletePolicy(policy)) recordSafetyPassRevocation()- continue- }- if (!desiredContextIds.has(contextId)) {- const name = policy.metadata?.name || ''- await runContextEffect(contextId, async () => {- if (- !(await this.contextOrphanDeleteAllowed(- contextId,- contextCleanupAuthoritative,- options.resolveCurrentContextById- ))- ) {- return- }- console.log(describePolicy(name, contextId))- if (await deletePolicy(policy)) recordSafetyPassRevocation()- })- }+ const allContextPolicies = await this.listAllContextPolicies()++ for (const policy of allContextPolicies) {+ const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]+ if (contextId && !desiredContextIds.has(contextId)) {+ const name = policy.metadata?.name || ''+ console.log(+ `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`+ )+ await this.deletePolicy(name)}}- let allContextPolicies = contextCleanupAuthoritative()- ? await this.listAllContextPolicies()- : []- let allContextEgressPolicies = contextCleanupAuthoritative()- ? await this.listAllContextEgressPolicies()- : []- let allRpcProxyEgressPolicies = contextCleanupAuthoritative()- ? await this.listAllRpcProxyEgressPolicies()- : []- let allExternalPolicies: k8s.V1NetworkPolicy[] = []- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {- allExternalPolicies = await this.listAllExternalEgressPolicies()- }-- // Label selectors alone cannot prove safety: removing either ownership- // marker would hide an active allow from the selected LIST. Audit the- // bounded namespaces, then merge only owned or conservatively repairable- // candidates into their lane. Ambiguous reserved names fail the pass.- const needsMcpServerBroadInventory =- contextCleanupAuthoritative() ||- (options.serverInventoryComplete !== false && serverCleanupAuthoritative())- const broadMcpServerPolicies = needsMcpServerBroadInventory- ? await this.listNamespacePoliciesForSafety(config.namespace)- : []- const broadHostPolicies = contextCleanupAuthoritative()- ? await this.listNamespacePoliciesForSafety(config.hostNamespace)- : []- const broadRpcProxyPolicies = contextCleanupAuthoritative()- ? await this.listNamespacePoliciesForSafety(config.rpcProxyNamespace)- : []-- if (contextCleanupAuthoritative()) {- allContextPolicies = this.mergeSafetyLaneInventory(- allContextPolicies,- broadMcpServerPolicies,- 'context-ingress'- )- allContextEgressPolicies = this.mergeSafetyLaneInventory(- allContextEgressPolicies,- broadHostPolicies,- 'context-host-egress'- )- allRpcProxyEgressPolicies = this.mergeSafetyLaneInventory(- allRpcProxyEgressPolicies,- broadRpcProxyPolicies,- 'context-rpc-egress'- )- }- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {- allExternalPolicies = this.mergeSafetyLaneInventory(- allExternalPolicies,- broadMcpServerPolicies,- 'external-egress'- )- }- safetyPassListedPolicies +=- allContextPolicies.length +- allContextEgressPolicies.length +- allRpcProxyEgressPolicies.length +- allExternalPolicies.length-- // Delete orphaned Context policies across every L2 lane. Each lane uses- // the canonical contextId effect key and rechecks authority plus live CRD- // presence immediately before its delete.- await cleanupOrphanedContextPolicies(- allContextPolicies,- policy =>- this.deleteSafetyPolicySnapshot(config.namespace, policy, contextCleanupAuthoritative),- (name, contextId) =>- `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`- )- await cleanupOrphanedContextPolicies(- allContextEgressPolicies,- policy =>- this.deleteSafetyPolicySnapshot(config.hostNamespace, policy, contextCleanupAuthoritative),- name => `[NetPol] Deleting orphaned L2 egress policy "${name}"`- )- await cleanupOrphanedContextPolicies(- allRpcProxyEgressPolicies,- policy =>- this.deleteSafetyPolicySnapshot(- config.rpcProxyNamespace,- policy,- contextCleanupAuthoritative- ),- name => `[NetPol] Deleting orphaned rpc-proxy egress policy "${name}"`- )-- let liveDesiredRevisionChanged = false- for (const context of contexts) {- const contextId = context.spec.contextId- await runContextEffect(contextId, async () => {- const current = options.resolveCurrentContext- ? options.resolveCurrentContext(context.name)- : context- if (!current || current.spec.contextId !== contextId) {- liveDesiredRevisionChanged = true- return- }- const contextEffectIsCurrent = (): boolean => {- if (!contextCleanupAuthoritative() || !serverCleanupAuthoritative()) return false- if (!options.resolveCurrentContext) return true- const latest = options.resolveCurrentContext(current.name)- return latest !== undefined && sameContextDesiredRevision(current, latest)- }- const forContext = (policy: k8s.V1NetworkPolicy): boolean =>- policy.metadata?.labels?.[CONTEXT_LABEL] === contextId- const completed = await this.revokeStaleContextAllows(- current,- contextEffectIsCurrent,- {- context: allContextPolicies.filter(forContext),- hostEgress: allContextEgressPolicies.filter(forContext),- rpcProxyEgress: allRpcProxyEgressPolicies.filter(forContext),- },- recordSafetyPassRevocation- )- if (!completed && contextCleanupAuthoritative() && serverCleanupAuthoritative()) {- liveDesiredRevisionChanged = true- }- })+ // Also clean up orphaned rpc-proxy egress policies+ const allRpcProxyPolicies = await this.listAllRpcProxyEgressPolicies()+ for (const policy of allRpcProxyPolicies) {+ const ctxId = policy.metadata?.labels?.[CONTEXT_LABEL]+ if (ctxId && !desiredContextIds.has(ctxId)) {+ const pName = policy.metadata?.name || ''+ console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${pName}"`)+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, pName)+ }}// Clean up external egress policies for servers that no longer exist.- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {+ if (options.serverInventoryComplete !== false) {const desiredServerNames = new Set(servers.map(s => s.name))+ const allExternalPolicies = await this.listAllExternalEgressPolicies()for (const policy of allExternalPolicies) {const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]- if (!serverName) {- if (!serverCleanupAuthoritative()) break- const name = policy.metadata?.name- if (!name) {- throw new Error(- 'HCC-managed external egress NetworkPolicy is missing its name and owner label'- )- }- console.warn(- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`- )- if (- await this.deleteSafetyPolicySnapshot(- config.namespace,- policy,- serverCleanupAuthoritative- )- ) {- recordSafetyPassRevocation()- }- continue- }- if (!desiredServerNames.has(serverName)) {+ if (serverName && !desiredServerNames.has(serverName)) {const name = policy.metadata?.name || ''- await runServerEffect(serverName, async () => {- if (- !(await confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: serverCleanupAuthoritative,- resolveCurrent: () => options.resolveCurrentServer?.(serverName),- readCurrent: () =>- this.customApi.getNamespacedCustomObject({- group: GROUP,- version: VERSION,- namespace: config.namespace,- plural: PLURAL_MCPSERVERS,- name: serverName,- }),- }))- ) {- return- }- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)- if (- await this.deleteSafetyPolicySnapshot(- config.namespace,- policy,- serverCleanupAuthoritative- )- ) {- recordSafetyPassRevocation()- }- })+ console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)+ await this.deletePolicyInNamespace(config.namespace, name)}}- } else if (options.serverInventoryComplete === false) {+ } else {console.warn('[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete')}- if (options.serverInventoryComplete !== false) {- for (const server of servers) {- await runServerEffect(server.name, async () => {- const current = options.resolveCurrentServer- ? options.resolveCurrentServer(server.name)- : server- if (!current) {- liveDesiredRevisionChanged = true- return- }- const serverEffectIsCurrent = (): boolean => {- if (!serverCleanupAuthoritative()) return false- if (!options.resolveCurrentServer) return true- const latest = options.resolveCurrentServer(current.name)- return latest !== undefined && sameMcpServerDesiredRevision(current, latest)- }- const completed = await this.reconcileExternalEgressSafety(- current,- allExternalPolicies.filter(- policy => policy.metadata?.labels?.[MCPSERVER_LABEL] === server.name- ),- serverEffectIsCurrent,- () => {- recordSafetyPassRevocation()- options.onExternalEgressRevoked?.(current)- }- )- if (!completed && serverCleanupAuthoritative()) {- liveDesiredRevisionChanged = true- }- })- }- }-- const inventoryDesiredRevisionChanged =- (selectedContextDesiredRevision !== undefined &&- options.contextDesiredRevision?.() !== selectedContextDesiredRevision) ||- (selectedServerDesiredRevision !== undefined &&- options.serverDesiredRevision?.() !== selectedServerDesiredRevision)- if (- (liveDesiredRevisionChanged || inventoryDesiredRevisionChanged) &&- contextCleanupAuthoritative() &&- serverCleanupAuthoritative()- ) {- throw new Error('Desired NetworkPolicy inventory changed during authoritative revocation')- }-- if (- options.serverInventoryComplete !== false &&- contextCleanupAuthoritative() &&- serverCleanupAuthoritative()- ) {- if (measureSafetyPass) {- networkPolicySafetyPassPoliciesTotal.inc({ operation: 'listed' }, safetyPassListedPolicies)- networkPolicySafetyPassPoliciesTotal.inc(- { operation: 'revoked' },- safetyPassRevokedPolicies- )- }- options.onAuthoritativeRevocationComplete?.()- }-- // Additive Context convergence happens only after every stale allow policy- // implied by the authoritative inventory has been revoked. A slow or- // failing positive effect therefore cannot extend a stale-permission- // window after readiness becomes available. External egress creation and- // DNS refresh intentionally do not run here:- // ExternalEgressConvergenceCoordinator is their sole startup, retry, and- // periodic-resync owner.- const contextAdditiveFailures: unknown[] = []- for (const context of contexts) {- const contextId = context.spec.contextId- try {- await runContextEffect(contextId, async () => {- const current = options.resolveCurrentContext- ? options.resolveCurrentContext(context.name)- : context- if (!current || current.spec.contextId !== contextId) return- const contextEffectIsCurrent = (): boolean => {- if (!contextInventoryIsCurrent()) return false- if (!options.resolveCurrentContext) return true- const latest = options.resolveCurrentContext(current.name)- return latest !== undefined && sameContextDesiredRevision(current, latest)- }- await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })- })- } catch (error) {- contextAdditiveFailures.push(error)- console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)- }- }-- if (contextAdditiveFailures.length > 0) {- throw new AggregateError(- contextAdditiveFailures,- 'One or more additive Context NetworkPolicy reconciliations failed'- )- }-console.log('[NetPol] Full reconciliation complete')}- private async contextOrphanDeleteAllowed(- contextId: string,- inventoryAuthoritative: () => boolean,- resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined- ): Promise<boolean> {- if (!inventoryAuthoritative() || resolveCurrentContextById?.(contextId)) return false-- const response = (await this.customApi.listNamespacedCustomObject({- group: GROUP,- version: VERSION,- namespace: config.namespace,- plural: PLURAL_CONTEXTS,- })) as { items?: Array<{ spec?: { contextId?: string } }> }- if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false-- return inventoryAuthoritative() && !resolveCurrentContextById?.(contextId)- }-// ─── L3 External Egress ────────────────────────────────────────────- private buildPublicWebEgressPolicy(server: McpServerCRD, name: string): k8s.V1NetworkPolicy {- return {- apiVersion: 'networking.k8s.io/v1',- kind: 'NetworkPolicy',- metadata: {- name,- namespace: server.namespace,- labels: {- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,- [MCPSERVER_LABEL]: server.name,- [EGRESS_CLASS_LABEL]: 'public-web',- },- },- spec: {- podSelector: {- matchLabels: { [MCPSERVER_LABEL]: server.name },- },- policyTypes: ['Egress'],- egress: [- {- to: [- {- ipBlock: {- cidr: '0.0.0.0/0',- except: PUBLIC_EGRESS_EXCEPT_CIDRS,- },- },- ],- ports: [- { port: 443, protocol: 'TCP' },- { port: 80, protocol: 'TCP' },- ],- },- ],- },- }- }-- private buildExactHostEgressPolicy(- server: McpServerCRD,- name: string,- binding: EgressBinding,- cidrs: string[]- ): k8s.V1NetworkPolicy {- return {- apiVersion: 'networking.k8s.io/v1',- kind: 'NetworkPolicy',- metadata: {- name,- namespace: server.namespace,- labels: {- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,- [MCPSERVER_LABEL]: server.name,- [EGRESS_CLASS_LABEL]: 'exact-host',- },- },- spec: {- podSelector: {- matchLabels: { [MCPSERVER_LABEL]: server.name },- },- policyTypes: ['Egress'],- egress: cidrs.map(cidr => ({- to: [{ ipBlock: { cidr } }],- ports: [{ port: binding.port!, protocol: binding.protocol ?? 'TCP' }],- })),- },- }- }-- private async resolveExternalEgressDns(- hostname: string- ): Promise<{ address: string; ttl: number }[]> {- // Resolve with TTL (issue #299 H2 ratchet needs the smallest observed TTL)- // while keeping the #205 bounded-timeout race so a silent resolver cannot- // block safety convergence indefinitely.- let timeout: ReturnType<typeof setTimeout> | undefined- try {- return await Promise.race([- dns.resolve4(hostname, { ttl: true }),- new Promise<never>((_, reject) => {- timeout = setTimeout(- () =>- reject(- new Error(- `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`- )- ),- config.externalEgressDnsResolveTimeoutMs- )- }),- ])- } finally {- if (timeout !== undefined) clearTimeout(timeout)- }- }-- private async reconcileExternalEgressSafety(- server: McpServerCRD,- existingPolicies: k8s.V1NetworkPolicy[],- isCurrent: () => boolean,- onRevoked?: () => void- ): Promise<boolean> {- if (!isCurrent()) return false- const desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()- for (const binding of server.spec.egressBindings ?? []) {- const name = this.externalEgressPolicyName(server.name, binding)- if (!name) continue- const egressClass = binding.egressClass ?? 'exact-host'- if (- egressClass === 'public-web' &&- !binding.dns &&- !binding.cidr &&- binding.port === undefined &&- binding.protocol === undefined- ) {- desired.set(name, { binding, policy: this.buildPublicWebEgressPolicy(server, name) })- } else if (- egressClass === 'exact-host' &&- binding.cidr &&- isAllowedExternalEgressCidr(binding.cidr) &&- binding.port !== undefined &&- Number.isInteger(binding.port) &&- binding.port >= 1 &&- binding.port <= 65535- ) {- desired.set(name, {- binding,- policy: this.buildExactHostEgressPolicy(server, name, binding, [binding.cidr]),- })- } else if (- egressClass === 'exact-host' &&- binding.dns &&- isPublicDnsHostname(binding.dns) &&- binding.port !== undefined &&- Number.isInteger(binding.port) &&- binding.port >= 1 &&- binding.port <= 65535- ) {- desired.set(name, { binding })- }- }-- const stale: k8s.V1NetworkPolicy[] = []- const unprovableDnsPolicies: k8s.V1NetworkPolicy[] = []- const deterministicReplacements: Array<{- snapshot: k8s.V1NetworkPolicy- desired: k8s.V1NetworkPolicy- }> = []- for (const existing of existingPolicies) {- const name = existing.metadata?.name- const selected = name ? desired.get(name) : undefined- if (!selected) {- stale.push(existing)- continue- }- desired.delete(name!)- if (!selected.policy) {- // DNS binding: the /32s cannot be proven without a fresh lookup, but- // the binding IDENTITY (protocol, spec-level ports, selector,- // ownership) can. Retain the live allow when identity is unchanged;- // revoke only on an identity change (a port move in the CRD changes the- // policy NAME and is handled by the stale lane above). The comparison is- // "modulo cidr": the desired policy is rebuilt WITH the live cidrs, so- // the cidrs compare against themselves and everything else must match- // exactly. A policy with no ipBlock peers is a deny disguised as an- // allow and is never retained.- const existingCidrs = (existing.spec?.egress ?? [])- .flatMap(rule => rule.to ?? [])- .map(peer => peer.ipBlock?.cidr)- .filter((cidr): cidr is string => typeof cidr === 'string')- const identityUnchanged =- existingCidrs.length > 0 &&- hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE) &&- sameNetworkPolicySpec(- existing,- this.buildExactHostEgressPolicy(server, name!, selected.binding, existingCidrs)- )- if (!identityUnchanged) unprovableDnsPolicies.push(existing)- continue- }- const retainable =- sameNetworkPolicySpec(existing, selected.policy) &&- hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE)- if (!retainable) {- deterministicReplacements.push({ snapshot: existing, desired: selected.policy })- }- }-- // Preserve traffic only with deterministic CIDR/public-web policies. A- // renamed deterministic binding is created before its stale predecessor is- // removed; a same-name conflict is converged only under the ownership- // guard, which still refuses to adopt a foreign live object.- if (stale.length > 0) {- for (const selected of desired.values()) {- if (!selected.policy) continue- if (- !(await this.createSafetyPolicy(- server.namespace,- selected.policy,- 'external-egress',- isCurrent- ))- ) {- return false- }- }- }- for (const replacement of deterministicReplacements) {- if (- !(await this.replaceSafetyPolicySnapshot(- server.namespace,- replacement.snapshot,- replacement.desired,- 'external-egress',- isCurrent- ))- ) {- return false- }- onRevoked?.()- }-- // DNS-derived /32s are unprovable without a fresh lookup. Revoke them- // immediately; replacement belongs to the additive lane and must not put- // external DNS latency back on the global readiness path.- await this.cleanupExternalEgress(- server.name,- server.namespace,- unprovableDnsPolicies,- async () => isCurrent(),- onRevoked- )- if (!isCurrent()) return false-- await this.cleanupExternalEgress(- server.name,- server.namespace,- stale,- async () => isCurrent(),- onRevoked- )- return isCurrent()- }-/*** Reconcile external egress policies for an McpServer with egressBindings.* For DNS bindings, resolves hostnames to IPs and creates ipBlock rules.* For CIDR bindings, creates ipBlock rules directly.*/- async reconcileExternalEgress(- server: McpServerCRD,- options: NetworkPolicyMutationOptions- ): Promise<void> {- const isCurrent = options.isCurrent- if (!isCurrent()) return+ async reconcileExternalEgress(server: McpServerCRD): Promise<void> {let existingPolicies: k8s.V1NetworkPolicy[]try {existingPolicies = await this.listExternalEgressPoliciesForServer(@@ -1918,48 +816,36 @@ export class NetworkPolicyReconciler {server.namespace)} catch (error) {- if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'False','InventoryListFailed',- `Failed to list existing external egress policies: ${this.errorMessage(error)}`,- isCurrent+ `Failed to list existing external egress policies: ${this.errorMessage(error)}`)throw error}- if (!isCurrent()) returnconst bindings = server.spec.egressBindingsif (!bindings || bindings.length === 0) {try {- await this.cleanupExternalEgress(- server.name,- server.namespace,- existingPolicies,- async () => isCurrent()- )+ await this.cleanupExternalEgress(server.name, server.namespace, existingPolicies)} catch (error) {- if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'False','CleanupFailed',- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,- isCurrent+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`)throw error}- if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,[],'True','NoEgressBindings',- 'No external egress bindings declared',- isCurrent+ 'No external egress bindings declared')return}@@ -1971,11 +857,9 @@ export class NetworkPolicyReconciler {const desiredPolicyNames = new Set<string>()const resolvedEgressIPs: McpServerResolvedEgressIP[] = []const failures: string[] = []- const resolvedAtMs = Date.now()- const resolvedAt = new Date(resolvedAtMs).toISOString()+ const resolvedAt = new Date().toISOString()for (const binding of bindings) {- if (!isCurrent()) returnconst egressClass = binding.egressClass ?? 'exact-host'if (egressClass !== 'exact-host' && egressClass !== 'public-web') {failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)@@ -2021,13 +905,47 @@ export class NetworkPolicyReconciler {desiredPolicyNames.add(name)- // public-web is a static policy. Preserve #205's ownership-fenced write- // (applyOwnedPolicy asserts ownership + honors the isCurrent generation- // fence) while keeping dev's F2 no-churn gate so the TTL-accelerated- // resync does not rewrite it every tick.- const policy = this.buildPublicWebEgressPolicy(server, name)+ const policy: k8s.V1NetworkPolicy = {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name,+ namespace: server.namespace,+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,+ [MCPSERVER_LABEL]: server.name,+ [EGRESS_CLASS_LABEL]: egressClass,+ },+ },+ spec: {+ podSelector: {+ matchLabels: { [MCPSERVER_LABEL]: server.name },+ },+ policyTypes: ['Egress'],+ egress: [+ {+ to: [+ {+ ipBlock: {+ cidr: '0.0.0.0/0',+ except: PUBLIC_EGRESS_EXCEPT_CIDRS,+ },+ },+ ],+ ports: [+ { port: 443, protocol: 'TCP' },+ { port: 80, protocol: 'TCP' },+ ],+ },+ ],+ },+ }++ // public-web is a static policy — write only when new/changed/drifted so+ // the TTL-accelerated resync does not rewrite it every tick (audit F2).if (this.externalEgressWriteNeeded(existingPolicy, policy)) {- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)+ await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')}continue}@@ -2075,8 +993,7 @@ export class NetworkPolicyReconciler {maxEntries: config.externalEgressMaxEntries,}try {- const records = await this.resolveExternalEgressDns(binding.dns)- if (!isCurrent()) return+ const records = await dns.resolve4(binding.dns, { ttl: true })const uniqueIps = [...new Set(records.map(r => r.address))].sort()if (uniqueIps.length === 0) {failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)@@ -2177,25 +1094,43 @@ export class NetworkPolicyReconciler {desiredPolicyNames.add(name)- // Build the exact-host policy from #205's helper, then fold in the #299- // sliding-window state annotations so the next reconcile rehydrates the- // accumulated set instead of collapsing to a single snapshot.- const policy = this.buildExactHostEgressPolicy(server, name, binding, cidrs)- if (stateAnnotations) {- policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }+ const policy: k8s.V1NetworkPolicy = {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name,+ namespace: server.namespace,+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,+ [MCPSERVER_LABEL]: server.name,+ [EGRESS_CLASS_LABEL]: egressClass,+ },+ // Persist the sliding-window state so the next reconcile rehydrates the+ // accumulated set instead of collapsing to a single snapshot (#299).+ ...(stateAnnotations ? { annotations: stateAnnotations } : {}),+ },+ spec: {+ podSelector: {+ matchLabels: { [MCPSERVER_LABEL]: server.name },+ },+ policyTypes: ['Egress'],+ egress: cidrs.map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: binding.port, protocol }],+ })),+ },}// Write only when the enforced rules changed / the policy is new / drifted// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.- // Route the write through applyOwnedPolicy so #205's ownership assertion- // and isCurrent generation fence still guard the mutation.if (this.externalEgressWriteNeeded(existingPolicy, policy) ||egressRenewalDue ||egressStateChanged) {- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)+ await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')} else {console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)}@@ -2208,23 +1143,18 @@ export class NetworkPolicyReconciler {existingPolicies.filter(policy => {const name = policy.metadata?.namereturn Boolean(name) && !desiredPolicyNames.has(name!)- }),- async () => isCurrent(),- options.onRevoked+ }))} catch (error) {- if (!isCurrent()) returnawait this.writeExternalEgressStatus(server,resolvedEgressIPs,'False','CleanupFailed',- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,- isCurrent+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`)throw error}- if (!isCurrent()) returnif (failures.length > 0) {const message = failures.join('; ')@@ -2233,8 +1163,7 @@ export class NetworkPolicyReconciler {resolvedEgressIPs,'False','ExternalEgressRejected',- message,- isCurrent+ message)throw new Error(`External egress reconciliation failed for "${server.name}": ${message}`)}@@ -2246,8 +1175,7 @@ export class NetworkPolicyReconciler {'Reconciled',bindings.some(binding => binding.egressClass === 'public-web')? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'- : 'External egress policies reconciled',- isCurrent+ : 'External egress policies reconciled')}@@ -2257,9 +1185,7 @@ export class NetworkPolicyReconciler {async cleanupExternalEgress(serverName: string,namespace: string,- policies?: k8s.V1NetworkPolicy[],- deleteAllowed?: () => Promise<boolean>,- onDeleted?: () => void+ policies?: k8s.V1NetworkPolicy[]): Promise<void> {const policiesToDelete =policies ?? (await this.listExternalEgressPoliciesForServer(serverName, namespace))@@ -2268,19 +1194,13 @@ export class NetworkPolicyReconciler {const name = policy.metadata?.nameif (!name) continuetry {- if (deleteAllowed && !(await deleteAllowed())) return- await this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {- console.log(`[NetPol] Deleted external egress policy "${name}"`)- onDeleted?.()- })+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })+ console.log(`[NetPol] Deleted external egress policy "${name}"`)} catch (error: unknown) {if (getErrorCode(error) !== 404) {console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)throw error}- // A concurrent actor may already have removed the policy. The safety- // invariant still holds, but this pass did not revoke it and must not- // inflate the authoritative safety-pass `revoked` counter.}}}@@ -2362,19 +1282,10 @@ export class NetworkPolicyReconciler {resolvedEgressIPs: McpServerResolvedEgressIP[],status: 'True' | 'False' | 'Unknown',reason: string,- message: string,- isCurrent: () => boolean = () => true+ message: string): Promise<void> {- if (!isCurrent()) returnlet currentStatus: McpServerCrdStatus = {}let hasStatusObject = false- let currentMetadata:- | {- uid?: string- generation?: number- resourceVersion?: string- }- | undefinedtry {const current = (await this.customApi.getNamespacedCustomObjectStatus({@@ -2383,15 +1294,7 @@ export class NetworkPolicyReconciler {namespace: server.namespace,plural: PLURAL_MCPSERVERS,name: server.name,- })) as {- metadata?: {- uid?: string- generation?: number- resourceVersion?: string- }- status?: McpServerCrdStatus- }- currentMetadata = current.metadata+ })) as { status?: McpServerCrdStatus }hasStatusObject = typeof current.status === 'object' && current.status !== nullcurrentStatus = current.status ?? {}} catch (error) {@@ -2407,20 +1310,6 @@ export class NetworkPolicyReconciler {)throw error}- if (!isCurrent()) return-- if (- (server.uid !== undefined && currentMetadata?.uid !== server.uid) ||- (server.generation !== undefined && currentMetadata?.generation !== server.generation)- ) {- throw new Error(- `Refusing to write external egress status for stale McpServer "${server.name}": ` +- `expected uid=${String(server.uid)} generation=${String(server.generation)}, ` +- `read uid=${String(currentMetadata?.uid)} generation=${String(- currentMetadata?.generation- )}`- )- }const nextConditions = this.mergeStatusCondition(currentStatus.conditions ?? [], {type: 'ExternalEgressReady',@@ -2445,36 +1334,14 @@ export class NetworkPolicyReconciler {return}- const identityFence: JsonPatchOperation[] = []- const expectedUid = server.uid ?? currentMetadata?.uid- if (expectedUid !== undefined) {- identityFence.push({ op: 'test', path: '/metadata/uid', value: expectedUid })- }- const expectedGeneration = server.generation ?? currentMetadata?.generation- if (expectedGeneration !== undefined) {- identityFence.push({- op: 'test',- path: '/metadata/generation',- value: expectedGeneration,- })- }- if (currentMetadata?.resourceVersion !== undefined) {- identityFence.push({- op: 'test',- path: '/metadata/resourceVersion',- value: currentMetadata.resourceVersion,- })- }- const statusMutation: JsonPatchOperation[] = hasStatusObject+ const statusPatch: JsonPatchOperation[] = hasStatusObject? [{ op: 'add', path: '/status/resolvedEgressIPs', value: stableResolvedEgressIPs },{ op: 'add', path: '/status/conditions', value: nextConditions },]: [{ op: 'add', path: '/status', value: nextStatus }]- const statusPatch = [...identityFence, ...statusMutation]try {- if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObjectStatus({group: GROUP,version: VERSION,@@ -2527,61 +1394,6 @@ export class NetworkPolicyReconciler {}}- /** Broad, namespace-bounded inventory used only by the safety pass. */- private async listNamespacePoliciesForSafety(namespace: string): Promise<k8s.V1NetworkPolicy[]> {- try {- const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })- return response.items || []- } catch (error) {- console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)- throw error- }- }-- private mergeSafetyLaneInventory(- selected: k8s.V1NetworkPolicy[],- broad: k8s.V1NetworkPolicy[],- lane: SafetyInventoryLane- ): k8s.V1NetworkPolicy[] {- const result = new Map<string, k8s.V1NetworkPolicy>()- const keyFor = (policy: k8s.V1NetworkPolicy, index: number): string =>- policy.metadata?.uid ?? policy.metadata?.name ?? `unnamed-${index}`-- selected.forEach((policy, index) => result.set(keyFor(policy, index), policy))- broad.forEach((policy, index) => {- const classification = classifySafetyInventoryPolicy(policy, lane)- if (classification === 'ambiguous') {- const name = policy.metadata?.name ?? '<unnamed>'- throw new Error(- `Ambiguous NetworkPolicy ownership for reserved HCC ${lane} policy "${name}"`- )- }- if (classification === 'owned' || classification === 'repairable') {- result.set(keyFor(policy, selected.length + index), policy)- } else {- // The broad LIST is later and authoritative for this object identity.- // Do not retain a selector snapshot after the same object was relabelled- // out of HCC ownership.- result.delete(keyFor(policy, selected.length + index))- }- })- return [...result.values()]- }-- /** List all L2 context-allow egress policies in the mcp-host namespace. */- private async listAllContextEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {- try {- const response = await this.networkingApi.listNamespacedNetworkPolicy({- namespace: config.hostNamespace,- labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=context-allow`,- })- return response.items || []- } catch (error) {- console.error('[NetPol] Failed to list all Context egress policies:', error)- throw error- }- }-/** List all external-egress policies in the MCP server namespace. */private async listAllExternalEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {try {@@ -2609,234 +1421,26 @@ export class NetworkPolicyReconciler {}/** Create or update a NetworkPolicy. */- private async applyFixedPolicy(- name: string,- namespace: string,- policy: k8s.V1NetworkPolicy,- policyType: string- ): Promise<void> {- await applyNetworkPolicy(- this.networkingApi,- name,- namespace,- policy,- '[NetPol]',- undefined,- existing => {- if (!hasExpectedPolicyOwnership(existing, policyType)) {- throw new Error(- `NetworkPolicy "${name}" has conflicting ownership for the ${policyType} lane`- )- }- }- )+ private async applyPolicy(name: string, policy: k8s.V1NetworkPolicy): Promise<void> {+ await applyNetworkPolicy(this.networkingApi, name, config.namespace, policy, '[NetPol]')}- /** Create or update a Context ingress NetworkPolicy. */- private async applyContextIngressPolicy(- name: string,- policy: k8s.V1NetworkPolicy,- isCurrent?: () => boolean- ): Promise<void> {- await this.applyOwnedPolicy(name, config.namespace, policy, 'context-ingress', isCurrent)- }-- private async applyOwnedPolicy(- name: string,- namespace: string,- policy: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane,- isCurrent?: () => boolean- ): Promise<void> {- await applyNetworkPolicy(- this.networkingApi,- name,- namespace,- policy,- '[NetPol]',- isCurrent,- existing => this.assertPolicyOwnership(name, existing, policy, lane)- )- }-- private assertPolicyOwnership(- name: string,- existing: k8s.V1NetworkPolicy,- desired: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane- ): void {- const classification = classifySafetyInventoryPolicy(existing, lane)- const existingLabels = existing.metadata?.labels ?? {}- const desiredLabels = desired.metadata?.labels ?? {}- const ownerLabels =- lane === 'external-egress' ? [MCPSERVER_LABEL] : [CONTEXT_LABEL, MCPSERVER_LABEL]- if (- (classification !== 'owned' && classification !== 'repairable') ||- ownerLabels.some(label => existingLabels[label] !== desiredLabels[label])- ) {- throw new Error(`NetworkPolicy "${name}" has conflicting ownership for the ${lane} lane`)- }- }-- private safetyPolicyIdentity(policy: k8s.V1NetworkPolicy): {- name: string- uid: string- resourceVersion: string- } {- const name = policy.metadata?.name- const uid = policy.metadata?.uid- const resourceVersion = policy.metadata?.resourceVersion- if (!name || !uid || !resourceVersion) {- throw new Error(- `NetworkPolicy safety mutation requires name, uid, and resourceVersion for "${name ?? '<unnamed>'}"`- )- }- return { name, uid, resourceVersion }- }-- /**- * Delete exactly the policy version classified by the authoritative safety- * inventory. Kubernetes rejects the preconditions if the name was recreated- * or changed after LIST, so HCC cannot delete a new or foreign object.- */- private async deleteSafetyPolicySnapshot(- namespace: string,- policy: k8s.V1NetworkPolicy,- isCurrent?: () => boolean,- onDeleted?: () => void,- lostFenceOutcome: 'throw' | 'report' = 'throw'- ): Promise<boolean> {- if (isCurrent && !isCurrent()) return false- const { name, uid, resourceVersion } = this.safetyPolicyIdentity(policy)+ /** Delete a NetworkPolicy in mcp-server namespace. */+ private async deletePolicy(name: string): Promise<void> {try {await this.networkingApi.deleteNamespacedNetworkPolicy({name,- namespace,- body: { preconditions: { uid, resourceVersion } },+ namespace: config.namespace,})- console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)- onDeleted?.()- return true+ console.log(`[NetPol] Deleted policy "${name}"`)} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)- return true- }- if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {- // Kubernetes rejected the uid/resourceVersion preconditions, so the- // object live under this name is not the one the authoritative- // inventory classified. Nothing was revoked, and nothing may be- // adopted. Callers that opt in act on this outcome by declining to- // certify, which is the expected racy result rather than a failure.- // Callers that cannot act on it keep the loud default.- console.warn(- `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`- )- return false- }- console.error(- `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,- error- )- throw error- }- }-- /**- * Replace only the exact live object classified by the safety inventory.- * The fresh read closes ownership drift before mutation; resourceVersion on- * the replace body closes the remaining read-to-write race.- */- private async replaceSafetyPolicySnapshot(- namespace: string,- snapshot: k8s.V1NetworkPolicy,- desired: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane,- isCurrent: () => boolean- ): Promise<boolean> {- if (!isCurrent()) return false- const expected = this.safetyPolicyIdentity(snapshot)- const live = await this.networkingApi.readNamespacedNetworkPolicy({- name: expected.name,- namespace,- })- if (!isCurrent()) return false- const actual = this.safetyPolicyIdentity(live)- if (actual.uid !== expected.uid || actual.resourceVersion !== expected.resourceVersion) {- throw new Error(- `NetworkPolicy "${expected.name}" changed identity or ownership after the authoritative safety inventory`- )- }- this.assertPolicyOwnership(expected.name, live, desired, lane)- await this.networkingApi.replaceNamespacedNetworkPolicy({- name: expected.name,- namespace,- body: {- ...desired,- metadata: {- ...desired.metadata,- name: expected.name,- namespace,- resourceVersion: actual.resourceVersion,- },- },- })- return isCurrent()- }-- private async createSafetyPolicy(- namespace: string,- desired: k8s.V1NetworkPolicy,- lane: SafetyInventoryLane,- isCurrent: () => boolean- ): Promise<boolean> {- if (!isCurrent()) return false- let created: k8s.V1NetworkPolicy- try {- created = await this.networkingApi.createNamespacedNetworkPolicy({- namespace,- body: desired,- })- } catch (error: unknown) {- if (getErrorCode(error) !== 409) throw error- // This is the only creation route that does not go through- // applyNetworkPolicy, so a same-name object used to abort the whole- // safety pass and hold readiness down. Converge it exactly like every- // other lane does: adopt it only under the ownership guard, which still- // refuses a foreign or ambiguous object.- const name = desired.metadata?.name- if (!name) {- throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)+ console.log(`[NetPol] Policy "${name}" already gone`)+ } else {+ console.error(`[NetPol] Failed to delete policy "${name}":`, error)+ throw error}- console.log(- `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`- )- // Deliberately not delegating to applyOwnedPolicy here: that helper owns- // the create-then-conflict flow, so calling it from inside our own- // create's catch would issue a second create. Delegating from the top is- // not an option either — the desired-state fence below needs the object- // this request created, which applyNetworkPolicy does not return. The- // wiring below is kept identical to it on purpose.- await replaceWithConflictRetry<k8s.V1NetworkPolicy>({- description: `policy "${name}" in ${namespace}`,- logPrefix: '[NetPol]',- body: desired,- read: () => this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),- replace: body =>- this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),- mutationAllowed: isCurrent,- validateExisting: existing => this.assertPolicyOwnership(name, existing, desired, lane),- })- return isCurrent()}- if (isCurrent()) return true-- // The mutation crossed its desired-state fence. Remove exactly the object- // this request created, even though the old reconciliation is no longer- // authoritative. The UID/resourceVersion preconditions ensure a- // replacement or foreign policy with the same name is never deleted.- await this.deleteSafetyPolicySnapshot(namespace, created)- return false}/** List L2 egress counterpart policies in mcp-host namespace for a given context. */@@ -2853,15 +1457,37 @@ export class NetworkPolicyReconciler {}}+ /** Delete a NetworkPolicy in mcp-host namespace (L2 egress counterpart). */+ private async deleteEgressPolicy(name: string): Promise<void> {+ try {+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name,+ namespace: config.hostNamespace,+ })+ console.log(`[NetPol] Deleted L2 egress policy "${name}"`)+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) {+ console.log(`[NetPol] L2 egress policy "${name}" already gone`)+ } else {+ console.error(`[NetPol] Failed to delete L2 egress policy "${name}":`, error)+ throw error+ }+ }+ }+// ─── RPC-Proxy Egress (L2) ──────────────────────────────────────────- private buildRpcProxyEgressPolicy(+ /**+ * Create or update an egress policy in the rpc-proxy namespace allowing+ * rpc-proxy pods to reach a specific MCP server on a given port.+ */+ private async ensureRpcProxyEgress(contextId: string,serverName: string,port: number- ): k8s.V1NetworkPolicy {+ ): Promise<void> {const name = `rpc-egress-${contextId}-${serverName}`- return {+ const policy: k8s.V1NetworkPolicy = {apiVersion: 'networking.k8s.io/v1',kind: 'NetworkPolicy',metadata: {@@ -2892,18 +1518,16 @@ export class NetworkPolicyReconciler {],},}+ await applyNetworkPolicy(this.networkingApi, name, config.rpcProxyNamespace, policy, '[NetPol]')}/** Delete all rpc-proxy egress policies for a given context. */- private async deleteRpcProxyPoliciesForContext(- contextId: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteRpcProxyPoliciesForContext(contextId: string): Promise<void> {try {const policies = await this.listRpcProxyEgressPoliciesForContext(contextId)for (const policy of policies) {- if (deleteAllowed && !(await deleteAllowed())) return- await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)+ const name = policy.metadata?.name || ''+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, name)}} catch (error) {console.error(@@ -2947,19 +1571,34 @@ export class NetworkPolicyReconciler {}}+ /** Delete a NetworkPolicy in an arbitrary namespace. */+ private async deletePolicyInNamespace(namespace: string, name: string): Promise<void> {+ try {+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })+ console.log(`[NetPol] Deleted rpc-proxy egress policy "${name}"`)+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) {+ console.log(`[NetPol] rpc-proxy egress policy "${name}" already gone`)+ } else {+ console.error(`[NetPol] Failed to delete policy "${name}" in ${namespace}:`, error)+ throw error+ }+ }+ }+/** Delete a formerly static NetworkPolicy that is now generated dynamically. */private async deleteLegacyStaticPolicy(namespace: string, name: string): Promise<void> {- let existing: k8s.V1NetworkPolicytry {- existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })+ console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)} catch (error: unknown) {- if (getErrorCode(error) === 404) return- throw error- }- if (existing.metadata?.labels?.[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE) {- throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)+ if (getErrorCode(error) !== 404) {+ console.error(+ `[NetPol] Failed to delete legacy static policy "${name}" in "${namespace}":`,+ error+ )+ throw error+ }}- await this.deleteSafetyPolicySnapshot(namespace, existing)- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)}}diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.tsdeleted file mode 100644index a9b522ab0..000000000--- a/host-context-controller/src/readinessGate.test.ts+++ /dev/null@@ -1,53 +0,0 @@-import { describe, expect, it } from 'vitest'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'--describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {- it('dev provider (no watcher) is unconditionally authoritative', () => {- // Regression for R3-B1: main.ts left this undefined for DevMcpServerProvider,- // so the ContextMapperServer fail-closed default (() => false) pinned /ready- // and every /api/v1/* endpoint at 503 forever in dev mode. There is no- // NetworkPolicy inventory to certify in dev, so authority must be granted.- expect(resolveProviderAuthoritativeFn(null)()).toBe(true)- })-- it('production watcher delegates to the live inventory certification', () => {- let asked = false- const watcher = {- isReadinessInventoryAuthoritative: () => {- asked = true- return false- },- }- // Delegates (returns the watcher's verdict) rather than a constant.- expect(resolveProviderAuthoritativeFn(watcher)()).toBe(false)- expect(asked).toBe(true)- watcher.isReadinessInventoryAuthoritative = () => true- expect(resolveProviderAuthoritativeFn(watcher)()).toBe(true)- })-})--describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {- it('dev provider (no watcher) is unconditionally authoritative', () => {- // Regression for R2-M1: main.ts left this undefined for DevMcpServerProvider,- // so the ContextMapperServer fail-closed default (() => false) pinned every- // /api/v1/desktop/* response at 503 after setReady(true) — where the base- // answered 200 {status:'inactive'}. There is no Host inventory to certify in- // dev, so authority must mirror the provider gate and be granted.- expect(resolveHostAuthoritativeFn(null)()).toBe(true)- })-- it('production watcher delegates to the live Host inventory certification', () => {- let asked = false- const watcher = {- isHostInventoryAuthoritative: () => {- asked = true- return false- },- }- // Delegates (returns the watcher's verdict) rather than a constant.- expect(resolveHostAuthoritativeFn(watcher)()).toBe(false)- expect(asked).toBe(true)- watcher.isHostInventoryAuthoritative = () => true- expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)- })-})diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.tsdeleted file mode 100644index 88ff77ee6..000000000--- a/host-context-controller/src/readinessGate.ts+++ /dev/null@@ -1,41 +0,0 @@-/**- * Resolve the readiness-authority gate for the server from the provider.- *- * Production (McpServerWatcher present): readiness follows the live inventory- * certification — /ready is 200 only once stale allows are revoked.- *- * Dev (DevMcpServerProvider — no watcher): there is NO NetworkPolicy inventory- * to certify, so authority is unconditional. The ContextMapperServer default- * for a missing gate is fail-closed (`() => false`), which is correct for a- * caller that forgets to wire one — but in dev it would pin /ready (and every- * data endpoint) at 503 forever. Dev must therefore pass an explicit- * always-authoritative gate; this function is that single, named decision so- * the affirmative literal never gets copy-pasted into a production path.- */-export function resolveProviderAuthoritativeFn(- watcher: { isReadinessInventoryAuthoritative(): boolean } | null-): () => boolean {- return watcher ? () => watcher.isReadinessInventoryAuthoritative() : () => true-}--/**- * Resolve the Host-inventory-authority gate the desktop route uses.- *- * Production (McpServerWatcher present): desktop status follows the live Host- * inventory certification — a degraded Host lane 503s desktop rather than- * answering a stale 200 'inactive'.- *- * Dev (DevMcpServerProvider — no watcher): there is NO Host inventory to- * certify, exactly as there is no NetworkPolicy inventory for the provider- * gate. The ContextMapperServer default for a missing gate is fail-closed- * (`() => false`), which is correct for a caller that forgets to wire one — but- * in dev it would pin every /api/v1/desktop/* response at 503 forever (R2-M1).- * Dev must therefore mirror the provider gate and pass an explicit- * always-authoritative gate; this function is that single, named decision so- * the affirmative literal never gets copy-pasted into a production path.- */-export function resolveHostAuthoritativeFn(- watcher: { isHostInventoryAuthoritative(): boolean } | null-): () => boolean {- return watcher ? () => watcher.isHostInventoryAuthoritative() : () => true-}diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.tsdeleted file mode 100644index 4bd14f5e6..000000000--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts+++ /dev/null@@ -1,513 +0,0 @@-/**- * Crosses the whole readiness chain with a real NetworkPolicyReconciler.- *- * Every other test of this behaviour mocks one side. `k8sClient.test.ts` mocks- * `./networkPolicyReconciler`, so it can prove the readiness predicate *reads*- * the certification getter but not that the reconciler ever *degrades* it.- * `networkPolicyReconciler.test.ts` drives the real fence but never reaches an- * HTTP response. Each half is verified by mutation; the seam between them was- * not, and the seam is where the guarantee actually lives.- *- * Here `@kubernetes/client-node` is the only mock. The reconciler, the watcher- * predicate, and the HTTP server are the production classes, so the assertion- * is end-to-end: a delete that loses its uid/resourceVersion precondition- * during the additive phase — the 409 a second writer produces, and every- * rollout opens a two-replica window for one — must turn /ready from 200 into- * 503, and a later clean pass must turn it back.- */-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import * as http from 'http'-import { McpServerWatcher } from './k8sClient'-import { ContextMapperServer } from './server'--const mocks = vi.hoisted(() => ({- listNamespacedNetworkPolicy: vi.fn(),- deleteNamespacedNetworkPolicy: vi.fn(),- readNamespacedNetworkPolicy: vi.fn(),- createNamespacedNetworkPolicy: vi.fn(),- replaceNamespacedNetworkPolicy: vi.fn(),- listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),- getNamespacedCustomObject: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),- getNamespacedCustomObjectStatus: vi- .fn()- .mockResolvedValue({ metadata: { generation: 1, resourceVersion: 'rv-1' }, status: {} }),- patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),- watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),-}))--vi.mock('./config', () => ({- config: {- devMode: false,- namespace: 'mcp-server',- hostNamespace: 'mcp-host',- rpcProxyNamespace: 'rpc-proxy',- channelsNamespace: 'channels',- port: 0,- runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],- hostK8sRequestTimeoutMs: 30_000,- hostResyncIntervalSec: 0,- externalEgressResyncIntervalSec: 0,- controlApiBaseUrl: 'http://control-api.test:8090',- governedTracingEnabled: false,- },-}))--vi.mock('@kubernetes/client-node', () => {- class CustomObjectsApi {}- class CoreV1Api {}- class NetworkingV1Api {}- class PolicyV1Api {}- class AppsV1Api {}- class BatchV1Api {}- class RbacAuthorizationV1Api {}- class Watch {- constructor(_kc: unknown) {}- watch = mocks.watch- }- class KubeConfig {- loadFromDefault(): void {}- loadFromOptions(_o: unknown): void {}- makeApiClient(api: unknown): unknown {- if (api === NetworkingV1Api) {- return {- listNamespacedNetworkPolicy: mocks.listNamespacedNetworkPolicy,- deleteNamespacedNetworkPolicy: mocks.deleteNamespacedNetworkPolicy,- readNamespacedNetworkPolicy: mocks.readNamespacedNetworkPolicy,- createNamespacedNetworkPolicy: mocks.createNamespacedNetworkPolicy,- replaceNamespacedNetworkPolicy: mocks.replaceNamespacedNetworkPolicy,- }- }- if (api === CustomObjectsApi) {- return {- listNamespacedCustomObject: mocks.listNamespacedCustomObject,- getNamespacedCustomObject: mocks.getNamespacedCustomObject,- getNamespacedCustomObjectStatus: mocks.getNamespacedCustomObjectStatus,- patchNamespacedCustomObjectStatus: mocks.patchNamespacedCustomObjectStatus,- }- }- return new Proxy({}, { get: () => vi.fn().mockResolvedValue({ items: [] }) })- }- }- return {- CustomObjectsApi,- CoreV1Api,- NetworkingV1Api,- PolicyV1Api,- AppsV1Api,- BatchV1Api,- RbacAuthorizationV1Api,- Watch,- KubeConfig,- }-})--const MANAGED = 'clerum.io/managed-by'-const POLICY_TYPE = 'clerum.io/policy-type'-const CONTEXT_LABEL = 'clerum.io/context'-const SERVER_LABEL = 'clerum.io/mcpserver'--/** A stale context-allow: HCC-owned, for a server the Context no longer lists. */-const stalePolicy = {- metadata: {- name: 'ctx-alpha-ghost',- namespace: 'mcp-server',- uid: 'ghost-uid',- resourceVersion: '7',- labels: {- [MANAGED]: 'host-context-controller',- [POLICY_TYPE]: 'context-allow',- [CONTEXT_LABEL]: 'alpha',- [SERVER_LABEL]: 'ghost',- },- },- spec: { podSelector: {}, policyTypes: ['Ingress'] },-}--const alphaContext = {- name: 'alpha',- namespace: 'mcp-server',- spec: { contextId: 'alpha', mcpServers: [] },-}--async function readyStatus(server: InstanceType<typeof ContextMapperServer>): Promise<number> {- const listener = (server as unknown as { server: http.Server | null }).server- const address = listener?.address()- if (!address || typeof address === 'string') throw new Error('readiness listener never bound')- return new Promise((resolve, reject) => {- http- .get({ host: '127.0.0.1', port: address.port, path: '/ready' }, res => {- res.resume()- res.on('end', () => resolve(res.statusCode ?? 0))- })- .on('error', reject)- })-}--/** Aligns every revocation counter so only the fence can gate readiness. */-function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>): void {- const w = watcher as unknown as Record<string, unknown>- w.mcpServerCacheSynced = true- w.contextCacheSynced = true- w.hostCacheSynced = true- w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration- w.networkPolicyRevocationServerGeneration = w.mcpWatchGeneration- w.networkPolicyRevocationContextRevision = w.contextDesiredRevision- w.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision-}--describe('readiness withdrawal on a real lost delete fence', () => {- let watcher: InstanceType<typeof McpServerWatcher>- let server: InstanceType<typeof ContextMapperServer>-- beforeEach(async () => {- vi.clearAllMocks()- mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })- mocks.getNamespacedCustomObject.mockRejectedValue(- Object.assign(new Error('not found'), { code: 404 })- )- mocks.createNamespacedNetworkPolicy.mockResolvedValue({})- mocks.replaceNamespacedNetworkPolicy.mockResolvedValue({})- mocks.readNamespacedNetworkPolicy.mockResolvedValue(stalePolicy)- watcher = new McpServerWatcher()- server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>- watcher.isReadinessInventoryAuthoritative()- )- await server.start()- server.setReady(true)- })-- afterEach(async () => {- await server.stop()- await watcher.stop()- })-- it('turns /ready from 200 to 503 when the additive phase loses a delete fence, and back on a clean pass', async () => {- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler- alignRevocationCounters(watcher)-- // Baseline: nothing stale anywhere, so the pass certifies and the gate opens.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(200)-- // The stale allow only becomes visible after the authoritative phase has- // certified, so the safety phase is clean by construction and the additive- // re-LIST is what finds it. Its delete then loses the uid/resourceVersion- // precondition — a 409, exactly what a concurrent writer produces.- let certified = false- mocks.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace }: { namespace?: string }) => ({- items: certified && namespace === 'mcp-server' ? [stalePolicy] : [],- })- )- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('the object has been modified'), { code: 409 })- )-- await expect(- reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {- certified = true- },- })- ).rejects.toThrow()-- // Nothing moved: no watch generation bumped, no desired revision changed.- // Only the fence knows this pass left an allow it classified as stale.- alignRevocationCounters(watcher)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- expect(await readyStatus(server)).toBe(503)-- // The retry lands: the allow is gone and the pass certifies again.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)- expect(await readyStatus(server)).toBe(200)- })-- it('withholds readiness when the authoritative pass loses a delete fence under a provided safety snapshot', async () => {- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler- alignRevocationCounters(watcher)-- // Baseline: a clean authoritative pass certifies and the gate opens.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(200)-- // Unlike the additive-phase test above, the stale allow is visible to the- // authoritative LISTs from the very start of the pass, so fullReconcile- // classifies it into the safety snapshot it provides to the revocation- // lane. Its identity-bound delete then loses the uid/resourceVersion- // precondition (409). The external-egress selector stays empty so the- // fence is lost in the provided-snapshot lane and nowhere else.- mocks.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({- items:- namespace === 'mcp-server' && !(labelSelector ?? '').includes('external-egress')- ? [stalePolicy]- : [],- })- )- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('the object has been modified'), { code: 409 })- )-- // The guarantee under test is that the fence degrades at the point of- // loss, while the condemned pass is still unwinding — not only when the- // pass finally reports. Sample it through the authority callback the pass- // keeps polling: every sample before the lost delete must be certified,- // and every later sample must already be withdrawn.- let certifications = 0- const fenceSeenDuringPass: boolean[] = []- await expect(- reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => {- fenceSeenDuringPass.push(reconciler.hasCertifiedSafetyInventory())- return true- },- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {- certifications += 1- },- })- ).rejects.toThrow()-- expect(certifications).toBe(0)- expect(fenceSeenDuringPass.length).toBeGreaterThan(1)- expect(fenceSeenDuringPass[0]).toBe(true)- expect(fenceSeenDuringPass[fenceSeenDuringPass.length - 1]).toBe(false)-- alignRevocationCounters(watcher)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- expect(await readyStatus(server)).toBe(503)-- // A clean authoritative pass re-certifies and reopens the gate.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([alphaContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)- expect(await readyStatus(server)).toBe(200)- })-- it('short-circuits a scoped delta at the certificate source while the last authoritative pass left the inventory uncertified', async () => {- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler- const w = watcher as unknown as Record<string, any>-- // Register the real Context watch so the production watch handler — the- // scoped-delta certification lane — can be driven through its callback.- await w.startContextWatch('ctx-rv-1')- const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>- String(path).endsWith('/namespaces/mcp-server/contexts')- )- if (!contextWatchCall) throw new Error('Context watch was never registered')- const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>-- // Seed the prior Context so the MODIFIED event below is a same-identity- // delta: same uid, same contextId, new generation.- const priorContext = {- name: 'alpha',- namespace: 'mcp-server',- uid: 'ctx-alpha-uid',- generation: 1,- spec: { contextId: 'alpha', mcpServers: [] },- }- w.contexts.set('alpha', priorContext)- alignRevocationCounters(watcher)-- // Baseline clean authoritative pass: certified, gate open.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([priorContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(200)-- // The authoritative pass loses its delete fence under the provided- // snapshot, exactly as in the test above: the broad safety LISTs see the- // stale allow and its delete keeps 409ing. Every context-scoped LIST- // stays empty so the scoped delta fired below completes its own- // revocation cleanly — and because the 409 never clears, the convergence- // pass the watch event kicks off in the background can never certify- // either.- mocks.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- const selector = labelSelector ?? ''- const scoped =- selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')- return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }- }- )- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('the object has been modified'), { code: 409 })- )- await expect(- reconciler.fullReconcile([priorContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- ).rejects.toThrow()- alignRevocationCounters(watcher)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- expect(await readyStatus(server)).toBe(503)-- // Fire a same-identity MODIFIED delta. Its scoped revocation completes- // (its label-scoped LISTs are clean) and it carries a delta certificate,- // so the only thing standing between it and- // recordNetworkPolicySafetyCertificate is the uncertified fence left by- // the authoritative pass.- const warnSpy = vi.spyOn(console, 'warn')- try {- await fireContextEvent('MODIFIED', {- metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },- spec: { contextId: 'alpha', mcpServers: [] },- })-- // M1: the fence was lost before this delta, so- // currentNetworkPolicySafetyCertificate() returns null at the capture and no- // delta certificate is ever minted — the delta short-circuits at the source- // and never reaches the per-delta certification branch. No warn is emitted;- // nothing is recorded; readiness stays withdrawn via the independent- // readiness fence even after every counter is realigned.- expect(- warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))- ).toBe(false)- expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(503)- } finally {- warnSpy.mockRestore()- }- })-- it('declines to certify a scoped delta whose certificate outlived a fence lost mid-flight, and warns', async () => {- // Coverage preservation for the per-delta fence check (k8sClient.ts :3801):- // the delta certificate is minted while the fence is still certified, then a- // CONCURRENT authoritative pass loses its delete fence mid-flight. M1's- // source short-circuit cannot catch this — only the per-delta check can. This- // test is green with AND without M1; it guards the TOCTOU window, not M1.- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler- const w = watcher as unknown as Record<string, any>-- await w.startContextWatch('ctx-rv-2')- const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>- String(path).endsWith('/namespaces/mcp-server/contexts')- )- if (!contextWatchCall) throw new Error('Context watch was never registered')- const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>-- const priorContext = {- name: 'alpha',- namespace: 'mcp-server',- uid: 'ctx-alpha-uid',- generation: 1,- spec: { contextId: 'alpha', mcpServers: [] },- }- w.contexts.set('alpha', priorContext)- alignRevocationCounters(watcher)-- // Baseline clean authoritative pass: certified, gate open.- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})- await reconciler.fullReconcile([priorContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(200)-- // The delta certificate is captured now, while the fence is still certified.- // The FIRST scoped LIST the delta issues runs a concurrent authoritative pass- // that loses its own delete fence — degrading the fence AFTER the delta minted- // its certificate but BEFORE the delta reaches its certification branch.- let fenceBrokenMidFlight = false- mocks.listNamespacedNetworkPolicy.mockImplementation(- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {- const selector = labelSelector ?? ''- const scoped =- selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')- if (scoped && !fenceBrokenMidFlight) {- fenceBrokenMidFlight = true- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(- Object.assign(new Error('the object has been modified'), { code: 409 })- )- await reconciler- .fullReconcile([priorContext], [], {- ensureDefaults: false,- contextInventoryAuthoritative: () => true,- serverInventoryAuthoritative: () => true,- onAuthoritativeRevocationComplete: () => {},- })- .then(- () => {- throw new Error('interleaved authoritative pass should have lost its fence')- },- () => {}- )- }- return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }- }- )- const warnSpy = vi.spyOn(console, 'warn')- try {- await fireContextEvent('MODIFIED', {- metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },- spec: { contextId: 'alpha', mcpServers: [] },- })-- // The interleave actually ran (no vacuous pass) and the per-delta check- // withheld certification: it warned, recorded nothing, readiness stays 503.- expect(fenceBrokenMidFlight).toBe(true)- expect(- warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))- ).toBe(true)- expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)- alignRevocationCounters(watcher)- expect(await readyStatus(server)).toBe(503)- } finally {- warnSpy.mockRestore()- }- })-})diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 26a7f5ab0..92e74d6e8 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -1,4 +1,4 @@-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { beforeEach, describe, expect, it, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'import {asAppsApi,@@ -14,17 +14,6 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './constimport { McpServerReconciler } from './reconciler'import { McpServerCRD } from './types'-function deferred<T = void>(): {- promise: Promise<T>- resolve: (value: T | PromiseLike<T>) => void-} {- let resolve!: (value: T | PromiseLike<T>) => void- const promise = new Promise<T>(res => {- resolve = res- })- return { promise, resolve }-}-/*** Tests for the managed:false guard in the reconciler (Risk 1.7).*@@ -41,7 +30,6 @@ vi.mock('./config', () => ({controlPlaneNamespace: 'control-plane',hostNamespace: 'mcp-host',rpcProxyNamespace: 'rpc-proxy',- egressProxyImage: 'clerum/nginx-egress-proxy:test',devMcpServers: [],devContexts: [],devAuthTokens: new Map(),@@ -76,7 +64,6 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -168,55 +155,6 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()})- it('completes a remote proxy reconcile without rewriting the desired image', async () => {- const server = makeServer({- name: 'remote-api',- image: 'vendor/original-image:1',- remote: { baseUrl: 'https://api.example.com' },- })- const originalSpec = structuredClone(server.spec)- let desiredRevisionIsCurrent = true- customApi.patchNamespacedCustomObject.mockImplementation(- ({ body }: { body?: { spec?: unknown } }) => {- if (body?.spec !== undefined) {- desiredRevisionIsCurrent = false- }- return Promise.resolve({})- }- )-- await reconciler.reconcile(server, {- isCurrent: () => desiredRevisionIsCurrent,- })-- expect(server.spec).toEqual(originalSpec)- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(- expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),- expect.anything()- )- expect(coreApi.createNamespacedConfigMap).toHaveBeenCalledOnce()- expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(- expect.objectContaining({- body: expect.objectContaining({- spec: expect.objectContaining({- template: expect.objectContaining({- spec: expect.objectContaining({- containers: [- expect.objectContaining({- name: 'egress-proxy',- image: 'clerum/nginx-egress-proxy:test',- }),- ],- }),- }),- }),- }),- })- )- })-it('should log WRC-owned ownership when managed: false', async () => {const consoleSpy = vi.spyOn(console, 'log')const server = makeServer({ name: 'workflow-recipes', managed: false })@@ -285,7 +223,6 @@ describe('PR-B B1 — validateSecret result shape', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -470,12 +407,7 @@ describe('PR-B B1 — writeStatusCondition', () => {const getPatchedConditions = () => {const patch = customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0]- const op = patch.body.find(- (candidate: { op?: string; path?: string }) =>- candidate.op === 'add' &&- (candidate.path === '/status' || candidate.path === '/status/conditions')- )- if (!op) throw new Error('status condition add operation was not emitted')+ const op = patch.body[0]const conditions = op.path === '/status' ? op.value.conditions : op.valuereturn { patch, op, conditions }}@@ -483,7 +415,6 @@ describe('PR-B B1 — writeStatusCondition', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -578,216 +509,6 @@ describe('PR-B B1 — writeStatusCondition', () => {expect(types).toContain('Ready')})- it('records the reconciled McpServer generation on status conditions', async () => {- const server = {- ...makeServer({ name: 'pg' }),- generation: 17,- }-- await reconciler.writeStatusCondition(server, {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'done',- })-- const { conditions } = getPatchedConditions()- expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(- expect.objectContaining({- status: 'True',- observedGeneration: 17,- })- )- })-- it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {- const networkReady = {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'network ready',- lastTransitionTime: '2020-01-01T00:00:00.000Z',- }- const externalEgressReady = {- type: 'ExternalEgressReady',- status: 'True',- reason: 'EgressPolicyApplied',- message: 'external egress ready',- lastTransitionTime: '2020-01-02T00:00:00.000Z',- }- customApi.getNamespacedCustomObjectStatus- .mockResolvedValueOnce({- metadata: { resourceVersion: '100' },- status: { conditions: [networkReady] },- })- .mockResolvedValueOnce({- metadata: { resourceVersion: '101' },- status: { conditions: [networkReady, externalEgressReady] },- })- customApi.patchNamespacedCustomObjectStatus- .mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))- .mockResolvedValueOnce({})-- const server: McpServerCRD = {- ...makeServer({ name: 'pg' }),- uid: 'uid-pg',- generation: 17,- }- await reconciler.writeStatusCondition(server, {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'done',- })-- expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)- expect(customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body).toContainEqual({- op: 'test',- path: '/metadata/resourceVersion',- value: '100',- })- const secondPatch = customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body- expect(secondPatch).toContainEqual({- op: 'test',- path: '/metadata/resourceVersion',- value: '101',- })- const conditionPatch = secondPatch.find(- (candidate: { op?: string; path?: string }) =>- candidate.op === 'add' && candidate.path === '/status/conditions'- )- expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([- 'NetworkReady',- 'ExternalEgressReady',- 'Ready',- ])- })-- it('bounds resourceVersion conflict retries', async () => {- const currentStatus = {- metadata: { resourceVersion: '100' },- status: { conditions: [] },- }- customApi.getNamespacedCustomObjectStatus- .mockResolvedValueOnce(currentStatus)- .mockResolvedValueOnce(currentStatus)- .mockResolvedValueOnce(currentStatus)- const conflict = Object.assign(new Error('resourceVersion conflict'), { code: 422 })- customApi.patchNamespacedCustomObjectStatus- .mockRejectedValueOnce(conflict)- .mockRejectedValueOnce(conflict)- .mockRejectedValueOnce(conflict)- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)-- await reconciler.writeStatusCondition(- {- ...makeServer({ name: 'pg' }),- uid: 'uid-pg',- generation: 17,- },- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'done',- }- )-- expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)- expect(warn).toHaveBeenCalledWith(- expect.stringContaining('Failed to write status condition Ready=True'),- conflict- )- warn.mockRestore()- })-- it('guards the status patch with the captured UID, generation, and resourceVersion', async () => {- const staleServer: McpServerCRD = {- ...makeServer({ name: 'recreated' }),- uid: 'uid-before-recreation',- generation: 1,- }- customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- metadata: { resourceVersion: '100' },- status: { conditions: [] },- })- const writeFailure = Object.assign(new Error('status API unavailable'), {- code: 500,- })- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async request => {- expect(request.body.slice(0, 3)).toEqual([- {- op: 'test',- path: '/metadata/uid',- value: 'uid-before-recreation',- },- {- op: 'test',- path: '/metadata/generation',- value: 1,- },- {- op: 'test',- path: '/metadata/resourceVersion',- value: '100',- },- ])- throw writeFailure- })- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)-- await reconciler.writeStatusCondition(staleServer, {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'stale reconcile result',- })-- expect(warn).toHaveBeenCalledWith(- expect.stringContaining('Failed to write status condition Ready=True'),- writeFailure- )- warn.mockRestore()- })-- it('refreshes observedGeneration even when the condition payload is otherwise unchanged', async () => {- customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({- status: {- conditions: [- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'done',- observedGeneration: 16,- lastTransitionTime: '2020-01-01T00:00:00.000Z',- },- ],- },- })- const server = {- ...makeServer({ name: 'pg' }),- generation: 17,- }-- await reconciler.writeStatusCondition(server, {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'done',- })-- const { conditions } = getPatchedConditions()- expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(- expect.objectContaining({- observedGeneration: 17,- lastTransitionTime: '2020-01-01T00:00:00.000Z',- })- )- })-it('creates the status object when the CRD has no status yet', async () => {customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({})@@ -932,170 +653,6 @@ describe('PR-B B1 — writeStatusCondition', () => {})})-describe('restart-safe discovery status', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()- let reconciler: McpServerReconciler-- beforeEach(() => {- vi.clearAllMocks()- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(coreApi),- customApi: asCustomApi(customApi),- })- })-- it('uses a persisted Ready condition only when it observes the current generation', () => {- const server: McpServerCRD = {- ...makeServer({ name: 'restart-safe' }),- generation: 7,- status: {- conditions: [- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'Deployment created',- observedGeneration: 7,- },- ],- },- }-- expect(reconciler.getStatus(server)).toEqual({- deployed: true,- ready: true,- message: 'Deployment created',- authoritative: true,- })- })-- it.each([- ['stale', 6],- ['unversioned', undefined],- ])('fails closed for a %s persisted Ready condition', (_label, observedGeneration) => {- const server: McpServerCRD = {- ...makeServer({ name: 'restart-unsafe' }),- generation: 7,- status: {- conditions: [- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'Deployment created',- observedGeneration,- },- ],- },- }-- expect(reconciler.getStatus(server)).toEqual({- deployed: false,- ready: false,- message: 'Not reconciled yet',- authoritative: false,- })- })-- it('fails closed when persisted Ready conditions conflict for the current generation', () => {- const server: McpServerCRD = {- ...makeServer({ name: 'restart-conflict' }),- generation: 7,- status: {- conditions: [- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- observedGeneration: 7,- },- {- type: 'Ready',- status: 'False',- reason: 'SecretValidationFailed',- observedGeneration: 7,- },- ],- },- }-- expect(reconciler.getStatus(server)).toEqual({- deployed: false,- ready: false,- message: 'Not reconciled yet',- authoritative: false,- })- })-- it('prefers fresh in-memory status over persisted status from the same generation', async () => {- const server: McpServerCRD = {- ...makeServer({ name: 'live-status' }),- generation: 3,- status: {- conditions: [- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- observedGeneration: 3,- },- ],- },- }- appsApi.readNamespacedDeployment.mockResolvedValueOnce({- status: { readyReplicas: 0 },- })-- await reconciler.reconcile(server)-- expect(reconciler.getStatus(server)).toEqual(- expect.objectContaining({- deployed: true,- ready: false,- authoritative: true,- })- )- })-- it.each([- ['generation changes', 'stable-uid', 4],- ['the same name is recreated', 'replacement-uid', 3],- ])('does not reuse in-memory status when %s', async (_case, uid, generation) => {- const original: McpServerCRD = {- ...makeServer({ name: 'identity-fenced' }),- uid: 'original-uid',- generation: 3,- }- await reconciler.reconcile(original)- // #223 (credential rotation) makes readiness immediately False until the poll- // upgrades it, so the original caches deployed/authoritative here — not ready.- // This test's subject is the identity fencing asserted below, not readiness- // timing; it only needs a real (non-default) cached status to prove it is not- // reused across an identity change.- expect(reconciler.getStatus(original)).toEqual(- expect.objectContaining({ deployed: true, ready: false, authoritative: true })- )-- const current: McpServerCRD = {- ...original,- uid,- generation,- status: undefined,- }-- expect(reconciler.getStatus(current)).toEqual({- deployed: false,- ready: false,- message: 'Not reconciled yet',- authoritative: false,- })- })-})-describe('ownership-safe fail-closed cleanup', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()@@ -1105,7 +662,6 @@ describe('ownership-safe fail-closed cleanup', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -1208,480 +764,6 @@ describe('ownership-safe fail-closed cleanup', () => {})})-describe('full reconciliation inventory authority', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()- let reconciler: McpServerReconciler-- beforeEach(() => {- vi.clearAllMocks()- customApi.getNamespacedCustomObject.mockRejectedValue(- Object.assign(new Error('not found'), { code: 404 })- )- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(coreApi),- customApi: asCustomApi(customApi),- })- })-- it('rejects when Deployment inventory LIST fails so startup retry remains possible', async () => {- const inventoryError = new Error('deployment inventory unavailable')- const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)- appsApi.listNamespacedDeployment.mockRejectedValueOnce(inventoryError)-- await expect(reconciler.fullReconcile([])).rejects.toBe(inventoryError)-- expect(error).toHaveBeenCalledWith(- '[Reconciler] Failed to list managed deployments:',- inventoryError- )- error.mockRestore()- })-- it('defaults to fail-closed orphan cleanup until an authority source is wired', async () => {- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- appsApi.listNamespacedDeployment.mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- })-- await reconciler.fullReconcile([])-- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))- warn.mockRestore()- })-- it('finishes desired reconciliation but skips orphan cleanup if inventory authority is lost mid-pass', async () => {- let authority = { known: true, generation: 7 }- reconciler.setInventoryAuthority(() => authority)- const desired = makeServer({ name: 'desired-server' })- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {- authority = { known: false, generation: 8 }- })- appsApi.listNamespacedDeployment.mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- })-- await reconciler.fullReconcile([desired])-- expect(reconcile).toHaveBeenCalledOnce()- expect(reconcile).toHaveBeenCalledWith(desired, {- isCurrent: expect.any(Function),- })- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))- warn.mockRestore()- })-- it('skips orphan cleanup if the authoritative inventory generation changes mid-pass', async () => {- let authority = { known: true, generation: 11 }- reconciler.setInventoryAuthority(() => authority)- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)- vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {- authority = { known: true, generation: 12 }- })- appsApi.listNamespacedDeployment.mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- })-- await reconciler.fullReconcile([makeServer({ name: 'desired-server' })])-- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))- warn.mockRestore()- })-- it('deletes an orphan when inventory authority remains known and stable', async () => {- reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))- appsApi.listNamespacedDeployment.mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- })-- await reconciler.fullReconcile([])-- expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({- name: 'orphan-server',- namespace: 'mcp-server',- })- expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({- name: 'orphan-server',- namespace: 'mcp-server',- })- })-- it('skips a startup runtime revision superseded before its keyed effect is admitted', async () => {- const snapshot: McpServerCRD = {- ...makeServer({ name: 'updated-server' }),- generation: 1,- spec: {- contextRef: 'default',- image: 'clerum/updated-server:old',- transport: { type: 'streamableHttp', port: 3000 },- },- }- const current: McpServerCRD = {- ...makeServer({ name: 'updated-server' }),- generation: 2,- spec: {- contextRef: 'default',- image: 'clerum/updated-server:new',- transport: { type: 'streamableHttp', port: 3000 },- egressBindings: [{ dns: 'new.example', port: 443 }],- },- }- reconciler.setInventoryAuthority(() => ({ known: true, generation: 40 }))- reconciler.setResolveCurrentServer(() => current)- const reconcile = vi.spyOn(reconciler, 'reconcile')-- await reconciler.fullReconcile([snapshot])-- expect(reconcile).not.toHaveBeenCalled()- })-- it('does not mutate runtime or status after startup authority retires during Secret validation', async () => {- let authority = { known: true, generation: 51 }- const server = {- ...makeServer({- name: 'secret-authority-race',- envSecret: {- name: 'runtime-secret',- keys: [{ secretKey: 'token', envVar: 'TOKEN' }],- },- }),- uid: 'secret-authority-race-uid',- generation: 4,- }- reconciler.setInventoryAuthority(() => authority)- reconciler.setResolveCurrentServer(name => (name === server.name ? server : undefined))- const secretReadStarted = deferred()- const releaseSecretRead = deferred<{ data: Record<string, string> }>()- coreApi.readNamespacedSecret.mockImplementationOnce(async () => {- secretReadStarted.resolve(undefined)- return releaseSecretRead.promise- })-- const convergence = reconciler.fullReconcile([server])- await secretReadStarted.promise-- authority = { known: true, generation: 53 }- releaseSecretRead.resolve({ data: { token: 'encoded-token' } })- await convergence-- expect(coreApi.createNamespacedService).not.toHaveBeenCalled()- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- })-- it('retires a same-watch runtime revision superseded during Secret validation', async () => {- let current: McpServerCRD- const original: McpServerCRD = {- ...makeServer({- name: 'same-watch-runtime',- image: 'clerum/same-watch-runtime:v1',- envSecret: {- name: 'runtime-secret',- keys: [{ secretKey: 'token', envVar: 'TOKEN' }],- },- }),- uid: 'same-watch-runtime-uid',- generation: 1,- }- const replacement: McpServerCRD = {- ...original,- generation: 2,- spec: {- ...original.spec,- image: 'clerum/same-watch-runtime:v2',- },- }- current = original- reconciler.setInventoryAuthority(() => ({ known: true, generation: 61 }))- reconciler.setResolveCurrentServer(name => (name === original.name ? current : undefined))-- const originalSecretReadStarted = deferred()- const releaseOriginalSecretRead = deferred<{ data: Record<string, string> }>()- coreApi.readNamespacedSecret- .mockImplementationOnce(async () => {- originalSecretReadStarted.resolve(undefined)- return releaseOriginalSecretRead.promise- })- .mockResolvedValueOnce({ data: { token: 'encoded-token' } })-- const originalReconcile = reconciler.reconcile(original)- await originalSecretReadStarted.promise- current = replacement- const replacementReconcile = reconciler.reconcile(replacement)- releaseOriginalSecretRead.resolve({ data: { token: 'encoded-token' } })-- await Promise.all([originalReconcile, replacementReconcile])-- expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({- namespace: replacement.namespace,- body: expect.objectContaining({- spec: expect.objectContaining({- template: expect.objectContaining({- spec: expect.objectContaining({- containers: expect.arrayContaining([- expect.objectContaining({- image: 'clerum/same-watch-runtime:v2',- }),- ]),- }),- }),- }),- }),- })- })-- it('admits independent desired effects while another server effect is blocked', async () => {- const blocked = makeServer({ name: 'blocked-server' })- const independent = makeServer({ name: 'independent-server' })- let markBlockedStarted!: () => void- let releaseBlocked!: () => void- const blockedStarted = new Promise<void>(resolve => {- markBlockedStarted = resolve- })- const blockedRelease = new Promise<void>(resolve => {- releaseBlocked = resolve- })- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)- const fullReconcile = reconciler.fullReconcile([blocked, independent], {- runEffect: async (serverName, work) => {- if (serverName === blocked.name) {- markBlockedStarted()- await blockedRelease- }- await work()- },- })-- await blockedStarted- await Promise.resolve()-- expect(reconcile).toHaveBeenCalledWith(independent, {- isCurrent: expect.any(Function),- })- expect(reconcile.mock.calls.some(([server]) => server === blocked)).toBe(false)-- releaseBlocked()- await fullReconcile-- expect(reconcile).toHaveBeenCalledWith(blocked, {- isCurrent: expect.any(Function),- })- })-- it('bounds desired-effect admission without serializing independent siblings', async () => {- const desired = Array.from({ length: 25 }, (_, index) =>- makeServer({ name: `bounded-server-${index}` })- )- let active = 0- let maxActive = 0- let startedBeforeRelease = 0- let releaseAll!: () => void- const release = new Promise<void>(resolve => {- releaseAll = resolve- })- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)- const options = {- maxConcurrency: 3,- runEffect: async (_serverName: string, work: () => Promise<void>) => {- active += 1- startedBeforeRelease += 1- maxActive = Math.max(maxActive, active)- await release- try {- await work()- } finally {- active -= 1- }- },- }-- const fullReconcile = reconciler.fullReconcile(desired, options)- await vi.waitFor(() => expect(active).toBeGreaterThanOrEqual(3))- await Promise.resolve()- const observedMaxActive = maxActive- const observedStartedBeforeRelease = startedBeforeRelease-- releaseAll()- await fullReconcile-- expect(observedMaxActive).toBe(3)- expect(observedStartedBeforeRelease).toBe(3)- expect(reconcile).toHaveBeenCalledTimes(desired.length)- })-- it('waits for every admitted desired effect before reporting a sibling failure', async () => {- const failed = makeServer({ name: 'failed-server' })- const slow = makeServer({ name: 'slow-server' })- let markSlowStarted!: () => void- let releaseSlow!: () => void- const slowStarted = new Promise<void>(resolve => {- markSlowStarted = resolve- })- const slowRelease = new Promise<void>(resolve => {- releaseSlow = resolve- })- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)-- const fullReconcile = reconciler.fullReconcile([failed, slow], {- runEffect: async (serverName, work) => {- if (serverName === failed.name) {- throw new Error('failed server gate')- }- markSlowStarted()- await slowRelease- await work()- },- })- let settled = false- const observed = fullReconcile.then(- () => undefined,- error => error- )- void observed.then(() => {- settled = true- })-- await slowStarted- await Promise.resolve()- await Promise.resolve()-- expect(settled).toBe(false)-- releaseSlow()- const error = await observed-- expect(error).toBeInstanceOf(AggregateError)- expect(error).toMatchObject({- message: 'Failed to reconcile desired McpServers',- })- expect(reconcile).toHaveBeenCalledWith(slow, {- isCurrent: expect.any(Function),- })- })-- it('aggregates every desired-effect failure after all bounded workers settle', async () => {- const failures = [- new Error('first desired failure'),- new Error('second desired failure'),- new Error('third desired failure'),- ]- const desired = failures.map((_, index) => makeServer({ name: `failed-${index}` }))- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)-- const error = await reconciler- .fullReconcile(desired, {- maxConcurrency: 2,- runEffect: async serverName => {- throw failures[Number(serverName.replace('failed-', ''))]- },- })- .then(- () => undefined,- reason => reason- )-- expect(error).toBeInstanceOf(AggregateError)- expect((error as AggregateError).errors).toEqual(failures)- expect(reconcile).not.toHaveBeenCalled()- })-- it('does not delete an orphan candidate that was added to the same-generation cache mid-pass', async () => {- const currentServers = new Map<string, McpServerCRD>()- ;(reconciler as any).setResolveCurrentServer((name: string) => currentServers.get(name))- reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))- appsApi.listNamespacedDeployment.mockImplementationOnce(async () => {- currentServers.set('orphan-server', makeServer({ name: 'orphan-server' }))- return {- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- }- })-- await reconciler.fullReconcile([])-- expect(customApi.getNamespacedCustomObject).not.toHaveBeenCalled()- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()- })-- it('does not delete an orphan candidate when a fresh API GET sees a same-name CRD before ADDED arrives', async () => {- ;(reconciler as any).setResolveCurrentServer(() => undefined)- reconciler.setInventoryAuthority(() => ({ known: true, generation: 32 }))- customApi.getNamespacedCustomObject.mockResolvedValueOnce({- metadata: { name: 'orphan-server', namespace: 'mcp-server', uid: 'recreated-uid' },- })- appsApi.listNamespacedDeployment.mockResolvedValueOnce({- items: [- {- metadata: {- name: 'orphan-server',- namespace: 'mcp-server',- },- },- ],- })-- await reconciler.fullReconcile([])-- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledWith(- expect.objectContaining({- namespace: 'mcp-server',- plural: 'mcpservers',- name: 'orphan-server',- })- )- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()- })-})-describe('updateStatusConditions', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()@@ -1691,7 +773,6 @@ describe('updateStatusConditions', () => {beforeEach(() => {vi.clearAllMocks()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),customApi: asCustomApi(customApi),@@ -1754,53 +835,3 @@ describe('updateStatusConditions', () => {expect(types).toContain('DeploymentReady')})})--describe('unconfigured inventory authority fails closed (G1)', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()- let errorSpy: ReturnType<typeof vi.spyOn>-- // Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO- // setInventoryAuthority/setResolveCurrentServer wiring: the fence must deny.- function makeUnwiredReconciler(): McpServerReconciler {- return new McpServerReconciler({} as k8s.KubeConfig, {- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(coreApi),- customApi: asCustomApi(customApi),- })- }-- beforeEach(() => {- vi.clearAllMocks()- errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- })-- afterEach(() => {- errorSpy.mockRestore()- })-- it('T-G1a: reconcile() denies (no Deployment) and warns once when authority is unwired', async () => {- const reconciler = makeUnwiredReconciler()- await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()- expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('fail closed'))- })-- it('T-G1b: fullReconcile() denies when authority is unwired', async () => {- const reconciler = makeUnwiredReconciler()- await reconciler.fullReconcile([makeServer({ name: 'g1-mcp', managed: true })])- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()- })-- it('T-G1c: the explicit opt-in restores assume-current', async () => {- const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {- appsApi: asAppsApi(appsApi),- coreApi: asCoreApi(coreApi),- customApi: asCustomApi(customApi),- assumeInventoryAuthorityWhenUnconfigured: true,- })- await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))- expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()- })-})diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex 4056c621c..caba6be91 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -16,10 +16,6 @@ import { classifyPluginImage } from '@clerum/image-policy'import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'import { config } from './config'import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'-import {- confirmAuthoritativeMcpServerAbsence,- sameMcpServerDesiredRevision,-} from './mcpServerSafety'import { mcpserverMissingSecret } from './metrics'import { McpServerCRD, McpServerStatus } from './types'import {@@ -66,7 +62,6 @@ export interface McpServerCondition {reason: stringmessage: stringlastTransitionTime: string- observedGeneration?: number}type McpServerReconcilerDeps = {@@ -74,28 +69,6 @@ type McpServerReconcilerDeps = {coreApi?: k8s.CoreV1ApicustomApi?: k8s.CustomObjectsApinetworkingApi?: k8s.NetworkingV1Api- /**- * TEST-ONLY escape hatch. When the inventory-authority fence is not wired- * (setInventoryAuthority / setResolveCurrentServer never called), mutation- * fences fail CLOSED by default. Unit tests that exercise the reconciler- * without a watcher opt in to the legacy assume-current behavior here.- * Production construction (k8sClient) must never set this.- */- assumeInventoryAuthorityWhenUnconfigured?: boolean-}--export type McpServerInventoryAuthoritySnapshot = {- known: boolean- generation: number-}--export type McpServerFullReconcileOptions = {- runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>- maxConcurrency?: number-}--export type McpServerMutationOptions = {- isCurrent?: () => boolean}// G2: Pre-deploy handshake constants@@ -115,13 +88,6 @@ const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/// Bound the value length so a pathological CRD cannot blow up the ConfigMap.const MAX_AUTH_HEADER_VALUE_LENGTH = 2048const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'-// Keep fleet admission aligned with the external-egress convergence lane:-// enough parallelism for independent servers without allocating one pending-// effect promise per fleet member.-const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10-// Status is a read/merge/write transaction shared with other controllers.-// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.-const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3function throwCleanupFailures(failures: unknown[], message: string): void {if (failures.length > 0) {@@ -129,37 +95,6 @@ function throwCleanupFailures(failures: unknown[], message: string): void {}}-async function mapSettledWithConcurrency<T, TResult>(- items: readonly T[],- maxConcurrency: number,- work: (item: T, index: number) => Promise<TResult>-): Promise<Array<PromiseSettledResult<TResult>>> {- if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {- throw new RangeError('maxConcurrency must be a positive integer')- }-- const results = new Array<PromiseSettledResult<TResult>>(items.length)- let nextIndex = 0- const worker = async (): Promise<void> => {- while (nextIndex < items.length) {- const index = nextIndex- nextIndex += 1- try {- results[index] = {- status: 'fulfilled',- value: await work(items[index], index),- }- } catch (reason) {- results[index] = { status: 'rejected', reason }- }- }- }-- const workerCount = Math.min(maxConcurrency, items.length)- await Promise.all(Array.from({ length: workerCount }, () => worker()))- return results-}-/*** State of the readiness poll window for one McpServer (issue #223).*@@ -211,35 +146,6 @@ interface ReadinessPollState {* WaitingForReplicas by the very reconcile the terminal write triggers.*/detail: string- /**- * Mutation fence and CRD object of the LATEST reconcile that owns this- * window. pollReadiness() REBINDS both on every call that finds the window- * live: a superseding reconcile (e.g. the fullReconcile after a routine- * McpServer watch recovery bumps the inventory-authority generation) becomes- * the window's owner, so the next tick evaluates the CURRENT fence instead of- * the arming reconcile's stale closure. Without the rebind the stale fence- * self-terminates the window with no re-arm and — status-only MODIFIED events- * being suppressed, with no periodic McpServer resync — DeploymentReady=False- * sticks forever for a healthy rollout (the #223 rotation-timeout symptom,- * reintroduced under a routine reconnect).- *- * Two ORTHOGONAL supersession guards protect this window; do not assume one- * covers the other (that assumption is exactly how the F1 race was born):- * - Identity retirement: beginStatusTracking()/clearStatus() retire the- * window on any CRD uid or CRD-generation change (spec edit, recreate),- * so a surviving window always has the caller's status identity — a- * rebind can never adopt a recreate or a superseded desired revision.- * - In-tick generation reset + recheckRequested: a credential rotation- * bumps neither CRD uid nor CRD generation (it changes the Deployment's- * metadata.generation only), so identity retirement does NOT fire; the- * rotation is instead guarded by the tick's live Deployment read (a new- * generation resets attempts/exhausted) and by recheckRequested deferring- * a mid-flight tick's terminal verdict past the rotation's write.- * A rebind never touches the budget fields (generation/attempts/exhausted/- * detail) above.- */- isCurrent: () => boolean- server: McpServerCRD}export class McpServerReconciler {@@ -248,26 +154,8 @@ export class McpServerReconciler {private readonly customApi: k8s.CustomObjectsApiprivate readonly networkingApi?: k8s.NetworkingV1Api- /**- * Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.- * The fail-closed default prevents orphan cleanup until the watcher wires- * the real authority source.- */- private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({- known: false,- generation: 0,- })- private inventoryAuthorityConfigured = false- private resolveCurrentServer?: (name: string) => McpServerCRD | undefined- /** See McpServerReconcilerDeps: test-only opt-in to assume-current when the- * inventory-authority fence is unwired. Production leaves it false. */- private readonly assumeInventoryAuthorityWhenUnconfigured: boolean- private warnedUnconfiguredAuthority = false-/** Tracks the deployment status of each managed McpServer. */private readonly statusMap: Map<string, McpServerStatus> = new Map()- private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =- new Map()/** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */private readonly managedSnapshot: Map<string, boolean> = new Map()@@ -302,120 +190,20 @@ export class McpServerReconciler {this.networkingApi =deps?.networkingApi ??(typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)- this.assumeInventoryAuthorityWhenUnconfigured =- deps?.assumeInventoryAuthorityWhenUnconfigured ?? false- }-- /**- * Late-bound authority source for destructive full-reconcile cleanup.- * A full pass may finish reconciling desired servers, but it may delete an- * orphan only while the same authoritative inventory generation remains- * active from pass start through delete admission.- */- setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {- this.inventoryAuthority = fn- this.inventoryAuthorityConfigured = true- }-- /**- * Resolve current cache membership at effect admission. Full-reconcile- * snapshots select candidates only; they must never overwrite or delete a- * same-name CRD accepted by the continuing watch.- */- setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {- this.resolveCurrentServer = fn- }-- /**- * Verdict for a mutation fence evaluated while the inventory authority is NOT- * wired. Fail closed (deny) unless a test explicitly opted in; deny loudly- * once, because an unconfigured fence in a running controller is a- * construction bug, never a runtime condition.- */- private currentWhenUnconfigured(): boolean {- if (this.assumeInventoryAuthorityWhenUnconfigured) return true- if (!this.warnedUnconfiguredAuthority) {- this.warnedUnconfiguredAuthority = true- console.error(- '[Reconciler] inventory authority not configured — mutation fences fail closed. ' +- 'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'- )- }- return false}// ─── Status ────────────────────────────────────────────────────────/*** Get deployment status for an McpServer.- * Returns current in-memory status when this process has reconciled the- * server. After a controller restart, a generation-matched persisted Ready- * condition is the only safe fallback: a legacy condition without- * observedGeneration, or a condition for an older spec generation, cannot- * prove that the current desired runtime is ready.+ * Returns unknown status if the server hasn't been reconciled yet.*/- getStatus(serverOrName: string | McpServerCRD): McpServerStatus {- const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name- const current = this.statusMap.get(name)- if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {- return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }- }-- if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {- const readyConditions =- serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []- const persistedReady =- readyConditions.length === 1 &&- readyConditions[0].observedGeneration === serverOrName.generation- ? readyConditions[0]- : undefined- if (persistedReady) {- const ready = persistedReady.status === 'True'- return {- deployed: ready,- ready,- message: persistedReady.message,- authoritative: true,- }- }- }-- return {- deployed: false,- ready: false,- message: 'Not reconciled yet',- authoritative: false,- }- }-- private statusIdentityMatches(server: McpServerCRD): boolean {- const identity = this.statusIdentities.get(server.name)+ getStatus(name: string): McpServerStatus {return (- identity !== undefined &&- identity.uid === server.uid &&- identity.generation === server.generation+ this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' })}- private beginStatusTracking(server: McpServerCRD): void {- if (!this.statusIdentityMatches(server)) {- this.statusMap.delete(server.name)- // A new CRD identity (uid/generation) retires any poll window armed for- // the previous desired revision: its verdicts would describe a rollout- // that no longer represents the desired state. The reconcile for the- // new identity arms its own window after its Deployment write.- const poll = this.readinessPolls.get(server.name)- if (poll) {- if (poll.timer !== undefined) clearTimeout(poll.timer)- this.readinessPolls.delete(server.name)- }- }- this.statusIdentities.set(server.name, {- uid: server.uid,- generation: server.generation,- })- }-/*** Update the status of an McpServer after reconciliation.*/@@ -428,7 +216,6 @@ export class McpServerReconciler {*/private clearStatus(name: string): void {this.statusMap.delete(name)- this.statusIdentities.delete(name)this.managedSnapshot.delete(name)const poll = this.readinessPolls.get(name)if (poll) {@@ -520,26 +307,14 @@ export class McpServerReconciler {* window with a superseded generation's numbers and the new rollout would* never be observed (see ReadinessPollState.recheckRequested).*/- private pollReadiness(- server: McpServerCRD,- intervalMs = 5000,- maxAttempts = 24,- isCurrent: () => boolean = () => true- ): void {+ private pollReadiness(server: McpServerCRD, intervalMs = 5000, maxAttempts = 24): void {const { name, namespace } = serverconst previous = this.readinessPolls.get(name)if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {- // A window is already running for this server — leave its TIMER and- // BUDGET alone (re-arming would reset the attempt budget: the E2- // status-update-loop). But this caller is now the window's owner: rebind- // the mutation fence (and the same-identity server object) so the next- // tick evaluates the CURRENT fence instead of the arming reconcile's- // stale closure, which would otherwise self-terminate the window with no- // re-arm after a routine watch-generation bump.- previous.isCurrent = isCurrent- previous.server = server- // If its tick is executing RIGHT NOW, the tick's Deployment read may+ // A window is already running for this server — leave it alone+ // (re-arming would reset the attempt budget: the E2 status-update-loop).+ // But if its tick is executing RIGHT NOW, the tick's Deployment read may// predate the Deployment write that triggered THIS call (a new// generation), so a silent no-op could let the tick end the window on// stale data. Record that a re-check is owed; the tick honors it before@@ -562,8 +337,6 @@ export class McpServerReconciler {attempts: 0,exhausted: previous?.exhausted ?? false,detail: previous?.detail ?? '',- isCurrent,- server,}this.readinessPolls.set(name, state)@@ -573,21 +346,7 @@ export class McpServerReconciler {// can tell "tick running" apart from "window ended" during our awaits.state.timer = undefinedstate.ticking = true- // Mutation fencing: a superseded reconcile (desired-revision or- // inventory-authority change) must not keep observing — or writing- // status — on behalf of a stale desired state. End the window quietly;- // the current reconcile arms its own window after its Deployment write.- if (!state.isCurrent()) {- state.ticking = false- if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)- return- }const rollout = await this.readDeploymentRollout(name, namespace)- if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {- state.ticking = false- if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)- return- }// The McpServer may have been deleted (clearStatus) while we were// reading. A cancelled window must not keep ticking or write status@@ -636,20 +395,7 @@ export class McpServerReconciler {// always false, so this is the ONLY place a converged rollout is ever// recorded — without it every successful credential rotation would sit// at DeploymentReady=False forever and the UI would report a timeout.- await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)- // Readiness is decoupled from fleet convergence: the poll also- // upgrades the persisted, generation-matched Ready condition so a- // controller restart can trust the recorded readiness (see getStatus).- await this.writeStatusCondition(- state.server,- {- type: 'Ready',- status: 'True',- reason: 'ReconcileSuccess',- message: 'Deployment created',- },- state.isCurrent- )+ await this.updateStatusConditions(server, true)return}@@ -669,7 +415,7 @@ export class McpServerReconciler {// Give up loudly, on the CRD: a stuck rollout must be visible to the// operator with the numbers that prove it, never left as a stale// condition or a silent Unknown.- await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)+ await this.updateStatusConditions(server, false, rollout.detail)return}@@ -933,12 +679,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure ConfigMap exists and is up to date (create or replace).*/- private async ensureConfigMap(- server: McpServerCRD,- isCurrent: () => boolean = () => true- ): Promise<void> {+ private async ensureConfigMap(server: McpServerCRD): Promise<void> {const cm = this.buildNginxConfigMap(server)- if (!isCurrent()) returntry {await this.coreApi.createNamespacedConfigMap({namespace: server.namespace,@@ -951,7 +693,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvname: `${server.name}-nginx-conf`,namespace: server.namespace,})- if (!isCurrent()) returnconst next = preserveObjectAnnotations({...cm,@@ -962,7 +703,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},existing)- if (!isCurrent()) returnawait this.coreApi.replaceNamespacedConfigMap({name: `${server.name}-nginx-conf`,namespace: server.namespace,@@ -1070,13 +810,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {- // ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──- // The caller retains this object as the LIST -> WATCH desired-state cache.- // Mutating it here makes a later status-only watch event look like a- // user-driven spec revision, which invalidates the authoritative policy- // revocation marker during startup. Sanitization remains mandatory for the- // generated PodSpec, but must never rewrite the observed CRD inventory.- server = structuredClone(server)+ // ── Platform hardening: sanitize CRD before building PodSpec ──this.sanitizeCrdSpec(server)const transportPort = server.spec.transport.port || 3000@@ -1472,14 +1206,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure a Deployment exists and is up to date.*/- private async ensureDeployment(- server: McpServerCRD,- credentialsRevision = '',- isCurrent: () => boolean = () => true- ): Promise<void> {+ private async ensureDeployment(server: McpServerCRD, credentialsRevision = ''): Promise<void> {const deployment = this.buildDeployment(server, credentialsRevision)- if (!isCurrent()) returntry {await this.appsApi.createNamespacedDeployment({namespace: server.namespace,@@ -1514,7 +1243,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvnamespace: server.namespace,body,}),- mutationAllowed: isCurrent,})}}@@ -1522,13 +1250,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv/*** Ensure a Service exists and is up to date.*/- private async ensureService(- server: McpServerCRD,- isCurrent: () => boolean = () => true- ): Promise<void> {+ private async ensureService(server: McpServerCRD): Promise<void> {const service = this.buildService(server)- if (!isCurrent()) returntry {await this.coreApi.createNamespacedService({namespace: server.namespace,@@ -1552,7 +1276,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvnamespace: server.namespace,body,}),- mutationAllowed: isCurrent,})} else {throw error@@ -1565,16 +1288,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* Name matches are not enough: WRC-owned `managed:false` runtimes can share* names with McpServer CRDs, so every delete verifies HCC ownership labels.*/- private async deleteResources(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteResources(name: string, namespace: string): Promise<void> {const failures: unknown[] = []for (const cleanup of [- () => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),- () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),- () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),+ () => this.deleteDeploymentIfHccOwned(name, namespace),+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name),+ () => this.deleteServiceIfHccOwned(name, namespace),]) {try {await cleanup()@@ -1596,11 +1315,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName}- private async deleteDeploymentIfHccOwned(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteDeploymentIfHccOwned(name: string, namespace: string): Promise<void> {let deployment: k8s.V1Deploymenttry {deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })@@ -1618,7 +1333,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)return}- if (deleteAllowed && !(await deleteAllowed())) returntry {await this.appsApi.deleteNamespacedDeployment({ name, namespace })@@ -1636,8 +1350,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvprivate async deleteConfigMapIfHccOwned(configMapName: string,namespace: string,- serverName: string,- deleteAllowed?: () => Promise<boolean>+ serverName: string): Promise<void> {let configMap: k8s.V1ConfigMaptry {@@ -1656,7 +1369,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)return}- if (deleteAllowed && !(await deleteAllowed())) returntry {await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })@@ -1671,11 +1383,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async deleteServiceIfHccOwned(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteServiceIfHccOwned(name: string, namespace: string): Promise<void> {let service: k8s.V1Servicetry {service = await this.coreApi.readNamespacedService({ name, namespace })@@ -1693,7 +1401,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)return}- if (deleteAllowed && !(await deleteAllowed())) returntry {await this.coreApi.deleteNamespacedService({ name, namespace })@@ -1708,11 +1415,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async deleteNetworkPoliciesForServer(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteNetworkPoliciesForServer(name: string, namespace: string): Promise<void> {if (!this.networkingApi) returnconst failures: unknown[] = []const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]@@ -1726,7 +1429,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconst policyName = policy.metadata.nameif (!policyName) continuetry {- if (deleteAllowed && !(await deleteAllowed())) returnawait this.networkingApi.deleteNamespacedNetworkPolicy({name: policyName,namespace: policyNamespace,@@ -1758,19 +1460,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvthrowCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)}- private async deleteRuntimeResources(- name: string,- namespace: string,- deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {const failures: unknown[] = []try {- await this.deleteResources(name, namespace, deleteAllowed)+ await this.deleteResources(name, namespace)} catch (error) {failures.push(error)}try {- await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)+ await this.deleteNetworkPoliciesForServer(name, namespace)} catch (error) {failures.push(error)}@@ -1783,17 +1481,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.* Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.*/- private async setNetworkReadyAnnotation(- server: McpServerCRD,- isCurrent: () => boolean = () => true- ): Promise<void> {+ private async setNetworkReadyAnnotation(server: McpServerCRD): Promise<void> {// Use merge-patch (plain object body) instead of JSON Patch (array body).// JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations// to already exist — it fails with 422 when the CRD was created without annotations.// Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)// knows the handshake was not completed — WRC will time out and fail rather than proceed// silently without NetworkPolicies in place.- if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObject({group: 'clerum.io',@@ -1815,6 +1509,74 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}+ /**+ * Keep remote egress proxy CRDs aligned with the platform-owned runtime image.+ *+ * Remote servers are rendered as nginx egress proxies and never run+ * `server.spec.image`, but stale CR specs can mislead scanners or future+ * tooling. Patch only the declarative image field; Deployment rendering still+ * uses `config.egressProxyImage` as the authority.+ *+ * This stays internally non-throwing because image canonicalization is+ * best-effort cleanup; a failed CRD patch must not block ConfigMap,+ * Service, or Deployment reconciliation for the actual proxy runtime.+ */+ private async canonicalizeRemoteEgressProxyImage(server: McpServerCRD): Promise<void> {+ if (!this.isRemote(server)) {+ return+ }+ if (server.spec.image === config.egressProxyImage) {+ await this.writeStatusCondition(server, {+ type: 'ImageCanonicalized',+ status: 'True',+ reason: 'RemoteEgressProxyImageMatches',+ message: 'Remote McpServer spec.image matches the platform egress proxy image',+ })+ return+ }++ try {+ await this.customApi.patchNamespacedCustomObject(+ {+ group: 'clerum.io',+ version: 'v1alpha1',+ namespace: server.namespace,+ plural: 'mcpservers',+ name: server.name,+ body: {+ spec: {+ image: config.egressProxyImage,+ },+ },+ },+ {+ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],+ }+ )+ server.spec.image = config.egressProxyImage+ console.log(+ `[Reconciler] Canonicalized remote McpServer "${server.name}" image to ${config.egressProxyImage}`+ )+ await this.writeStatusCondition(server, {+ type: 'ImageCanonicalized',+ status: 'True',+ reason: 'RemoteEgressProxyImageMatches',+ message: 'Remote McpServer spec.image matches the platform egress proxy image',+ })+ } catch (err) {+ console.warn(+ `[Reconciler] Failed to canonicalize remote McpServer "${server.name}" image to ${config.egressProxyImage}:`,+ err+ )+ await this.writeStatusCondition(server, {+ type: 'ImageCanonicalized',+ status: 'False',+ reason: 'RemoteEgressProxyImagePatchFailed',+ message: 'Failed to patch remote McpServer spec.image to the platform egress proxy image',+ })+ }+ }+/*** G11: Update status conditions on McpServer CRD.* Sets NetworkReady and DeploymentReady conditions so WRC can watch@@ -1828,38 +1590,29 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* message so a stuck rollout reports WHAT is stuck (issue #223). Callers* that only know the verdict omit it.*/- rolloutDetail?: string,- isCurrent: () => boolean = () => true+ rolloutDetail?: string): Promise<void> {try {- await this.writeStatusCondition(- server,- {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'NetworkPolicies and Service created',- },- isCurrent- )- await this.writeStatusCondition(- server,- {- type: 'DeploymentReady',- status: deploymentReady ? 'True' : 'False',- reason: deploymentReady- ? 'ReplicasAvailable'- : rolloutDetail- ? 'RolloutIncomplete'- : 'WaitingForReplicas',- message: rolloutDetail- ? `Rollout did not converge — ${rolloutDetail}`- : deploymentReady- ? 'Deployment has ready replicas'- : 'Waiting for pods to become ready',- },- isCurrent- )+ await this.writeStatusCondition(server, {+ type: 'NetworkReady',+ status: 'True',+ reason: 'NetworkPoliciesApplied',+ message: 'NetworkPolicies and Service created',+ })+ await this.writeStatusCondition(server, {+ type: 'DeploymentReady',+ status: deploymentReady ? 'True' : 'False',+ reason: deploymentReady+ ? 'ReplicasAvailable'+ : rolloutDetail+ ? 'RolloutIncomplete'+ : 'WaitingForReplicas',+ message: rolloutDetail+ ? `Rollout did not converge — ${rolloutDetail}`+ : deploymentReady+ ? 'Deployment has ready replicas'+ : 'Waiting for pods to become ready',+ })console.log(`[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`)@@ -1884,29 +1637,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv** Failure modes:* - 404 (CRD deleted mid-reconcile) → log + swallow.- * - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.* - Other errors → log + swallow (status writes are best-effort).*/async writeStatusCondition(server: McpServerCRD,- condition: Omit<McpServerCondition, 'lastTransitionTime'>,- isCurrent: () => boolean = () => true+ condition: Omit<McpServerCondition, 'lastTransitionTime'>): Promise<void> {- if (!isCurrent()) return- const now = new Date().toISOString()-- // Update the missing-secret gauge for SecretResolved conditions.- // Do this before any no-op short-circuit so the gauge stays current- // across reconciles even when we skip the API patch.- if (condition.type === 'SecretResolved' && server.spec.envSecret) {- const labels = {- namespace: server.namespace,- name: server.name,- secret_name: server.spec.envSecret.name,- }- mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)- }-// Optimistic-concurrency loop (M2). The patch REPLACES the whole// /status/conditions array with a value built from a snapshot read just// before. Without a version guard, a concurrent writer — the readiness poll@@ -1916,11 +1652,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// op on metadata.resourceVersion makes the API server reject the patch (422)// if anything changed since our read; we then re-read and retry, so no// writer's condition is lost.- for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {- if (!isCurrent()) return- // Re-read on every optimistic-conflict retry. Merging against the- // latest resourceVersion preserves conditions written by peer- // controllers between our read and patch.+ const MAX_ATTEMPTS = 4+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {+ const now = new Date().toISOString()++ // Fetch current status (and its resourceVersion) to preserve existing+ // conditions and to fence the write below.let existingConditions: McpServerCondition[] = []let hasStatusObject = falselet resourceVersion: string | undefined@@ -1935,9 +1672,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvmetadata?: { resourceVersion?: string }status?: { conditions?: McpServerCondition[] }}+ resourceVersion = current.metadata?.resourceVersionhasStatusObject = typeof current.status === 'object' && current.status !== nullexistingConditions = current.status?.conditions ?? []- resourceVersion = current.metadata?.resourceVersion} catch (error) {if (getErrorCode(error) === 404) {console.warn(@@ -1951,16 +1688,26 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)return}- if (!isCurrent()) returnconst prior = existingConditions.find(c => c.type === condition.type)- const observedGeneration = condition.observedGeneration ?? server.generationconst unchanged =prior !== undefined &&prior.status === condition.status &&prior.reason === condition.reason &&- prior.message === condition.message &&- prior.observedGeneration === observedGeneration+ prior.message === condition.message++ // Update the missing-secret gauge for SecretResolved conditions. Do this+ // BEFORE the no-op short-circuit so the gauge stays current across+ // reconciles even when we skip the API patch. Idempotent, so a retry+ // re-running it is harmless.+ if (condition.type === 'SecretResolved' && server.spec.envSecret) {+ const labels = {+ namespace: server.namespace,+ name: server.name,+ secret_name: server.spec.envSecret.name,+ }+ mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)+ }// Skip the patch when the condition is identical to what's already on// the CRD. Without this, every reconcile bumps the McpServer's@@ -1980,49 +1727,26 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreason: condition.reason,message: condition.message,lastTransitionTime,- ...(observedGeneration !== undefined && { observedGeneration }),}const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]- const writePreconditions: Array<{- op: 'test'- path: string- value: string | number- }> = []- if (server.uid !== undefined) {- writePreconditions.push({- op: 'test',- path: '/metadata/uid',- value: server.uid,- })- }- if (server.generation !== undefined) {- writePreconditions.push({- op: 'test',- path: '/metadata/generation',- value: server.generation,- })- }- if (resourceVersion !== undefined) {- writePreconditions.push({- op: 'test',- path: '/metadata/resourceVersion',- value: resourceVersion,- })- }+ // The `test` op fences the write to the exact revision we read. If a+ // concurrent writer bumped resourceVersion in between, the API server+ // fails the patch (422) and we retry with a fresh read.const statusPatch = [- ...writePreconditions,- ...(hasStatusObject- ? [{ op: 'add', path: '/status/conditions', value: nextConditions }]- : [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),+ ...(resourceVersion+ ? [{ op: 'test', path: '/metadata/resourceVersion', value: resourceVersion }]+ : []),+ hasStatusObject+ ? { op: 'add', path: '/status/conditions', value: nextConditions }+ : { op: 'add', path: '/status', value: { conditions: nextConditions } },]try {// The generated client prefers JSON Patch for this endpoint. Send an// actual patch document so status writes keep working across client// upgrades instead of relying on merge-patch object bodies.- if (!isCurrent()) returnawait this.customApi.patchNamespacedCustomObjectStatus({group: 'clerum.io',version: 'v1alpha1',@@ -2033,25 +1757,22 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv})return} catch (error) {- const errorCode = getErrorCode(error)- if (errorCode === 404) {+ if (getErrorCode(error) === 404) {console.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`)return}- // 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer- // won the race. Re-read and retry rather than lose our condition. Any- // other error, or the last attempt, falls through to the best-effort- // log-and-swallow below.- if (- (errorCode === 409 || errorCode === 422) &&- attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS- ) {+ // 409 (Conflict) or 422 (the resourceVersion `test` op failed) mean a+ // concurrent writer won the race. Re-read and retry rather than lose+ // our condition. Any other error, or the last attempt, falls through to+ // the best-effort log-and-swallow below.+ const code = getErrorCode(error)+ if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) {continue}console.warn(- `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,+ `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${MAX_ATTEMPTS}):`,error)return@@ -2069,24 +1790,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* A prior failure does not block the next call; the chain absorbs it so* the next reconcile still runs.*/- async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {- const capturedAuthority = this.inventoryAuthority()- const callerIsCurrent = options.isCurrent ?? (() => true)- const isCurrent = () => {- if (!callerIsCurrent()) return false- if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()- if (!this.authorityMatches(capturedAuthority)) return false- const current = this.resolveCurrentServer?.(server.name)- return current !== undefined && sameMcpServerDesiredRevision(server, current)- }+ async reconcile(server: McpServerCRD): Promise<void> {const key = server.nameconst prev = this.inFlight.get(key) ?? Promise.resolve()- const next = prev- .catch(() => undefined)- .then(async () => {- if (!isCurrent()) return- await this.performReconcile(server, isCurrent)- })+ const next = prev.catch(() => undefined).then(() => this.performReconcile(server))this.inFlight.set(key, next)try {await next@@ -2097,12 +1804,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}}- private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {- if (!isCurrent()) return- this.beginStatusTracking(server)-+ private async performReconcile(server: McpServerCRD): Promise<void> {// G7: Enforce managed field immutability — once set, cannot change- if (!isCurrent()) returnconst currentManaged = server.spec.managed ?? trueconst previousManaged = this.managedSnapshot.get(server.name)if (previousManaged !== undefined && previousManaged !== currentManaged) {@@ -2110,7 +1813,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +`This is not allowed — delete and recreate the McpServer to change ownership.`)- if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -2118,7 +1820,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv})return}- if (!isCurrent()) returnthis.managedSnapshot.set(server.name, currentManaged)// If disabled, respect ownership before any cleanup. managed:false means@@ -2128,72 +1829,57 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.log(`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`)- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())+ await this.deleteRuntimeResources(server.name, server.namespace)} else {console.log(`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`)}- if (!isCurrent()) returnthis.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })- await this.writeStatusCondition(- server,- {- type: 'Ready',- status: 'False',- reason: 'Disabled',- message: 'McpServer is disabled',- },- isCurrent- )+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: 'False',+ reason: 'Disabled',+ message: 'McpServer is disabled',+ })return}if (!currentManaged) {- await this.reconcileWrcOwnedServer(server, isCurrent)+ await this.reconcileWrcOwnedServer(server)return}// Validate secret before creating/updatingconst secretResult = await this.validateSecret(server)- if (!isCurrent()) returnif (!secretResult.ok) {console.error(`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`)if (this.shouldFailClosedForSecretFailure(secretResult)) {- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())+ await this.deleteRuntimeResources(server.name, server.namespace)} else {console.warn(`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`)}- if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,message: secretResult.message,})- await this.writeStatusCondition(- server,- {- type: 'SecretResolved',- status: 'False',- reason: secretResult.reason,- message: secretResult.message,- },- isCurrent- )- await this.writeStatusCondition(- server,- {- type: 'Ready',- status: 'False',- reason: 'SecretValidationFailed',- message: secretResult.message,- },- isCurrent- )+ await this.writeStatusCondition(server, {+ type: 'SecretResolved',+ status: 'False',+ reason: secretResult.reason,+ message: secretResult.message,+ })+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: 'False',+ reason: 'SecretValidationFailed',+ message: secretResult.message,+ })return}@@ -2201,11 +1887,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvtry {// Remote egress proxy: create nginx ConfigMap before Deploymentif (this.isRemote(server)) {- // The CRD image remains desired-state input owned by its author. Remote- // workloads use the platform-owned proxy image selected in- // buildDeployment(); rewriting spec.image here would bump generation- // and retire this reconcile before it can create the runtime resources.- await this.ensureConfigMap(server, isCurrent)+ await this.canonicalizeRemoteEgressProxyImage(server)+ await this.ensureConfigMap(server)} else {// Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a// would-be denial and still builds; enforce mode blocks the workload.@@ -2217,17 +1900,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconst message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`if (config.enforcePluginImageAllowlist) {console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)- await this.writeStatusCondition(- server,- {- type: 'Ready',- status: 'False',- reason: 'ImageNotAllowed',- message,- },- isCurrent- )- if (!isCurrent()) return+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: 'False',+ reason: 'ImageNotAllowed',+ message,+ })this.setStatus(server.name, { deployed: false, ready: false, message })return}@@ -2236,12 +1914,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}}- if (!isCurrent()) return- await this.ensureService(server, isCurrent)- if (!isCurrent()) return- await this.ensureDeployment(server, secretResult.revision, isCurrent)+ await this.ensureService(server)+ await this.ensureDeployment(server, secretResult.revision)} catch (err) {- if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -2258,9 +1933,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvserver.annotations?.[NETWORK_READY_ANNOTATION] !== 'true') {try {- await this.setNetworkReadyAnnotation(server, isCurrent)+ await this.setNetworkReadyAnnotation(server)} catch (err) {- if (!isCurrent()) returnthis.setStatus(server.name, {deployed: false,ready: false,@@ -2272,7 +1946,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// Check readiness after reconciliationconst rollout = await this.readDeploymentRollout(server.name, server.namespace)- if (!isCurrent()) returnconst ready = rollout.readythis.setStatus(server.name, {deployed: true,@@ -2297,36 +1970,28 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvpollState.generation === rollout.generation? pollState.detail: undefined- await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)+ await this.updateStatusConditions(server, ready, exhaustedDetail)// PR-B B1: record SecretResolved=True + Ready=True after a successful// reconcile. This is additive to updateStatusConditions (NetworkReady /// DeploymentReady); the merge logic in writeStatusCondition preserves// the existing conditions by type.if (server.spec.envSecret) {- await this.writeStatusCondition(- server,- {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'Secret resolved and validated',- },- isCurrent- )+ await this.writeStatusCondition(server, {+ type: 'SecretResolved',+ status: 'True',+ reason: 'SecretFound',+ message: 'Secret resolved and validated',+ })}- await this.writeStatusCondition(- server,- {- type: 'Ready',- status: ready ? 'True' : 'Unknown',- reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',- message: ready- ? 'Deployment created'- : 'Deployment created, waiting for pods to become ready',- },- isCurrent- )+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: ready ? 'True' : 'Unknown',+ reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',+ message: ready+ ? 'Deployment created'+ : 'Deployment created, waiting for pods to become ready',+ })// If not ready yet, poll until it becomes ready. With generation-aware// readiness this is the normal path right after any Deployment write —@@ -2343,8 +2008,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// is a deployment whose replica tuple perpetually OSCILLATES, and that is// bounded to one reconcile per poll window (~120s) — not a tight loop, and (per// pollReadiness's atomic check-then-set) not a timer leak.- if (!ready && isCurrent()) {- this.pollReadiness(server, 5000, 24, isCurrent)+ if (!ready) {+ this.pollReadiness(server)}}@@ -2358,18 +2023,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv)}- private async reconcileWrcOwnedServer(- server: McpServerCRD,- isCurrent: () => boolean- ): Promise<void> {- if (!isCurrent()) return+ private async reconcileWrcOwnedServer(server: McpServerCRD): Promise<void> {console.log(`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +'skipping HCC runtime creation/deletion and updating discovery status.')const secretResult = await this.validateSecret(server)- if (!isCurrent()) returnif (!secretResult.ok) {this.setStatus(server.name, {deployed: true,@@ -2377,58 +2037,41 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvmessage: secretResult.message,})if (server.spec.envSecret) {- await this.writeStatusCondition(- server,- {- type: 'SecretResolved',- status: 'False',- reason: secretResult.reason,- message: secretResult.message,- },- isCurrent- )- }- await this.writeStatusCondition(- server,- {- type: 'Ready',+ await this.writeStatusCondition(server, {+ type: 'SecretResolved',status: 'False',- reason: 'SecretValidationFailed',+ reason: secretResult.reason,message: secretResult.message,- },- isCurrent- )+ })+ }+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: 'False',+ reason: 'SecretValidationFailed',+ message: secretResult.message,+ })return}- if (!isCurrent()) returnthis.setStatus(server.name, {deployed: true,ready: true,message: 'WRC-owned runtime registered',})if (server.spec.envSecret) {- await this.writeStatusCondition(- server,- {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'Secret resolved and validated',- },- isCurrent- )- }- await this.writeStatusCondition(- server,- {- type: 'Ready',+ await this.writeStatusCondition(server, {+ type: 'SecretResolved',status: 'True',- reason: WRC_OWNED_RUNTIME_READY_REASON,- message: 'WRC-owned runtime registered for HCC discovery',- },- isCurrent- )+ reason: 'SecretFound',+ message: 'Secret resolved and validated',+ })+ }+ await this.writeStatusCondition(server, {+ type: 'Ready',+ status: 'True',+ reason: WRC_OWNED_RUNTIME_READY_REASON,+ message: 'WRC-owned runtime registered for HCC discovery',+ })}/**@@ -2436,55 +2079,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv*/async reconcileDelete(name: string, namespace: string): Promise<void> {console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)- const capturedAuthority = this.inventoryAuthority()- await this.deleteRuntimeResources(name, namespace, () =>- this.orphanDeleteAllowed(name, namespace, capturedAuthority)- )+ await this.deleteRuntimeResources(name, namespace)this.clearStatus(name)}- private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {- const current = this.inventoryAuthority()- return captured.known && current.known && captured.generation === current.generation- }-- /**- * A cache miss is not proof of absence: the API watch may have accepted an- * ADDED event that has not reached this process yet. Confirm a 404 from the- * API and then recheck both cache membership and LIST→WATCH authority- * immediately before every destructive delete.- */- private async orphanDeleteAllowed(- name: string,- namespace: string,- capturedAuthority: McpServerInventoryAuthoritySnapshot- ): Promise<boolean> {- return confirmAuthoritativeMcpServerAbsence({- inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),- resolveCurrent: () => this.resolveCurrentServer?.(name),- readCurrent: () =>- this.customApi.getNamespacedCustomObject({- group: 'clerum.io',- version: 'v1alpha1',- namespace,- plural: 'mcpservers',- name,- }),- })- }-/*** Full reconciliation pass — sync desired state with actual state.* Called on startup to handle CRDs that were created while the skill-mapper was down.*/- async fullReconcile(- desiredServers: McpServerCRD[],- options: McpServerFullReconcileOptions = {}- ): Promise<void> {+ async fullReconcile(desiredServers: McpServerCRD[]): Promise<void> {console.log(`[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`)- const capturedAuthority = this.inventoryAuthority()// Get all deployments managed by skill-mapperconst existingDeployments = await this.listManagedDeployments()@@ -2494,62 +2100,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv.map(s => s.name))- const runEffect =- options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())- const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY-- // Create or update desired servers independently. runEffect owns each- // per-server lane, so a slow fleet member cannot become a process-wide- // startup barrier. The finite worker pool admits no more than the configured- // number of effects at once, while still waiting for every desired result- // before reporting aggregated failures or moving to orphan cleanup.- const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>- runEffect(server.name, async () => {- const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server- // Cache mutation happens synchronously before its keyed watch effect is- // queued. If this startup item was superseded, skip it and let the- // newer queued event perform external-egress-before-runtime using its- // own spec; reconciling the newer runtime here could otherwise race- // ahead of that safety policy.- if (!current || !sameMcpServerDesiredRevision(server, current)) return- const isCurrent = () => {- if (!this.inventoryAuthorityConfigured) {- if (!this.currentWhenUnconfigured()) return false- } else if (!this.authorityMatches(capturedAuthority)) {- return false- }- const latest = this.resolveCurrentServer?.(current.name)- return latest- ? sameMcpServerDesiredRevision(current, latest)- : this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()- }- await this.reconcile(current, { isCurrent })- })- )- throwCleanupFailures(- desiredResults- .filter((result): result is PromiseRejectedResult => result.status === 'rejected')- .map(result => result.reason),- 'Failed to reconcile desired McpServers'- )+ // Create or update desired servers+ for (const server of desiredServers) {+ await this.reconcile(server)+ }// Delete orphaned deployments (managed by us but no longer desired)for (const deployment of existingDeployments) {const name = deployment.metadata?.name || ''if (!desiredNames.has(name)) {const namespace = deployment.metadata?.namespace || config.namespace- await runEffect(name, async () => {- if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {- console.warn(- `[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`- )- return- }- console.log(`[Reconciler] Removing orphaned resources for "${name}"`)- await this.deleteRuntimeResources(name, namespace, () =>- this.orphanDeleteAllowed(name, namespace, capturedAuthority)- )- })+ console.log(`[Reconciler] Removing orphaned resources for "${name}"`)+ await this.deleteRuntimeResources(name, namespace)}}@@ -2568,7 +2130,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvreturn response.items || []} catch (error) {console.error('[Reconciler] Failed to list managed deployments:', error)- throw error+ return []}}}diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.tsindex 0ab7d03aa..633a48992 100644--- a/host-context-controller/src/server.test.ts+++ b/host-context-controller/src/server.test.ts@@ -1,6 +1,5 @@-import { afterEach, describe, expect, it, vi } from 'vitest'+import { afterEach, describe, expect, it } from 'vitest'import { McpServerProvider } from './k8sClient'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'import { ContextMapperServer } from './server'class FakeProvider implements McpServerProvider {@@ -76,30 +75,14 @@ describe('ContextMapperServer readiness', () => {afterEach(async () => {await server?.stop()server = null- vi.restoreAllMocks()- })-- it('withholds readiness when no inventory-authority gate is wired', async () => {- // The gate decides whether the provider's inventory is authoritative. A- // caller that forgets to wire it must not get a permanently Ready server:- // this endpoint is the contract that no stale allow is live, so the default- // has to fail closed. There is no type error to catch the omission.- server = new ContextMapperServer(new FakeProvider(), 0)- server.setReady(true)-- const response = await invoke(server, '/ready')- expect(response.statusCode).toBe(503)})it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)+ server = new ContextMapperServer(new FakeProvider(), 0)let response = await invoke(server, '/health')expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({- status: 'ok',- ready: false,- })+ expect(JSON.parse(response.body)).toEqual({ status: 'ok', ready: false })response = await invoke(server, '/ready')expect(response.statusCode).toBe(503)@@ -125,188 +108,4 @@ describe('ContextMapperServer readiness', () => {contextRef: '*',})})-- it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {- let providerAuthoritative = true- server = new ContextMapperServer(- new FakeProvider(),- 0,- undefined,- undefined,- () => providerAuthoritative- )-- let response = await invoke(server, '/ready')- expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })-- server.setReady(true)-- response = await invoke(server, '/ready')- expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })-- providerAuthoritative = false-- response = await invoke(server, '/ready')- expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })-- response = await invoke(server, '/health')- expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({- status: 'ok',- ready: false,- })-- response = await invoke(server, '/api/v1/mcpservers')- expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({- error: 'Service Unavailable',- message: 'Context mapper provider inventory is not authoritative',- })-- providerAuthoritative = true-- response = await invoke(server, '/ready')- expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })-- response = await invoke(server, '/api/v1/mcpservers')- expect(response.statusCode).toBe(200)- })-- it('fails readiness closed when the provider authority check throws', async () => {- const authorityError = new Error('authority unavailable')- const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {- throw authorityError- })- server.setReady(true)-- const response = await invoke(server, '/ready')-- expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })- expect(errorLog).toHaveBeenCalledWith(- '[Server] Provider authority readiness check failed:',- authorityError- )- })-- it('withdraws readiness as soon as shutdown begins', async () => {- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)- server.setReady(true)-- expect((await invoke(server, '/ready')).statusCode).toBe(200)-- await server.stop()-- const response = await invoke(server, '/ready')- expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })- })-})--describe('ContextMapperServer desktop status authority', () => {- let server: ContextMapperServer | null = null-- // A Host reconciler that reports a running desktop, so the only thing that can- // withhold a 200 is the Host-authority gate under test.- const runningReconciler = {- getStatus: () => ({ deployed: true, ready: true, message: '' }),- hasDesktop: () => true,- } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]- const hasDesktopFn = () => true-- afterEach(async () => {- await server?.stop()- server = null- vi.restoreAllMocks()- })-- it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {- // The heart of H1: desktop needs only Host authority. A degraded- // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,- // while /api/v1/mcpservers still 503s on the same server.- server = new ContextMapperServer(- new FakeProvider(),- 0,- runningReconciler,- hasDesktopFn,- () => false, // provider inventory NOT authoritative- () => true // Host inventory authoritative- )- server.setReady(true)-- const desktop = await invoke(server, '/api/v1/desktop/host-a')- expect(desktop.statusCode).toBe(200)- expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })-- const mcpservers = await invoke(server, '/api/v1/mcpservers')- expect(mcpservers.statusCode).toBe(503)- })-- it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {- server = new ContextMapperServer(- new FakeProvider(),- 0,- runningReconciler,- hasDesktopFn,- () => true, // provider authoritative — must not rescue desktop- () => false // Host inventory NOT authoritative- )- server.setReady(true)-- const desktop = await invoke(server, '/api/v1/desktop/host-a')- expect(desktop.statusCode).toBe(503)- expect(JSON.parse(desktop.body)).toEqual({- error: 'Service Unavailable',- message: 'Host inventory is not authoritative',- })- })-- it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {- // Wiring regression for R2-M1: main.ts resolves both authority gates from the- // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left- // the Host gate undefined, so the server's fail-closed default pinned every- // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200- // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly- // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.- const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)- const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)- server = new ContextMapperServer(- new FakeProvider(),- 0,- runningReconciler,- hasDesktopFn,- providerAuthoritativeFn,- hostAuthoritativeFn- )- server.setReady(true)-- const desktop = await invoke(server, '/api/v1/desktop/host-a')- expect(desktop.statusCode).toBe(200)- expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })- })-- it('fails desktop closed when no Host-authority gate is wired (default)', async () => {- // Omitting the 6th arg must default to fail-closed, exactly like the- // provider gate — no type error catches the omission.- server = new ContextMapperServer(- new FakeProvider(),- 0,- runningReconciler,- hasDesktopFn,- () => true- )- server.setReady(true)-- const desktop = await invoke(server, '/api/v1/desktop/host-a')- expect(desktop.statusCode).toBe(503)- expect(JSON.parse(desktop.body)).toEqual({- error: 'Service Unavailable',- message: 'Host inventory is not authoritative',- })- })})diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.tsindex 03f576dcf..7be800a8e 100644--- a/host-context-controller/src/server.ts+++ b/host-context-controller/src/server.ts@@ -8,53 +8,24 @@ import { McpServerProvider } from './k8sClient'import { registry } from './metrics'import { ErrorResponse, McpServersResponse } from './types'-type ReadinessState =- | { ready: true; status: 'ready' }- | { ready: false; status: 'starting' | 'degraded'; message: string }-export class ContextMapperServer {private server: http.Server | null = nullprivate provider: McpServerProviderprivate port: numberprivate hostReconciler: HostReconciler | nullprivate hasDesktopFn: ((hostRef: string) => boolean) | null- private providerAuthoritativeFn: () => boolean- private hostAuthoritativeFn: () => booleanprivate ready = falseconstructor(provider: McpServerProvider,port: number = config.port,hostReconciler?: HostReconciler,- hasDesktopFn?: (hostRef: string) => boolean,- // Fails closed. /ready is the assertion that no stale allow is live, and a- // caller that omits this gate gets no type error, so the default must- // withhold readiness rather than grant it.- providerAuthoritativeFn: () => boolean = () => false,- // Desktop status needs ONLY Host inventory authority, not the full- // readiness inventory. Fails closed like providerAuthoritativeFn: a caller- // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.- hostAuthoritativeFn: () => boolean = () => false+ hasDesktopFn?: (hostRef: string) => boolean) {this.provider = providerthis.port = portthis.hostReconciler = hostReconciler ?? nullthis.hasDesktopFn = hasDesktopFn ?? null- this.providerAuthoritativeFn = providerAuthoritativeFn- this.hostAuthoritativeFn = hostAuthoritativeFn- }-- /**- * Host inventory authority, fail-closed on error — the desktop route's only- * authority gate. Same shape as the provider check in getReadinessState().- */- private safeHostAuthoritative(): boolean {- try {- return this.hostAuthoritativeFn()- } catch (err) {- console.error('[Server] Host authority check failed:', err)- return false- }}/**@@ -64,30 +35,6 @@ export class ContextMapperServer {this.ready = ready}- private getReadinessState(): ReadinessState {- if (!this.ready) {- return {- ready: false,- status: 'starting',- message: 'Context mapper is still starting',- }- }-- try {- if (this.providerAuthoritativeFn()) {- return { ready: true, status: 'ready' }- }- } catch (err) {- console.error('[Server] Provider authority readiness check failed:', err)- }-- return {- ready: false,- status: 'degraded',- message: 'Context mapper provider inventory is not authoritative',- }- }-private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {// CORS headersres.setHeader('Access-Control-Allow-Origin', '*')@@ -110,20 +57,15 @@ export class ContextMapperServer {// Health checkif (req.method === 'GET' && url.pathname === '/health') {- const readiness = this.getReadinessState()- this.sendJson(res, 200, {- status: 'ok',- ready: readiness.ready,- })+ this.sendJson(res, 200, { status: 'ok', ready: this.ready })return}// Readiness checkif (req.method === 'GET' && url.pathname === '/ready') {- const readiness = this.getReadinessState()- this.sendJson(res, readiness.ready ? 200 : 503, {- status: readiness.status,- ready: readiness.ready,+ this.sendJson(res, this.ready ? 200 : 503, {+ status: this.ready ? 'ready' : 'starting',+ ready: this.ready,})return}@@ -136,30 +78,10 @@ export class ContextMapperServer {return}- // Desktop status: Host inventory is the ONLY authority this route needs, so- // it is gated here — ABOVE the blanket readiness gate — on Host authority- // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a- // degraded Host lane must 503 it (never answer 200 'inactive' from a stale- // cache). See the E3 design note's endpoint→authority matrix.- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {- if (!this.ready || !this.safeHostAuthoritative()) {- this.sendJson(res, 503, {- error: 'Service Unavailable',- message: 'Host inventory is not authoritative',- })- return- }- if (!this.checkDesktopAuth(req, res)) return- const hostRef = url.pathname.replace('/api/v1/desktop/', '')- await this.handleDesktopStatus(res, hostRef)- return- }-- const readiness = this.getReadinessState()- if (!readiness.ready) {+ if (!this.ready) {this.sendJson(res, 503, {error: 'Service Unavailable',- message: readiness.message,+ message: 'Context mapper is still starting',})return}@@ -188,6 +110,14 @@ export class ContextMapperServer {return}+ // Desktop status (requires service token)+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {+ if (!this.checkDesktopAuth(req, res)) return+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')+ await this.handleDesktopStatus(res, hostRef)+ return+ }+// 404this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })}@@ -268,8 +198,7 @@ export class ContextMapperServer {endpoints: {'GET /': 'This information page','GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",- 'GET /ready':- 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative',+ 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes','GET /api/v1/mcpservers': 'List all McpServer resources','GET /api/v1/mcpservers/context/:contextRef':'List McpServers filtered by contextRef (only enabled servers)',@@ -417,12 +346,9 @@ export class ContextMapperServer {* Stop the server.*/stop(): Promise<void> {- this.ready = false- const activeServer = this.server- this.server = nullreturn new Promise(resolve => {- if (activeServer) {- activeServer.close(() => {+ if (this.server) {+ this.server.close(() => {console.log('[Server] Context Mapper stopped')resolve()})diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.tsindex f731391ca..81542832a 100644--- a/host-context-controller/src/statelessLifecycleExecutor.ts+++ b/host-context-controller/src/statelessLifecycleExecutor.ts@@ -42,21 +42,6 @@ const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'*/const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'-/**- * H2: reflect a committed lifecycle outcome onto the CURRENT canonical cache- * entry for `name` — but only if that entry still has the admitted `uid`.- * The receiver (McpServerWatcher) owns the lookup and the uid guard; the- * executor never touches the cache directly. A same-name recreation with a- * different uid MUST be skipped silently (the mutation already committed- * server-side under the old uid; corrupting the new object would be worse- * than staleness, which the watch/resync repairs).- */-export type ReflectHostOutcomeFn = (- name: string,- uid: string | undefined,- apply: (target: HostCRD) => void-) => void-/** Constructor-injected seams from HostReconciler (late-bound where needed). */export interface StatelessLifecycleExecutorDeps {appsApi: k8s.AppsV1Api@@ -86,16 +71,7 @@ export interface StatelessLifecycleExecutorDeps {* serialization slot, so re-entering the per-host chain would* self-deadlock.*/- reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>- /**- * Capture Host inventory authority at heartbeat dispatch time and return an- * admission closure that revalidates that authority and resolves the current- * same-identity Host inside this executor's per-Host serializer. The two-step- * shape is intentional: capturing only after the queue drains cannot detect- * an authority generation change that happened while the work was waiting.- */- prepareHostMutationAdmission?: (action: string, host: HostCRD) => () => HostCRD- reflectHostOutcome?: ReflectHostOutcomeFn+ reconcileCore: (host: HostCRD) => Promise<void>onLifecycleStatusCommitted?: (host: HostCRD, lifecycle: HostLifecycleStatus) => void}@@ -106,9 +82,7 @@ export class StatelessLifecycleExecutor {private readonly now: () => Dateprivate readonly countCommunicationChannels: (hostName: string) => numberprivate readonly isCommunicationChannelCacheSynced: () => boolean- private readonly reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>- private readonly prepareHostMutationAdmission: (action: string, host: HostCRD) => () => HostCRD- private readonly reflectHostOutcome: ReflectHostOutcomeFn+ private readonly reconcileCore: (host: HostCRD) => Promise<void>private readonly onLifecycleStatusCommitted: (host: HostCRD,lifecycle: HostLifecycleStatus@@ -158,9 +132,6 @@ export class StatelessLifecycleExecutor {this.countCommunicationChannels = deps.countCommunicationChannelsthis.isCommunicationChannelCacheSynced = deps.isCommunicationChannelCacheSyncedthis.reconcileCore = deps.reconcileCore- this.prepareHostMutationAdmission =- deps.prepareHostMutationAdmission ?? ((_action, host) => () => host)- this.reflectHostOutcome = deps.reflectHostOutcome ?? (() => {})this.onLifecycleStatusCommitted = deps.onLifecycleStatusCommitted ?? (() => {})StatelessLifecycleExecutor.warnSingleReplicaInvariantOnce()}@@ -700,19 +671,15 @@ export class StatelessLifecycleExecutor {metadata: {name: stringnamespace?: string- uid?: string- generation?: numberresourceVersion?: stringannotations?: Record<string, string>}spec: HostCRD['spec']status?: HostCRD['status']}- const fresh: HostCRD = {+ return {name: obj.metadata.name,namespace: obj.metadata.namespace ?? host.namespace,- uid: obj.metadata.uid,- generation: obj.metadata.generation,// Preserve resourceVersion so heartbeat-path status writers can pass it// back as an optimistic-concurrency precondition (D3). Dropping it here// would force every write to be an unconditioned last-write-wins patch.@@ -721,8 +688,6 @@ export class StatelessLifecycleExecutor {spec: obj.spec,status: obj.status,}- this.requireSameHostIdentity(host, fresh)- return fresh}/**@@ -751,24 +716,17 @@ export class StatelessLifecycleExecutor {reason: string,entryWakeHandledGeneration: number): Promise<SuspendFromHeartbeatOutcome> {- return this.serializeHeartbeatMutation(- 'heartbeat suspension',- host,- (admittedHost, revalidate) =>- this.suspendHostFromHeartbeatCore(- admittedHost,- reason,- entryWakeHandledGeneration,- revalidate- )+ // L2: serialize on the host chain so a tracker-driven suspend never+ // interleaves with a watch-driven wake reconcile for the same host.+ return this.serializeByHost(host.name, () =>+ this.suspendHostFromHeartbeatCore(host, reason, entryWakeHandledGeneration))}private async suspendHostFromHeartbeatCore(host: HostCRD,reason: string,- entryWakeHandledGeneration: number,- revalidate?: () => HostCRD+ entryWakeHandledGeneration: number): Promise<SuspendFromHeartbeatOutcome> {// FRESH GET (see readFreshHost): guard against the CURRENT server-side// lifecycle, not the caller's watch-cache snapshot. The guard is re-run@@ -783,17 +741,6 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {- // UID proves object identity, not decision freshness. A same-identity- // Host may have advanced from stateless to stateful while the watch- // cache trails the API. Re-evaluate the complete effective stateless- // policy from every fresh read/retry before honoring old heartbeat- // evidence.- if (- !this.sameHostSpecRevision(host, fresh) ||- !this.effectiveLifecycleFromCache(fresh).stateless- ) {- return { skip: true }- }const freshLifecycle = fresh.status?.lifecycleif (freshLifecycle?.state !== 'draining') {// Every legitimate suspend follows a durable `draining` write (the@@ -855,36 +802,12 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh,- revalidate+ initialFresh)if ('skipped' in result || writtenStatus === undefined) {return skipOutcome}- // Revalidate authority/UID immediately before follow-on effects, then read- // once more from the API. The successful PATCH attempt's fresh object is- // the exact spec on which the status write committed, but a wake/spec- // update can land immediately AFTER that commit and BEFORE runtime- // reconciliation. A linearizable follow-on GET observes both our durable- // suspended write and any such newer same-UID update; the watch cache can- // legitimately trail either one.- const currentAtFollowOn = revalidate?.()- if (currentAtFollowOn) this.requireSameHostIdentity(host, currentAtFollowOn)- this.reflectOutcome(host, target => {- target.status = writtenStatus- })- const reconcileHost = await this.readFreshHost(result.fresh)- if (!this.sameHostSpecRevision(host, reconcileHost)) {- const error = new Error(- `Host spec generation changed before heartbeat suspension follow-on for "${host.name}"`- )- error.name = 'HostMutationSpecRevisionChangedError'- throw error- }- const currentAfterFollowOnRead = revalidate?.()- if (currentAfterFollowOnRead) {- this.requireSameHostIdentity(host, currentAfterFollowOnRead)- }+ host.status = writtenStatusthis.lastWrittenLifecycleStatus.delete(host.name)this.logSuspendedApplied(host.name)console.log(@@ -893,32 +816,18 @@ export class StatelessLifecycleExecutor {// L2: call reconcileCore (not the serialized reconcile) — this method// already runs inside this host's serialization slot, so re-entering the// per-host chain here would self-deadlock.- await this.reconcileCore(reconcileHost, revalidate)- // Preserve the historical caller-visible reflection contract even when- // follow-on work correctly used a fresher API object. The tracker may hold- // the originally admitted instance until the watch delivers the next- // MODIFIED event; reflect the final authoritative outcome immediately so- // it cannot make a second decision from the pre-follow-on status.- this.reflectOutcome(host, target => {- target.uid = reconcileHost.uid- target.generation = reconcileHost.generation- target.resourceVersion = reconcileHost.resourceVersion- target.annotations = reconcileHost.annotations- target.spec = reconcileHost.spec- target.status = reconcileHost.status- })+ await this.reconcileCore(host)// The wake fast-path / drained-pre-scale guard inside reconcile() may// have flipped the host back to active (a wake raced the suspension) —// in that case no scale-down happened, so no phase/metric is emitted.const wakePending =- this.wakeRequestedGeneration(reconcileHost) >- (reconcileHost.status?.lifecycle?.wakeHandledGeneration ?? 0)- if (reconcileHost.status?.lifecycle?.state !== 'suspended' || wakePending) {+ this.wakeRequestedGeneration(host) > (host.status?.lifecycle?.wakeHandledGeneration ?? 0)+ if (host.status?.lifecycle?.state !== 'suspended' || wakePending) {// The durable suspended write DID commit (a racing wake revived the// host afterwards), so the drained evidence was honored: 'suspended'.return 'suspended'}- this.recordSuspendedApplied(reconcileHost.name)+ this.recordSuspendedApplied(host.name)return 'suspended'}@@ -929,19 +838,12 @@ export class StatelessLifecycleExecutor {* Throws on API failure — the tracker logs it loudly.*/async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {- return this.serializeHeartbeatMutation(- 'heartbeat suspend-blocked reason publication',- host,- (admittedHost, revalidate) =>- this.publishSuspendBlockedReasonCore(admittedHost, reason, revalidate)- )+ // L2: serialize on the host chain (see serializeByHost) so this status+ // write never interleaves with a concurrent reconcile for the same host.+ return this.serializeByHost(host.name, () => this.publishSuspendBlockedReasonCore(host, reason))}- private async publishSuspendBlockedReasonCore(- host: HostCRD,- reason: string,- revalidate?: () => HostCRD- ): Promise<void> {+ private async publishSuspendBlockedReasonCore(host: HostCRD, reason: string): Promise<void> {// FRESH GET (see readFreshHost): this is a suspend-BLOCKED reason// ANNOTATOR, not a state writer — it must NEVER change state, only stamp// the reason on top of the CURRENT server-side state. AP-1 (the 9th@@ -964,12 +866,6 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {- if (- !this.sameHostSpecRevision(host, fresh) ||- !this.effectiveLifecycleFromCache(fresh).stateless- ) {- return { skip: true }- }const freshLifecycle = fresh.status?.lifecycleconst freshState = freshLifecycle?.state ?? 'active'if (freshState !== 'active') {@@ -993,16 +889,12 @@ export class StatelessLifecycleExecutor {writtenLifecycle = lifecyclereturn [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]},- initialFresh,- revalidate+ initialFresh)if ('skipped' in result || writtenLifecycle === undefined) {return}- const reflectedStatus = { ...(host.status ?? {}), lifecycle: writtenLifecycle }- this.reflectOutcome(host, target => {- target.status = reflectedStatus- })+ host.status = { ...(host.status ?? {}), lifecycle: writtenLifecycle }this.lastWrittenLifecycleStatus.delete(host.name)}@@ -1034,18 +926,17 @@ export class StatelessLifecycleExecutor {host: HostCRD,entryWakeHandledGeneration: number): Promise<void> {- return this.serializeHeartbeatMutation(- 'heartbeat draining transition',- host,- (admittedHost, revalidate) =>- this.markHostDrainingFromHeartbeatCore(admittedHost, entryWakeHandledGeneration, revalidate)+ // L2: serialize on the host chain (see serializeByHost) so the drain+ // status write never interleaves with a concurrent reconcile for the+ // same host.+ return this.serializeByHost(host.name, () =>+ this.markHostDrainingFromHeartbeatCore(host, entryWakeHandledGeneration))}private async markHostDrainingFromHeartbeatCore(host: HostCRD,- entryWakeHandledGeneration: number,- revalidate?: () => HostCRD+ entryWakeHandledGeneration: number): Promise<void> {// FRESH GET (see readFreshHost): a stale `active` snapshot over an// actually-`suspended` CR would write `draining` (a running state) and@@ -1057,12 +948,6 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {- if (- !this.sameHostSpecRevision(host, fresh) ||- !this.effectiveLifecycleFromCache(fresh).stateless- ) {- return { skip: true }- }const freshLifecycle = fresh.status?.lifecycleif ((freshLifecycle?.state ?? 'active') !== 'active') {return { skip: true }@@ -1098,15 +983,12 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh,- revalidate+ initialFresh)if ('skipped' in result || writtenStatus === undefined) {return}- this.reflectOutcome(host, target => {- target.status = writtenStatus- })+ host.status = writtenStatusthis.lastWrittenLifecycleStatus.delete(host.name)console.log(`[HostReconciler] Marked stateless Host "${host.name}" draining — control-api now answers {drain:true} from Host.status`@@ -1128,17 +1010,13 @@ export class StatelessLifecycleExecutor {* heartbeat retries.*/async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {- return this.serializeHeartbeatMutation(- 'heartbeat cancel-drain transition',- host,- (admittedHost, revalidate) => this.markHostActiveFromHeartbeatCore(admittedHost, revalidate)- )+ // L2: serialize on the host chain (see serializeByHost) so the+ // cancel-drain status write never interleaves with a concurrent+ // reconcile for the same host.+ return this.serializeByHost(host.name, () => this.markHostActiveFromHeartbeatCore(host))}- private async markHostActiveFromHeartbeatCore(- host: HostCRD,- revalidate?: () => HostCRD- ): Promise<void> {+ private async markHostActiveFromHeartbeatCore(host: HostCRD): Promise<void> {// FRESH GET (see readFreshHost): a stale `draining` snapshot over an// actually-`suspended` CR must never resurrect it (and would regress// wakeHandledGeneration). Proceed only when the CURRENT server-side@@ -1151,12 +1029,6 @@ export class StatelessLifecycleExecutor {const result = await this.patchStatusWithPrecondition(host,fresh => {- if (- !this.sameHostSpecRevision(host, fresh) ||- !this.effectiveLifecycleFromCache(fresh).stateless- ) {- return { skip: true }- }const freshLifecycle = fresh.status?.lifecycleif (freshLifecycle?.state !== 'draining') {skippedNotDraining = freshLifecycle?.state ?? 'active'@@ -1169,8 +1041,7 @@ export class StatelessLifecycleExecutor {writtenStatus = { ...(fresh.status ?? {}), lifecycle }return this.fullStatusOps(writtenStatus)},- initialFresh,- revalidate+ initialFresh)if ('skipped' in result || writtenStatus === undefined) {console.log(@@ -1178,9 +1049,7 @@ export class StatelessLifecycleExecutor {)return}- this.reflectOutcome(host, target => {- target.status = writtenStatus- })+ host.status = writtenStatusthis.lastWrittenLifecycleStatus.delete(host.name)console.log(`[StatelessLifecycle] host=${host.name} cancel-drain: activity evidence while draining — reverting status to active`@@ -1296,10 +1165,8 @@ export class StatelessLifecycleExecutor {// Reflect the fresh truth on the in-memory object so the heavy// reconcile body derives replicas from the current durable state// instead of the stale event payload.- this.reflectOutcome(host, target => {- target.annotations = initialFresh.annotations- target.status = initialFresh.status- })+ host.annotations = initialFresh.annotations+ host.status = initialFresh.statusreturn false}@@ -1358,10 +1225,8 @@ export class StatelessLifecycleExecutor {initialFresh)if ('skipped' in result || writtenLifecycle === undefined) {- this.reflectOutcome(host, target => {- target.annotations = latestFresh.annotations- target.status = latestFresh.status- })+ host.annotations = latestFresh.annotations+ host.status = latestFresh.statusreturn false}} catch (err) {@@ -1380,11 +1245,8 @@ export class StatelessLifecycleExecutor {// (assessLifecycle reads host.status) derives replicas from the woken// state instead of re-suspending from the stale event payload. The base// is the FRESH status the write committed over, never the cached spread.- const reflectedStatus = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }- this.reflectOutcome(host, target => {- target.annotations = latestFresh.annotations- target.status = reflectedStatus- })+ host.annotations = latestFresh.annotations+ host.status = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }console.log(`[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=status_flipped ts=${this.now().getTime()}`)@@ -1555,31 +1417,19 @@ export class StatelessLifecycleExecutor {private async patchStatusWithPrecondition(host: HostCRD,build: (fresh: HostCRD) => Array<{ op: 'add'; path: string; value: unknown }> | { skip: true },- initialFresh?: HostCRD,- revalidate?: () => HostCRD+ initialFresh?: HostCRD): Promise<{ fresh: HostCRD } | { skipped: true }> {const maxAttempts = 5let fresh = initialFreshfor (let attempt = 1; attempt <= maxAttempts; attempt += 1) {- const currentAtAttempt = revalidate?.()- if (currentAtAttempt) this.requireSameHostIdentity(host, currentAtAttempt)if (fresh === undefined) {fresh = await this.readFreshHost(host)}- if (host.uid !== undefined && fresh.resourceVersion === undefined) {- const error = new Error(- `Host status mutation for "${host.name}" has no fresh resourceVersion; refusing an unconditioned write`- )- error.name = 'HostMutationPreconditionUnavailableError'- throw error- }const built = build(fresh)if ('skip' in built) {return { skipped: true }}try {- const currentBeforePatch = revalidate?.()- if (currentBeforePatch) this.requireSameHostIdentity(host, currentBeforePatch)await this.customApi.patchNamespacedCustomObjectStatus({group: GROUP,version: VERSION,@@ -1649,79 +1499,6 @@ export class StatelessLifecycleExecutor {return result}- /**- * Capture admission before enqueueing, then resolve the current Host only- * after this mutation reaches the front of its own serializer. Keeping the- * check here avoids wrapping a lifecycle call in HostReconciler's serializer,- * which would recursively acquire the same Host chain and deadlock.- */- private serializeHeartbeatMutation<T>(- action: string,- host: HostCRD,- work: (admittedHost: HostCRD, revalidate: () => HostCRD) => Promise<T>- ): Promise<T> {- const revalidate = this.prepareHostMutationAdmission(action, host)- return this.serializeByHost(host.name, () => work(revalidate(), revalidate))- }-- /**- * A resourceVersion conflict can be caused by ordinary writes or by a- * delete/recreate. Every fresh read, including every bounded 409 retry, must- * remain on the Kubernetes UID that was admitted. Name equality is not an- * identity fence. Production watch snapshots always carry UID; if only one- * side carries it, fail closed rather than guessing.- */- /**- * H2: apply a committed outcome to BOTH visible surfaces:- * 1. the local working object (`host`) — with a resolver wired this is the- * admitted CLONE (serializeHeartbeatMutation), so this keeps same-pass- * logic (assessLifecycle, drained-pre-scale guard, follow-on reconcile)- * reading the outcome exactly as before; standalone callers (no resolver)- * get the legacy caller-visible reflection unchanged;- * 2. the CURRENT cache entry, via the uid-guarded reflector (no-op unless- * McpServerWatcher wired it).- * `host.uid` is the admitted identity (admission proved requested.uid ===- * current.uid; readFreshHost re-proves it), so passing it after mutate() ran- * is equivalent to passing it before.- *- * `mutate` closures MUST only ASSIGN precomputed values — never derive from- * `target` — so both surfaces receive the identical outcome.- */- private reflectOutcome(host: HostCRD, mutate: (target: HostCRD) => void): void {- mutate(host)- this.reflectHostOutcome(host.name, host.uid, mutate)- }-- private requireSameHostIdentity(expected: HostCRD, current: HostCRD): void {- if (expected.uid === undefined && current.uid === undefined) return- if (expected.uid === undefined || current.uid === undefined || expected.uid !== current.uid) {- const error = new Error(- `Host identity changed while mutating "${expected.name}"; refusing to cross a same-name recreation`- )- error.name = 'HostMutationIdentityChangedError'- throw error- }- }-- /**- * Heartbeat evidence belongs to one Host spec epoch. UID equality allows- * status-only updates and retries, but a generation change can replace the- * runtime inputs (context, Secret, image, lifecycle policy, etc.) while- * remaining stateless. Never apply an older runtime's heartbeat decision to- * that new spec. UID-less standalone fixtures retain their legacy behavior;- * production objects (which carry UID) fail closed on a missing generation.- */- private sameHostSpecRevision(expected: HostCRD, current: HostCRD): boolean {- if (expected.generation === undefined && current.generation === undefined) {- return expected.uid === undefined && current.uid === undefined- }- return (- expected.generation !== undefined &&- current.generation !== undefined &&- expected.generation === current.generation- )- }-/*** Drained-pre-scale guard (Stage 4.3): a pending wake must abort the* suspension IMMEDIATELY before replicas:0 derives from the assessment.@@ -1802,10 +1579,8 @@ export class StatelessLifecycleExecutor {// A wake is pending in the fresh read — run the wake transition// instead of the scale-down. Reflect the fresh wake generation on the// in-memory object so downstream writes preserve it.- this.reflectOutcome(host, target => {- target.annotations = fresh.annotations- target.status = fresh.status- })+ host.annotations = fresh.annotations+ host.status = fresh.statusreturn {effective: { stateless: true, state: 'active' },lifecycle: { state: 'active', wakeHandledGeneration: freshWakeRequested },@@ -1831,10 +1606,8 @@ export class StatelessLifecycleExecutor {// Reflect fresh onto the in-memory host (mirroring the wake fast-path's// epoch guard) so every downstream write derives from the current// server-side truth instead of the stale event payload.- this.reflectOutcome(host, target => {- target.annotations = fresh.annotations- target.status = fresh.status- })+ host.annotations = fresh.annotations+ host.status = fresh.statusconst freshReason = fresh.status?.lifecycle?.reasonconst preservedReason = isHeartbeatManagedLifecycleReason(freshReason) ? freshReason : undefinedreturn {diff --git a/host-context-controller/src/types.ts b/host-context-controller/src/types.tsindex 552ed3f0f..a7488dc7d 100644--- a/host-context-controller/src/types.ts+++ b/host-context-controller/src/types.ts@@ -350,8 +350,6 @@ export interface ContextStatus {export interface ContextCRD {name: stringnamespace: string- uid?: string- generation?: numberspec: ContextSpecstatus?: ContextStatus}@@ -511,12 +509,6 @@ export interface McpServerStatus {ready: boolean/** Human-readable status message. */message?: string- /**- * Whether HCC can tie this status to the current McpServer identity and- * generation. False means discovery is authoritative but runtime status has- * not yet been re-established after controller startup.- */- authoritative?: boolean}/**diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.tsindex a6992eeb8..22d63b8b2 100644--- a/host-context-controller/src/utils.ts+++ b/host-context-controller/src/utils.ts@@ -36,10 +36,6 @@ export async function replaceWithConflictRetry<* when a meaningful change is still required.*/isUpToDate?: (body: T, existing: T) => boolean- /** Reject an object whose identity or ownership is unsafe to replace. */- validateExisting?: (existing: T) => void- /** Rechecked immediately before every Kubernetes write attempt. */- mutationAllowed?: () => booleanmaxAttempts?: number}): Promise<void> {const {@@ -51,13 +47,10 @@ export async function replaceWithConflictRetry<replace,mergeExisting,isUpToDate,- validateExisting,- mutationAllowed,maxAttempts = 3,} = optsfor (let attempt = 1; attempt <= maxAttempts; attempt++) {const existing = await read()- validateExisting?.(existing)const desired = resolveBody ? await resolveBody() : bodyconst base: T = {...desired,@@ -65,7 +58,6 @@ export async function replaceWithConflictRetry<}const next = mergeExisting ? mergeExisting(base, existing) : baseif (isUpToDate?.(next, existing)) return- if (mutationAllowed && !mutationAllowed()) returntry {await replace(next)const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''@@ -179,11 +171,8 @@ export async function applyNetworkPolicy(name: string,namespace: string,policy: k8s.V1NetworkPolicy,- logPrefix = '[NetPol]',- mutationAllowed?: () => boolean,- validateExisting?: (existing: k8s.V1NetworkPolicy) => void+ logPrefix = '[NetPol]'): Promise<void> {- if (mutationAllowed && !mutationAllowed()) returntry {await api.createNamespacedNetworkPolicy({ namespace, body: policy })console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)@@ -199,8 +188,6 @@ export async function applyNetworkPolicy(body: policy,read: () => api.readNamespacedNetworkPolicy({ name, namespace }),replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),- mutationAllowed,- validateExisting,})}diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.tsindex 17ac53f0f..4e6593024 100644--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts@@ -136,20 +136,6 @@ function createReconciler() {return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }}-function deferred<T = void>(): {- promise: Promise<T>- resolve: (value: T) => void- reject: (reason?: unknown) => void-} {- let resolve!: (value: T) => void- let reject!: (reason?: unknown) => void- const promise = new Promise<T>((res, rej) => {- resolve = res- reject = rej- })- return { promise, resolve, reject }-}-/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {const calls = [@@ -221,755 +207,7 @@ function freshHostRead(lifecycle?: {}}-describe('HostReconciler heartbeat mutation admission', () => {- const heartbeatMutations = [- {- name: 'suspend',- core: 'suspendHostFromHeartbeatCore',- coreArgs: ['idle', 0] as const,- freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,- invoke: (reconciler: HostReconciler, host: HostCRD) =>- reconciler.suspendHostFromHeartbeat(host, 'idle', 0),- },- {- name: 'suspend-blocked reason',- core: 'publishSuspendBlockedReasonCore',- coreArgs: ['suspend-blocked: activeTask'] as const,- freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,- invoke: (reconciler: HostReconciler, host: HostCRD) =>- reconciler.publishSuspendBlockedReason(host, 'suspend-blocked: activeTask'),- },- {- name: 'draining',- core: 'markHostDrainingFromHeartbeatCore',- coreArgs: [0] as const,- freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,- invoke: (reconciler: HostReconciler, host: HostCRD) =>- reconciler.markHostDrainingFromHeartbeat(host, 0),- },- {- name: 'cancel-drain',- core: 'markHostActiveFromHeartbeatCore',- coreArgs: [] as const,- freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,- invoke: (reconciler: HostReconciler, host: HostCRD) =>- reconciler.markHostActiveFromHeartbeat(host),- },- ] as const-- it.each(heartbeatMutations)(- 'rejects queued $name work when Host authority changes before serializer admission',- async ({ core, invoke }) => {- const { reconciler } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 1 }- const authority = { current: { known: true, generation: 7 } }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- const lifecycle = (reconciler as any).lifecycle- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = lifecycle.serializeByHost(host.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const pending = invoke(reconciler, host)- authority.current = { known: false, generation: 8 }- releaseBlocker.resolve(undefined)- await occupied-- await expect(pending).rejects.toThrow(/Host inventory authority/)- expect(coreSpy).not.toHaveBeenCalled()- }- )-- it.each(heartbeatMutations)(- 'rejects queued $name work when the Host UID changes before serializer admission',- async ({ core, invoke }) => {- const { reconciler } = createReconciler()- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-old-uid', generation: 4 }- let current = requested- const authority = { current: { known: true, generation: 9 } }- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- const lifecycle = (reconciler as any).lifecycle- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = lifecycle.serializeByHost(requested.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const pending = invoke(reconciler, requested)- current = { ...requested, uid: 'heartbeat-host-new-uid', generation: 1 }- releaseBlocker.resolve(undefined)- await occupied-- await expect(pending).rejects.toThrow(/Host identity/)- expect(coreSpy).not.toHaveBeenCalled()- }- )-- it.each(heartbeatMutations)(- 'admits the latest same-UID same-spec Host snapshot for $name inside the heartbeat serializer',- async ({ core, coreArgs, invoke }) => {- const { reconciler } = createReconciler()- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- const current = {- ...requested,- resourceVersion: 'cache-rv-latest',- status: { lifecycle: { state: 'active' as const, wakeHandledGeneration: 2 } },- }- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))- const lifecycle = (reconciler as any).lifecycle- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)-- await invoke(reconciler, requested)-- expect(coreSpy).toHaveBeenCalledOnce()- expect(coreSpy).toHaveBeenCalledWith(current, ...coreArgs, expect.any(Function))- }- )-- it.each(heartbeatMutations)(- 'rejects a $name 409 retry that resolves to a recreated same-name Host',- async ({ freshLifecycle, invoke }) => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),- uid: 'heartbeat-host-old-uid',- generation: 4,- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 15 }))- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead(freshLifecycle),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-old',- },- })- .mockResolvedValueOnce({- ...freshHostRead(freshLifecycle),- metadata: {- name: host.name,- namespace: host.namespace,- uid: 'heartbeat-host-new-uid',- generation: 1,- resourceVersion: 'rv-new',- },- })- customApi.patchNamespacedCustomObjectStatus- .mockRejectedValueOnce({ code: 409 })- .mockResolvedValueOnce(undefined)-- await expect(invoke(reconciler, host)).rejects.toThrow(/Host identity/)- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)- }- )-- it.each(heartbeatMutations)(- 'rejects queued $name work when the same Host advances to a new spec generation',- async ({ core, invoke }) => {- const { reconciler } = createReconciler()- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- let current = requested- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))- const lifecycle = (reconciler as any).lifecycle- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)- const blockerStarted = deferred()- const releaseBlocker = deferred()- const occupied = lifecycle.serializeByHost(requested.name, async () => {- blockerStarted.resolve(undefined)- await releaseBlocker.promise- }) as Promise<void>- await blockerStarted.promise-- const pending = invoke(reconciler, requested)- current = {- ...requested,- generation: 5,- spec: { ...requested.spec, contextRef: 'new-runtime-context' },- }- releaseBlocker.resolve(undefined)- await occupied-- await expect(pending).rejects.toThrow(/Host spec generation/)- expect(coreSpy).not.toHaveBeenCalled()- }- )-- it.each(heartbeatMutations)(- 'does not apply stale $name evidence to a newer same-UID stateless spec',- async ({ freshLifecycle, invoke }) => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),- uid: 'heartbeat-host-uid',- generation: 4,- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead(freshLifecycle),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-new-spec',- },- spec: { ...host.spec, contextRef: 'new-runtime-context' },- })- const reconcileCore = vi- .spyOn(reconciler as any, 'reconcileCore')- .mockResolvedValue(undefined)-- await invoke(reconciler, host)-- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- expect(reconcileCore).not.toHaveBeenCalled()- }- )-- it.each(heartbeatMutations)(- 'does not apply stale $name evidence after the same-UID Host becomes stateful',- async ({ invoke }) => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 16 }))- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-stateful',- },- spec: {- ...host.spec,- lifecycle: { stateless: false },- },- })- const reconcileCore = vi- .spyOn(reconciler as any, 'reconcileCore')- .mockResolvedValue(undefined)-- await invoke(reconciler, host)-- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- expect(reconcileCore).not.toHaveBeenCalled()- }- )-- it('does not retry stale heartbeat evidence after a 409 reveals a same-UID stateful spec', async () => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-stateless',- },- })- .mockResolvedValueOnce({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-stateful',- },- spec: {- ...host.spec,- lifecycle: { stateless: false },- },- })- customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })-- await reconciler.markHostDrainingFromHeartbeat(host, 0)-- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)- })-- it('does not retry stale heartbeat evidence after a 409 reveals a newer stateless spec', async () => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-old-spec',- },- })- .mockResolvedValueOnce({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-new-spec',- },- spec: { ...host.spec, contextRef: 'new-runtime-context' },- })- customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })-- await reconciler.markHostDrainingFromHeartbeat(host, 0)-- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)- })-- it('revalidates authority after the fresh read and before the first status patch', async () => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- const authority = { current: { known: true, generation: 18 } }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- customApi.getNamespacedCustomObject.mockImplementation(async () => {- authority.current = { known: false, generation: 19 }- return {- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- resourceVersion: 'rv-authority-lost',- },- }- })-- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(- /Host inventory authority/- )- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- })-- it('revalidates authority before a 409 retry performs another fresh read', async () => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- const authority = { current: { known: true, generation: 21 } }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => authority.current)- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-before-conflict',- },- })- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {- authority.current = { known: false, generation: 22 }- throw { code: 409 }- })-- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(- /Host inventory authority/- )- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)- })-- it('does not reconcile runtime effects after suspend when the Host UID changes', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-old-uid',- generation: 4,- }- let current = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 24 }))- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-suspend',- },- })- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {- current = { ...host, uid: 'heartbeat-host-new-uid', generation: 1 }- })-- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(- /Host identity/- )- expect(reconcileCore).not.toHaveBeenCalled()- })-- it('reconciles suspend follow-on effects from the successful fresh same-UID Host spec', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- spec: { contextRef: 'latest-context' },- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-uid',- generation: 5,- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 26 }))- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-suspend-current',- },- spec: { ...host.spec, contextRef: 'latest-context' },- })- .mockResolvedValue({- ...freshHostRead({- state: 'suspended',- wakeHandledGeneration: 0,- reason: 'idle',- }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-suspend-committed',- },- spec: { ...host.spec, contextRef: 'latest-context' },- })-- await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)-- expect(reconcileCore).toHaveBeenCalledOnce()- const reconciledHost = reconcileCore.mock.calls[0][0] as HostCRD- expect(reconciledHost).toMatchObject({- uid: host.uid,- generation: 5,- spec: expect.objectContaining({ contextRef: 'latest-context' }),- })- expect(reconciledHost.status?.lifecycle).toEqual({- state: 'suspended',- wakeHandledGeneration: 0,- reason: 'idle',- })- })-- it('rejects a known-identity status write without a fresh resourceVersion', async () => {- const { reconciler, customApi } = createReconciler()- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 28 }))- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: { name: host.name, namespace: host.namespace, uid: host.uid },- })-- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(- /no fresh resourceVersion/- )- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()- })-- it('retries a 409 on the same UID and uses the latest Host for suspend follow-on effects', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-uid',- generation: 4,- }- let current = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 30 }))- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-1',- },- })- .mockResolvedValueOnce({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-2',- },- spec: host.spec,- })- .mockResolvedValueOnce({- ...freshHostRead({- state: 'suspended',- wakeHandledGeneration: 0,- reason: 'idle',- }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-3',- },- spec: host.spec,- })- customApi.patchNamespacedCustomObjectStatus- .mockImplementationOnce(async () => {- current = {- ...host,- resourceVersion: 'cache-rv-after-conflict',- }- throw { code: 409 }- })- .mockResolvedValueOnce(undefined)-- await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)-- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(3)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)- expect(reconcileCore.mock.calls[0][0]).toMatchObject({- uid: host.uid,- generation: 4,- spec: expect.objectContaining({ contextRef: 'context-a' }),- })- })-- it('does not apply or record suspension when only the follow-on GET observes a newer spec', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-uid',- generation: 4,- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 33 }))- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-suspend-write',- },- })- .mockResolvedValueOnce({- ...freshHostRead({- state: 'suspended',- wakeHandledGeneration: 0,- reason: 'idle',- }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 5,- resourceVersion: 'rv-new-spec',- },- spec: {- ...host.spec,- contextRef: 'replacement-context',- lifecycle: { stateless: false },- },- })- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)- const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')-- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(- /Host spec generation/- )-- expect(reconcileCore).not.toHaveBeenCalled()- expect(recordSuspended).not.toHaveBeenCalled()- })-- it('rejects stale suspend follow-on work when the Host spec changes during Secret validation', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-uid',- generation: 4,- }- let current = host- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 34 }))- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-suspend-write',- },- })- .mockResolvedValueOnce({- ...freshHostRead({- state: 'suspended',- wakeHandledGeneration: 0,- reason: 'idle',- }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-suspend-follow-on',- },- })- const secretReadStarted = deferred()- const releaseSecretRead = deferred()- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {- secretReadStarted.resolve(undefined)- await releaseSecretRead.promise- return { ok: true }- })- const staleMutation = vi- .spyOn(reconciler as any, 'ensureHostServiceAccount')- .mockRejectedValue(new Error('stale follow-on mutation was admitted'))-- const pending = reconciler.suspendHostFromHeartbeat(host, 'idle', 0)- await secretReadStarted.promise- current = {- ...host,- generation: 5,- spec: {- ...host.spec,- contextRef: 'replacement-context',- lifecycle: { stateless: false },- },- }- releaseSecretRead.resolve(undefined)-- await expect(pending).rejects.toThrow(/Host spec generation/)- expect(staleMutation).not.toHaveBeenCalled()- })-- it('does not record a suspended scale-down when follow-on reconcile handles a wake', async () => {- const { reconciler, customApi } = createReconciler()- const host = {- ...makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },- }),- uid: 'heartbeat-host-uid',- generation: 4,- }- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 32 }))- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),- metadata: {- name: host.name,- namespace: host.namespace,- uid: host.uid,- generation: 4,- resourceVersion: 'rv-wake-follow-on',- },- })- vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async (...args: unknown[]) => {- const reconcileHost = args[0] as HostCRD- reconcileHost.annotations = { 'clerum.io/wake-requested': '1' }- reconcileHost.status = {- lifecycle: { state: 'active', wakeHandledGeneration: 1 },- }- })- const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')-- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).resolves.toBe('suspended')-- expect(recordSuspended).not.toHaveBeenCalled()- })-})-describe('HostReconciler.suspendHostFromHeartbeat', () => {- it('does not mutate the admitted watch-cache snapshot while applying a suspension', async () => {- const { reconciler, customApi } = createReconciler()- const host = makeStatelessHost({- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },- })- host.uid = 'stateless-host-uid'- host.generation = 1- host.resourceVersion = 'cache-rv'- const snapshot = structuredClone(host)- Object.freeze(host)- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))- reconciler.setHostMutationAuthority(() => ({- known: true,- hostGeneration: 1,- contextGeneration: 1,- }))- const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })- Object.assign(draining.metadata, {- uid: host.uid,- generation: host.generation,- resourceVersion: 'draining-rv',- })- const suspended = freshHostRead({- state: 'suspended',- wakeHandledGeneration: 2,- reason: 'idle',- })- Object.assign(suspended.metadata, {- uid: host.uid,- generation: host.generation,- resourceVersion: 'suspended-rv',- })- customApi.getNamespacedCustomObject.mockResolvedValueOnce(draining).mockResolvedValue(suspended)-- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')- expect(host).toEqual(snapshot)- })-it("writes state='suspended' + reason 'idle' durably, then reconciles to replicas=0", async () => {const { reconciler, appsApi, customApi } = createReconciler()const host = makeStatelessHost({@@ -2075,89 +1313,3 @@ describe('HostReconciler reconcile — stateless replicas derive from FRESH stat}})})--describe('H2: guarded cache reflection', () => {- it('reflects a committed lifecycle outcome onto the current cache entry without a watch event', async () => {- const { reconciler, customApi } = createReconciler()- const entry = {- ...makeStatelessHost({- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },- }),- uid: 'uid-1',- generation: 1,- resourceVersion: 'rv-1',- }- const hosts = new Map<string, HostCRD>([[entry.name, entry]])- reconciler.setResolveCurrentHost(name => hosts.get(name))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))- // Exactly the k8sClient.ts wiring shape (uid-guarded apply onto the cache).- reconciler.setReflectHostOutcome((name, uid, apply) => {- const cached = hosts.get(name)- if (!cached || uid === undefined || cached.uid !== uid) return- apply(cached)- })- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: entry.name,- namespace: entry.namespace,- uid: 'uid-1',- generation: 1,- resourceVersion: 'fresh-rv',- },- })-- await reconciler.markHostDrainingFromHeartbeat(entry, 0)-- // The status write committed to the API…- expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')- // …and — the H2 fix — the committed outcome is visible on the CANONICAL cache- // entry without waiting for a watch MODIFIED. With the resolver wired, the- // heartbeat core mutates the admitted clone (Shape A), so this can ONLY be- // true if the guarded reflector routed the outcome onto the cache entry.- expect(hosts.get(entry.name)?.status?.lifecycle?.state).toBe('draining')- })-- it('never reflects a committed outcome onto a same-name recreation with a different uid', async () => {- const { reconciler, customApi } = createReconciler()- const entry = {- ...makeStatelessHost({- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },- }),- uid: 'uid-1',- generation: 1,- resourceVersion: 'rv-1',- }- const hosts = new Map<string, HostCRD>([[entry.name, entry]])- const replacement = { ...makeStatelessHost({ name: entry.name }), uid: 'uid-2' }- reconciler.setResolveCurrentHost(name => hosts.get(name))- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))- reconciler.setReflectHostOutcome((name, uid, apply) => {- const cached = hosts.get(name)- if (!cached || uid === undefined || cached.uid !== uid) return- apply(cached)- })- customApi.getNamespacedCustomObject.mockResolvedValue({- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),- metadata: {- name: entry.name,- namespace: entry.namespace,- uid: 'uid-1',- generation: 1,- resourceVersion: 'fresh-rv',- },- })- // A watch DELETE+ADD recreates the entry (new uid) mid-commit.- customApi.patchNamespacedCustomObjectStatus.mockImplementation(async () => {- hosts.set(entry.name, replacement)- return {}- })-- await reconciler.markHostDrainingFromHeartbeat(entry, 0)-- // The write committed…- expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')- // …but the recreation (uid-2) was NOT corrupted by the stale reflection.- expect(replacement.status?.lifecycle).toBeUndefined()- })-})diff --git a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.tsindex f8696e57e..a06465eeb 100644--- a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts@@ -34,12 +34,9 @@ describe('ensureK8sApiEgress', () => {mockApply.mockClear()// ensureDefaultPolicies() also prunes legacy static policies via the// networking API directly (not applyNetworkPolicy), so the mocked client- // must expose the read-before-delete legacy cleanup path.+ // must expose deleteNamespacedNetworkPolicy.const mockKc = {makeApiClient: () => ({- readNamespacedNetworkPolicy: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),}),} as unknown as k8s.KubeConfigdiff --git a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.tsindex 04e7b3b33..6a89edc00 100644--- a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts@@ -26,9 +26,7 @@ const mockApply = vi.mocked(applyNetworkPolicy)function createMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),diff --git a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.tsindex 9d0db5860..ae59d2dbd 100644--- a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts@@ -32,9 +32,7 @@ const mockApply = vi.mocked(applyNetworkPolicy)function createMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi- .fn()- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),diff --git a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.tsindex 6acb165b3..a93e20eed 100644--- a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts+++ b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts@@ -83,7 +83,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('generates egress policy in rpc-proxy namespace when context has servers', async () => {const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx, { isCurrent: () => true })+ await reconciler.reconcileContext(ctx)const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)@@ -91,7 +91,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('egress policy targets correct server pod by mcpserver label', async () => {const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx, { isCurrent: () => true })+ await reconciler.reconcileContext(ctx)const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)@@ -103,7 +103,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('egress policy uses correct port from McpServer spec', async () => {const ctx = makeContext('ctx-beta', ['airtable-mcp'])- await reconciler.reconcileContext(ctx, { isCurrent: () => true })+ await reconciler.reconcileContext(ctx)const rpcProxyCalls = mockApply.mock.calls.filter(call => call[1] === 'rpc-egress-ctx-beta-airtable-mcp'@@ -122,8 +122,6 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {metadata: {name: 'rpc-egress-ctx-gamma-mongodb-mcp',namespace: 'rpc-proxy',- uid: 'rpc-delete-uid',- resourceVersion: '7',labels: {'clerum.io/managed-by': 'host-context-controller','clerum.io/policy-type': 'rpc-proxy-egress',@@ -146,22 +144,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('deletes orphaned egress policies when server removed from context', async () => {// First reconcile with both serversconst ctx1 = makeContext('ctx-delta', ['mongodb-mcp', 'airtable-mcp'])- await reconciler.reconcileContext(ctx1, { isCurrent: () => true })- const existingRpcProxyPolicies = mockApply.mock.calls- .filter(call => call[2] === 'rpc-proxy')- .map(call => {- const policy = call[3] as k8s.V1NetworkPolicy- const name = policy.metadata?.name- return {- ...policy,- metadata: {- ...policy.metadata,- uid: `rpc-proxy:${name}:uid`,- resourceVersion: '1',- },- }- })- expect(existingRpcProxyPolicies).toHaveLength(2)+ await reconciler.reconcileContext(ctx1)vi.clearAllMocks()mockNetApi.listNamespacedNetworkPolicy.mockImplementation(@@ -171,7 +154,10 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {String(labelSelector).includes('clerum.io/context=ctx-delta')) {return {- items: existingRpcProxyPolicies,+ items: [+ { metadata: { name: 'rpc-egress-ctx-delta-mongodb-mcp' } },+ { metadata: { name: 'rpc-egress-ctx-delta-airtable-mcp' } },+ ],}}return { items: [] }@@ -180,7 +166,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {// Now reconcile with only one server — orphaned rpc-proxy policy should be cleanedconst ctx2 = makeContext('ctx-delta', ['mongodb-mcp'])- await reconciler.reconcileContext(ctx2, { isCurrent: () => true })+ await reconciler.reconcileContext(ctx2)// applyNetworkPolicy should be called for mongodb-mcp in rpc-proxy but not airtable-mcpconst rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')@@ -190,12 +176,6 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {expect(mockNetApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({name: 'rpc-egress-ctx-delta-airtable-mcp',namespace: 'rpc-proxy',- body: {- preconditions: {- uid: 'rpc-proxy:rpc-egress-ctx-delta-airtable-mcp:uid',- resourceVersion: '1',- },- },})expect(mockNetApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({name: 'rpc-egress-ctx-delta-mongodb-mcp',@@ -205,7 +185,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {it('skips egress generation when server not found in cache', async () => {const ctx = makeContext('ctx-epsilon', ['nonexistent-server'])- await reconciler.reconcileContext(ctx, { isCurrent: () => true })+ await reconciler.reconcileContext(ctx)const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')expect(rpcProxyCalls.length).toBe(0)diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 8519e148d..15f46834e 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.239",+ "version": "0.1.240","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.239",+ "version": "0.1.240","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 2dcb39dec..0fd114fe5 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.239",+ "version": "0.1.240","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/main.mcpInitialization.test.ts b/mcp-host/src/__tests__/main.mcpInitialization.test.tsindex 02e9e9c75..c90bad869 100644--- a/mcp-host/src/__tests__/main.mcpInitialization.test.ts+++ b/mcp-host/src/__tests__/main.mcpInitialization.test.ts@@ -22,12 +22,7 @@ function readyServer(overrides: Partial<McpServerInfo> = {}): McpServerInfo {transport: { type: 'streamableHttp', url: 'http://secured-server.test/mcp' },auth: { type: 'bearer', secretRef: 'secured-server-auth' },enabled: true,- // Mirror the real producer contract: HCC's reconciler.getStatus() (surfaced- // through k8sClient.toServerInfo) always stamps `authoritative` explicitly —- // a ready, identity-matched snapshot carries authoritative: true. The legacy- // shape that omitted it is never emitted for a live server, so the default- // fixture must carry it too.- status: { deployed: true, ready: true, authoritative: true },+ status: { deployed: true, ready: true },...overrides,}}diff --git a/nginx-egress-proxy/README.md b/nginx-egress-proxy/README.mdindex 372f8e5aa..b4c205974 100644--- a/nginx-egress-proxy/README.md+++ b/nginx-egress-proxy/README.md@@ -53,10 +53,10 @@ reference is platform-controlled:| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | host-context-controller | `clerum/nginx-egress-proxy:0.1.0` || `CONTROL_API_REMOTE_MCP_EGRESS_PROXY_IMAGE` | control-api (registry install stamps `spec.image` for remote entries) | `clerum/nginx-egress-proxy:0.1.0` |-HCC does not rewrite `spec.image` during reconciliation: that field remains-desired state owned by the CRD author. Deployment rendering always stamps the-configured platform image for remote egress proxies, so a remote CRD's-`spec.image` is never what runs.+HCC additionally canonicalizes `spec.image` of remote McpServers back to the+platform image (`canonicalizeRemoteEgressProxyImage`) as best-effort cleanup;+the enforcement is that Deployment rendering always stamps the configured+platform image, so a CRD's `spec.image` is never what runs.## Portsdiff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex 38afc5704..fde59308e 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.65",+ "version": "0.1.64","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.65",+ "version": "0.1.64","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex a17ac4862..2c7a4f3ed 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.65",+ "version": "0.1.64","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/config.hccBaseUrl.test.ts b/rpc-proxy/src/config.hccBaseUrl.test.tsindex bc3543a8d..b1cedb606 100644--- a/rpc-proxy/src/config.hccBaseUrl.test.ts+++ b/rpc-proxy/src/config.hccBaseUrl.test.ts@@ -27,33 +27,16 @@ describe('rpc-proxy HCC base URL default', () => {)const configmapMatch = configmap.match(/RPC_PROXY_HCC_BASE_URL:\s*"?([^"\n]+)"?/)- // Derive the real endpoint from the api-gateway manifest itself, so renaming- // the Service or moving its port breaks this test unless the dev default and- // configmap are updated in lockstep — no hand-copied literal to drift.- const gatewayManifest = readFileSync(- path.join(repoRoot, 'deploy/base/control-plane/host-context-controller-api-gateway.yaml'),- 'utf8'- )- const serviceDoc = gatewayManifest.split(/^---$/m).find(doc => /\bkind:\s*Service\b/.test(doc))- const serviceName = serviceDoc?.match(/\bmetadata:[\s\S]*?\bname:\s*([\w.-]+)/)?.[1]- const serviceNamespace = serviceDoc?.match(/\bnamespace:\s*([\w.-]+)/)?.[1]- const servicePort = serviceDoc?.match(/\bports:[\s\S]*?\bport:\s*(\d+)/)?.[1]-- it('parses all three sources', () => {+ it('parses both sources', () => {expect(defaultMatch?.[1]).toBeTruthy()expect(configmapMatch?.[1]).toBeTruthy()- expect(serviceName).toBeTruthy()- expect(serviceNamespace).toBeTruthy()- expect(servicePort).toBeTruthy()})it('dev default matches the deployed configmap endpoint', () => {expect(defaultMatch?.[1]).toBe(configmapMatch?.[1]?.trim())})- it('targets the api-gateway Service exactly as the gateway manifest declares it', () => {- const url = new URL(defaultMatch![1])- expect(url.hostname).toBe(`${serviceName}.${serviceNamespace}.svc.cluster.local`)- expect(url.port).toBe(servicePort)+ it('targets the api-gateway in control-plane, not a nonexistent Service', () => {+ expect(defaultMatch?.[1]).toContain('host-context-controller-api-gateway.control-plane')})})diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.shindex ecb97cbdc..bc77ee0f1 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh@@ -1,82 +1,28 @@#!/usr/bin/env bash-# Safety helpers shared by disruptive HCC fault-injection fixtures.--profile_env_value() {- local key=$1 file=$2- awk -v key="$key" '- index($0, key "=") == 1 {- print substr($0, length(key) + 2)- exit- }- ' "$file"-}+# Safety helpers for the HCC CommunicationChannel watch-recovery fault fixture.require_branch_owned_hcc_gate() {- local marker_namespace="${1:-control-plane}"- local actual_worktree_id actual_head actual_cluster_fingerprint actual_gate- local worktree_dirty expected_branch profile_env- local pre_gate_state_root cluster_fingerprint_file expected_cluster_fingerprint+ local repo_root expected_worktree_id expected_head actual_worktree_id actual_head worktree_dirtyis_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||die "fault injection requires a generated branch-scoped context, got '${E2E_KUBECONTEXT}'"- HCC_BRANCH_GATE_REPO_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"- worktree_dirty="$(- git -C "$HCC_BRANCH_GATE_REPO_ROOT" status --porcelain --untracked-files=normal- )"+ repo_root="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"+ worktree_dirty="$(git -C "$repo_root" status --porcelain --untracked-files=normal)"[ -z "$worktree_dirty" ] ||die "worktree has uncommitted changes; commit and re-sync before runtime proof"- HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID="$(- printf '%s' "$HCC_BRANCH_GATE_REPO_ROOT" | shasum | awk '{print $1}'- )"- HCC_BRANCH_GATE_EXPECTED_HEAD="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" rev-parse HEAD)"- expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"- [ -n "$expected_branch" ] ||- die "branch-owned runtime proof requires a named branch, not detached HEAD"- [ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ] ||- die "MINIKUBE_PROFILE must explicitly select context ${E2E_KUBECONTEXT}, got ${MINIKUBE_PROFILE:-missing}"-- profile_env="${E2E_BRANCH_PROFILE_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/profile.env}"- [ -r "$profile_env" ] ||- die "branch-profile helper evidence is missing at ${profile_env}; run a state-writing branch-profile helper action from this worktree"- [ "$(profile_env_value PROFILE "$profile_env")" = "$E2E_KUBECONTEXT" ] ||- die "branch-profile helper output does not select context ${E2E_KUBECONTEXT}"- [ "$(profile_env_value REPO_DIR "$profile_env")" = "$HCC_BRANCH_GATE_REPO_ROOT" ] ||- die "branch-profile helper output belongs to a different worktree"- [ "$(profile_env_value BRANCH "$profile_env")" = "$expected_branch" ] ||- die "branch-profile helper output belongs to a different branch"- [ "$(profile_env_value DIRTY "$profile_env")" = false ] ||- die "branch-profile helper recorded a dirty worktree; refresh it after committing"-- pre_gate_state_root="${E2E_PRE_GATE_STATE_ROOT:-${TMPDIR:-/tmp}/clerum-pre-gate-sync}"- cluster_fingerprint_file="${pre_gate_state_root}/${HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID}/cluster.sha"- [ -r "$cluster_fingerprint_file" ] ||- die "pre-gate fingerprint evidence is missing at ${cluster_fingerprint_file}"- expected_cluster_fingerprint="$(sed -n '1p' "$cluster_fingerprint_file")"- [[ "$expected_cluster_fingerprint" =~ ^[0-9a-f]{40}$ ]] ||- die "pre-gate fingerprint evidence is malformed"- [ -n "${E2E_EXPECTED_PRE_GATE_GATE:-}" ] ||- die "E2E_EXPECTED_PRE_GATE_GATE must name the gate recorded by the branch-owned pre-gate sync"-- HCC_BRANCH_GATE_SYNC_MARKER="$(- kctl get configmap clerum-pre-gate-sync-state -n "$marker_namespace" -o json 2>/dev/null- )" || die "cluster has no readable pre-gate sync marker"- actual_worktree_id="$(jq -r '.data.worktreeId // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"- actual_head="$(jq -r '.data.gitHead // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"- actual_cluster_fingerprint="$(- jq -r '.data.clusterFingerprint // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER"- )"- actual_gate="$(jq -r '.data.gate // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"+ expected_worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"+ expected_head="$(git -C "$repo_root" rev-parse HEAD)"+ actual_worktree_id="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \+ -o jsonpath='{.data.worktreeId}' 2>/dev/null || true)"+ actual_head="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \+ -o jsonpath='{.data.gitHead}' 2>/dev/null || true)"- [ "$actual_worktree_id" = "$HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID" ] ||+ [ "$actual_worktree_id" = "$expected_worktree_id" ] ||die "cluster ownership marker does not match this worktree; run minikube-pre-gate-sync first"- [ "$actual_head" = "$HCC_BRANCH_GATE_EXPECTED_HEAD" ] ||- die "cluster HEAD marker ${actual_head:-missing} does not match ${HCC_BRANCH_GATE_EXPECTED_HEAD}"- [ "$actual_cluster_fingerprint" = "$expected_cluster_fingerprint" ] ||- die "cluster fingerprint marker does not match this worktree's last pre-gate sync"- [ "$actual_gate" = "$E2E_EXPECTED_PRE_GATE_GATE" ] ||- die "cluster gate marker ${actual_gate:-missing} does not match expected ${E2E_EXPECTED_PRE_GATE_GATE}"+ [ "$actual_head" = "$expected_head" ] ||+ die "cluster HEAD marker ${actual_head:-missing} does not match ${expected_head}"}create_hcc_api_proxy() {@@ -338,10 +284,10 @@ delete_hcc_proxy_fixture() {}restore_hcc_after_fault_injection() {- local host_override port_override api_cidrs_override host_aliases desired ready+ local host_override port_override host_aliases desired readykctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- >/dev/null || return 1kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=merge \-p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null || return 1@@ -350,12 +296,10 @@ restore_hcc_after_fault_injection() {-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"- api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"host_aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')"desired="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"ready="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.status.readyReplicas}')"- [ -z "$host_override$port_override$api_cidrs_override$host_aliases" ] && [ "$desired" = "$ready" ]+ [ -z "$host_override$port_override$host_aliases" ] && [ "$desired" = "$ready" ]}print_hcc_repair_instructions() {@@ -364,7 +308,7 @@ HCC restoration failed. Proxy resources were retained for repair.Context: ${E2E_KUBECONTEXT}Proxy: ${HCC_NS}/${PROXY_NAME}Repair commands:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT-kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180sAfter HCC is healthy, delete ${PROXY_NAME}, ${PROXY_EGRESS_NP}, ${HCC_PROXY_NP}, and ${PROBE_EGRESS_NP} in ${HCC_NS}.diff --git a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh b/scripts/e2e/_lib/hcc-watch-recovery-logs.shindex f0857c3d5..b227248a6 100755--- a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh+++ b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh@@ -2,18 +2,6 @@# Log capture and causal recovery-cycle assertions for the HCC watch gate.-hcc_log_snapshot_contains() {- local snapshot=$1 marker=$2- grep -Fq -- "$marker" <<<"$snapshot"-}--hcc_initial_pass_snapshot_is_active() {- local snapshot=$1 start_marker=$2 complete_marker=$3 fail_marker=$4- hcc_log_snapshot_contains "$snapshot" "$start_marker" &&- ! hcc_log_snapshot_contains "$snapshot" "$complete_marker" &&- ! hcc_log_snapshot_contains "$snapshot" "$fail_marker"-}-hcc_recovery_logs() {[ -n "$START_TIME" ] || return 0grep -E 'CommunicationChannel|Listing all Hosts|Reconciling [0-9]+ Host|Completed Host reconciliation|Periodic resync' \@@ -71,8 +59,7 @@ recovery_cycle_used_fresh_host_inventory() {hcc_recovery_logs | awk \-v target="$cycle" -v host_namespace="$HOST_NS" -v expected="$expected_count" 'BEGIN {- channel_inventory = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"- host_inventory = "Reconciling " expected " Host(s) after Host watch recovery convergence"+ inventory_line = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"succeeded = 0}/CommunicationChannel watch ended;/ {@@ -81,7 +68,7 @@ recovery_cycle_used_fresh_host_inventory() {if (inside) {listed = 0recovered = 0- completion_mode = ""+ inventory_matched = 0}next}@@ -91,20 +78,11 @@ recovery_cycle_used_fresh_host_inventory() {inside && /Recovered [0-9]+ CommunicationChannel\(s\) into cache/ {recovered = 1}- inside && listed && recovered && index($0, channel_inventory) > 0 {- completion_mode = "channel"+ inside && index($0, inventory_line) > 0 {+ inventory_matched = 1}- inside && listed && recovered && index($0, host_inventory) > 0 {- completion_mode = "host"- }- inside && completion_mode == "channel" &&- /Completed Host reconciliation after CommunicationChannel recovery$/ {- succeeded = 1- exit- }- inside && completion_mode == "host" &&- /Completed Host reconciliation after Host watch recovery convergence$/ {- succeeded = 1+ inside && /Completed Host reconciliation after CommunicationChannel recovery$/ {+ succeeded = listed && recovered && inventory_matchedexit}END {@@ -112,15 +90,3 @@ recovery_cycle_used_fresh_host_inventory() {}'}--wait_recovery_cycle_fresh_host_inventory() {- local cycle=$1 expected_count=$2 timeout=$3 elapsed=0- while [ "$elapsed" -lt "$timeout" ]; do- require_hcc_recovery_log_stream || return 1- recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_count" && return 0- sleep 2- elapsed=$((elapsed + 2))- done- hcc_recovery_logs >&2- return 1-}diff --git a/scripts/e2e/e2e-crd-field-injection.sh b/scripts/e2e/e2e-crd-field-injection.shindex d4fe358f8..af8a25829 100755--- a/scripts/e2e/e2e-crd-field-injection.sh+++ b/scripts/e2e/e2e-crd-field-injection.sh@@ -55,7 +55,6 @@ require_safe_kube_context || exit 1cleanup_e2e_resourcesensure_e2e_resources_absent ||fail "Stale e2e-sec resources remain after cleanup; refusing stale-positive E2E run"-# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.[ "$e2e_fail" -gt 0 ] && exit 1# ═══ Test 1: imagePullPolicy override prevention ═════════════════════@@ -79,11 +78,8 @@ spec:url: http://e2e-sec-pullpolicy.mcp-server.svc:3000/mcpEOF-if wait_for_deployment "$MCP_NS" e2e-sec-pullpolicy "$TIMEOUT_POD"; then- ok "imagePullPolicy fixture rendered a ready Deployment"-else- fail "imagePullPolicy fixture did not render a ready Deployment"-fi+log "Waiting for HCC to create Deployment..."+sleep 15PULL_POLICY=$(kctl get deployment e2e-sec-pullpolicy -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.containers[0].imagePullPolicy}' 2>/dev/null || echo "NOT_FOUND")@@ -241,11 +237,7 @@ spec:value: postgres://safeEOF-if wait_for_deployment "$MCP_NS" e2e-sec-envvars "$TIMEOUT_POD"; then- ok "Dangerous-env fixture rendered a ready Deployment"-else- fail "Dangerous-env fixture did not render a ready Deployment"-fi+sleep 15ENV_JSON=$(kctl get deployment e2e-sec-envvars -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.containers[0].env}' 2>/dev/null || echo "[]")@@ -284,11 +276,7 @@ spec:- DAC_OVERRIDEEOF-if wait_for_deployment "$MCP_NS" e2e-sec-uid70 "$TIMEOUT_POD"; then- ok "Non-root UID fixture rendered a ready Deployment"-else- fail "Non-root UID fixture did not render a ready Deployment"-fi+sleep 15UID_70=$(kctl get deployment e2e-sec-uid70 -n "$MCP_NS" \-o jsonpath='{.spec.template.spec.securityContext.runAsUser}' 2>/dev/null || echo "NOT_FOUND")@@ -304,6 +292,5 @@ ensure_e2e_resources_absent && ok "All e2e-sec resources cleaned up" || fail "e2# ═══ Summary ═════════════════════════════════════════════════════════header "Summary"-# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.echo -e " ${GREEN}PASS${NC}: ${e2e_pass} ${RED}FAIL${NC}: ${e2e_fail} Total: ${e2e_total}"[ "$e2e_fail" -gt 0 ] && exit 1 || exit 0diff --git a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.shindex 480b9abd9..d1a38ed5b 100755--- a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh+++ b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh@@ -35,15 +35,13 @@ command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }exit 1}kctl get nodes -o json | jq -e \- --arg context "$E2E_KUBECONTEXT" \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null || {+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] != null)' >/dev/null || {echo "Refusing HCC fault injection: target is not a minikube cluster." >&2exit 1}HCC_NS="${HCC_NS:-control-plane}"HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"-MCP_NS="${MCP_NS:-mcp-server}"SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm-stateless}"HOST_NS="${MCP_HOST_NS:-mcp-host}"CHANNEL_NS="${CHANNELS_NS:-channels}"@@ -63,7 +61,7 @@ HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-stream.XXXXXX")"NON_FIXTURE_BASELINE="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-baseline.XXXXXX")"NON_FIXTURE_AFTER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-after.XXXXXX")"CLEANUP_KINDS="deployments,services,secrets,serviceaccounts,roles,rolebindings,networkpolicies,persistentvolumeclaims"-START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY="" ORIGINAL_REPLICAS=""+START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY=""HCC_LOG_STREAM_PID=""# Mirrors COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS in k8sClient.ts.CC_RECOVERY_RETRY_SECONDS=5@@ -77,35 +75,6 @@ HCC_GATE_FINALIZATION_FAILURE=""die() { fail "$*"; exit 1; }now_rfc3339() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }-# This gate deliberately changes only the HCC Kubernetes API path. If its-# rollout fails before the normal recovery log stream starts, preserve the-# bounded pod state and both container log buffers that distinguish a proxy,-# policy, or process-startup failure. Cleanup still restores HCC afterwards.-print_hcc_proxy_rollout_diagnostics() {- local pod-- echo "HCC proxy rollout diagnostics (before restoration):" >&2- kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o json |- jq -r '- .items[] |- .metadata.name as $pod |- .status.phase as $phase |- .status.containerStatuses[]? |- "pod=\($pod) phase=\($phase // "unknown") ready=\(.ready) restarts=\(.restartCount) waiting=\(.state.waiting.reason // "none") terminated=\(.state.terminated.reason // "none") exitCode=\(.state.terminated.exitCode // "none") lastTerminated=\(.lastState.terminated.reason // "none") lastExitCode=\(.lastState.terminated.exitCode // "none")"- ' >&2 || true-- while IFS= read -r pod; do- [ -n "$pod" ] || continue- echo "HCC proxy rollout previous logs for pod/${pod}:" >&2- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \- --previous --tail=120 >&2 || true- echo "HCC proxy rollout current logs for pod/${pod}:" >&2- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \- --tail=120 >&2 || true- done < <(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)-}-cleanup() {local status=$? cleanup_failed=0 restore_ok=1 host remainingset +e@@ -157,8 +126,6 @@ cleanup() {elsecleanup_failed=1print_hcc_repair_instructions- echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}" >&2- echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s" >&2fifinalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1rm -f "$HCC_LOG_BUFFER" "$NON_FIXTURE_BASELINE" "$NON_FIXTURE_AFTER"@@ -176,35 +143,19 @@ header "HCC CommunicationChannel watch recovery"require_branch_owned_hcc_gateacquire_hcc_watch_gate_lock || die "another HCC watch-recovery gate owns this profile"kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" >/dev/null-ORIGINAL_REPLICAS="$(- kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}'-)"-[ "$ORIGINAL_REPLICAS" = 1 ] ||- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"kctl get host "$SOURCE_HOST" -n "$HOST_NS" >/dev/nullhost_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"-api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"-[ -z "$host_override$port_override$api_cidrs_override" ] ||- die "HCC already has an explicit Kubernetes service or API CIDR override"+[ -z "$host_override$port_override" ] || die "HCC already has an explicit Kubernetes service override"[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||die "HCC already has hostAliases; refusing a non-restorable fault injection"HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"-K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- -c host-context-controller -- printenv KUBERNETES_SERVICE_HOST)" ||- die "could not read the Kubernetes-injected API service IP"-kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \- node -e 'const { isIP } = require("node:net"); process.exit(isIP(process.argv[1]) === 4 ? 0 : 1)' \- "$K8S_API_SERVICE_HOST" >/dev/null ||- die "Kubernetes-injected API service host is not an IPv4 address"-K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"log "Creating an isolated TCP proxy for the Kubernetes API"create_hcc_api_proxy@@ -213,27 +164,19 @@ verify_hcc_proxy_network_policyPROXY_IP="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"[ -n "$PROXY_IP" ] || die "proxy Service has no ClusterIP"HCC_PATCHED=1-hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" --arg api_cidr "$K8S_API_CIDR" '{spec:{template:{spec:{+hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" '{spec:{template:{spec:{hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],containers:[{name:"host-context-controller",env:[{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},- {name:"KUBERNETES_SERVICE_PORT",value:"443"},- {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr}+ {name:"KUBERNETES_SERVICE_PORT",value:"443"}]}]}}}}')"kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$hcc_proxy_patch" >/dev/null-if ! kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null; then- print_hcc_proxy_rollout_diagnostics+kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||die "HCC did not become ready through the API proxy"-fiinitial_hcc_identity="$(wait_for_hcc_identity 30)" || die "could not capture HCC pod identity"read -r HCC_UID HCC_RESTARTS <<<"$initial_hcc_identity"[ -n "$HCC_UID" ] && [ "$HCC_UID" != invalid ] || die "could not capture HCC pod identity"-proxy_policy_cidrs="$(kctl get networkpolicy "allow-k8s-api-egress-${MCP_NS}" -n "$MCP_NS" -o json |- jq -r '[.spec.egress[0].to[]?.ipBlock.cidr] | join(",")')" ||- die "could not read the proxy-run API egress policy"-[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ] ||- die "proxy-run API egress policy did not retain the injected Kubernetes API CIDR"ok "HCC is healthy through the proxy (pod=${HCC_UID}, restarts=${HCC_RESTARTS})"log "Creating one stateful control and two stateless Host fixtures"@@ -271,13 +214,14 @@ ok "isolated Host fleet reached its baseline"run_recovery_cycle() {local cycle=$1 cycle_logs recovery_logs outage_started outage_seconds max_retry_failures list_pattern- local host_list_pattern ended_before started_before recovered_before failed_before+ local host_list_pattern ended_before started_before recovered_before completed_before failed_beforelocal list_before host_list_before expected_host_countlocal failed_after retry_failurescycle_logs="$(hcc_recovery_logs)"ended_before="$(log_count_from "$cycle_logs" 'CommunicationChannel watch ended;')"started_before="$(log_count_from "$cycle_logs" 'Starting CommunicationChannel watch')"recovered_before="$(log_count_from "$cycle_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"+ completed_before="$(log_count_from "$cycle_logs" 'Completed Host reconciliation after CommunicationChannel recovery$')"failed_before="$(log_count_from "$cycle_logs" 'cache recovery failed;')"list_pattern="Listing all CommunicationChannels in namespace ${CHANNEL_NS}"host_list_pattern="Listing all Hosts in namespace ${HOST_NS}"@@ -321,7 +265,10 @@ run_recovery_cycle() {max_retry_failures=$(((outage_seconds + CC_RECOVERY_RETRY_SECONDS - 1) / CC_RECOVERY_RETRY_SECONDS + 2))- wait_recovery_cycle_fresh_host_inventory "$cycle" "$expected_host_count" 180 ||+ wait_log_count 'Completed Host reconciliation after CommunicationChannel recovery$' \+ "$((completed_before + 1))" 180 ||+ die "Host fleet did not converge after snapshot recovery in cycle ${cycle}"+ recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_host_count" ||die "cycle ${cycle} did not reconcile the fresh ${expected_host_count}-Host inventory"recovery_logs="$(hcc_recovery_logs)"failed_after="$(log_count_from "$recovery_logs" 'cache recovery failed;')"@@ -369,18 +316,14 @@ ended="$(log_count_from "$final_logs" 'CommunicationChannel watch ended;')"started="$(log_count_from "$final_logs" 'Starting CommunicationChannel watch')"recovered="$(log_count_from "$final_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"failed="$(log_count_from "$final_logs" 'cache recovery failed;')"-fleet_passes="$(- log_count_from "$final_logs" \- 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'-)"+fleet_passes="$(log_count_from "$final_logs" 'Completed Host reconciliation after CommunicationChannel')"[ "$ended" = 2 ] && [ "$started" = 2 ] && [ "$recovered" = 2 ] && \[ "$failed" = "$TOTAL_RETRY_FAILURES" ] && [ "$failed" -le "$TOTAL_RETRY_FAILURE_LIMIT" ] ||die "unexpected watch lifecycle counts: ended=${ended}, started=${started}, recovered=${recovered}, failed=${failed}"[ "$fleet_passes" -ge 2 ] && [ "$fleet_passes" -le 4 ] ||die "unexpected successful fleet reconcile count ${fleet_passes}; expected 2..4 passes"-# Each recovered snapshot requires one causally ordered successful pass; a-# Host-watch recovery pass may cover the same fresh CC+Host inventories. At-# most one already queued covering pass may also finish per cycle. The deterministic+# Each recovered snapshot requires one successful pass; at most one already+# queued fail-closed pass may also finish per cycle. The deterministic# max-concurrency=1 proof stays in k8sClient unit tests where overlapping passes# can be barrier-controlled.diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.shdeleted file mode 100755index cccb17538..000000000--- a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh+++ /dev/null@@ -1,1790 +0,0 @@-#!/usr/bin/env bash-# Proves that HCC readiness is independent from the exact McpServer ->-# external-egress -> runtime -> Context NetworkPolicy initial convergence path.-#-# The gate runs only on an exact-HEAD, branch-owned Minikube profile. It routes-# HCC DNS through an isolated in-cluster proxy that forwards every query except-# one fixture hostname. The gate first lets HCC create a real external-egress-# policy. A same-intent restart then proves the safety pass RETAINS that-# identity-stable DNS-derived allow without attempting DNS and certifies global-# readiness with the allow still present — no deny window — while the affected-# runtime stays blocked. DNS release lets the additive lane refresh the current-# TCP policy and runtime. A second phase changes that binding's protocol while-# HCC is stopped and proves the divergent contract: an identity-changing-# TCP-to-UDP replacement IS revoked before readiness. A real Context watch update-# still converges its three-# NetworkPolicies. The fleet is wider than the bounded worker width, covering-# the original large-fleet symptom.-set -euo pipefail--SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-# shellcheck source=scripts/e2e/e2e-lib.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/e2e-lib.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"--[ -n "$E2E_KUBECONTEXT" ] || {- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2- exit 1-}-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {- echo "Refusing HCC MCP readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2- exit 1-}-require_safe_kube_context-[ "${E2E_HCC_MCP_READINESS_FAULT_INJECTION:-0}" = 1 ] || {- echo "Set E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC DNS fault injection." >&2- exit 1-}-command -v jq >/dev/null 2>&1 || {- echo "jq is required" >&2- exit 1-}--HCC_NS="${HCC_NS:-control-plane}"-HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"-MCP_NS="${MCP_SERVER_NS:-mcp-server}"-HOST_NS="${MCP_HOST_NS:-mcp-host}"-RPC_PROXY_NS="${RPC_PROXY_NS:-rpc-proxy}"-RUN_ID="$(date +%s)-$$"-SUITE_NAME="hcc-mcp-readiness"-RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"-MCP_NAME="$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}")"-CONTEXT_NAME="$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}")"-CONTEXT_ID="$CONTEXT_NAME"-MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"-TARGET_DNS="$(truncate_rfc1123 "hcc-readiness-${RUN_ID}").example.com"-DNS_BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-dns-${RUN_ID}")"-DNS_BLOCKER_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-policy")"-HCC_DNS_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-from-hcc")"-CONTEXT_POLICY="ctx-${CONTEXT_ID}-${MCP_NAME}"-CONTEXT_EGRESS_POLICY="${CONTEXT_POLICY}-egress"-RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_ID}-${MCP_NAME}"-MANAGED_BY_LABEL="clerum.io/managed-by"-MANAGED_BY_VALUE="host-context-controller"-MCP_SERVER_LABEL="clerum.io/mcpserver"-POLICY_TYPE_LABEL="clerum.io/policy-type"-CONTEXT_LABEL="clerum.io/context"-EXTERNAL_EGRESS_CIDR="93.184.216.34/32"-EXTERNAL_EGRESS_PORT=443-INITIAL_EXTERNAL_EGRESS_PROTOCOL=TCP-EXTERNAL_EGRESS_PROTOCOL=UDP-ORIGINAL_REPLICAS=""-ORIGINAL_DNS_POLICY=""-ORIGINAL_DNS_CONFIG=""-DNS_BLOCKER_IP=""-DNS_RELEASE_COUNT_AT_HOLD=0-DNS_HOLD_COUNT_AT_ARM=0-DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=0-UPSTREAM_DNS_IP=""-NEW_HCC_POD=""-HCC_UID=""-HCC_RESTARTS=""-HCC_PORT=""-HCC_MUTATED=0-DNS_BLOCKER_CREATED=0-MCP_CREATED=0-CONTEXT_CREATED=0-PEER_FLEET_CREATED=0-MCP_FLEET_NAMES=("$MCP_NAME")-CONTEXT_FLEET_NAMES=("$CONTEXT_NAME")-# Read and mutated by the sourced compare-and-swap lock helper.-# shellcheck disable=SC2034-HCC_GATE_LOCK_ACQUIRED=0-# shellcheck disable=SC2034-HCC_GATE_LOCK_NAME=""-# shellcheck disable=SC2034-HCC_GATE_LOCK_UID=""-# shellcheck disable=SC2034-HCC_GATE_FINALIZATION_FAILURE=""-HCC_READY_PROBE_DIAGNOSTIC=""-BASELINE_POLICY_SNAPSHOT=""-SAME_INTENT_EGRESS_SNAPSHOT=""--# Clean-window negative markers. Consumed ONLY while the DNS fault injection is-# NOT applied (pre-injection pre-checks, post-restore cleanup): under the-# injection the additive NetworkPolicy lane legitimately fails and schedules-# retries, so these negatives false-fail inside the injected window (removed-# there by 1abcfb1/15997353/e7064a7f). Inventoried in-# scripts/tests/test-hcc-watch-recovery-gate.sh marker_bindings.-NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'-NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'-NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'-NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '-clean_window_checks_run=0--[[ "$MCP_FLEET_SIZE" =~ ^[0-9]+$ ]] && [ "$MCP_FLEET_SIZE" -gt 10 ] &&- [ "$MCP_FLEET_SIZE" -le 24 ] || {- echo "E2E_HCC_MCP_FLEET_SIZE must be an integer from 11 through 24; got ${MCP_FLEET_SIZE}." >&2- exit 1-}-for ((index = 2; index <= MCP_FLEET_SIZE; index++)); do- MCP_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}-${index}")")- CONTEXT_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}-${index}")")-done--die() {- if [ -n "$NEW_HCC_POD" ]; then- echo "Recent HCC logs from ${NEW_HCC_POD}:" >&2- kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller \- --tail=300 >&2 || true- fi- if [ "$DNS_BLOCKER_CREATED" = 1 ]; then- echo "Recent DNS blocker logs:" >&2- kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" --tail=120 >&2 || true- fi- fail "$*"- exit 1-}--wait_until() {- local timeout=$1 description=$2- shift 2- local deadline now- deadline=$(( $(date +%s) + timeout ))- while :; do- "$@" && return 0- now="$(date +%s)"- [ "$now" -lt "$deadline" ] || break- sleep 1- done- echo "Timed out after ${timeout}s waiting for ${description}" >&2- return 1-}--wait_until_fast() {- local timeout=$1 description=$2- shift 2- local deadline now rc- deadline=$(( $(date +%s) + timeout ))- while :; do- "$@" && return 0- rc=$?- # rc=2 is a fatal, non-retryable ordering violation (e.g. 200 observed with- # a stale policy still present): stop polling immediately so a later clean- # poll cannot mask it. rc=1 is benign "not ready yet".- if [ "$rc" -eq 2 ]; then- echo "FATAL ordering violation while waiting for ${description}: ${HCC_READY_PROBE_DIAGNOSTIC:-unset}" >&2- return 2- fi- now="$(date +%s)"- [ "$now" -lt "$deadline" ] || break- sleep 0.5- done- echo "Timed out after ${timeout}s waiting for ${description}" >&2- return 1-}--running_hcc_pod() {- local rows- rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- --field-selector=status.phase=Running \- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \- 2>/dev/null)" || return 1- awk -F '\t' '$1 != "" && $4 == "" { print $1; exit }' <<<"$rows"-}--capture_hcc() {- NEW_HCC_POD="$(running_hcc_pod)"- [ -n "$NEW_HCC_POD" ]-}--hcc_pods_absent() {- local pods- pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||- return 1- [ -z "$pods" ]-}--hcc_log_contains() {- local marker=$1 logs- [ -n "$NEW_HCC_POD" ] || return 1- logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \- -c host-context-controller 2>/dev/null)" || return 1- grep -Fq "$marker" <<<"$logs"-}--dns_log_contains() {- local marker=$1 logs- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||- return 1- grep -Fq "$marker" <<<"$logs"-}--dns_release_count() {- local marker="released DNS A ${TARGET_DNS}" logs- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||- return 1- grep -Fc "$marker" <<<"$logs" || true-}--dns_hold_count() {- local marker="holding DNS A ${TARGET_DNS}" logs- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||- return 1- grep -Fc "$marker" <<<"$logs" || true-}--dns_hold_observed_since_arm() {- [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_ARM" ]-}--dns_has_not_released_since_hold() {- [ "$(dns_release_count)" = "$DNS_RELEASE_COUNT_AT_HOLD" ]-}--dns_released_since_hold() {- [ "$(dns_release_count)" -gt "$DNS_RELEASE_COUNT_AT_HOLD" ]-}--external_egress_retry_progress_is_observed() {- local logs marker- [ -n "$NEW_HCC_POD" ] || return 1- logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \- -c host-context-controller 2>/dev/null)" || return 1- marker="Scheduling external egress retry "- grep -Fq \- "Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};" <<<"$logs" &&- awk \- -v marker="$marker" \- -v server="for McpServer \"${MCP_NAME}\"" '- index($0, marker) && index($0, server) { found = 1 }- END { exit found ? 0 : 1 }- ' <<<"$logs"-}--dns_retry_query_observed_since_schedule() {- [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_RETRY_SCHEDULE" ]-}--hcc_identity_is_stable() {- local row uid restarts- row="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}' \- 2>/dev/null)" || return 1- read -r uid restarts <<<"$row"- [ "$uid" = "$HCC_UID" ] && [ "$restarts" = "$HCC_RESTARTS" ]-}--hcc_kubernetes_readiness_is_exact() {- [ -n "$NEW_HCC_POD" ] && [ -n "$HCC_UID" ] || return 1- kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" -o json 2>/dev/null |- jq -e --arg uid "$HCC_UID" '- .metadata.uid == $uid and- any(.status.conditions[]?; .type == "Ready" and .status == "True") and- any(.status.containerStatuses[]?;- .name == "host-context-controller" and .ready == true- )- ' >/dev/null || return 1- kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json 2>/dev/null |- jq -e '- .status.observedGeneration == .metadata.generation and- (.spec.replicas // 0) == 1 and- (.status.updatedReplicas // 0) == 1 and- (.status.readyReplicas // 0) == 1 and- (.status.availableReplicas // 0) == 1 and- (.status.unavailableReplicas // 0) == 0- ' >/dev/null-}--resource_absent() {- local kind=$1 name=$2 namespace=$3 found- found="$(- kctl get "$kind" "$name" -n "$namespace" --ignore-not-found -o name 2>/dev/null- )" || return 1- [ -z "$found" ]-}--fixture_mcp_runtime_absent() {- local external_egress_policies- fixture_runtime_absent || return 1- external_egress_policies="$(- kctl get networkpolicy -n "$MCP_NS" \- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \- -o name 2>/dev/null- )" || return 1- [ -z "$external_egress_policies" ]-}--fixture_runtime_absent() {- resource_absent deployment "$MCP_NAME" "$MCP_NS" &&- resource_absent service "$MCP_NAME" "$MCP_NS"-}--mcpserver_current_ready() {- local name=$1- kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null |- jq -e '- .metadata.generation as $generation |- any(- .status.conditions[]?;- .type == "Ready" and .status == "True" and- (.observedGeneration // -1) == $generation- )- ' >/dev/null-}--mcpserver_current_ready_absent() {- local name=$1 resource- resource="$(kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null)" ||- return 1- jq -e '- .metadata.generation as $generation |- any(- .status.conditions[]?;- .type == "Ready" and .status == "True" and- (.observedGeneration // -1) == $generation- ) | not- ' <<<"$resource" >/dev/null-}--fixture_runtime_and_ready_absent() {- fixture_runtime_absent && mcpserver_current_ready_absent "$MCP_NAME"-}--context_policies_absent() {- resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&- resource_absent networkpolicy "$CONTEXT_EGRESS_POLICY" "$HOST_NS" &&- resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS"-}--context_policies_converged() {- local context_id=${1:-$CONTEXT_ID}- local context_name=${2:-$CONTEXT_NAME}- local server=${3:-$MCP_NAME}- local ingress_policy="ctx-${context_id}-${server}"- local host_egress_policy="${ingress_policy}-egress"- local rpc_egress_policy="rpc-egress-${context_id}-${server}"- kctl get networkpolicy "$ingress_policy" -n "$MCP_NS" -o json |- jq -e \- --arg contextId "$context_id" \- --arg contextName "$context_name" \- --arg server "$server" \- --arg hostNs "$HOST_NS" \- --arg rpcNs "$RPC_PROXY_NS" \- --arg mcpNs "$MCP_NS" \- --arg managedByLabel "$MANAGED_BY_LABEL" \- --arg managedByValue "$MANAGED_BY_VALUE" \- --arg contextLabel "$CONTEXT_LABEL" \- --arg serverLabel "$MCP_SERVER_LABEL" \- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '- .metadata.labels[$managedByLabel] == $managedByValue and- .metadata.labels[$policyTypeLabel] == "context-allow" and- .metadata.labels[$contextLabel] == $contextId and- .metadata.labels[$serverLabel] == $server and- .spec.podSelector.matchLabels[$serverLabel] == $server and- .spec.policyTypes == ["Ingress"] and- (.spec.ingress | length) == 1 and- .spec.ingress[0].ports == [{port: 3000, protocol: "TCP"}] and- (.spec.ingress[0].from | length) == 3 and- any(.spec.ingress[0].from[];- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $hostNs and- .podSelector.matchLabels[$managedByLabel] == $managedByValue and- .podSelector.matchLabels[$contextLabel] == $contextName- ) and- any(.spec.ingress[0].from[];- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $rpcNs and- .podSelector.matchLabels.app == "rpc-proxy"- ) and- any(.spec.ingress[0].from[];- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $mcpNs and- .podSelector.matchLabels.app == "mcp-proxy"- )- ' >/dev/null || return 1- kctl get networkpolicy "$host_egress_policy" -n "$HOST_NS" -o json |- jq -e \- --arg contextId "$context_id" \- --arg contextName "$context_name" \- --arg server "$server" \- --arg mcpNs "$MCP_NS" \- --arg managedByLabel "$MANAGED_BY_LABEL" \- --arg managedByValue "$MANAGED_BY_VALUE" \- --arg contextLabel "$CONTEXT_LABEL" \- --arg serverLabel "$MCP_SERVER_LABEL" \- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '- .metadata.labels[$managedByLabel] == $managedByValue and- .metadata.labels[$policyTypeLabel] == "context-allow" and- .metadata.labels[$contextLabel] == $contextId and- .metadata.labels[$serverLabel] == $server and- .spec.podSelector.matchLabels[$managedByLabel] == $managedByValue and- .spec.podSelector.matchLabels[$contextLabel] == $contextName and- .spec.policyTypes == ["Egress"] and- (.spec.egress | length) == 1 and- .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and- (.spec.egress[0].to | length) == 1 and- .spec.egress[0].to[0].namespaceSelector.matchLabels[- "kubernetes.io/metadata.name"- ] == $mcpNs and- .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server- ' >/dev/null || return 1- kctl get networkpolicy "$rpc_egress_policy" -n "$RPC_PROXY_NS" -o json |- jq -e \- --arg contextId "$context_id" \- --arg server "$server" \- --arg mcpNs "$MCP_NS" \- --arg managedByLabel "$MANAGED_BY_LABEL" \- --arg managedByValue "$MANAGED_BY_VALUE" \- --arg contextLabel "$CONTEXT_LABEL" \- --arg serverLabel "$MCP_SERVER_LABEL" \- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '- .metadata.labels[$managedByLabel] == $managedByValue and- .metadata.labels[$policyTypeLabel] == "rpc-proxy-egress" and- .metadata.labels[$contextLabel] == $contextId and- .metadata.labels[$serverLabel] == $server and- .spec.podSelector.matchLabels.app == "rpc-proxy" and- .spec.policyTypes == ["Egress"] and- (.spec.egress | length) == 1 and- .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and- (.spec.egress[0].to | length) == 1 and- .spec.egress[0].to[0].namespaceSelector.matchLabels[- "kubernetes.io/metadata.name"- ] == $mcpNs and- .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server- ' >/dev/null-}--external_egress_policy_converged_with_protocol() {- local protocol=$1- kctl get networkpolicy -n "$MCP_NS" \- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \- -o json |- jq -e \- --arg server "$MCP_NAME" \- --arg cidr "$EXTERNAL_EGRESS_CIDR" \- --arg managedByLabel "$MANAGED_BY_LABEL" \- --arg managedByValue "$MANAGED_BY_VALUE" \- --arg serverLabel "$MCP_SERVER_LABEL" \- --arg policyTypeLabel "$POLICY_TYPE_LABEL" \- --arg protocol "$protocol" \- --argjson port "$EXTERNAL_EGRESS_PORT" '- (.items | length) == 1 and- .items[0].metadata.labels[$managedByLabel] == $managedByValue and- .items[0].metadata.labels[$policyTypeLabel] == "external-egress" and- .items[0].metadata.labels[$serverLabel] == $server and- .items[0].spec.podSelector.matchLabels[$serverLabel] == $server and- .items[0].spec.policyTypes == ["Egress"] and- (.items[0].spec.egress | length) == 1 and- .items[0].spec.egress[0].ports == [{port: $port, protocol: $protocol}] and- (.items[0].spec.egress[0].to | length) == 1 and- .items[0].spec.egress[0].to[0].ipBlock == {cidr: $cidr}- ' >/dev/null-}--external_egress_policy_converged() {- external_egress_policy_converged_with_protocol "$EXTERNAL_EGRESS_PROTOCOL"-}--external_egress_policy_absent() {- local policies- policies="$(- kctl get networkpolicy -n "$MCP_NS" \- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \- -o name 2>/dev/null- )" || return 1- [ -z "$policies" ]-}--clean_window_netpol_log_negatives_hold() {- # Family 1, fail-closed snapshot: read ONE authoritative log snapshot so a- # kubectl logs failure cannot make the negative assertions pass vacuously.- local pod=$1 logs- [ -n "$pod" ] || return 1- logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" \- -c host-context-controller 2>/dev/null)" || return 1- ! grep -Fq "$NETPOL_INITIAL_FAILED_MARKER" <<<"$logs" &&- ! grep -Fq "$NETPOL_ADDITIVE_FAILED_MARKER" <<<"$logs" &&- ! grep -Fq "$NETPOL_RETRY_SCHEDULED_MARKER" <<<"$logs" &&- ! grep -Fq "${NETPOL_CONTEXT_FAILED_MARKER}${CONTEXT_NAME}:" <<<"$logs"-}--clean_window_runtime_and_ready_absent() {- # Family 3, clean-window form. fixture_runtime_and_ready_absent requires the- # McpServer object to EXIST (mcpserver_current_ready_absent fails on a missing- # object); in clean windows the fixture may not exist at all, which is the- # strongest form of absence. An API error still fails closed.- fixture_runtime_absent || return 1- if ! resource_absent mcpserver "$MCP_NAME" "$MCP_NS"; then- mcpserver_current_ready_absent "$MCP_NAME" || return 1- fi- external_egress_policy_absent-}--fixture_runtime_converged_with_protocol() {- local protocol=$1 desired available- desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \- -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1- available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \- -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1- kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1- mcpserver_current_ready "$MCP_NAME" || return 1- external_egress_policy_converged_with_protocol "$protocol"-}--fixture_resources_absent() {- fixture_mcp_runtime_absent &&- context_policies_absent &&- peer_fleet_runtime_absent &&- peer_fleet_policies_absent-}--fixture_inputs_absent() {- resource_absent context "$CONTEXT_NAME" "$MCP_NS" &&- resource_absent mcpserver "$MCP_NAME" "$MCP_NS" && peer_fleet_inputs_absent-}--peer_fleet_inputs_absent() {- local index- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- resource_absent context "${CONTEXT_FLEET_NAMES[index]}" "$MCP_NS" || return 1- resource_absent mcpserver "${MCP_FLEET_NAMES[index]}" "$MCP_NS" || return 1- done-}--peer_fleet_runtime_absent() {- local index server- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- server="${MCP_FLEET_NAMES[index]}"- resource_absent deployment "$server" "$MCP_NS" || return 1- resource_absent service "$server" "$MCP_NS" || return 1- done-}--peer_fleet_policies_absent() {- local index server context context_policy external_policies- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- server="${MCP_FLEET_NAMES[index]}"- context="${CONTEXT_FLEET_NAMES[index]}"- context_policy="ctx-${context}-${server}"- resource_absent networkpolicy "$context_policy" "$MCP_NS" || return 1- resource_absent networkpolicy "${context_policy}-egress" "$HOST_NS" || return 1- resource_absent networkpolicy "rpc-egress-${context}-${server}" "$RPC_PROXY_NS" || return 1- external_policies="$(- kctl get networkpolicy -n "$MCP_NS" \- -l "${MCP_SERVER_LABEL}=${server},${POLICY_TYPE_LABEL}=external-egress" \- -o name 2>/dev/null- )" || return 1- [ -z "$external_policies" ] || return 1- done-}--peer_fleet_converged() {- local index server context desired available- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- server="${MCP_FLEET_NAMES[index]}"- context="${CONTEXT_FLEET_NAMES[index]}"- desired="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1- available="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1- kctl get service "$server" -n "$MCP_NS" >/dev/null 2>&1 || return 1- mcpserver_current_ready "$server" || return 1- context_policies_converged "$context" "$context" "$server" || return 1- done-}--baseline_policy_snapshot() {- kctl get \- "networkpolicy/deny-all-${MCP_NS}" \- "networkpolicy/allow-dns-egress-${MCP_NS}" \- networkpolicy/allow-host-context-controller-api \- -n "$MCP_NS" -o json 2>/dev/null |- jq -ceS \- --arg managedBy "$MANAGED_BY_VALUE" \- --arg deny "deny-all-${MCP_NS}" \- --arg dns "allow-dns-egress-${MCP_NS}" '- [.items[] | {- name: .metadata.name,- managedBy: .metadata.labels["clerum.io/managed-by"],- policyType: .metadata.labels["clerum.io/policy-type"],- spec- }]- | sort_by(.name)- | select(length == 3)- | select(all(.[];- .managedBy == $managedBy and- ((.name == $deny and .policyType == "default-deny") or- (.name == $dns and .policyType == "infrastructure") or- (.name == "allow-host-context-controller-api" and .policyType == "allow-api"))))- '-}--baseline_policies_unchanged() {- local current- [ -n "$BASELINE_POLICY_SNAPSHOT" ] || return 1- current="$(baseline_policy_snapshot)" || return 1- [ "$current" = "$BASELINE_POLICY_SNAPSHOT" ]-}--context_watch_has_latest_spec() {- kctl get context "$CONTEXT_NAME" -n "$MCP_NS" -o json |- jq -e --arg server "$MCP_NAME" '.spec.mcpServers == [$server]' >/dev/null-}--mcp_egress_binding_is_udp() {- kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json |- jq -e --arg dns "$TARGET_DNS" '- (.spec.egressBindings | length) == 1 and- .spec.egressBindings[0].egressClass == "exact-host" and- .spec.egressBindings[0].dns == $dns and- .spec.egressBindings[0].port == 443 and- .spec.egressBindings[0].protocol == "UDP"- ' >/dev/null-}--create_peer_fleet() {- local index server context- # Arm cleanup before the first write so a partial apply cannot strand input- # CRDs or leave HCC deliberately scaled down during failure recovery.- PEER_FLEET_CREATED=1- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- server="${MCP_FLEET_NAMES[index]}"- context="${CONTEXT_FLEET_NAMES[index]}"- kctl apply -f - >/dev/null <<EOF-apiVersion: clerum.io/v1alpha1-kind: Context-metadata:- name: ${context}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- contextId: ${context}- description: Gate-owned peer Context for HCC readiness fleet coverage.- mcpServers: [${server}]-----apiVersion: clerum.io/v1alpha1-kind: McpServer-metadata:- name: ${server}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- contextRef: ${context}- description: Gate-owned peer McpServer for bounded HCC fleet coverage.- image: clerum/mock-mcp-server:test- imagePullPolicy: IfNotPresent- transport:- type: streamableHttp- url: http://${server}.${MCP_NS}.svc.cluster.local:3000/mcp- port: 3000-EOF- done-}--fixture_converged() {- local desired available- desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \- -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1- available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \- -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1- kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1- mcpserver_current_ready "$MCP_NAME" || return 1- external_egress_policy_converged || return 1- context_policies_converged-}--probe_hcc_final_context() {- local probe_script- probe_script="$(cat <<'NODE'-const http = require('http');-const context = process.argv[1];-const expected = process.argv[2];-function get(path) {- return new Promise((resolve, reject) => {- const request = http.get({- host: '127.0.0.1',- port: Number(process.env.HCC_E2E_PORT),- path- }, response => {- let body = '';- response.setEncoding('utf8');- response.on('data', chunk => { body += chunk; });- response.on('end', () => resolve({status: response.statusCode, body}));- });- request.setTimeout(5000, () => request.destroy(new Error('timeout')));- request.once('error', reject);- });-}-(async () => {- const ready = await get('/ready');- const scoped = await get('/api/v1/mcpservers/context/' + encodeURIComponent(context));- const body = JSON.parse(scoped.body);- const fixture = body.servers?.find(server => server.name === expected);- if (ready.status !== 200 || JSON.parse(ready.body).ready !== true) {- throw new Error('readiness regressed: ' + ready.status + ' ' + ready.body);- }- if (scoped.status !== 200 || !fixture || fixture.status?.ready !== true) {- throw new Error('latest Context did not converge in discovery: ' + scoped.status + ' ' + scoped.body);- }- console.log(JSON.stringify({- readyStatus: ready.status,- scopedDiscoveryStatus: scoped.status,- contextRef: body.contextRef,- fixture: fixture.name,- fixtureReady: fixture.status.ready- }));-})().catch(error => { console.error(error.message); process.exit(1); });-NODE-)"- kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" \- node -e "$probe_script" "$CONTEXT_NAME" "$MCP_NAME"-}--probe_hcc_ready_pod() {- local pod- pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1- kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" \- node -e '- const http = require("node:http");- const request = http.get(- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},- response => {- let body = "";- response.setEncoding("utf8");- response.on("data", chunk => { body += chunk; });- response.on("end", () => {- let parsed;- try { parsed = JSON.parse(body); } catch { process.exit(2); }- process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);- });- }- );- request.setTimeout(5000, () => request.destroy(new Error("timeout")));- request.once("error", () => process.exit(4));- ' >/dev/null 2>&1-}--probe_new_hcc_ready_endpoint() {- local output first_line- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=pod-not-selected"- [ -n "$NEW_HCC_POD" ] || return 1- if output="$(- kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" \- node -e '- const http = require("node:http");- let finished = false;- const finish = (code, category, detail = "") => {- if (finished) return;- finished = true;- if (category) {- console.error(`HCC_READY_PROBE category=${category}${detail}`);- }- process.exitCode = code;- };- const request = http.get(- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},- response => {- let body = "";- response.setEncoding("utf8");- response.on("data", chunk => { body += chunk; });- response.on("end", () => {- if (response.statusCode !== 200) {- finish(3, "http-status", ` status=${response.statusCode ?? "unknown"}`);- return;- }- let parsed;- try {- parsed = JSON.parse(body);- } catch {- finish(2, "json-invalid");- return;- }- if (parsed?.ready !== true) {- finish(3, "json-ready-false");- return;- }- finish(0);- });- }- );- request.setTimeout(2000, () => {- finish(4, "timeout");- request.destroy();- });- request.once("error", () => finish(5, "transport"));- ' 2>&1- )"; then- HCC_READY_PROBE_DIAGNOSTIC=""- return 0- fi- first_line="${output%%$'\n'*}"- case "$first_line" in- "HCC_READY_PROBE category="*)- printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s' "$first_line"- ;;- *)- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=kubectl-exec"- ;;- esac- return 1-}--# Third, independent signal of the safety contract (M6): the authoritative pass-# must actually have MEASURED and REVOKED. The unit tests assert the readiness-# predicate over vi.fn() mocks, so a metric rename would leave them green; these-# family-name literals live in the gate, so a rename turns the REAL gate red.-# Pure adjudication over a /metrics scrape — unit-testable without a cluster.-safety_pass_metrics_are_certified() {- local metrics=$1- grep -Fq -- 'clerum_hcc_networkpolicy_safety_pass_duration_seconds' <<<"$metrics" || return 1- # At least one policy revoked by an authoritative pass ($NF is the sample- # value; tolerant of extra/reordered labels).- awk '$0 ~ /^clerum_hcc_networkpolicy_safety_pass_policies_total\{/ && /operation="revoked"/ { if ($NF + 0 >= 1) found = 1 } END { exit(found ? 0 : 1) }' <<<"$metrics"-}--# Scrapes the running replacement pod's /metrics over the same in-pod node/http-# channel the /ready probes use, then adjudicates. The sub-0.5s window B4 leaves-# unobservable on /ready is covered here: the counter is monotonic.-safety_pass_metrics_certify() {- local metrics- metrics="$(kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" \- node -e '- const http = require("node:http");- const request = http.get(- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/metrics"},- response => {- let body = "";- response.setEncoding("utf8");- response.on("data", chunk => { body += chunk; });- response.on("end", () => {- process.stdout.write(body);- process.exit(response.statusCode === 200 ? 0 : 1);- });- }- );- request.setTimeout(2000, () => { request.destroy(); process.exit(1); });- request.on("error", () => process.exit(1));- ')" || return 1- safety_pass_metrics_are_certified "$metrics"-}--hcc_ready_after_stale_policy_revoked() {- # A single poll that sees 200 WHILE the stale policy is still present is a- # safety-contract violation and must be FATAL, never retried: retrying lets a- # later poll (policy already revoked) turn the violation into a false PASS.- # rc=2 signals that fatal ordering violation to wait_until_fast; rc=1 is a- # benign "not ready yet, keep polling". The sub-0.5s window where revocation- # races the probe inside one poll stays unobservable on THIS signal — the- # third signal (M6, /metrics) covers it.- local policy_absent_before=0- external_egress_policy_absent && policy_absent_before=1- probe_new_hcc_ready_endpoint || return 1- if external_egress_policy_absent; then- [ "$policy_absent_before" -eq 1 ] && return 0- # Revocation raced the probe inside this poll: the final state is clean but- # this poll cannot adjudicate the order. Retry to adjudicate cleanly.- return 1- fi- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=ready-200-with-stale-policy-present"- return 2-}--# Canonical, order-stable projection of the single external-egress policy for the-# fixture server: name, ownership labels, and full spec. Used to prove a-# same-intent restart RETAINS the identity-stable DNS allow byte-for-byte (B3-# retain-identity-stable contract), the inverse of the divergent revoke path.-external_egress_policy_snapshot() {- kctl get networkpolicy -n "$MCP_NS" \- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \- -o json 2>/dev/null |- jq -ceS \- --arg managedByLabel "$MANAGED_BY_LABEL" \- --arg managedByValue "$MANAGED_BY_VALUE" \- --arg policyTypeLabel "$POLICY_TYPE_LABEL" \- --arg serverLabel "$MCP_SERVER_LABEL" \- --arg server "$MCP_NAME" '- [.items[] | {- name: .metadata.name,- managedBy: .metadata.labels[$managedByLabel],- policyType: .metadata.labels[$policyTypeLabel],- serverLabel: .metadata.labels[$serverLabel],- spec- }]- | select(length == 1)- | .[0]- | select(.managedBy == $managedByValue- and .policyType == "external-egress"- and .serverLabel == $server)- '-}--# True when the external-egress policy is present and identical (spec +-# ownership) to the snapshot captured before the same-intent restart. An API-# error or a missing/renamed policy fails closed.-identity_stable_policy_retained() {- local current- [ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] || return 1- current="$(external_egress_policy_snapshot)" || return 1- [ -n "$current" ] || return 1- [ "$current" = "$SAME_INTENT_EGRESS_SNAPSHOT" ]-}--hcc_ready_with_identity_stable_policy_retained() {- # INVERSE of the pre-B3 revoke-before-ready contract. B3 retains an- # identity-stable DNS allow across a same-intent restart with NO deny window and- # certifies /ready=200 WHILE that allow is still present and unchanged. A single- # poll that reaches /ready=200 with the identity-stable allow ABSENT from the- # start is the retain regression (a deny window) and must be FATAL, never- # retried: retrying would let a later poll — after the held-refresh timeout- # legitimately fail-closed deletes the allow — mask the regression as a PASS.- # rc=2 signals that fatal violation to wait_until_fast; rc=1 is a benign- # "not ready yet, keep polling".- local retained_before=0- identity_stable_policy_retained && retained_before=1- probe_new_hcc_ready_endpoint || return 1- if identity_stable_policy_retained; then- [ "$retained_before" -eq 1 ] && return 0- # The allow reappeared within this poll: adjudicate cleanly on the next poll.- return 1- fi- if [ "$retained_before" -eq 1 ]; then- # The allow was present immediately before this probe and is gone now: a- # legitimate fail-closed refresh timeout can race the probe inside one poll.- # Retry to adjudicate — a real retain regression never presents the allow at- # readiness, so a later poll still returns rc=2.- return 1- fi- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=identity-stable-policy-revoked"- return 2-}--hcc_restore_is_verified() {- local deployment- deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1- jq -e \- --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \- --argjson replicas "${ORIGINAL_REPLICAS:-1}" '- .spec.replicas == $replicas and- (.status.readyReplicas // 0) == $replicas and- (.status.availableReplicas // 0) == $replicas and- (.spec.template.spec.dnsPolicy // "ClusterFirst") == $dnsPolicy and- (.spec.template.spec.dnsConfig // null) == null- ' <<<"$deployment" >/dev/null || return 1- probe_hcc_ready_pod-}--release_dns_blocker() {- local release_script- release_script="$(cat <<'NODE'-const http = require('http');-const request = http.request(- {host: '127.0.0.1', port: 8090, path: '/release', method: 'POST'},- response => {- response.resume();- response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));- }-);-request.setTimeout(5000, () => request.destroy(new Error('timeout')));-request.once('error', error => { console.error(error.message); process.exit(3); });-request.end();-NODE-)"- kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \- node -e "$release_script"-}--hold_dns_blocker() {- local hold_script- DNS_HOLD_COUNT_AT_ARM="$(dns_hold_count)" ||- return 1- hold_script="$(cat <<'NODE'-const http = require('http');-const request = http.request(- {host: '127.0.0.1', port: 8090, path: '/hold', method: 'POST'},- response => {- response.resume();- response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));- }-);-request.setTimeout(5000, () => request.destroy(new Error('timeout')));-request.once('error', error => { console.error(error.message); process.exit(3); });-request.end();-NODE-)"- kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \- node -e "$hold_script"-}--delete_fixture_runtime_leaving_dns_policy_for_safety() {- local failed=0- kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete service "$MCP_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- [ "$failed" = 0 ] && fixture_runtime_absent-}--delete_dns_blocker_fixture() {- local failed=0- kctl delete networkpolicy "$HCC_DNS_POLICY" "$DNS_BLOCKER_POLICY" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- [ "$failed" = 0 ]-}--delete_residual_fixture_runtime() {- local failed=0- kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete service "$MCP_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy "$CONTEXT_EGRESS_POLICY" -n "$HOST_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy "$RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${MCP_NAME}" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- delete_peer_fleet_runtime || failed=1- [ "$failed" = 0 ]-}--delete_peer_fleet_runtime() {- local index server context context_policy failed=0- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- server="${MCP_FLEET_NAMES[index]}"- context="${CONTEXT_FLEET_NAMES[index]}"- context_policy="ctx-${context}-${server}"- kctl delete deployment "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \- >/dev/null 2>&1 || failed=1- kctl delete service "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \- >/dev/null 2>&1 || failed=1- kctl delete configmap "${server}-nginx-conf" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \- >/dev/null 2>&1 || failed=1- kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${server}" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy "${context_policy}-egress" -n "$HOST_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete networkpolicy "rpc-egress-${context}-${server}" -n "$RPC_PROXY_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- done- [ "$failed" = 0 ]-}--delete_peer_fleet_inputs() {- local index failed=0- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do- kctl delete context "${CONTEXT_FLEET_NAMES[index]}" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete mcpserver "${MCP_FLEET_NAMES[index]}" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- done- [ "$failed" = 0 ]-}--suite_resources_absent() {- local namespaced cluster_crds- namespaced="$(kctl get deployment,service,networkpolicy -A \- -l "e2e.clerum.io/suite=${SUITE_NAME}" -o name 2>/dev/null)" || return 1- cluster_crds="$(- kctl get mcpserver,context -A -l "e2e.clerum.io/suite=${SUITE_NAME}" \- -o name 2>/dev/null- )" || return 1- [ -z "$namespaced$cluster_crds" ]-}--print_repair_instructions() {- cat >&2 <<EOF-HCC MCP readiness gate cleanup could not restore a verified clean state.-Context: ${E2E_KUBECONTEXT}-HCC: ${HCC_NS}/${HCC_DEPLOY}-DNS blocker: ${HCC_NS}/${DNS_BLOCKER_NAME}-Fixtures: ${MCP_NS}/${MCP_NAME}, ${MCP_NS}/${CONTEXT_NAME}--Inspect before changing anything:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml- kubectl --context=${E2E_KUBECONTEXT} get deployment,service,networkpolicy -A -l e2e.clerum.io/suite=${SUITE_NAME}- kubectl --context=${E2E_KUBECONTEXT} get mcpserver,context -A -l e2e.clerum.io/suite=${SUITE_NAME}--Do not remove the HCC gate lock until HCC health and fixture absence are verified.--Restore the original HCC replica count before verifying rollout and /ready:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=240s-EOF-}--cleanup() {- local status=$? cleanup_failed=0 restore_ok=1 fixture_inputs_removed=1 restore_patch- local clean_window_pod- trap - EXIT- set +e-- if [ "$HCC_MUTATED" = 1 ]; then- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1 ||- restore_ok=0- if [ "$restore_ok" = 1 ]; then- wait_until 120 "HCC pods to stop before fixture cleanup" \- hcc_pods_absent >/dev/null 2>&1 || restore_ok=0- fi- fi-- if [ "$CONTEXT_CREATED" = 1 ]; then- kctl delete context "$CONTEXT_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||- {- cleanup_failed=1- fixture_inputs_removed=0- }- fi- if [ "$MCP_CREATED" = 1 ]; then- kctl delete mcpserver "$MCP_NAME" -n "$MCP_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||- {- cleanup_failed=1- fixture_inputs_removed=0- }- fi- if [ "$PEER_FLEET_CREATED" = 1 ]; then- delete_peer_fleet_inputs || {- cleanup_failed=1- fixture_inputs_removed=0- }- fi- if ! wait_until 60 "gate-owned Context and McpServer inputs to disappear" \- fixture_inputs_absent >/dev/null 2>&1; then- cleanup_failed=1- fixture_inputs_removed=0- fi-- if [ "$HCC_MUTATED" = 1 ]; then- if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then- restore_patch="$(jq -cn --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \- '{spec:{template:{spec:{dnsPolicy:$dnsPolicy,dnsConfig:null}}}}')"- kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \- -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \- --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0- kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \- --timeout=240s >/dev/null 2>&1 || restore_ok=0- if [ "$restore_ok" = 1 ]; then- wait_until 60 "restored HCC DNS, replicas, Ready pod, and /ready endpoint" \- hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0- fi- else- restore_ok=0- fi- fi-- if [ "$restore_ok" = 1 ]; then- if ! wait_until 180 "HCC to remove all MCP/Context fixture runtime resources" \- fixture_resources_absent >/dev/null 2>&1; then- cleanup_failed=1- delete_residual_fixture_runtime || cleanup_failed=1- fi- if [ "$DNS_BLOCKER_CREATED" = 1 ]; then- delete_dns_blocker_fixture || cleanup_failed=1- fi- wait_until 60 "all HCC MCP readiness suite resources to disappear" \- suite_resources_absent >/dev/null 2>&1 || cleanup_failed=1-- # Clean window B: DNS restored and fixture inputs gone. The restored pod is- # new (scale-0 preceded the restore), so its log is scoped to this window.- clean_window_pod="$(running_hcc_pod)"- if [ -n "$clean_window_pod" ] &&- clean_window_netpol_log_negatives_hold "$clean_window_pod"; then- clean_window_checks_run=$((clean_window_checks_run + 1))- else- fail "post-restore clean-window NetworkPolicy log negatives did not hold"- cleanup_failed=1- fi- if fixture_mcp_runtime_absent; then- clean_window_checks_run=$((clean_window_checks_run + 1))- else- fail "post-restore fixture runtime or external-egress policy survived"- cleanup_failed=1- fi- if clean_window_runtime_and_ready_absent; then- clean_window_checks_run=$((clean_window_checks_run + 1))- else- fail "post-restore McpServer Ready status or external-egress policy survived"- cleanup_failed=1- fi- else- cleanup_failed=1- fi-- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then- cleanup_failed=1- fi-- if [ "$status" = 0 ]; then- [ "${clean_window_checks_run:-0}" -gt 0 ] ||- die "no clean-window negative checks ran (clean_window_checks_run=0)"- fi-- if [ "$cleanup_failed" -ne 0 ]; then- echo "HCC MCP readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2- [ "$restore_ok" = 1 ] || print_repair_instructions- [ "$status" = 0 ] && status=1- fi- if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then- header "HCC McpServer/Context readiness gate passed"- echo "$final_probe"- fi- exit "$status"-}-trap cleanup EXIT--header "HCC readiness during initial McpServer and Context convergence"--require_branch_owned_hcc_gate "$HCC_NS"-kctl get nodes -o json |- jq -e --arg context "$E2E_KUBECONTEXT" \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||- die "target context is not a minikube cluster"-suite_resources_absent ||- die "stale ${SUITE_NAME} resources exist; inspect them before starting a new fault injection"-acquire_hcc_watch_gate_lock ||- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"-ok "branch helper, profile, exact HEAD/fingerprint/gate, and exclusive HCC lock verified"--# Clean window A: the DNS fault injection is not applied yet. The branch-deployed-# HCC must show a clean NetworkPolicy lane and zero fixture residue before the-# gate mutates anything.-clean_window_pod="$(running_hcc_pod)" && [ -n "$clean_window_pod" ] ||- die "no Running HCC pod before fault injection"-clean_window_netpol_log_negatives_hold "$clean_window_pod" ||- die "pre-injection HCC logs already contain NetworkPolicy-lane failure or retry markers"-clean_window_checks_run=$((clean_window_checks_run + 1))-fixture_mcp_runtime_absent ||- die "pre-injection fixture runtime or external-egress policy already exists"-clean_window_checks_run=$((clean_window_checks_run + 1))-clean_window_runtime_and_ready_absent ||- die "pre-injection McpServer Ready status or external-egress policy already exists"-clean_window_checks_run=$((clean_window_checks_run + 1))-ok "all three clean-window negative families hold before DNS fault injection"--ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.replicas}')"-[ "$ORIGINAL_REPLICAS" = 1 ] ||- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"-ORIGINAL_DNS_POLICY="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.template.spec.dnsPolicy}')"-ORIGINAL_DNS_POLICY="${ORIGINAL_DNS_POLICY:-ClusterFirst}"-[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] ||- die "HCC already has non-default dnsPolicy ${ORIGINAL_DNS_POLICY}; refusing a non-restorable fixture"-ORIGINAL_DNS_CONFIG="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |- jq -c '.spec.template.spec.dnsConfig // null')"-[ "$ORIGINAL_DNS_CONFIG" = null ] ||- die "HCC already has dnsConfig ${ORIGINAL_DNS_CONFIG}; refusing a non-restorable fixture"-HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"-[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"-HCC_PORT="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json | jq -r '- .spec.template.spec.containers[]? |- select(.name == "host-context-controller") as $container |- (- [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //- [$container.ports[]? | select(.name == "http") | .containerPort][0] //- empty- )-')"-[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||- die "could not resolve a valid HCC HTTP port from the Deployment"-UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"-[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-system/kube-dns ClusterIP"-running_dns_pods="$(kctl get pods -n kube-system -l k8s-app=kube-dns \- --field-selector=status.phase=Running -o name)"-[ -n "$running_dns_pods" ] ||- die "no Running kube-dns pod is available"--DNS_BLOCKER_CREATED=1-kctl apply -f - >/dev/null <<EOF-apiVersion: apps/v1-kind: Deployment-metadata:- name: ${DNS_BLOCKER_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- replicas: 1- selector:- matchLabels:- app: ${DNS_BLOCKER_NAME}- template:- metadata:- labels:- app: ${DNS_BLOCKER_NAME}- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"- spec:- automountServiceAccountToken: false- terminationGracePeriodSeconds: 1- containers:- - name: dns-blocker- image: ${HCC_IMAGE}- imagePullPolicy: IfNotPresent- command: [node, -e]- args:- - >-- const dgram=require('node:dgram'),http=require('node:http'),- net=require('node:net');- const target=process.env.TARGET_DNS,upstream=process.env.UPSTREAM_DNS;- const udp=dgram.createSocket('udp4');let released=process.env.START_RELEASED==='1';const held=[];- function question(message){let offset=12,labels=[];while(offset<message.length){- const length=message[offset++];if(length===0)break;if(offset+length>message.length)- return null;labels.push(message.subarray(offset,offset+length).toString('ascii'));- offset+=length;}if(offset+4>message.length)return null;return- {name:labels.join('.'),type:message.readUInt16BE(offset),end:offset+4};}- function answer(message,parsed){const header=Buffer.from(message.subarray(0,12));- header.writeUInt16BE(0x8180,2);header.writeUInt16BE(1,6);- header.writeUInt16BE(0,8);header.writeUInt16BE(0,10);- const record=Buffer.from([0xc0,0x0c,0x00,0x01,0x00,0x01,0x00,0x00,- 0x00,0x1e,0x00,0x04,93,184,216,34]);- return Buffer.concat([header,message.subarray(12,parsed.end),record]);}- function tcpFrame(message){const frame=Buffer.alloc(message.length+2);- frame.writeUInt16BE(message.length,0);message.copy(frame,2);return frame;}- function respond(entry){const response=answer(entry.message,entry.parsed);- if(entry.transport==='udp'){udp.send(response,entry.rinfo.port,- entry.rinfo.address);console.log('released DNS A '+target+' via UDP');return;}- if(!entry.socket.destroyed){entry.socket.write(tcpFrame(response));- console.log('released DNS A '+target+' via TCP');}}- function forwardUdp(message,rinfo){const relay=dgram.createSocket('udp4');- const timer=setTimeout(()=>relay.close(),5000);relay.once('message',response=>{- clearTimeout(timer);udp.send(response,rinfo.port,rinfo.address);relay.close();});- relay.once('error',()=>{clearTimeout(timer);relay.close();});- relay.send(message,53,upstream);}- function forwardTcp(message,socket){const relay=net.createConnection(- {host:upstream,port:53},()=>relay.write(tcpFrame(message)));- let pending=Buffer.alloc(0);relay.setTimeout(5000,()=>relay.destroy());- relay.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);- if(pending.length<2)return;const length=pending.readUInt16BE(0);- if(pending.length<length+2)return;if(!socket.destroyed)- socket.write(pending.subarray(0,length+2));relay.destroy();});- relay.once('error',()=>relay.destroy());}- function handle(message,entry){const parsed=question(message);- if(parsed&&parsed.name===target&&parsed.type===1){entry.message=message;- entry.parsed=parsed;if(released)respond(entry);else{held.push(entry);- console.log('holding DNS A '+target+' via '+entry.transport.toUpperCase());}- return;}if(entry.transport==='udp')forwardUdp(message,entry.rinfo);- else forwardTcp(message,entry.socket);}- let listeners=0;function listening(){listeners+=1;if(listeners===2)- console.log('dns hold proxy ready for '+target+' over UDP and TCP');}- udp.on('message',(message,rinfo)=>handle(message,{transport:'udp',rinfo}));- udp.bind(8053,'0.0.0.0',listening);- const tcp=net.createServer(socket=>{let pending=Buffer.alloc(0);- socket.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);- while(pending.length>=2){const length=pending.readUInt16BE(0);- if(pending.length<length+2)return;const message=pending.subarray(2,length+2);- pending=pending.subarray(length+2);handle(message,{transport:'tcp',socket});}});- });tcp.listen(8053,'0.0.0.0',listening);- http.createServer((request,response)=>{if(request.method==='POST'&&- request.url==='/release'){released=true;while(held.length)respond(held.shift());- response.writeHead(200);response.end('released');return;}if(request.method==='POST'&&- request.url==='/hold'){released=false;response.writeHead(200);response.end('holding');return;}- response.writeHead(404);response.end();}).listen(8090,'127.0.0.1');- env:- - name: TARGET_DNS- value: ${TARGET_DNS}- - name: START_RELEASED- value: "1"- - name: UPSTREAM_DNS- value: ${UPSTREAM_DNS_IP}- resources:- requests: {cpu: 5m, memory: 20Mi}- limits: {cpu: 75m, memory: 64Mi}- securityContext:- allowPrivilegeEscalation: false- capabilities: {drop: [ALL]}- readOnlyRootFilesystem: true- runAsNonRoot: true- runAsUser: 1000- runAsGroup: 1000- seccompProfile: {type: RuntimeDefault}-----apiVersion: v1-kind: Service-metadata:- name: ${DNS_BLOCKER_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- selector:- app: ${DNS_BLOCKER_NAME}- ports:- - name: dns-udp- protocol: UDP- port: 53- targetPort: 8053- - name: dns-tcp- protocol: TCP- port: 53- targetPort: 8053-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${DNS_BLOCKER_POLICY}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- podSelector:- matchLabels:- app: ${DNS_BLOCKER_NAME}- policyTypes: [Ingress, Egress]- ingress:- - from:- - podSelector:- matchLabels:- app: ${HCC_DEPLOY}- ports:- - {protocol: UDP, port: 8053}- - {protocol: UDP, port: 53}- - {protocol: TCP, port: 8053}- - {protocol: TCP, port: 53}- egress:- - to:- - namespaceSelector:- matchLabels:- kubernetes.io/metadata.name: kube-system- podSelector:- matchLabels:- k8s-app: kube-dns- ports:- - {protocol: UDP, port: 53}- - {protocol: TCP, port: 53}-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${HCC_DNS_POLICY}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- podSelector:- matchLabels:- app: ${HCC_DEPLOY}- policyTypes: [Egress]- egress:- - to:- - podSelector:- matchLabels:- app: ${DNS_BLOCKER_NAME}- ports:- - {protocol: UDP, port: 53}- - {protocol: UDP, port: 8053}- - {protocol: TCP, port: 53}- - {protocol: TCP, port: 8053}-EOF-kctl rollout status deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \- --timeout=90s >/dev/null || die "DNS blocker did not become ready"-wait_until 30 "DNS blocker process to listen" \- dns_log_contains "dns hold proxy ready for ${TARGET_DNS}" ||- die "DNS blocker process did not start"-DNS_BLOCKER_IP="$(kctl get service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \- -o jsonpath='{.spec.clusterIP}')"-[ -n "$DNS_BLOCKER_IP" ] || die "DNS blocker Service has no ClusterIP"-ok "isolated DNS hold proxy is ready and forwards non-fixture UDP/TCP queries to CoreDNS"--[ "${clean_window_checks_run:-0}" -ge 3 ] ||- die "pre-injection clean-window negatives did not run before the DNS mutation"--HCC_MUTATED=1-dns_patch="$(jq -cn --arg nameserver "$DNS_BLOCKER_IP" '- {spec:{template:{spec:{- dnsPolicy:"None",- dnsConfig:{- nameservers:[$nameserver],- options:[- {name:"ndots",value:"1"},- {name:"timeout",value:"30"},- {name:"attempts",value:"1"}- ]- }- }}}}-')"-kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \- -p "$dns_patch" >/dev/null-kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||- die "HCC did not become Ready with the deterministic DNS proxy"-wait_until 60 "HCC /ready endpoint with the released DNS proxy" probe_hcc_ready_pod ||- die "HCC readiness was unavailable before the stale-policy fixture"--CONTEXT_CREATED=1-kctl apply -f - >/dev/null <<EOF-apiVersion: clerum.io/v1alpha1-kind: Context-metadata:- name: ${CONTEXT_NAME}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- contextId: ${CONTEXT_ID}- description: E2E HCC readiness Context updated while initial MCP convergence is blocked.- mcpServers: []-EOF--MCP_CREATED=1-kctl apply -f - >/dev/null <<EOF-apiVersion: clerum.io/v1alpha1-kind: McpServer-metadata:- name: ${MCP_NAME}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- contextRef: ${CONTEXT_NAME}- description: E2E MCP fixture for initial fleet readiness.- image: clerum/mock-mcp-server:test- imagePullPolicy: IfNotPresent- transport:- type: streamableHttp- url: http://${MCP_NAME}.${MCP_NS}.svc.cluster.local:3000/mcp- port: 3000- egressBindings:- - dns: ${TARGET_DNS}- port: 443- protocol: ${INITIAL_EXTERNAL_EGRESS_PROTOCOL}-EOF--wait_until 180 "HCC to create the real initial TCP external-egress policy" \- external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||- die "HCC never created the initial external-egress policy for the stale-policy fixture"-ok "HCC created a real initial TCP external-egress policy"-BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)" ||- die "could not capture canonical baseline NetworkPolicy ownership and specs"--# Same-intent safety phase: restart with the exact same desired binding. The-# safety pass must RETAIN the identity-stable DNS-derived policy without resolving-# DNS (B3 retain-identity-stable contract), certify global readiness with that-# allow still present — no deny window — and leave the affected runtime blocked-# for the additive lane. Retention-at-readiness is adjudicated promptly: a-# held-refresh timeout later fail-closed deletes the retained allow, so the proof-# must land before any refresh timeout.-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null-wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||- die "existing HCC pod did not stop"-delete_fixture_runtime_leaving_dns_policy_for_safety ||- die "could not remove the initial MCP runtime while retaining its external-egress policy"-external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||- die "the initial TCP policy disappeared before the same-intent safety pass"-SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)" ||- die "could not capture the identity-stable external-egress policy snapshot"-[ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] ||- die "the identity-stable external-egress policy snapshot was empty before the same-intent restart"-create_peer_fleet || die "could not create the gate-owned MCP/Context peer fleet"-ok "created ${#MCP_FLEET_NAMES[@]} gate-owned MCP/Context fixtures before fleet bootstrap"--hold_dns_blocker || die "could not hold the next fixture DNS query"-DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||- die "could not capture the DNS release baseline before the same-intent restart"-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null-wait_until 120 "same-intent HCC pod to enter Running" capture_hcc ||- die "same-intent HCC pod did not start"-identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"-read -r HCC_UID HCC_RESTARTS <<<"$identity"-[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||- die "could not capture same-intent HCC pod identity"-# Adjudicate retention-at-readiness FIRST — at the earliest possible moment —-# because the certifying safety pass retains WITHOUT resolving DNS, so /ready=200-# and the retained allow flip together. The separate additive refresh reaches the-# DNS hold and, on its timeout, fail-closed deletes the retained allow; latching-# the retained+Ready observation here beats that timeout.-wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained" \- hcc_ready_with_identity_stable_policy_retained ||- die "HCC did not certify readiness with the same-intent identity-stable DNS policy retained (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"-# Retention-at-readiness is latched. Corroborate promptly — present, unchanged,-# under a still-held DNS, with no restart and the baseline intact — before any-# held-refresh timeout can fail-closed delete the retained allow.-dns_has_not_released_since_hold ||- die "the DNS blocker released before the same-intent retention assertion"-identity_stable_policy_retained ||- die "the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)"-fixture_runtime_and_ready_absent ||- die "the affected runtime or Ready status escaped the held same-intent DNS boundary"-hcc_identity_is_stable ||- die "HCC restarted while the same-intent identity-stable DNS policy was retained"-baseline_policies_unchanged ||- die "baseline safety NetworkPolicy ownership or specs changed during same-intent retention"-wait_until 30 "Kubernetes to corroborate same-intent HCC readiness" \- hcc_kubernetes_readiness_is_exact ||- die "Kubernetes did not corroborate same-intent HCC readiness"-wait_until 120 "the same-intent external-egress refresh to reach the DNS hold" \- dns_hold_observed_since_arm ||- die "the same-intent external-egress refresh never reached the DNS hold"-ok "same-intent safety retained the identity-stable DNS policy while HCC stayed Ready and runtime stayed blocked"--release_dns_blocker || die "could not release the same-intent DNS query"-wait_until 30 "DNS blocker to answer the same-intent query" \- dns_released_since_hold ||- die "DNS blocker did not answer the same-intent query"-wait_until 120 "current TCP external-egress policy after same-intent DNS release" \- external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||- die "the current TCP external-egress policy did not converge after DNS release"-wait_until 180 "same-intent runtime convergence after DNS release" \- fixture_runtime_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||- die "same-intent runtime did not converge after DNS release"--# Security phase: a protocol change invalidates the current TCP policy. The old-# TCP allow must be revoked even when DNS prevents constructing the UDP policy.-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null-wait_until 120 "same-intent HCC pod to stop" hcc_pods_absent ||- die "same-intent HCC pod did not stop"-delete_fixture_runtime_leaving_dns_policy_for_safety ||- die "could not remove the same-intent runtime before the divergent revision"-external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||- die "the current TCP policy disappeared before the divergent safety pass"-hold_dns_blocker || die "could not hold the divergent fixture DNS query"-DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||- die "could not capture the DNS release baseline before the divergent revision"-egress_patch="$(jq -cn --arg dns "$TARGET_DNS" \- '{spec:{egressBindings:[{dns:$dns,port:443,protocol:"UDP"}]}}')"-kctl patch mcpserver "$MCP_NAME" -n "$MCP_NS" --type=merge \- -p "$egress_patch" >/dev/null-wait_until 60 "McpServer API to expose the revised UDP external-egress binding" \- mcp_egress_binding_is_udp ||- die "McpServer external-egress revision did not persist"--kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null-wait_until 120 "replacement HCC pod to enter Running" capture_hcc ||- die "replacement HCC pod did not start"-identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"-read -r HCC_UID HCC_RESTARTS <<<"$identity"-[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||- die "could not capture replacement HCC pod identity"--wait_until 120 "fixture external-egress DNS query to reach the hold proxy" \- dns_hold_observed_since_arm ||- die "NetworkPolicy safety convergence never reached the divergent DNS hold"-wait_until_fast 120 "HCC readiness after revoking the divergent stale policy" \- hcc_ready_after_stale_policy_revoked ||- die "HCC did not become globally ready after revoking the divergent stale policy (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"-wait_until 30 "Kubernetes to corroborate HCC readiness during divergent DNS hold" \- hcc_kubernetes_readiness_is_exact ||- die "Kubernetes did not corroborate HCC readiness during divergent DNS hold"-dns_log_contains "holding DNS A ${TARGET_DNS}" ||- die "DNS hold ended before the fail-closed revocation assertion"-if ! dns_has_not_released_since_hold; then- die "DNS blocker released before the fail-closed revocation assertion"-fi-fixture_runtime_and_ready_absent ||- die "fixture runtime or Ready status appeared while divergent DNS remained held"-external_egress_policy_absent ||- die "a divergent external-egress policy survived the fail-closed revocation"-context_policies_absent ||- die "Context policy existed before the Context watch update"-baseline_policies_unchanged ||- die "baseline safety NetworkPolicy ownership or specs changed during safety convergence"-hcc_identity_is_stable ||- die "HCC restarted while the MCP external-egress query was held"-wait_until_fast 30 "authoritative safety pass metrics to certify the revocation" \- safety_pass_metrics_certify ||- die "safety_pass_* metrics did not corroborate the stale-policy revocation (families renamed or pass never measured?)"-ok "/ready is 200 while the divergent stale policy and affected runtime remain absent"-ok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"-ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"--initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"-wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \- hcc_log_contains "$initial_empty_context_log" ||- die "initial NetworkPolicy pass never reconciled the fixture's empty Context"-wait_until 120 "a real divergent external-egress retry to be scheduled" \- external_egress_retry_progress_is_observed ||- die "the held divergent refresh did not schedule a real retry"-DNS_HOLD_COUNT_AT_RETRY_SCHEDULE="$(dns_hold_count)" ||- die "could not snapshot DNS hold evidence after the divergent retry was scheduled"-wait_until 180 "the scheduled divergent external-egress retry to execute" \- dns_retry_query_observed_since_schedule ||- die "the scheduled divergent external-egress retry did not execute"-context_policies_absent ||- die "Context policy appeared before the isolated MODIFIED event"--context_patch="$(jq -cn --arg server "$MCP_NAME" '{spec:{mcpServers:[$server]}}')"-kctl patch context "$CONTEXT_NAME" -n "$MCP_NS" --type=merge \- -p "$context_patch" >/dev/null-wait_until 60 "Context API to expose the latest server membership" context_watch_has_latest_spec ||- die "Context update did not persist"-wait_until 60 "HCC Context watch to observe the update during the blocked fleet pass" \- hcc_log_contains "Context watch event: MODIFIED for ${CONTEXT_NAME}" ||- die "HCC did not observe the real Context update while initial MCP convergence was blocked"-wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold" \- context_policies_converged ||- die "Context policies remained blocked behind the held McpServer convergence"-fixture_runtime_and_ready_absent ||- die "McpServer runtime or Ready status escaped the deterministic DNS hold boundary"-external_egress_policy_absent ||- die "a divergent external-egress policy reappeared while DNS remained held"-dns_log_contains "holding DNS A ${TARGET_DNS}" ||- die "DNS hold evidence disappeared before Context policy convergence"-if ! dns_has_not_released_since_hold; then- die "DNS blocker released the McpServer query before the Context isolation assertion"-fi-probe_new_hcc_ready_endpoint ||- die "HCC lost global readiness while the affected runtime remained safely blocked (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"-hcc_kubernetes_readiness_is_exact ||- die "Kubernetes lost HCC readiness during isolated Context convergence"-hcc_identity_is_stable ||- die "HCC restarted while Context policies converged around the held MCP entity"-ok "the MODIFIED handler, not the initial empty-Context pass, converged all three policies"-ok "global readiness and Context progress continued while the affected runtime stayed blocked"--release_dns_blocker ||- die "could not release the fixture DNS query"-wait_until 30 "DNS blocker to answer the held query" \- dns_released_since_hold ||- die "DNS blocker did not answer the held query"-wait_until 120 "current UDP external-egress policy after DNS release" \- external_egress_policy_converged_with_protocol UDP ||- die "the current UDP external-egress policy did not converge after DNS release"-probe_new_hcc_ready_endpoint ||- die "HCC lost readiness while the released UDP policy converged (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"-wait_until 300 "McpServer runtime, status, and latest Context NetworkPolicies to converge" \- fixture_converged ||- die "McpServer/Context/NetworkPolicy convergence did not complete after DNS release"-wait_until 300 "all peer MCP/Context runtimes to converge after releasing the held primary egress" \- peer_fleet_converged ||- die "peer MCP/Context fleet did not converge after the held primary egress was released"-hcc_identity_is_stable ||- die "HCC restarted instead of completing the released background convergence"-final_probe="$(probe_hcc_final_context)" ||- die "HCC final Context-scoped discovery did not expose the converged fixture"-echo "$final_probe" | jq -e \- --arg fixture "$MCP_NAME" --arg context "$CONTEXT_NAME" '- .readyStatus == 200 and .scopedDiscoveryStatus == 200 and- .fixture == $fixture and .fixtureReady == true and .contextRef == $context- ' >/dev/null ||- die "HCC final probe returned an invalid result: ${final_probe}"-baseline_policies_unchanged ||- die "baseline safety NetworkPolicy ownership or specs changed before final convergence"-ok "DNS release created the current UDP policy and affected runtime without gating HCC readiness"-ok "the full worker-width-crossing peer fleet converged after releasing the held primary egress"-ok "Context ingress, mcp-host egress, and rpc-proxy egress policies reflect the latest watch state"-ok "HCC stayed Ready through divergent revocation while the affected runtime remained fail-closed"--header "HCC McpServer/Context assertions complete; restoring branch-owned runtime"diff --git a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh b/scripts/e2e/e2e-hcc-readiness-bootstrap.shdeleted file mode 100755index eb485a590..000000000--- a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh+++ /dev/null@@ -1,785 +0,0 @@-#!/usr/bin/env bash-# Proves that HCC exposes its real readiness and discovery endpoints while the-# initial Host fleet reconciliation is still demonstrably in progress.-#-# The gate temporarily routes HCC's control-api token issuance to an isolated-# in-cluster HTTP blocker, creates a gate-owned fleet while HCC is stopped, and-# restarts HCC. There are more fixture Hosts than two complete bounded worker-# waves, so the initial pass remains deterministically active even though each-# individual held token request has its own client timeout. No real Host or-# runtime-token Secret is modified.-set -euo pipefail--SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-# shellcheck source=scripts/e2e/e2e-lib.sh-source "${SCRIPT_DIR}/e2e-lib.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"--[ -n "$E2E_KUBECONTEXT" ] || {- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2- exit 1-}-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {- echo "Refusing HCC readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2- exit 1-}-require_safe_kube_context-[ "${E2E_HCC_READINESS_FAULT_INJECTION:-0}" = 1 ] || {- echo "Set E2E_HCC_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC fault injection." >&2- exit 1-}-command -v jq >/dev/null 2>&1 || {- echo "jq is required" >&2- exit 1-}--HCC_NS="${HCC_NS:-control-plane}"-HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"-HCC_GATEWAY_DEPLOY="${HCC_GATEWAY_DEPLOY:-host-context-controller-api-gateway}"-HOST_NS="${MCP_HOST_NS:-mcp-host}"-MCP_NS="${MCP_SERVER_NS:-mcp-server}"-TOKEN_REQUEST_PATH="/api/v1/auth/mcp-host/${HOST_NS}/standalone/tokens"-TOKEN_REQUEST_LOG_PREFIX="holding-token POST ${TOKEN_REQUEST_PATH} host="-RUN_ID="$(date +%s)-$$"-BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-readiness-blocker-${RUN_ID}")"-BLOCKER_EGRESS_POLICY_NAME="$(truncate_rfc1123 "${BLOCKER_NAME}-egress")"-SUITE_NAME="hcc-readiness-bootstrap"-FIXTURE_HOST_PREFIX="$(truncate_rfc1123 "e2e-hcc-ready-${RUN_ID}")"-FIXTURE_CONTEXT="$(truncate_rfc1123 "e2e-hcc-ready-context-${RUN_ID}")"-FIXTURE_SECRET="$(truncate_rfc1123 "e2e-hcc-ready-llm-${RUN_ID}")"-START_MARKER='Starting initial Host background convergence'-COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'-FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'-ORIGINAL_CONTROL_API_BASE_URL=""-ORIGINAL_REPLICAS=""-HCC_PORT=""-HOST_RECONCILE_CONCURRENCY=""-FIXTURE_HOST_COUNT=0-FIXTURE_HOST_NAMES=()-HCC_MUTATED=0-BLOCKER_CREATED=0-FIXTURES_CREATED=0-# Read and mutated by the sourced compare-and-swap lock helper.-# shellcheck disable=SC2034-HCC_GATE_LOCK_ACQUIRED=0-# shellcheck disable=SC2034-HCC_GATE_LOCK_NAME=""-# shellcheck disable=SC2034-HCC_GATE_LOCK_UID=""-# shellcheck disable=SC2034-HCC_GATE_FINALIZATION_FAILURE=""--die() {- if [ -n "${new_hcc_pod:-}" ]; then- echo "Recent HCC logs from ${new_hcc_pod}:" >&2- kctl logs "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller --tail=300 >&2 || true- fi- fail "$*"- exit 1-}--wait_until() {- local timeout=$1 description=$2- shift 2- local deadline now- deadline=$(( $(date +%s) + timeout ))- while :; do- "$@" && return 0- now="$(date +%s)"- [ "$now" -lt "$deadline" ] || break- sleep 1- done- echo "Timed out after ${timeout}s waiting for ${description}" >&2- return 1-}--running_hcc_pod() {- local rows- rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- --field-selector=status.phase=Running \- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \- 2>/dev/null)" || return 1- awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"-}--hcc_pods_absent() {- local pods- pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||- return 1- [ -z "$pods" ]-}--hcc_logs() {- local pod=$1- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller 2>/dev/null-}--hcc_log_contains() {- local pod=$1 marker=$2 logs- logs="$(hcc_logs "$pod")" || return 1- hcc_log_snapshot_contains "$logs" "$marker"-}--blocker_holds_token_request() {- local logs- logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1- hcc_log_snapshot_contains "$logs" \- "${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-"-}--blocker_fixture_request_count() {- local logs- logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1- awk -v prefix="${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-" '- index($0, prefix) {- host = substr($0, index($0, "host=") + length("host="))- sub(/[[:space:]].*$/, "", host)- if (!(host in seen)) {- seen[host] = 1- count += 1- }- }- END { print count + 0 }- ' <<<"$logs"-}--blocker_observed_more_than_one_worker_wave() {- local count- count="$(blocker_fixture_request_count)" || return 1- [ "$count" -gt "$HOST_RECONCILE_CONCURRENCY" ]-}--initial_host_pass_is_active() {- local pod=$1 logs- logs="$(hcc_logs "$pod")" || return 1- hcc_initial_pass_snapshot_is_active \- "$logs" "$START_MARKER" "$COMPLETE_MARKER" "$FAIL_MARKER"-}--control_api_service_excludes_blocker() {- kctl get endpointslice -n "$HCC_NS" -l 'kubernetes.io/service-name=control-api' -o json |- jq -e --arg blocker "$BLOCKER_NAME" \- 'all(.items[]?.endpoints[]?; .targetRef.name != $blocker)' >/dev/null-}--readiness_blocker_suite_absent() {- local resources- resources="$(- kctl get deployment,service,networkpolicy -n "$HCC_NS" \- -l 'e2e.clerum.io/hcc-readiness' -o name 2>/dev/null- )" || return 1- [ -z "$resources" ]-}--probe_hcc_ready_pod() {- local pod- pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1- kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" \- node -e '- const http = require("node:http");- const request = http.get(- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},- response => {- let body = "";- response.setEncoding("utf8");- response.on("data", chunk => { body += chunk; });- response.on("end", () => {- let parsed;- try { parsed = JSON.parse(body); } catch { process.exit(2); }- process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);- });- }- );- request.setTimeout(5000, () => request.destroy(new Error("timeout")));- request.once("error", () => process.exit(4));- ' >/dev/null 2>&1-}--running_hcc_gateway_pod() {- ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"-}--hcc_gateway_deployment_ready() {- kctl get deployment "$HCC_GATEWAY_DEPLOY" -n "$HCC_NS" -o json |- jq -e '- .spec.replicas == 1 and- .status.observedGeneration == .metadata.generation and- (.status.updatedReplicas // 0) == 1 and- (.status.readyReplicas // 0) == 1 and- (.status.availableReplicas // 0) == 1 and- (.status.unavailableReplicas // 0) == 0- ' >/dev/null-}--hcc_gateway_local_health_ok() {- local pod- pod="$(running_hcc_gateway_pod)" || return 1- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \- wget -qO- http://127.0.0.1:8081/health |- jq -e '.status == "ok"' >/dev/null-}--hcc_gateway_ready_proxy_unavailable() {- local pod output status- pod="$(running_hcc_gateway_pod)" || return 1- output="$(- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \- wget -T 70 -t 1 -O /dev/null -S http://127.0.0.1:8081/ready 2>&1- )" && return 1- status="$(- awk '- /^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]([[:space:]]|$)/ {- code = $2- }- END {- if (code != "") print code- }- ' <<<"$output"- )"- case "$status" in- 502|503|504) return 0 ;;- *) return 1 ;;- esac-}--hcc_gateway_remains_ready_without_hcc() {- local duration=$1 deadline- deadline=$(( $(date +%s) + duration ))- while [ "$(date +%s)" -lt "$deadline" ]; do- hcc_gateway_deployment_ready &&- hcc_gateway_local_health_ok &&- hcc_gateway_ready_proxy_unavailable &&- hcc_gateway_deployment_ready &&- hcc_gateway_local_health_ok ||- return 1- sleep 1- done-}--hcc_gateway_ready_proxy_recovers() {- local pod- pod="$(running_hcc_gateway_pod)" || return 1- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \- wget -qO- http://127.0.0.1:8081/ready |- jq -e '.ready == true and .status == "ready"' >/dev/null-}--fixture_inputs_absent() {- local hosts contexts secrets- hosts="$(kctl get host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- -o name 2>/dev/null)" || return 1- contexts="$(kctl get context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- -o name 2>/dev/null)" || return 1- secrets="$(kctl get secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- -o name 2>/dev/null)" || return 1- [ -z "${hosts}${contexts}${secrets}" ]-}--fixture_runtime_absent() {- local host resources- [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0- for host in "${FIXTURE_HOST_NAMES[@]}"; do- resources="$(- kctl get deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \- -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \- -o name 2>/dev/null- )" || return 1- [ -z "$resources" ] || return 1- done-}--fixture_resources_absent() {- fixture_inputs_absent && fixture_runtime_absent-}--hcc_restore_is_verified() {- local deployment- deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1- jq -e \- --arg controlApiBaseUrl "$ORIGINAL_CONTROL_API_BASE_URL" \- --argjson replicas "${ORIGINAL_REPLICAS:-1}" '- .spec.replicas == $replicas and- .status.observedGeneration == .metadata.generation and- (.status.updatedReplicas // 0) == $replicas and- (.status.readyReplicas // 0) == $replicas and- (.status.availableReplicas // 0) == $replicas and- (.status.unavailableReplicas // 0) == 0 and- any(.spec.template.spec.containers[]?;- .name == "host-context-controller" and- any(.env[]?;- .name == "CONTROL_API_BASE_URL" and .value == $controlApiBaseUrl- )- )- ' <<<"$deployment" >/dev/null || return 1- probe_hcc_ready_pod && fixture_resources_absent-}--delete_blocker_fixture() {- local failed=0- kctl delete networkpolicy "$BLOCKER_EGRESS_POLICY_NAME" "$BLOCKER_NAME" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete service "$BLOCKER_NAME" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete deployment "$BLOCKER_NAME" -n "$HCC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- [ "$failed" = 0 ]-}--create_host_fixtures() {- local index host- FIXTURES_CREATED=1-- kctl apply -f - >/dev/null <<EOF || return 1-apiVersion: v1-kind: Secret-metadata:- name: ${FIXTURE_SECRET}- namespace: ${HOST_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_ID}"-type: Opaque-stringData:- # Test-only placeholder: Host secret validation checks existence, while token- # issuance is intentionally held before any mcp-host Deployment is created.- OPENAI_API_KEY: "e2e-readiness-not-a-real-key"-----apiVersion: clerum.io/v1alpha1-kind: Context-metadata:- name: ${FIXTURE_CONTEXT}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_ID}"-spec:- contextId: ${FIXTURE_CONTEXT}- description: Gate-owned Context for HCC Host bootstrap readiness.- mcpServers: []-EOF-- for ((index = 1; index <= FIXTURE_HOST_COUNT; index++)); do- host="$(printf '%s-%02d' "$FIXTURE_HOST_PREFIX" "$index")"- FIXTURE_HOST_NAMES+=("$host")- kctl apply -f - >/dev/null <<EOF || return 1-apiVersion: clerum.io/v1alpha1-kind: Host-metadata:- name: ${host}- namespace: ${HOST_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_ID}"-spec:- host: ${host}- contextRef: ${FIXTURE_CONTEXT}- secretRef: ${FIXTURE_SECRET}- model:- provider: openai- name: e2e-readiness-EOF- done-}--delete_host_fixtures() {- local failed=0 host- kctl delete host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1- kctl delete context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"- for host in "${FIXTURE_HOST_NAMES[@]}"; do- kctl delete \- deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \- -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \- --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1- done- [ "$failed" = 0 ] || return 1- wait_until 60 "all gate-owned Host inputs and managed runtime resources to disappear" \- fixture_resources_absent-}--print_repair_instructions() {- cat >&2 <<EOF-HCC readiness bootstrap cleanup could not restore a verified clean state.-Context: ${E2E_KUBECONTEXT}-HCC: ${HCC_NS}/${HCC_DEPLOY}-Fixture Hosts: ${HOST_NS}/${FIXTURE_HOST_PREFIX}-*-Fixture Context: ${MCP_NS}/${FIXTURE_CONTEXT}-Fixture LLM Secret: ${HOST_NS}/${FIXTURE_SECRET}-Retained blocker: ${HCC_NS}/${BLOCKER_NAME}--Inspect before changing anything:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml- kubectl --context=${E2E_KUBECONTEXT} -n ${HOST_NS} get host,secret -l e2e.clerum.io/suite=${SUITE_NAME}- kubectl --context=${E2E_KUBECONTEXT} -n ${MCP_NS} get context -l e2e.clerum.io/suite=${SUITE_NAME}- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/hcc-readiness--Do not remove the blocker or HCC gate lock until HCC configuration, rollout, Pod readiness,-/ready, and fixture absence are verified.--Restore the original HCC replica count before verifying rollout and /ready:- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s-EOF-}--cleanup() {- local status=$? cleanup_failed=0 restore_ok=1 hcc_stopped=1- trap - EXIT- set +e-- # Remove every gate-owned input while HCC is stopped. Restarting HCC with a- # partially deleted fixture fleet would let the real control-api path mutate- # those leftovers and make cleanup non-deterministic.- if [ "$HCC_MUTATED" = 1 ]; then- hcc_stopped=0- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 \- >/dev/null 2>&1 || restore_ok=0- if [ "$restore_ok" = 1 ]; then- wait_until 120 "HCC pods to stop before fixture deletion" \- hcc_pods_absent >/dev/null 2>&1 || restore_ok=0- fi- [ "$restore_ok" = 1 ] && hcc_stopped=1- fi-- if [ "$FIXTURES_CREATED" = 1 ]; then- if [ "$hcc_stopped" = 1 ]; then- delete_host_fixtures >/dev/null 2>&1 || restore_ok=0- else- restore_ok=0- fi- fi-- if [ "$HCC_MUTATED" = 1 ] && [ "$restore_ok" = 1 ] &&- [ -n "$ORIGINAL_CONTROL_API_BASE_URL" ]; then- kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- "CONTROL_API_BASE_URL=${ORIGINAL_CONTROL_API_BASE_URL}" >/dev/null 2>&1 ||- restore_ok=0- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \- --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 ||- restore_ok=0- kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s \- >/dev/null 2>&1 || restore_ok=0- if [ "$restore_ok" = 1 ]; then- wait_until 240 \- "restored HCC config, rollout, Ready pod, /ready, and fixture absence" \- hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0- fi- fi-- if [ "$restore_ok" = 1 ]; then- if [ "$BLOCKER_CREATED" = 1 ]; then- delete_blocker_fixture || cleanup_failed=1- fi- wait_until 60 "all HCC readiness blocker resources to disappear" \- readiness_blocker_suite_absent >/dev/null 2>&1 || cleanup_failed=1- else- cleanup_failed=1- fi-- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then- cleanup_failed=1- fi-- if [ "$cleanup_failed" -ne 0 ]; then- echo "HCC readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2- [ "$restore_ok" = 1 ] || print_repair_instructions- [ "$status" = 0 ] && status=1- fi- if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then- header "HCC readiness bootstrap gate passed"- fi- exit "$status"-}-trap cleanup EXIT--header "HCC readiness during initial Host fleet reconciliation"--require_branch_owned_hcc_gate "$HCC_NS"-ok "branch helper, profile, exact HEAD/fingerprint/gate verified"--kctl get nodes -o json |- jq -e --arg context "$E2E_KUBECONTEXT" \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||- die "target context is not a minikube cluster"-readiness_blocker_suite_absent ||- die "stale HCC readiness blocker resources exist; inspect them before fault injection"-fixture_inputs_absent ||- die "stale ${SUITE_NAME} Host, Context, or Secret fixtures exist; inspect them before fault injection"-acquire_hcc_watch_gate_lock ||- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"-ok "exclusive HCC fault-injection lock acquired"--HCC_DEPLOYMENT_JSON="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" ||- die "could not read HCC Deployment"-ORIGINAL_REPLICAS="$(jq -r '.spec.replicas // empty' <<<"$HCC_DEPLOYMENT_JSON")"-[ "$ORIGINAL_REPLICAS" = 1 ] ||- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"-ORIGINAL_CONTROL_API_BASE_URL="$(jq -r '- first(- .spec.template.spec.containers[]? |- select(.name == "host-context-controller") |- .env[]? |- select(.name == "CONTROL_API_BASE_URL") |- .value- ) // empty-' <<<"$HCC_DEPLOYMENT_JSON")"-[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ] ||- die "HCC CONTROL_API_BASE_URL is missing"-HCC_PORT="$(jq -r '- .spec.template.spec.containers[]? |- select(.name == "host-context-controller") as $container |- (- [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //- [$container.ports[]? | select(.name == "http") | .containerPort][0] //- empty- )-' <<<"$HCC_DEPLOYMENT_JSON")"-[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||- die "could not resolve a valid HCC HTTP port from the Deployment"-HOST_RECONCILE_CONCURRENCY="$(jq -r '- first(- .spec.template.spec.containers[]? |- select(.name == "host-context-controller") |- .env[]? |- select(.name == "HCC_HOST_FULL_RECONCILE_CONCURRENCY") |- .value- ) // "2"-' <<<"$HCC_DEPLOYMENT_JSON")"-[[ "$HOST_RECONCILE_CONCURRENCY" =~ ^[1-8]$ ]] ||- die "HCC_HOST_FULL_RECONCILE_CONCURRENCY must be 1..8, found ${HOST_RECONCILE_CONCURRENCY}"-FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))-HCC_IMAGE="$(jq -r '- first(- .spec.template.spec.containers[]? |- select(.name == "host-context-controller") |- .image- ) // empty-' <<<"$HCC_DEPLOYMENT_JSON")"-[ -n "$HCC_IMAGE" ] || die "could not resolve the HCC image"-ok "planned ${FIXTURE_HOST_COUNT} gate-owned Hosts for concurrency ${HOST_RECONCILE_CONCURRENCY}"--BLOCKER_CREATED=1-kctl apply -f - >/dev/null <<EOF-apiVersion: apps/v1-kind: Deployment-metadata:- name: ${BLOCKER_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"-spec:- replicas: 1- selector:- matchLabels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"- template:- metadata:- labels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"- spec:- automountServiceAccountToken: false- terminationGracePeriodSeconds: 1- containers:- - name: blocker- image: ${HCC_IMAGE}- imagePullPolicy: IfNotPresent- env:- - name: EXPECTED_TOKEN_PATH- value: ${TOKEN_REQUEST_PATH}- command: [node, -e]- args:- - >-- require('http').createServer((request,response) => {- if(request.method!=='POST'||request.url!==process.env.EXPECTED_TOKEN_PATH){- console.log('rejecting '+request.method+' '+request.url);- request.resume();- request.on('end',()=>{response.statusCode=404;response.end()});- return;- }- let body='';- request.setEncoding('utf8');- request.on('data',chunk=>{if(body.length<65536)body+=chunk;});- request.on('end',()=>{- let host='unknown';- try{const parsed=JSON.parse(body);if(typeof parsed.host==='string')- host=parsed.host;}catch{}- console.log('holding-token '+request.method+' '+request.url+' host='+host);- });- }).listen(8090, '0.0.0.0');- readinessProbe:- tcpSocket: {port: 8090}- periodSeconds: 1- resources:- requests: {cpu: 5m, memory: 16Mi}- limits: {cpu: 50m, memory: 48Mi}- securityContext:- allowPrivilegeEscalation: false- capabilities: {drop: [ALL]}- readOnlyRootFilesystem: true- runAsNonRoot: true- runAsUser: 1000- runAsGroup: 1000- seccompProfile: {type: RuntimeDefault}-----apiVersion: v1-kind: Service-metadata:- name: ${BLOCKER_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"-spec:- selector:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"- ports:- - name: http- port: 8090- targetPort: 8090-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${BLOCKER_EGRESS_POLICY_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"-spec:- podSelector:- matchLabels:- app: ${HCC_DEPLOY}- policyTypes: [Egress]- egress:- - to:- - podSelector:- matchLabels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"- ports:- - protocol: TCP- port: 8090-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${BLOCKER_NAME}- namespace: ${HCC_NS}- labels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"-spec:- podSelector:- matchLabels:- e2e.clerum.io/hcc-readiness: "${RUN_ID}"- policyTypes: [Ingress]- ingress:- - from:- - podSelector:- matchLabels:- app: ${HCC_DEPLOY}- ports:- - protocol: TCP- port: 8090-EOF-kctl rollout status deployment "$BLOCKER_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||- die "isolated token-issuance blocker did not become ready"-control_api_service_excludes_blocker ||- die "isolated blocker was selected by the real control-api Service"-ok "isolated in-cluster blocker is ready"--HCC_MUTATED=1-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null-wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||- die "existing HCC pod did not stop"-hcc_gateway_remains_ready_without_hcc 20 ||- die "HCC API gateway did not remain Ready on local /health while proxied /ready was unavailable"-ok "API gateway remained Ready for a complete probe-failure window while /ready reflected HCC unavailability"-create_host_fixtures ||- die "could not create the gate-owned Host fleet while HCC was stopped"-ok "created ${FIXTURE_HOST_COUNT} gate-owned Hosts without modifying any real Host or Secret"-kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \- "CONTROL_API_BASE_URL=http://${BLOCKER_NAME}.${HCC_NS}.svc.cluster.local:8090" >/dev/null-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null--new_hcc_pod=""-capture_running_hcc() {- new_hcc_pod="$(running_hcc_pod)"- [ -n "$new_hcc_pod" ]-}-wait_until 120 "replacement HCC pod to enter Running" capture_running_hcc ||- die "replacement HCC pod did not start"-wait_until 60 "initial Host reconciliation to start" \- hcc_log_contains "$new_hcc_pod" "$START_MARKER" ||- die "HCC never started the initial Host reconciliation"-wait_until 120 "a gate-owned Host token request to reach the blocker" blocker_holds_token_request ||- die "initial Host reconciliation never reached the deterministic blocker"-wait_until 180 "more than one bounded Host worker wave to reach the blocker" \- blocker_observed_more_than_one_worker_wave ||- die "fixture fleet did not prove progress beyond the first bounded Host worker wave"-initial_host_pass_is_active "$new_hcc_pod" ||- die "initial Host reconciliation failed or completed before the readiness assertion"-ok "initial Host reconciliation is active beyond one bounded worker wave"--probe_script="$(cat <<'NODE'-const http = require('http');-function get(path) {- return new Promise((resolve, reject) => {- const request = http.get(- {host: '127.0.0.1', port: Number(process.env.HCC_E2E_PORT), path},- response => {- let body = '';- response.setEncoding('utf8');- response.on('data', chunk => { body += chunk; });- response.on('end', () => resolve({status: response.statusCode, body}));- }- );- request.setTimeout(5000, () => request.destroy(new Error('timeout')));- request.once('error', reject);- });-}-(async () => {- const ready = await get('/ready');- const readyBody = JSON.parse(ready.body);- if (ready.status !== 200 || readyBody.status !== 'ready' || readyBody.ready !== true) {- throw new Error('unexpected readiness response: ' + ready.status + ' ' + ready.body);- }- const discovery = await get('/api/v1/mcpservers');- if (discovery.status !== 200) {- throw new Error('unexpected discovery response: ' + discovery.status);- }- const discoveryBody = JSON.parse(discovery.body);- if (!Array.isArray(discoveryBody.servers) || discoveryBody.contextRef !== '*') {- throw new Error('discovery response does not match the live API contract');- }- console.log(JSON.stringify({readyStatus: ready.status, discoveryStatus: discovery.status,- discoveredServers: discoveryBody.servers.length}));-})().catch(error => { console.error(error.message); process.exit(1); });-NODE-)"-probe_result="$(kctl exec "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller -- \- env "HCC_E2E_PORT=${HCC_PORT}" node -e "$probe_script")" ||- die "real HCC readiness/discovery HTTP probe failed while the fleet pass was active"-echo "$probe_result" | jq -e \- '.readyStatus == 200 and .discoveryStatus == 200 and (.discoveredServers | type == "number")' \- >/dev/null ||- die "HCC HTTP probe returned an invalid result: ${probe_result}"-hcc_gateway_ready_proxy_recovers ||- die "HCC API gateway /ready did not recover after HCC became ready"-initial_host_pass_is_active "$new_hcc_pod" ||- die "initial Host reconciliation failed or completed before the HTTP assertions finished"-ok "/ready returned 200 while the initial Host fleet pass remained active"-ok "API gateway /ready recovered without changing its local readiness"-ok "/api/v1/mcpservers returned a valid live discovery response during that same pass"--header "HCC readiness assertions passed; restoring branch-owned runtime"-echo "$probe_result"diff --git a/scripts/e2e/e2e-host-storm-gate.sh b/scripts/e2e/e2e-host-storm-gate.shindex 1015dd276..095bb8c93 100755--- a/scripts/e2e/e2e-host-storm-gate.sh+++ b/scripts/e2e/e2e-host-storm-gate.sh@@ -26,14 +26,16 @@# construction, so A1 is primarily a RECORDING of that evidence — it# can only fail if the harness itself degenerates into sequential# phases; the load-bearing falsifiable claims are A2-A8.-# A2 urgent queue-wait coverage has two independent same-pod windows:-# (a) the supported wake window immediately before the HCC kill requires-# every completed urgent sample <= 5s, after proving the wake reconcile-# finished and the urgent lane returned idle; and-# (b) ten dedicated Host ADDED events issued only after the replacement-# HCC completed LIST→WATCH recovery use a D - C1 p95 delta. A slow wake-# can therefore never be diluted by later fast post-recovery probes, and-# neither window crosses the intentional process restart.+# A2 urgent queue-wait p95 <= 5s DURING the storm window, computed from+# the clerum_hcc_host_reconcile_queue_wait_seconds{lane="urgent"}+# histogram buckets (§14.1) across two counter windows: pre-kill pod+# delta (scrape B - scrape A) plus post-kill pod absolute (scrape C1,+# taken IMMEDIATELY after created-Host convergence so post-storm quiet+# samples cannot dilute the p95 — a fresh process starts at zero, and+# the C1 window includes replacement-pod recovery traffic). Samples in+# flight between scrape B and process death are unrecoverable (counter+# reset) and are recorded as a documented measurement gap, never+# silently backfilled.# A3 no starvation: EVERY created Host's Deployment materializes within# STORM_MATERIALIZATION_BUDGET_S (20s, §14.2 "New Host# materialization") of that Host's OWN metadata.creationTimestamp,@@ -57,16 +59,13 @@# All-zero is legitimate (deletes fully handled by the direct# watch-DELETE path, which by design does not touch this counter) —# A5 remains the falsifiable cleanup proof.-# A7 recovered fleet convergence is bounded, from-# clerum_hcc_host_fleet_requests_total (scrape C2) plus the replacement-# pod's explicit initial-fleet COMPLETED log: failed == 0 in both windows,-# the exact replacement pod completes its startup fleet pass, and-# trailing <= coalesced. The pod identity, completion marker, and A4's-# successful recovery histogram form the causal fence; started is not-# capped because an independent request may legitimately begin after the-# recovery pass settles. Requiring a coalesced request here would invent-# a second overlapping signal; the concurrent-request coalescing contract-# is covered deterministically by hostFleetScheduler.test.ts.+# A7 fleet coalescing held (one active + one trailing slot), from+# clerum_hcc_host_fleet_requests_total (scrape C2): failed == 0 in+# both windows; on the post-kill pod started >= 1 (the recovery+# convergence pass ran), coalesced >= 1 (the recovery-triggered+# request landed against the active initial pass), and trailing <=+# coalesced (every trailing drain corresponds to at least one request+# that was queued behind the single pending slot).# A8 wake ingredient integrity: the woken Host leaves state=suspended and# reaches a Ready pod within STORM_WAKE_READY_BUDGET_S of the wake# POST. This is an ingredient-integrity bound (a silently no-op wake@@ -98,26 +97,23 @@## ----------------------------------------------------------------------# INPUTS (environment; fail-loud if missing/invalid)-# KUBECONTEXT / E2E_K8S_CONTEXT-# Exact generated branch-owned minikube context.-# MINIKUBE_PROFILE-# Must equal that context.-# E2E_EXPECTED_PRE_GATE_GATE-# Exact gate recorded by the branch-owned pre-gate sync.-# E2E_HCC_HOST_STORM_FAULT_INJECTION=1-# Explicit acknowledgement of the disruptive HCC pod restart.+# CONTEXT kubectl context. REFUSED unless it matches+# ^clerum-(test|codex-|detached-|claude-) — a local minikube+# profile. Any GKE / *prod* context is rejected outright.## OPTIONAL INPUTS# E2E_STORM_WAKE_HOST_REF suspended-wake target (default chatllm-stateless,# the seeded stateless Host — it must exist with# spec.lifecycle.stateless=true and be associated to the E2E# user; seed via scripts/e2e/seed-stateless-host.sh).+# EXTERNAL_REST_API_BASE_URL default http://127.0.0.1:8091+# RPC_PROXY_BASE_URL default http://127.0.0.1:8094# E2E_DEV_LOGIN_EMAIL default [email protected]-# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-changeme123!}+# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-admin123!}# STORM_CONTEXT_REF Context CR fixtures reference (default context1)# STORM_MODEL_PROVIDER / STORM_MODEL_NAME fixture model (default zai/glm-5.1)-# CLERUM_PROFILE_PORTS_ENV optional explicit path to the selected branch-# profile's generated ports.env.+# HCC_METRICS_LOCAL_PORT ephemeral scrape port (default 18082)+# KEEP_FIXTURES set to 1 to skip teardown (diagnosis only)## Prereqs (each a HARD FAIL with the concrete reason):# - branch HCC image deployed (clerum_hcc_host_delete_cleanup_total@@ -126,10 +122,7 @@# - external-rest-api + rpc-proxy reachable (make minikube-pf-all)## Usage:-# MINIKUBE_PROFILE=<profile> KUBECONTEXT=<profile> \-# E2E_EXPECTED_PRE_GATE_GATE=<gate> \-# E2E_HCC_HOST_STORM_FAULT_INJECTION=1 \-# bash scripts/e2e/e2e-host-storm-gate.sh+# CONTEXT=clerum-test bash scripts/e2e/e2e-host-storm-gate.sh## EXIT CODES# 0 every storm assertion passed and teardown left zero fixtures@@ -138,17 +131,6 @@# ======================================================================set -euo pipefail-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-# shellcheck source=scripts/e2e/e2e-lib.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/e2e-lib.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh-# shellcheck disable=SC1091-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"-# ─── Constants mirrored from host-context-controller/src/{constants,config}.ts ──readonly MANAGED_BY_LABEL='clerum.io/managed-by'readonly MANAGED_BY_VALUE='host-context-controller'@@ -162,10 +144,6 @@ readonly CONTROL_NS='control-plane'readonly HCC_DEPLOY='host-context-controller'readonly HCC_PORT='8081' # containerPort http (deploy/base/control-plane)readonly HCC_METRICS_PATH='/metrics' # server.ts GET /metrics-readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'-readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'-# Shared HCC ownership/lock helpers use HCC_NS.-HCC_NS="$CONTROL_NS"# Owned-child kinds + namespaces of a per-Host bundle (mirrors# measure-host-bundle-reconcile.sh — includes per-Host NetworkPolicies).@@ -178,7 +156,6 @@ readonly BUNDLE_NAMESPACES="${MCP_HOST_NS} ${CHANNELS_NS} ${RPC_PROXY_NS}"# (`QUEUE_WAIT_METRIC="$QUEUE_WAIT_METRIC" python3 …`) would fail as a readonly# reassignment under `set -e` — export once here and drop the inline prefixes.readonly QUEUE_WAIT_METRIC='clerum_hcc_host_reconcile_queue_wait_seconds'-readonly IN_FLIGHT_METRIC='clerum_hcc_host_reconcile_in_flight'readonly WATCH_RECOVERY_METRIC='clerum_hcc_host_watch_recovery_seconds'readonly FLEET_METRIC='clerum_hcc_host_fleet_requests_total'readonly DELETE_CLEANUP_METRIC='clerum_hcc_host_delete_cleanup_total'@@ -187,20 +164,11 @@ export QUEUE_WAIT_METRIC WATCH_RECOVERY_METRIC FLEET_METRIC DELETE_CLEANUP_METRI# ─── Storm shape + budgets (each constant states its contract) ────────# Addendum 5.1: ">=10" created fixture Hosts, mixed stateful/stateless.readonly STORM_CREATE_COUNT=10-# A2's direct-watch probe is deliberately separate from the restart window:-# a cold-start LIST legitimately converges objects on the fleet lane. These-# ten ADDED events are emitted only once the replacement's LIST→WATCH recovery-# has succeeded, so their queue histogram is an unambiguous urgent-lane SLO.-readonly STORM_URGENT_PROBE_COUNT=10# Addendum 5.1: ">=3" pre-created fixture Hosts deleted mid-storm.readonly STORM_DELETE_COUNT=3# §14.2 "urgent new/wake queue wait": local hard gate <= 5s (histogram# bucket boundary le="5" exists in RECONCILE_LATENCY_BUCKETS).readonly STORM_URGENT_P95_BUDGET_S=5-# The queue value itself is adjudicated against the 5s hard gate. This larger-# deadline bounds collection of the completed reconcile evidence and produces-# a loud operational failure if the wake never settles or telemetry is absent.-readonly STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S=60# §14.2 "New Host materialization": Host CR -> Deployment <= 20s.readonly STORM_MATERIALIZATION_BUDGET_S=20# §14.2 "Watch recovery": recovery requested -> WATCH installed <= 15s@@ -272,44 +240,26 @@ print_results() {log "fairness and convergence under realistic concurrency, not API degradation."}+# ─── kubectl wrapper — EVERY call carries --context (CLAUDE.md mandate) ──+kctl() { kubectl --context "$CONTEXT" "$@"; }+# ─── Portable time helper (python3; mirrors measure script) ───────────now_ms() { python3 -c 'import time;print(int(time.time()*1000))'; }# ─── Strict local-profile context guard (fail-loud) ───────────────────require_storm_context() {- [[ -n "${E2E_KUBECONTEXT:-}" ]] ||- die "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." 2- is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||- die "Refusing Host storm fault injection on non-branch context '${E2E_KUBECONTEXT}'." 2- require_safe_kube_context ||- die "context '${E2E_KUBECONTEXT}' is not an allowed local E2E context." 2+ [[ -n "${CONTEXT:-}" ]] || die "CONTEXT is required (a local minikube profile context)." 2+ case "$CONTEXT" in+ *gke_*|*prod*)+ die "Refusing: CONTEXT='${CONTEXT}' looks like a GKE/production context. Local profiles only." 2 ;;+ esac+ if [[ ! "$CONTEXT" =~ ^clerum-(test|codex-|detached-|claude-) ]]; then+ die "Refusing: CONTEXT='${CONTEXT}' is not an allowed local profile. Must match ^clerum-(test|codex-|detached-|claude-)." 2+ fiif ! kctl get nodes -o name >/dev/null 2>&1; then- die "context '${E2E_KUBECONTEXT}' is not reachable (kubectl get nodes failed)." 1+ die "CONTEXT='${CONTEXT}' is not reachable (kubectl get nodes failed). Start the profile and its port-forwards." 1fi- kctl get nodes -o json | jq -e --arg context "$E2E_KUBECONTEXT" \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||- die "target context is not the exact selected minikube profile '${E2E_KUBECONTEXT}'." 2- log "branch-owned minikube context OK: ${E2E_KUBECONTEXT}"-}--load_branch_profile_endpoints() {- local ports_env external_url rpc_url- ports_env="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/ports.env}"- [[ -r "$ports_env" ]] ||- die "branch profile ports evidence is missing at ${ports_env}; refresh the branch-profile helper." 2- external_url="$(profile_env_value EXTERNAL_REST_API_BASE_URL "$ports_env")"- [[ -n "$external_url" ]] ||- external_url="$(profile_env_value EXTERNAL_REST_API_URL "$ports_env")"- rpc_url="$(profile_env_value RPC_PROXY_BASE_URL "$ports_env")"- [[ -n "$rpc_url" ]] ||- rpc_url="$(profile_env_value RPC_PROXY_URL "$ports_env")"- [[ "$external_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||- die "branch ports.env has no valid loopback External REST URL." 2- [[ "$rpc_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||- die "branch ports.env has no valid loopback RPC proxy URL." 2- EXT_BASE="$external_url"- RPC_BASE="$rpc_url"- log "using branch-profile endpoints from ${ports_env}"+ log "context OK: ${CONTEXT}"}require_deps() {@@ -345,12 +295,11 @@ RUN_ID=''; WORK_DIR=''FIXTURE_SECRET=''STORM_CREATED_HOSTS=()STORM_DELETE_HOSTS=()-STORM_URGENT_PROBE_HOSTS=()WAKE_HOST_REF="${E2E_STORM_WAKE_HOST_REF:-chatllm-stateless}"-EXT_BASE=''-RPC_BASE=''+EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"+RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"-DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"+DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-admin123!}}"STORM_CTX_REF="${STORM_CONTEXT_REF:-context1}"STORM_PROVIDER="${STORM_MODEL_PROVIDER:-zai}"STORM_MODEL="${STORM_MODEL_NAME:-glm-5.1}"@@ -362,14 +311,6 @@ T_CREATE_FIRST_MS=''; T_DELETE_MS=''; T_WAKE_MS=''; T_KILL_MS=''; T_CREATE_LAST_PF_PID=''HCC_ENV_SAVED=''TEARDOWN_DONE=0-TEARDOWN_CLEANUP_FAILED=0-HCC_RESTORE_OK=1-STORM_FIXTURES_CREATED=0-# Read and mutated by the sourced compare-and-swap lock helper.-HCC_GATE_LOCK_ACQUIRED=0-HCC_GATE_LOCK_NAME=''-HCC_GATE_LOCK_UID=''-HCC_GATE_FINALIZATION_FAILURE=''cleanup_portforward() {if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then@@ -399,16 +340,10 @@ restore_hcc_env() {_hcc_restore_args[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }log "Restoring HCC cadence env on deployment/${HCC_DEPLOY}"- if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then- warn "failed to restore HCC env (manual check required)"- HCC_RESTORE_OK=0- return 1- fi- if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then+ kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1 || \+ warn "failed to restore HCC env (manual check advised)"+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1 || \warn "HCC rollout did not settle after env restore"- HCC_RESTORE_OK=0- return 1- fiHCC_ENV_SAVED=''}@@ -423,12 +358,10 @@ restore_hcc_env_strict() {[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }log "Restoring HCC cadence env on deployment/${HCC_DEPLOY} (strict)"if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then- HCC_RESTORE_OK=0fail "cadence restore FAILED — the profile is left with test cadences on deployment/${HCC_DEPLOY} (the EXIT trap retries best-effort; verify manually)"return 0fiif ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then- HCC_RESTORE_OK=0fail "HCC rollout did not settle after the cadence restore — verify deployment/${HCC_DEPLOY} manually"return 0fi@@ -438,11 +371,10 @@ restore_hcc_env_strict() {# shellcheck disable=SC2329 # invoked via the EXIT trap belowcleanup_on_exit() {- local status=$? cleanup_failed=0+ local status=$?set +ecleanup_portforward- if [[ "$TEARDOWN_DONE" != '1' && "$STORM_FIXTURES_CREATED" == '1' && -n "$RUN_ID" ]]; then- cleanup_failed=1+ if [[ "$TEARDOWN_DONE" != '1' && -n "$RUN_ID" && "${KEEP_FIXTURES:-}" != '1' ]]; thenlocal sel="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"warn "abnormal exit — best-effort deletion of storm fixtures (${sel})"kctl -n "$MCP_HOST_NS" delete host -l "$sel" --wait=false >/dev/null 2>&1 || \@@ -450,10 +382,7 @@ cleanup_on_exit() {kctl -n "$MCP_HOST_NS" delete secret -l "$sel" --wait=false >/dev/null 2>&1 || \warn "best-effort fixture Secret deletion failed (manual check advised)"fi- restore_hcc_env || HCC_RESTORE_OK=0- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$HCC_RESTORE_OK"; then- status=1- fi+ restore_hcc_envexit "$status"}trap cleanup_on_exit EXIT@@ -476,9 +405,6 @@ init_run() {for i in $(seq 1 "$STORM_DELETE_COUNT"); doSTORM_DELETE_HOSTS+=("$(printf 's791-%s-d%d' "$RUN_ID" "$i")")done- for i in $(seq 1 "$STORM_URGENT_PROBE_COUNT"); do- STORM_URGENT_PROBE_HOSTS+=("$(printf 's791-%s-u%02d' "$RUN_ID" "$i")")- donelog "run-id ${RUN_ID}; work dir ${WORK_DIR}"log "create fixtures: ${STORM_CREATED_HOSTS[*]}"log "delete fixtures: ${STORM_DELETE_HOSTS[*]}"@@ -605,30 +531,14 @@ storm_diagnostics() {# different pod (e.g. a terminating one during a rollout) than the one the# identity guard checked; cross-pod counter deltas would false-PASS A4.scrape_hcc_metrics() {- local out="$1" pod="$2" lport='' url pf_log- pf_log="${WORK_DIR}/pf-${pod}.log"- # Let kubectl allocate a process-owned loopback port. A fixed port is- # machine-global and can silently collide with another branch/profile gate.- kctl -n "$CONTROL_NS" port-forward --address=127.0.0.1 \- "pod/${pod}" ":${HCC_PORT}" >"$pf_log" 2>&1 &+ local out="$1" pod="$2" lport="${HCC_METRICS_LOCAL_PORT:-18082}"+ local url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"+ kctl -n "$CONTROL_NS" port-forward "pod/${pod}" "${lport}:${HCC_PORT}" >"${WORK_DIR}/pf.log" 2>&1 &PF_PID=$!# shellcheck disable=SC2329 # invoked by name via wait_until- _pf_ready_probe() {- kill -0 "$PF_PID" 2>/dev/null || return 1- lport="$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> .*/\1/p' "$pf_log" | head -n1)"- [[ "$lport" =~ ^[0-9]+$ ]]- }- # shellcheck disable=SC2329 # invoked by name via wait_until- _pf_diag() { { echo '----- port-forward log -----'; cat "$pf_log" 2>/dev/null; } >&2; }- if ! wait_until 30 1 "random HCC metrics port-forward" _pf_ready_probe _pf_diag; then- cleanup_portforward- return 1- fi- url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"- # shellcheck disable=SC2329 # invoked by name via wait_until_scrape_probe() { curl -fsS --max-time 3 "$url" -o "$out" 2>/dev/null; }# shellcheck disable=SC2329 # invoked by name via wait_until- _scrape_diag() { _pf_diag; }+ _scrape_diag() { { echo '----- port-forward log -----'; cat "${WORK_DIR}/pf.log" 2>/dev/null; } >&2; }if ! wait_until 30 2 "HCC metrics scrape -> ${out}" _scrape_probe _scrape_diag; thencleanup_portforwardreturn 1@@ -641,23 +551,6 @@ wake_lifecycle_state() {kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true}-# shellcheck disable=SC2329 # invoked from the wait_until callback below-wake_handled_generation() {- kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" \- -o jsonpath='{.status.lifecycle.wakeHandledGeneration}' 2>/dev/null || true-}--# Correlate the public wake request with HCC's durable lifecycle status. This-# predicate is deliberately pure so the static gate test can exercise the-# same ordering contract used by the cluster E2E.-# shellcheck disable=SC2329 # invoked from the wait_until callback below-wake_generation_is_handled() {- local requested="$1" handled="$2"- [[ "$requested" =~ ^[1-9][0-9]*$ ]] || return 1- [[ "$handled" =~ ^[0-9]+$ ]] || return 1- (( handled >= requested ))-}-# shellcheck disable=SC2329 # invoked from _wake_converged_probe (runs via wait_until)wake_ready_pod_exists() {kctl -n "$MCP_HOST_NS" get pods -l "app=${WAKE_HOST_REF}" \@@ -696,31 +589,19 @@ mint_rpc_token() {# The wake route is the user-facing rpc-proxy endpoint Desktop uses;# rpc-proxy owns the control-api hop. Never kubectl scale.-WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''+WAKE_STATUS=''; WAKE_BODY=''post_wake() {- local resp response_status response_generation- WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''+ local respmint_rpc_token || return 1resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \"${RPC_BASE}/api/v1/rpc/hosts/${WAKE_HOST_REF}/wake" \-H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json') || {echo "wake POST failed at ${RPC_BASE}" >&2; return 1; }WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"- [[ "$WAKE_STATUS" == '202' ]] || {- echo "wake POST returned HTTP ${WAKE_STATUS}, expected 202: ${WAKE_BODY}" >&2- return 1- }- response_status="$(jq -r '.status // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"- response_generation="$(jq -r '.wakeGeneration // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"- [[ "$response_status" == 'wake-requested' ]] || {- echo "wake POST returned unexpected status '${response_status:-<missing>}': ${WAKE_BODY}" >&2- return 1- }- [[ "$response_generation" =~ ^[1-9][0-9]*$ ]] || {- echo "wake POST returned invalid wakeGeneration '${response_generation:-<missing>}': ${WAKE_BODY}" >&2- return 1- }- WAKE_GENERATION="$response_generation"+ case "$WAKE_STATUS" in+ 200|202|204) return 0 ;;+ *) echo "wake POST returned HTTP ${WAKE_STATUS}: ${WAKE_BODY}" >&2; return 1 ;;+ esac}# ─── HCC cadence shortening (only when the wake target must suspend) ──@@ -773,15 +654,8 @@ _wake_target_diag() {require_depsrequire_storm_contextinit_run-[[ "${KEEP_FIXTURES:-0}" != '1' ]] ||- die "KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result; refusing before mutation." 2-[[ "${E2E_HCC_HOST_STORM_FAULT_INJECTION:-0}" == '1' ]] ||- die "Set E2E_HCC_HOST_STORM_FAULT_INJECTION=1 to acknowledge the temporary HCC pod restart." 2-require_branch_owned_hcc_gate "$CONTROL_NS"-load_branch_profile_endpointsheader "Prerequisites"-ok "branch helper, profile, exact HEAD/fingerprint/gate, and minikube node label verified"if kctl get deployment "$HCC_DEPLOY" -n "$CONTROL_NS" >/dev/null 2>&1; thenok "HCC deployment present in ${CONTROL_NS}"else@@ -821,18 +695,12 @@ elsefail "cannot mint RPC token for hostRef=${WAKE_HOST_REF} -- is ${DEV_EMAIL} associated to it? Seed: bash scripts/e2e/seed-stateless-host.sh"print_results; exit 1fi-if acquire_hcc_watch_gate_lock; then- ok "exclusive shared HCC fault-injection lock acquired"-else- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"-fi# ======================================================================# Prep 1 — storm fixtures that will be DELETED mid-storm (bundles must# exist first: deleting a bundle-less Host proves nothing)# ======================================================================header "Prep — pre-create ${STORM_DELETE_COUNT} delete-target fixture Hosts"-STORM_FIXTURES_CREATED=1ensure_fixture_secretfor h in "${STORM_DELETE_HOSTS[@]}"; doapply_storm_host "$h" false@@ -880,9 +748,6 @@ SCRAPE_A="${WORK_DIR}/metrics-A.prom"SCRAPE_B="${WORK_DIR}/metrics-B.prom"SCRAPE_C1="${WORK_DIR}/metrics-C1.prom"SCRAPE_C2="${WORK_DIR}/metrics-C2.prom"-SCRAPE_D="${WORK_DIR}/metrics-D.prom"-SCRAPE_WAKE_PRE="${WORK_DIR}/metrics-wake-pre.prom"-SCRAPE_WAKE_POST="${WORK_DIR}/metrics-wake-post.prom"HCC_POD_A="$(hcc_pod_name)"if [[ -z "$HCC_POD_A" ]]; thenfail "no Running HCC pod found for baseline identity"; storm_diagnostics; print_results; exit 1@@ -916,89 +781,22 @@ for h in "${STORM_DELETE_HOSTS[@]}"; dodonelog "storm: ${STORM_DELETE_COUNT} mid-storm Host deletions issued"-# Isolate the supported wake on the pre-kill pod. Samples from the first half-# of Host creation may still settle inside this window; they are valid urgent-# traffic and the hard-gate assertion below requires every one of them, not-# only a percentile, to remain within budget.-HCC_POD_WAKE="$(hcc_pod_name)"-if [[ -z "$HCC_POD_WAKE" || "$HCC_POD_WAKE" != "$HCC_POD_A" ]]; then- fail "storm: HCC pod changed before the wake metric baseline (${HCC_POD_A} -> ${HCC_POD_WAKE:-<none>})"- storm_diagnostics; print_results; exit 1-fi-if ! scrape_hcc_metrics "$SCRAPE_WAKE_PRE" "$HCC_POD_WAKE"; then- fail "storm: could not capture the same-pod wake metric baseline"- print_results; exit 1-fi-T_WAKE_MS="$(now_ms)"if post_wake; then- log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS}, generation ${WAKE_GENERATION})"+ log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS})"elsefail "storm: mid-storm wake POST failed for ${WAKE_HOST_REF}"; print_results; exit 1fi-# Prove that this specific supported wake reached HCC: its response generation-# must be durably reflected in lifecycle status. Then require a new urgent-# queue sample and an idle urgent lane on the same pod. Prometheus collectors-# are read independently, so this deliberately uses a two-scrape fence instead-# of assuming cross-metric atomicity: once the first scrape observes the urgent-# lane idle, HCC's observe-before-in-flight-decrement ordering guarantees the-# wake observation is durable and therefore visible to the second scrape.-# shellcheck disable=SC2329 # invoked by name via wait_until-_wake_urgent_evidence_recorded() {- local current_pod lifecycle_state handled_generation- local fence_in_flight pre_count post_count in_flight- current_pod="$(hcc_pod_name)"- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_WAKE" ]] || return 1- lifecycle_state="$(wake_lifecycle_state)"- [[ -n "$lifecycle_state" && "$lifecycle_state" != 'suspended' ]] || return 1- handled_generation="$(wake_handled_generation)"- wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation" || return 1- scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1- fence_in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_WAKE_POST")"- awk -v active="$fence_in_flight" 'BEGIN { exit !(active == 0) }' || return 1-- # The wake histogram observation precedes the decrement seen by the fence.- # Re-scrape so the adjudicated histogram cannot predate that observation.- scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1- pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_WAKE_PRE")"- post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_WAKE_POST")"- in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_WAKE_POST")"- awk -v before="$pre_count" -v after="$post_count" -v active="$in_flight" \- 'BEGIN { exit !((after - before) >= 1 && active == 0) }'-}-if wait_until "$STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S" 1 \- "supported wake urgent reconcile + same-pod queue evidence" \- _wake_urgent_evidence_recorded _wake_target_diag; then- ok "A2 wake input: supported wake reconcile completed and published same-pod urgent queue evidence"-else- fail "A2 wake input: wake did not complete with durable same-pod urgent queue evidence within ${STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S}s"- print_results; exit 1-fi--# Capture A7's fleet-failure baseline immediately before the kill. The wake-# scrape remains the A2 wake-window oracle; a separate fresh scrape prevents-# fleet failures after wake adjudication from falling outside A7's delta.+# Scrape B: last look at the pre-kill pod's counters, immediately before+# the kill (samples still in flight at process death are a recorded gap).HCC_POD_B="$(hcc_pod_name)"if [[ "$HCC_POD_B" != "$HCC_POD_A" ]]; thenfail "storm: HCC pod changed unexpectedly before the kill (${HCC_POD_A} -> ${HCC_POD_B:-<none>}) — counter deltas would be invalid"storm_diagnostics; print_results; exit 1fiif ! scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"; then- fail "storm: could not capture the fresh pre-kill metric baseline"- print_results; exit 1+ fail "storm: could not scrape HCC metrics immediately before the kill"; print_results; exit 1filog "storm: pre-kill scrape B captured (pod ${HCC_POD_B})"@@ -1050,41 +848,6 @@ elseprint_results; exit 1fi-# A7's causal business signal is the replacement pod's own completion log for-# its startup fleet reconciliation. A scheduled/started counter is insufficient:-# it can increment before the pass fails, stalls, or is superseded.-FLEET_COMPLETION_PROVEN=0-replacement_fleet_state=''-# shellcheck disable=SC2329 # invoked by name via wait_until-_replacement_fleet_completed_probe() {- local logs current_pod- current_pod="$(hcc_pod_name)"- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1- logs="$(kctl -n "$CONTROL_NS" logs "pod/${HCC_POD_C}" \- -c host-context-controller 2>/dev/null)" || return 1- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then- replacement_fleet_state='failed'- return 0- fi- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then- replacement_fleet_state='completed'- return 0- fi- return 1-}-if ! wait_until "$STORM_CONVERGENCE_DEADLINE_S" 2 \- "replacement HCC initial Host fleet reconciliation completion" \- _replacement_fleet_completed_probe storm_diagnostics; then- fail "A7: replacement pod did not complete its initial Host fleet reconciliation within ${STORM_CONVERGENCE_DEADLINE_S}s"- print_results; exit 1-fi-if [[ "$replacement_fleet_state" != 'completed' ]]; then- fail "A7: replacement pod's initial Host fleet reconciliation failed"- storm_diagnostics; print_results; exit 1-fi-FLEET_COMPLETION_PROVEN=1-ok "A7 input: replacement pod logged completed initial Host fleet reconciliation"-# shellcheck disable=SC2329 # invoked by name via wait_until_all_created_deploys_probe() {local h@@ -1117,68 +880,6 @@ if ! scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"; thenfilog "scrape C1 captured (replacement pod ${HCC_POD_C})"-# ─── A2 input: direct-watch urgent probe after replacement LIST→WATCH ──-# The storm's second half is intentionally issued immediately after pod-# deletion, when a cold-start LIST may legitimately own those Hosts on the-# fleet lane. C1 proves the replacement has completed LIST→WATCH recovery;-# issue a fresh, independent set now so every observed sample is a direct-# watch ADDED admission on the urgent lane.-header "A2 input — urgent direct-watch probe (${STORM_URGENT_PROBE_COUNT} Hosts)"-for i in $(seq 1 "${#STORM_URGENT_PROBE_HOSTS[@]}"); do- h="${STORM_URGENT_PROBE_HOSTS[$((i - 1))]}"- if (( i % 2 == 0 )); then apply_storm_host "$h" true; else apply_storm_host "$h" false; fi-done-log "urgent probe: ${STORM_URGENT_PROBE_COUNT} Host ADDED events issued after replacement LIST→WATCH"--# shellcheck disable=SC2329 # invoked by name via wait_until-_urgent_probe_deploys_probe() {- local h- for h in "${STORM_URGENT_PROBE_HOSTS[@]}"; do- kctl -n "$MCP_HOST_NS" get deployment "$h" >/dev/null 2>&1 || return 1- done-}-if wait_until "$STORM_CONVERGENCE_DEADLINE_S" 3 \- "all ${STORM_URGENT_PROBE_COUNT} urgent-probe Hosts' Deployments to exist" \- _urgent_probe_deploys_probe storm_diagnostics; then- ok "A2 input: every direct-watch urgent-probe Host materialized"-else- fail "A2 input: urgent-probe Host materialization incomplete after ${STORM_CONVERGENCE_DEADLINE_S}s"- print_results; exit 1-fi-HCC_POD_C_NOW="$(hcc_pod_name)"-if [[ -z "$HCC_POD_C_NOW" || "$HCC_POD_C_NOW" != "$HCC_POD_C" ]]; then- fail "replacement HCC pod changed before urgent-probe scrape D (${HCC_POD_C} -> ${HCC_POD_C_NOW:-<none>}) — urgent counter window is invalid"- storm_diagnostics; print_results; exit 1-fi--# Deployment existence can precede the end of reconcileCore, where the queue-# histogram is observed. Poll the metric itself instead of sleeping, otherwise-# a fast API server can make this gate race its own telemetry.-# shellcheck disable=SC2329 # invoked by name via wait_until-_urgent_probe_metrics_recorded() {- local current_pod pre_count post_count- current_pod="$(hcc_pod_name)"- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1- scrape_hcc_metrics "$SCRAPE_D" "$HCC_POD_C" || return 1- pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_C1")"- post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }- END { print total + 0 }- ' "$SCRAPE_D")"- awk -v before="$pre_count" -v after="$post_count" -v required="$STORM_URGENT_PROBE_COUNT" \- 'BEGIN { exit !((after - before) >= required) }'-}-if ! wait_until 60 2 \- "at least ${STORM_URGENT_PROBE_COUNT} direct-watch urgent metric samples" \- _urgent_probe_metrics_recorded storm_diagnostics; then- fail "urgent-probe reconciles did not publish ${STORM_URGENT_PROBE_COUNT} queue samples within 60s"- print_results; exit 1-fi-log "scrape D captured (replacement pod ${HCC_POD_C}; direct-watch urgent window C1→D)"-# ─── A3: per-host materialization <= 20s from ITS OWN creation ───────header "A3 — per-host materialization (budget ${STORM_MATERIALIZATION_BUDGET_S}s, §14.2)"MAT_ROWS="${WORK_DIR}/materialization.rows"@@ -1290,14 +991,13 @@ if ! scrape_hcc_metrics "$SCRAPE_C2" "$HCC_POD_C"; thenfilog "scrape C2 captured (replacement pod ${HCC_POD_C}; A6/A7 counters settled post-A5)"-# One python pass performs the five metric assertions and emits one+# One python pass performs the four metric assertions and emits one# VERDICT|<name>|PASS/FAIL|<detail> line each. python exits nonzero only# when it cannot parse its inputs (a hard failure of the gate itself).METRIC_VERDICTS="${WORK_DIR}/metric-verdicts.txt"-if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" SCRAPE_D="$SCRAPE_D" \- SCRAPE_WAKE_PRE="$SCRAPE_WAKE_PRE" SCRAPE_WAKE_POST="$SCRAPE_WAKE_POST" \+if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" \P95_BUDGET_S="$STORM_URGENT_P95_BUDGET_S" RECOVERY_BUDGET_S="$STORM_WATCH_RECOVERY_BUDGET_S" \- MIN_URGENT_SAMPLES="$STORM_URGENT_PROBE_COUNT" FLEET_COMPLETION_PROVEN="$FLEET_COMPLETION_PROVEN" \+ MIN_URGENT_SAMPLES="$STORM_CREATE_COUNT" \python3 >"$METRIC_VERDICTS" <<'PY'import os, re, sysfrom collections import defaultdict@@ -1333,9 +1033,6 @@ a, _ = load(os.environ["SCRAPE_A"])b, _ = load(os.environ["SCRAPE_B"])c1, _ = load(os.environ["SCRAPE_C1"])c2, c2_types = load(os.environ["SCRAPE_C2"])-d, _ = load(os.environ["SCRAPE_D"])-wake_pre, _ = load(os.environ["SCRAPE_WAKE_PRE"])-wake_post, _ = load(os.environ["SCRAPE_WAKE_POST"])def buckets(samples, metric, match):"""Cumulative histogram buckets {le: count} summed over matching series."""@@ -1360,60 +1057,30 @@ def le_key(le):verdicts = []-# ── A2a: every completed urgent reconcile in the supported wake window ─+# ── A2: urgent queue-wait p95 <= budget across (B-A) + C1 ─────────────qm = os.environ["QUEUE_WAIT_METRIC"]budget = float(os.environ["P95_BUDGET_S"])-wake_before = buckets(wake_pre, qm, {"lane": "urgent"})-wake_after = buckets(wake_post, qm, {"lane": "urgent"})-wake_delta = defaultdict(float)-wake_reset = False-for le in set(wake_before) | set(wake_after):- delta = wake_after.get(le, 0.0) - wake_before.get(le, 0.0)- if delta < 0:- wake_reset = True- wake_delta[le] = delta-if wake_reset:- verdicts.append(("A2-wake-urgent-hard-gate", False,- "negative bucket delta in the same-pod wake window"))-else:- wake_total = wake_delta.get("+Inf", 0.0)- wake_within = wake_delta.get(str(int(budget)), wake_delta.get(str(budget), 0.0))- wake_dist = " ".join(- f"le{le}={wake_delta[le]:.0f}" for le in sorted(wake_delta, key=le_key)- )- if wake_total < 1:- verdicts.append(("A2-wake-urgent-hard-gate", False,- "no completed urgent queue sample in the supported wake window"))- elif wake_within != wake_total:- verdicts.append(("A2-wake-urgent-hard-gate", False,- f"{wake_total - wake_within:.0f} of {wake_total:.0f} urgent sample(s) exceeded "- f"{budget:.0f}s in the supported wake window ({wake_dist})"))- else:- verdicts.append(("A2-wake-urgent-hard-gate", True,- f"all {wake_total:.0f} urgent sample(s) <= {budget:.0f}s in the supported "- f"same-pod wake window ({wake_dist})"))--# ── A2b: urgent queue-wait p95 across direct-watch window D-C1 ────────-window_note = "window: replacement-pod direct-watch probe delta (D-C1) after LIST→WATCH recovery"-pre = buckets(c1, qm, {"lane": "urgent"})-post = buckets(d, qm, {"lane": "urgent"})+window_note = "windows: pre-kill delta (B-A) + replacement pod C1, which includes replacement-pod recovery traffic"+pre_a = buckets(a, qm, {"lane": "urgent"})+pre_b = buckets(b, qm, {"lane": "urgent"})+post = buckets(c1, qm, {"lane": "urgent"})combined = defaultdict(float)reset = False-for le in set(pre) | set(post):- delta = post.get(le, 0.0) - pre.get(le, 0.0)+for le in set(pre_a) | set(pre_b) | set(post):+ delta = pre_b.get(le, 0.0) - pre_a.get(le, 0.0)if delta < 0:reset = True- combined[le] = delta+ combined[le] = delta + post.get(le, 0.0)if reset:- verdicts.append(("A2-direct-watch-urgent-p95", False,- "negative bucket delta between scrapes C1 and D (unexpected counter reset)"))+ verdicts.append(("A2-urgent-queue-wait-p95", False,+ "negative bucket delta between scrapes A and B (unexpected counter reset)"))else:total = combined.get("+Inf", 0.0)min_samples = float(os.environ["MIN_URGENT_SAMPLES"])if total < min_samples:- verdicts.append(("A2-direct-watch-urgent-p95", False,- f"only {total:.0f} direct-watch urgent samples (need >= {min_samples:.0f} — "- "the urgent probe did not exercise its declared lane)"))+ verdicts.append(("A2-urgent-queue-wait-p95", False,+ f"only {total:.0f} urgent samples in the storm window (need >= {min_samples:.0f} — "+ "the storm did not exercise the urgent lane, cannot claim a p95)"))else:p95_ub = Nonefor le in sorted(combined, key=le_key):@@ -1422,10 +1089,11 @@ else:breakdist = " ".join(f"le{le}={combined[le]:.0f}" for le in sorted(combined, key=le_key))if p95_ub is not None and le_key(p95_ub) <= budget:- verdicts.append(("A2-direct-watch-urgent-p95", True,- f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note})"))+ verdicts.append(("A2-urgent-queue-wait-p95", True,+ f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note}; "+ "gap: samples in flight between scrape B and process death are unrecoverable)"))else:- verdicts.append(("A2-direct-watch-urgent-p95", False,+ verdicts.append(("A2-urgent-queue-wait-p95", False,f"p95 bucket upper bound {p95_ub} exceeds {budget:.0f}s over {total:.0f} samples ({dist}; {window_note})"))# ── A4: watch recovery <= budget on the replacement pod (C1) ──────────@@ -1449,36 +1117,28 @@ else:verdicts.append(("A4-watch-recovery", True,f"{succ_total:.0f} recovery sample(s), all <= {bound}s (budget {rbudget}s)"))-# ── A7: recovered fleet convergence remains bounded (settled C2) ─────+# ── A7: fleet coalescing held (settled window C2) ─────────────────────fm = os.environ["FLEET_METRIC"]-# Presence check (A6 pattern): counter() returns 0 for an absent family, so-# without this a renamed/unregistered FLEET_METRIC makes every count 0 and A7-# passes vacuously. The causal leg (FLEET_COMPLETION_PROVEN) alone cannot catch-# a metric rename — the metric itself must be registered on the running image.-if fm not in c2_types:- verdicts.append(("A7-fleet-recovery-bounded", False,- f"{fm} not registered on the running image — is the branch HCC image deployed?"))+pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")+post_started = counter(c2, fm, "result", "started")+post_coalesced = counter(c2, fm, "result", "coalesced")+post_trailing = counter(c2, fm, "result", "trailing")+post_failed = counter(c2, fm, "result", "failed")+problems = []+if pre_failed > 0 or post_failed > 0:+ problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")+if post_started < 1:+ problems.append("no fleet pass started on the replacement pod (recovery convergence missing)")+if post_coalesced < 1:+ problems.append("no coalesced fleet request on the replacement pod (recovery request should land against the active initial pass)")+if post_trailing > post_coalesced:+ problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")+detail = (f"started={post_started:.0f} coalesced={post_coalesced:.0f} "+ f"trailing={post_trailing:.0f} failed={post_failed:.0f} (pre-kill failed delta={pre_failed:.0f})")+if problems:+ verdicts.append(("A7-fleet-coalescing", False, "; ".join(problems) + f" [{detail}]"))else:- pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")- post_started = counter(c2, fm, "result", "started")- post_coalesced = counter(c2, fm, "result", "coalesced")- post_trailing = counter(c2, fm, "result", "trailing")- post_failed = counter(c2, fm, "result", "failed")- problems = []- if pre_failed != 0 or post_failed > 0:- problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")- if os.environ.get("FLEET_COMPLETION_PROVEN") != "1":- problems.append("replacement-pod initial fleet completion was not causally proven")- if post_trailing > post_coalesced:- problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")- detail = (f"completion-proven={os.environ.get('FLEET_COMPLETION_PROVEN', 'missing')} "- f"started={post_started:.0f} coalesced={post_coalesced:.0f} "- f"trailing={post_trailing:.0f} failed={post_failed:.0f} "- f"(replacement pod; A4 recovery-success fenced; pre-kill failed delta={pre_failed:.0f})")- if problems:- verdicts.append(("A7-fleet-recovery-bounded", False, "; ".join(problems) + f" [{detail}]"))- else:- verdicts.append(("A7-fleet-recovery-bounded", True, detail))+ verdicts.append(("A7-fleet-coalescing", True, detail))# ── A6: delete-cleanup counters consistent (settled window C2) ────────dm = os.environ["DELETE_CLEANUP_METRIC"]@@ -1518,8 +1178,8 @@ while IFS='|' read -r tag name verdict detail; dofail "${name}: ${detail}"fidone <"$METRIC_VERDICTS"-if [[ "$VERDICT_COUNT" -ne 5 ]]; then- fail "metric adjudication emitted ${VERDICT_COUNT}/5 verdicts — truncated output is a failure, not a pass"+if [[ "$VERDICT_COUNT" -ne 4 ]]; then+ fail "metric adjudication emitted ${VERDICT_COUNT}/4 verdicts — truncated output is a failure, not a pass"fi# ======================================================================@@ -1527,7 +1187,11 @@ fi# The profile and seeded data (including the wake target) stay.# ======================================================================header "Teardown — storm fixtures (${FIXTURE_RUN_KEY}=${RUN_ID})"-STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"+if [[ "${KEEP_FIXTURES:-}" == '1' ]]; then+ warn "KEEP_FIXTURES=1 — leaving fixtures in place for diagnosis (run-id ${RUN_ID})"+ TEARDOWN_DONE=1+else+ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"kctl -n "$MCP_HOST_NS" delete host -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \warn "teardown: label-selector Host deletion returned nonzero (leftover check below is authoritative)"kctl -n "$MCP_HOST_NS" delete secret -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \@@ -1550,7 +1214,6 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_IDok "teardown: all storm fixture Hosts deleted via the supported CR path"elsefail "teardown: storm fixture Hosts still present after ${STORM_TEARDOWN_DEADLINE_S}s (run-id ${RUN_ID})"- TEARDOWN_CLEANUP_FAILED=1fi# FAIL-CLOSED (B2): an unverifiable read counts as a leftover — the teardown@@ -1564,7 +1227,7 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_IDteardown_converge_deadline=$((SECONDS + STORM_TEARDOWN_CONVERGE_S))while :; doTEARDOWN_LEFTOVER=0; teardown_orphan=0; teardown_unverifiable=0; teardown_report=''- for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_URGENT_PROBE_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do+ for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; doif ! leftovers="$(collect_children "$h")"; thenteardown_report+=" ${h}: kubectl read FAILED (unverifiable, fail-closed)"$'\n'TEARDOWN_LEFTOVER=$((TEARDOWN_LEFTOVER + 1)); teardown_unverifiable=1@@ -1617,19 +1280,14 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_IDok "teardown: zero storm-fixture leftovers across ${BUNDLE_NAMESPACES} (profile and seeded data untouched)"elsefail "teardown: ${TEARDOWN_LEFTOVER} storm fixture subject(s) left resources behind (run-id ${RUN_ID}) — investigate before trusting results"- TEARDOWN_CLEANUP_FAILED=1fi-TEARDOWN_DONE=1+ TEARDOWN_DONE=1+fi# M3: on the success path a failed cadence restore must FAIL the gate# (restore_hcc_env_strict registers fail() and leaves HCC_ENV_SAVED for# the trap's best-effort retry); the EXIT trap keeps the lenient restore.restore_hcc_env_strict-if ! finalize_hcc_watch_gate_lock "$TEARDOWN_CLEANUP_FAILED" "$HCC_RESTORE_OK"; then- fail "shared HCC fault-injection lock was retained fail-closed; follow the printed repair instructions"-else- ok "exclusive shared HCC fault-injection lock finalized"-fiprint_resultsif [[ "$g_fail" -gt 0 ]]; thendiff --git a/scripts/e2e/e2e-wrc-hcc-contracts.sh b/scripts/e2e/e2e-wrc-hcc-contracts.shindex 570f0f923..e0b4c5029 100755--- a/scripts/e2e/e2e-wrc-hcc-contracts.sh+++ b/scripts/e2e/e2e-wrc-hcc-contracts.sh@@ -271,8 +271,6 @@ spec:agent:provider: zaimodel: glm-4.7- triggers:- onDemand: {}steps:- id: tool-stepinstruction: "Synthetic status-contract step for toolsCalled args validation."@@ -298,7 +296,7 @@ elsefirm -f "$running_response_file"-if wait_for_running_workflowrecipe_status >/dev/null; then+if running_status_json="$(wait_for_running_workflowrecipe_status)"; thenok "WorkflowRecipe becomes active when workflow execution is running"elsefail "WorkflowRecipe did not become active on running workflow execution"@@ -429,7 +427,7 @@ elsefirm -f "$pending_response_file"-if wait_for_pending_preserves_non_terminal_recipe_status >/dev/null; then+if pending_status_json="$(wait_for_pending_preserves_non_terminal_recipe_status)"; thenok "WorkflowRecipe stays non-terminal when workflow execution is pending"elsefail "WorkflowRecipe did not stay non-terminal on pending workflow execution"diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.shindex f72cbca00..35a61e913 100755--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh@@ -18,8 +18,6 @@ BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"-SOURCE_DEPLOYMENT="$SOURCE_ID"-BACKEND_DEPLOYMENT="$BACKEND_ID"CREATED=0cleanup() {@@ -30,9 +28,9 @@ cleanup() {--ignore-not-found --wait=false >/dev/null 2>&1 || status=1wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \>/dev/null 2>&1 || status=1- kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \+ kctl delete deployment "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \--ignore-not-found --wait=false >/dev/null 2>&1 || status=1- kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \+ kctl delete service "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \--ignore-not-found >/dev/null 2>&1 || status=1kctl delete networkpolicy -n "$SANDBOX_NS" \-l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \@@ -87,21 +85,6 @@ wait_internal_ready() {return 1}-wait_for_workload_instance() {- local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance- while [ "$elapsed" -lt "$timeout" ]; do- instance="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- -o "jsonpath={.status.workloadInstances.${workload_id}}" 2>/dev/null || true)"- if [ -n "$instance" ]; then- printf '%s\n' "$instance"- return 0- fi- sleep "$POLL_INTERVAL"- elapsed=$((elapsed + POLL_INTERVAL))- done- return 1-}-policy_refs() {kctl get networkpolicy -A -l "$1" \-o go-template='{{range .items}}{{.metadata.namespace}}/{{.metadata.name}}{{"\n"}}{{end}}' \@@ -231,26 +214,18 @@ fiok "Fixture has no egressBindings shortcut"header "Phase 2 - WRC reconciliation"-SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID" "$TIMEOUT_POD")" || {- fail "Source workload instance was not assigned"- exit 1-}-BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")" || {- fail "Backend workload instance was not assigned"- exit 1-}-wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {+wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID" "$TIMEOUT_POD" && ok "Source deployment ready" || {fail "Source deployment not ready"exit 1}-wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {+wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID" "$TIMEOUT_POD" && ok "Backend deployment ready" || {fail "Backend deployment not ready"exit 1}wait_internal_ready "$TIMEOUT_POD"-resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"-expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"+resolved_target="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"+expected_target="http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {fail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"exit 1@@ -265,7 +240,7 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"header "Phase 4 - Positive packet flow"-positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \+positive_output="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- \sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \ok "WRC source reached backend through inferred internal dependency" || {@@ -307,7 +282,7 @@ wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod}if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \--timeout="$CONNECT_TIMEOUT" --tries=1 \- "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then+ "http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; thenfail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"exit 1fidiff --git a/scripts/e2e/measure-host-bundle-reconcile.sh b/scripts/e2e/measure-host-bundle-reconcile.shindex c1f7ee5a7..1eb66d287 100755--- a/scripts/e2e/measure-host-bundle-reconcile.sh+++ b/scripts/e2e/measure-host-bundle-reconcile.sh@@ -151,10 +151,9 @@ readonly DELPASS_RESURRECTION_SETTLE_S=30 # quiet window before re-asserting# termination grace. A child with NO deletionTimestamp never gets this# extension (the delete was never issued → immediate fail).readonly MEASURE_TEARDOWN_CONVERGE_S=120-# The startup marker is emitted immediately before the cold-start Host-# inventory/watch recovery. COMPLETE/FAIL below retain the precise-# 'initial Host reconciliation' reason supplied to the fleet pass.-readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'+# Exact log lines emitted by host-context-controller/src/k8sClient.ts for+# the startup full fleet pass (reason string 'initial Host reconciliation').+readonly HCC_PASS_STARTED_MARKER='Running initial Host reconciliation'readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'@@ -965,9 +964,9 @@ phase_delete_during_pass() {_delpass_pass_active_probe() {local logslogs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then pass_state='completed'; return 0; fi- if grep -Fq -- "$HCC_PASS_STARTED_MARKER" <<<"$logs"; then pass_state='active'; return 0; fi+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then pass_state='completed'; return 0; fi+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_STARTED_MARKER"; then pass_state='active'; return 0; fireturn 1}# shellcheck disable=SC2329 # invoked by name via wait_until@@ -1016,7 +1015,7 @@ phase_delete_during_pass() {fail "delete-during-pass: could not re-read HCC logs immediately after the delete — overlap cannot be verified (fail-closed)"return 1fi- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$post_delete_logs"; then+ if printf '%s' "$post_delete_logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; thenfail "delete-during-pass: pass logged COMPLETED within the probe->delete window — overlap NOT established; re-run"return 1fi@@ -1028,8 +1027,8 @@ phase_delete_during_pass() {_delpass_pass_completed_probe() {local logslogs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi- grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs" && pass_state='completed'+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi+ printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER" && pass_state='completed'}if ! wait_until "$DELPASS_PASS_COMPLETE_DEADLINE_S" 2 "initial full pass completion after the delete" \_delpass_pass_completed_probe _delpass_logs_diag; thendiff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 0186ed584..0f9247a86 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -1012,7 +1012,7 @@ if [ "$RESET_DB" = true ]; then$KC -n control-plane rollout status deployment/trace-maintenance-worker --timeout=180s >/dev/nullfiif [ "$RESET_HCC_REPLICAS" -gt 0 ]; then- $KC -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null+ $KC -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/nullfiok "Control-api database and GFS roles converged after reset"elsediff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.shindex 590c749c0..71f036828 100755--- a/scripts/tests/test-converge-control-db-after-reset.sh+++ b/scripts/tests/test-converge-control-db-after-reset.sh@@ -51,7 +51,7 @@ case "$*" in*'rollout status deployment/workflow-recipes --timeout=180s') exit 0 ;;*'rollout status deployment/trace-maintenance-worker --timeout=180s') exit 0 ;;*'scale deployment/host-context-controller --replicas=4') exit 0 ;;- *'rollout status deployment/host-context-controller --timeout=900s') exit 0 ;;+ *'rollout status deployment/host-context-controller --timeout=180s') exit 0 ;;*'get pods -l app=workflow-recipes -o name') printf 'pod/workflow-old\n' ;;*'get pods -l app=trace-maintenance-worker -o name') printf 'pod/trace-old\n' ;;*'wait --for=delete pod -l app=workflow-recipes --timeout=180s') exit 0 ;;diff --git a/scripts/tests/test-hcc-api-gateway-readiness.sh b/scripts/tests/test-hcc-api-gateway-readiness.shdeleted file mode 100755index ce2930e86..000000000--- a/scripts/tests/test-hcc-api-gateway-readiness.sh+++ /dev/null@@ -1,184 +0,0 @@-#!/usr/bin/env bash-# Functions under test are evaluated from the E2E source, so ShellCheck cannot-# see their use of the mock environment variables declared by each harness.-# shellcheck disable=SC2034-set -euo pipefail--ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-DEPLOYMENT="${ROOT}/deploy/base/control-plane/host-context-controller-api-gateway.yaml"-CONFIGMAP="${ROOT}/deploy/base/control-plane/configmaps.yaml"-BOOTSTRAP_E2E="${ROOT}/scripts/e2e/e2e-hcc-readiness-bootstrap.sh"--fail() {- echo "FAIL: $1" >&2- exit 1-}--readiness_probe="$(- awk '- /^[[:space:]]*readinessProbe:/ { in_probe = 1; next }- in_probe && /^[[:space:]]*resources:/ { exit }- in_probe { print }- ' "${DEPLOYMENT}"-)"--if ! grep -Eq '^[[:space:]]*path:[[:space:]]*/health[[:space:]]*$' <<<"${readiness_probe}"; then- fail "HCC API gateway readiness probe must use the gateway-local /health endpoint"-fi-echo "PASS: HCC API gateway readiness probe uses the gateway-local /health endpoint"--hcc_config="$(- awk '- BEGIN { RS = "---" }- /name:[[:space:]]*host-context-controller-api-gateway/ { print; exit }- ' "${CONFIGMAP}"-)"--if [[ -z "${hcc_config}" ]]; then- fail "HCC API gateway nginx ConfigMap is missing"-fi--health_location="$(- awk '- /^[[:space:]]*location = \/health \{/ { in_location = 1 }- in_location { print }- in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }- ' <<<"${hcc_config}"-)"--if [[ "${health_location}" != *'return 200'* ]] ||- [[ "${health_location}" == *'proxy_pass'* ]]; then- fail "HCC API gateway /health must remain a local nginx health response"-fi-echo "PASS: HCC API gateway /health remains local to nginx"--ready_location="$(- awk '- /^[[:space:]]*location = \/ready \{/ { in_location = 1 }- in_location { print }- in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }- ' <<<"${hcc_config}"-)"--if [[ "${ready_location}" != *'proxy_pass http://host_context_controller_upstream;'* ]]; then- fail "HCC API gateway /ready must continue proxying the HCC readiness contract"-fi-echo "PASS: HCC API gateway /ready continues proxying HCC readiness"--runtime_hold_function="$(- sed -n '/^hcc_gateway_remains_ready_without_hcc() {$/,/^}$/p' "${BOOTSTRAP_E2E}"-)"-unavailable_probe_function="$(- sed -n '/^hcc_gateway_ready_proxy_unavailable() {$/,/^}$/p' "${BOOTSTRAP_E2E}"-)"-gateway_pod_function="$(- sed -n '/^running_hcc_gateway_pod() {$/,/^}$/p' "${BOOTSTRAP_E2E}"-)"-if [[ "${gateway_pod_function}" != *'ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"'* ]] ||- [[ "${gateway_pod_function}" == *'running_pod_name'* ]]; then- fail "gateway runtime proof must select a Ready pod through the shared E2E helper"-fi-echo "PASS: gateway runtime proof selects a Ready pod through the shared helper"--if [[ "${runtime_hold_function}" != *'hcc_gateway_deployment_ready'* ]] ||- [[ "${runtime_hold_function}" != *'hcc_gateway_local_health_ok'* ]] ||- [[ "${runtime_hold_function}" != *'hcc_gateway_ready_proxy_unavailable'* ]] ||- ! grep -Fq 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}"; then- fail "bootstrap E2E must hold HCC unavailable across a full gateway probe-failure window"-fi-echo "PASS: bootstrap E2E proves gateway readiness remains local while HCC is unavailable"--# Exercise the full conjunction with a bounded synthetic clock. Each required-# observation must independently fail the hold; this kills a permissive-# && -> || mutation without requiring a cluster or a real sleep.-gateway_hold_case() (- local failed_observation=$1 clock_file- clock_file="$(mktemp "${TMPDIR:-/tmp}/hcc-gateway-clock.XXXXXX")"- printf '%s' 0 >"$clock_file"- trap 'rm -f "$clock_file"' EXIT- HCC_NS=control-plane- date() {- local calls- calls="$(<"$clock_file")"- calls=$((calls + 1))- printf '%s' "$calls" >"$clock_file"- [ "$calls" -le 2 ] && printf '%s\n' 0 || printf '%s\n' 1- }- sleep() { :; }- hcc_gateway_deployment_ready() {- [ "$failed_observation" != deployment ]- }- hcc_gateway_local_health_ok() {- [ "$failed_observation" != health ]- }- hcc_gateway_ready_proxy_unavailable() {- [ "$failed_observation" != proxy ]- }- eval "${runtime_hold_function}"- hcc_gateway_remains_ready_without_hcc 1-)--if gateway_hold_case none &&- ! gateway_hold_case deployment &&- ! gateway_hold_case health &&- ! gateway_hold_case proxy; then- echo "PASS: gateway hold requires every readiness, health, and proxy-unavailable observation"-else- fail "gateway hold can accept a failed readiness, health, or proxy observation"-fi--for unavailable_status in 502 503 504; do- if (- HCC_NS=control-plane- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }- kctl() {- printf ' HTTP/1.1 %s upstream unavailable\n' "${unavailable_status}"- return 8- }- eval "${unavailable_probe_function}"- hcc_gateway_ready_proxy_unavailable- ); then- echo "PASS: gateway hold accepts explicit upstream-unavailable HTTP ${unavailable_status}"- else- fail "gateway hold rejects expected upstream-unavailable HTTP ${unavailable_status}"- fi-done--if (- HCC_NS=control-plane- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }- kctl() {- printf '%s\n' 'error: unable to upgrade connection: container not found'- return 1- }- eval "${unavailable_probe_function}"- ! hcc_gateway_ready_proxy_unavailable-); then- echo "PASS: gateway hold rejects kubectl or container transport failure"-else- fail "gateway hold can mistake kubectl or container transport failure for upstream unavailability"-fi--if (- HCC_NS=control-plane- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }- kctl() {- printf '%s\n' 'healthy upstream response'- return 0- }- eval "${unavailable_probe_function}"- ! hcc_gateway_ready_proxy_unavailable-); then- echo "PASS: gateway hold rejects a successful proxied /ready response"-else- fail "gateway hold can accept a successful proxied /ready response as unavailable"-fi--hold_line="$(grep -nF 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}" | cut -d: -f1)"-restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "${BOOTSTRAP_E2E}" | cut -d: -f1)"-recovery_line="$(grep -nF 'hcc_gateway_ready_proxy_recovers ||' "${BOOTSTRAP_E2E}" | cut -d: -f1)"-if [[ -z "${hold_line}" || -z "${restart_line}" || -z "${recovery_line}" ]] ||- (( hold_line >= restart_line || restart_line >= recovery_line )); then- fail "gateway unavailable/ready/recovery assertions must surround the HCC restart"-fi-echo "PASS: bootstrap E2E proves proxied /ready failure and recovery in causal order"diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.shindex 379d362e6..50104032d 100755--- a/scripts/tests/test-hcc-watch-recovery-gate.sh+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh@@ -1,39 +1,21 @@#!/usr/bin/env bash-# Values in the eval-based fixture harness are consumed by extracted gate-# functions, which static shell analysis cannot follow.-# shellcheck disable=SC2034set -uFAIL=0ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"SCRIPT_DIR="${ROOT}/scripts/e2e"-WATCH_GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"-READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-readiness-bootstrap.sh"-MCP_READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-mcp-context-readiness.sh"-HOST_BUNDLE_MEASURE="${SCRIPT_DIR}/measure-host-bundle-reconcile.sh"-HOST_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"-GATES=("$WATCH_GATE" "$READINESS_GATE" "$MCP_READINESS_GATE")+GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"LOCK_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"LOG_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"-FIXTURE_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"-HCC_K8S_CLIENT="${ROOT}/host-context-controller/src/k8sClient.ts"-HCC_EGRESS_COORDINATOR="$(- printf '%s' "${ROOT}/host-context-controller/src/externalEgressConvergenceCoordinator.ts"-)"-HCC_NETWORK_POLICY_RECONCILER="$(- printf '%s' "${ROOT}/host-context-controller/src/networkPolicyReconciler.ts"-)"-WATCH_GATE_TARGET="$(sed -n '/^test-e2e-hcc-communicationchannel-watch-recovery:/,/^$/p' "${ROOT}/Makefile")"MOCK_STATE_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-lock-test.XXXXXX")"MOCK_LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-log-test.XXXXXX")"-MOCK_PROFILE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hcc-profile-test.XXXXXX")"rm -f "$MOCK_STATE_FILE"-trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"; rm -rf "$MOCK_PROFILE_ROOT"' EXIT+trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"' EXITpass() { echo "PASS: $1"; }fail() { echo "FAIL: $1"; FAIL=1; }-for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; do+for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; doif bash -n "$script"; thenpass "$(basename "$script") has valid bash syntax"else@@ -41,1282 +23,12 @@ for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; dofidone-# A node carrying a minikube label from another profile must not satisfy the-# destructive-gate boundary. Each gate binds the label value to the explicit-# context instead of accepting any non-null label.-matching_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"branch-profile"}}}]}'-mismatched_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}}}]}'-if jq -e --arg context branch-profile \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \- <<<"$matching_nodes" >/dev/null &&- ! jq -e --arg context branch-profile \- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \- <<<"$mismatched_nodes" >/dev/null &&- [ "$(grep -Fl 'minikube.k8s.io/name"] == $context' "${GATES[@]}" | wc -l | tr -d ' ')" = 3 ] &&- ! grep -Fq 'minikube.k8s.io/name"] != null' "${GATES[@]}"; then- pass "all destructive HCC gates bind the minikube node label to the explicit profile"-else- fail "an HCC gate can accept a minikube node owned by another profile"-fi--# Literal source-code assertions.-# shellcheck disable=SC2016-if [[ "$WATCH_GATE_TARGET" == *'E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"'* ]] &&- [[ "$WATCH_GATE_TARGET" == *'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)'* ]]; then- pass "CommunicationChannel gate target propagates exact pre-gate and profile ownership"-else- fail "CommunicationChannel gate target omits exact pre-gate or profile ownership"-fi--proxy_rollout_function="$(- sed -n '/^print_hcc_proxy_rollout_diagnostics() {$/,/^}$/p' "$WATCH_GATE"-)"-# Literal source-code assertions.-# shellcheck disable=SC2016-if [[ "$proxy_rollout_function" == *'kctl get pods'* ]] &&- [[ "$proxy_rollout_function" == *'--previous --tail=120'* ]] &&- [[ "$proxy_rollout_function" == *'status.containerStatuses'* ]] &&- [[ "$proxy_rollout_function" == *'// "unknown"'* ]] &&- [[ "$proxy_rollout_function" != *'\\"unknown\\"'* ]] &&- grep -Fq 'if ! kctl rollout status deployment "$HCC_DEPLOY"' "$WATCH_GATE" &&- [ "$(grep -Fc 'print_hcc_proxy_rollout_diagnostics' "$WATCH_GATE")" = 2 ]; then- pass "proxy rollout failure emits bounded HCC pod and previous-container diagnostics"-else- fail "proxy rollout failure can discard the distinguishing HCC startup evidence"-fi--if (- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- if [ "$1" = get ] && [ "$2" = pods ] && [[ "$*" == *"-o jsonpath="* ]]; then- printf '%s\n' hcc-pod- elif [ "$1" = get ] && [ "$2" = pods ]; then- printf '%s\n' '{"items":[{"metadata":{"name":"hcc-pod"},"status":{"phase":"Pending","containerStatuses":[{"name":"host-context-controller","ready":false,"restartCount":2,"state":{"waiting":{"reason":"CrashLoopBackOff"}},"lastState":{}}]}}]}'- elif [ "$1" = logs ]; then- printf '%s\n' 'bounded diagnostic log'- else- return 1- fi- }- eval "$proxy_rollout_function"- output="$(print_hcc_proxy_rollout_diagnostics 2>&1)"- [[ "$output" == *"pod=hcc-pod phase=Pending"* ]] &&- [[ "$output" != *"pod=null"* ]] &&- [[ "$output" != *"phase=null"* ]]-); then- pass "proxy rollout diagnostics retain the parent pod identity and phase"-else- fail "proxy rollout diagnostics lose pod metadata inside container status iteration"-fi--# The proxy changes only the HCC client's target hostname. The HCC's-# allow-k8s-api-egress policies must retain the Kubernetes-injected API IP,-# which is the explicit fail-closed configuration seam for CIDR policy data.-# shellcheck disable=SC2016-if grep -Fq 'K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY"' "$WATCH_GATE" &&- grep -Fq 'K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"' "$WATCH_GATE" &&- grep -Fq -- '--arg api_cidr "$K8S_API_CIDR"' "$WATCH_GATE" &&- grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr' "$WATCH_GATE" &&- grep -Fq 'allow-k8s-api-egress-${MCP_NS}' "$WATCH_GATE" &&- grep -Fq '[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ]' "$WATCH_GATE" &&- grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS-' "$FIXTURE_HELPER"; then- pass "proxy fault injection preserves an explicit valid API egress CIDR and restores it"-else- fail "proxy fault injection can turn a DNS hostname into an invalid API egress CIDR"-fi--if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' "$READINESS_GATE" &&- [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$READINESS_GATE")" = 1 ] &&- ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$READINESS_GATE" &&- ! grep -Fq 'cluster_fingerprint_file=' "$READINESS_GATE" &&- ! grep -Fq 'profile_env=' "$READINESS_GATE"; then- pass "readiness gate delegates ownership, HEAD, profile, fingerprint, and gate proof once"-else- fail "readiness gate duplicates, bypasses, or splits the shared branch-owned proof"-fi--bootstrap_repair_function="$(- sed -n '/^print_repair_instructions() {$/,/^}$/p' "$READINESS_GATE"-)"-mcp_repair_function="$(- sed -n '/^print_repair_instructions() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-# Literal source-code assertions: every fail-closed repair path must restore-# the captured replica count before treating rollout health as repair proof.-# shellcheck disable=SC2016-if [[ "$bootstrap_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&- [[ "$bootstrap_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=180s'* ]] &&- [[ "$mcp_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&- [[ "$mcp_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=240s'* ]] &&- grep -Fq 'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}' "$WATCH_GATE" &&- grep -Fq 'rollout status deployment/${HCC_DEPLOY} --timeout=180s' "$WATCH_GATE"; then- pass "all HCC repair instructions restore the original replica count and verify rollout"-else- fail "an HCC repair path can leave the controller scaled to zero"-fi--fixture_gate_function="$(sed -n '/^require_branch_owned_hcc_gate() {$/,/^}$/p' "$FIXTURE_HELPER")"-marker_get_count="$(- grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' <<<"$fixture_gate_function"-)"-# Literal source-code assertions.-# shellcheck disable=SC2016-if [ "$marker_get_count" = 1 ] &&- [[ "$fixture_gate_function" == *'-o json'* ]] &&- [[ "$fixture_gate_function" == *'.data.worktreeId'* ]] &&- [[ "$fixture_gate_function" == *'.data.gitHead'* ]] &&- [[ "$fixture_gate_function" == *'<<<"$HCC_BRANCH_GATE_SYNC_MARKER"'* ]]; then- pass "shared branch gate derives ownership and HEAD from one exported ConfigMap JSON snapshot"-else- fail "shared branch gate can split ownership and HEAD across marker reads"-fi--# Literal source-code assertions.-# shellcheck disable=SC2016-if [[ "$fixture_gate_function" == *'expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"'* ]] &&- [[ "$fixture_gate_function" == *'[ -n "$expected_branch" ]'* ]] &&- [[ "$fixture_gate_function" == *'[ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ]'* ]] &&- [[ "$fixture_gate_function" == *'profile_env="${E2E_BRANCH_PROFILE_ENV:-'* ]] &&- [[ "$fixture_gate_function" == *'profile_env_value PROFILE'* ]] &&- [[ "$fixture_gate_function" == *'profile_env_value REPO_DIR'* ]] &&- [[ "$fixture_gate_function" == *'profile_env_value BRANCH'* ]] &&- [[ "$fixture_gate_function" != *'profile_env_value SHA_SHORT'* ]] &&- [[ "$fixture_gate_function" == *'profile_env_value DIRTY'* ]] &&- [[ "$fixture_gate_function" == *'cluster_fingerprint_file='* ]] &&- [[ "$fixture_gate_function" == *'.data.clusterFingerprint'* ]] &&- [[ "$fixture_gate_function" == *'.data.gate'* ]] &&- [[ "$fixture_gate_function" == *'E2E_EXPECTED_PRE_GATE_GATE'* ]]; then- pass "shared branch gate owns the complete profile, fingerprint, and expected-gate proof"-else- fail "shared branch gate omits a required profile, fingerprint, or expected-gate boundary"-fi--for gate in "${GATES[@]}"; do- marker_reads="$(grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' "$gate")"- if [ "$marker_reads" = 0 ] &&- ! grep -Fq 'profile_env=' "$gate" &&- ! grep -Fq 'cluster_fingerprint_file=' "$gate" &&- grep -Fq 'require_branch_owned_hcc_gate' "$gate"; then- pass "$(basename "$gate") delegates its branch-owned proof to the shared authority"- else- fail "$(basename "$gate") duplicates or bypasses the shared branch-owned proof"- fi-done--run_branch_gate_case() (- local mode=$1- local repo_root="${MOCK_PROFILE_ROOT}/repo"- local context="clerum-codex-profile-test-1234abcd"- local head="1111111111111111111111111111111111111111"- local short_head="11111111"- local branch="codex/profile-test"- local fingerprint="2222222222222222222222222222222222222222"- local marker_fingerprint="$fingerprint"- local expected_gate="full"- local marker_gate="$expected_gate"- local minikube_profile="$context"- local profile="$context"- local profile_repo="$repo_root"- local profile_branch="$branch"- local profile_sha="$short_head"- local profile_dirty=false- local mock_worktree_dirty=""- local worktree_id- worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"-- case "$mode" in- success) ;;- detached) branch="" ;;- minikube-profile-missing) minikube_profile="" ;;- minikube-profile) minikube_profile="clerum-other-profile-1234abcd" ;;- profile-name) profile="clerum-other-profile-1234abcd" ;;- profile-repo) profile_repo="${repo_root}-other" ;;- profile-branch) profile_branch="codex/other" ;;- reused-profile) profile_sha="aaaaaaaa" ;;- profile-dirty) profile_dirty=true ;;- dirty-worktree) mock_worktree_dirty=" M changed.ts" ;;- fingerprint) marker_fingerprint="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;- gate) marker_gate="targeted" ;;- *) exit 98 ;;- esac-- mkdir -p "${MOCK_PROFILE_ROOT}/state/${worktree_id}"- printf '%s\n' "$fingerprint" >"${MOCK_PROFILE_ROOT}/state/${worktree_id}/cluster.sha"- printf '%s\n' \- "PROFILE=${profile}" \- "REPO_DIR=${profile_repo}" \- "BRANCH=${profile_branch}" \- "SHA_SHORT=${profile_sha}" \- "DIRTY=${profile_dirty}" >"${MOCK_PROFILE_ROOT}/profile.env"-- E2E_KUBECONTEXT="$context"- MINIKUBE_PROFILE="$minikube_profile"- E2E_BRANCH_PROFILE_ENV="${MOCK_PROFILE_ROOT}/profile.env"- E2E_PRE_GATE_STATE_ROOT="${MOCK_PROFILE_ROOT}/state"- E2E_EXPECTED_PRE_GATE_GATE="$expected_gate"- HCC_NS=control-plane- SCRIPT_DIR="${ROOT}/scripts/e2e"- MOCK_MARKER="$(- jq -cn --arg worktreeId "$worktree_id" --arg gitHead "$head" \- --arg clusterFingerprint "$marker_fingerprint" --arg gate "$marker_gate" \- '{data:{worktreeId:$worktreeId,gitHead:$gitHead,- clusterFingerprint:$clusterFingerprint,gate:$gate}}'- )"-- is_branch_scoped_e2e_context() { return 0; }- die() { exit 97; }- git() {- local command=$3- shift 3- case "$command $*" in- "rev-parse --show-toplevel") printf '%s\n' "$repo_root" ;;- "status --porcelain --untracked-files=normal") printf '%s\n' "$mock_worktree_dirty" ;;- "rev-parse HEAD") printf '%s\n' "$head" ;;- "rev-parse --short=8 HEAD") printf '%s\n' "$short_head" ;;- "branch --show-current") printf '%s\n' "$branch" ;;- *) return 96 ;;- esac- }- kctl() { printf '%s\n' "$MOCK_MARKER"; }- # shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh- source "$FIXTURE_HELPER"- require_branch_owned_hcc_gate "$HCC_NS"-)--if run_branch_gate_case success; then- pass "shared branch gate accepts matching branch-profile evidence"-else- fail "shared branch gate rejects matching branch-profile evidence"-fi-if run_branch_gate_case reused-profile; then- pass "shared branch gate accepts a reused same-branch profile after an exact-head pre-gate sync"-else- fail "shared branch gate rejects a reused same-branch profile despite exact-head pre-gate evidence"-fi-for case_name in detached minikube-profile-missing minikube-profile profile-name profile-repo profile-branch \- profile-dirty dirty-worktree fingerprint gate; do- if run_branch_gate_case "$case_name" >/dev/null 2>&1; then- fail "shared branch gate accepted mismatched ${case_name} evidence"- else- pass "shared branch gate rejects mismatched ${case_name} evidence"- fi-done--retry_progress_function="$(- sed -n '/^external_egress_retry_progress_is_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-retry_query_function="$(- sed -n '/^dns_retry_query_observed_since_schedule() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-if (- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- CONTEXT_NAME=e2e-context- MOCK_HCC_LOG='-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout-[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms-'- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }- eval "$retry_progress_function"- external_egress_retry_progress_is_observed-); then- pass "DNS failure is attributed to the dedicated external-egress retry lane"-else- fail "the injected DNS hold cannot prove dedicated external-egress retry progress"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- CONTEXT_NAME=e2e-context- MOCK_HCC_LOG='-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout-[K8s] Scheduling external egress retry 1/3 for McpServer "different-server" in 5000ms-[K8s] unrelated message for McpServer "e2e-held-server"-'- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }- eval "$retry_progress_function"- ! external_egress_retry_progress_is_observed-); then- pass "external-egress retry evidence cannot combine a peer retry with the fixture server marker"-else- fail "external-egress retry evidence can be assembled from different log lines or servers"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- CONTEXT_NAME=e2e-context- MOCK_HCC_LOG='-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout-[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms-'- for ((line = 0; line < 4096; line++)); do- MOCK_HCC_LOG+=$'\n[K8s] unrelated high-volume log line'- done- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }- eval "$retry_progress_function"- external_egress_retry_progress_is_observed-); then- pass "an early matching retry line survives a high-volume log buffer without SIGPIPE ambiguity"-else- fail "a valid early retry marker is lost in a high-volume log buffer"-fi-if (- DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3- dns_hold_count() { printf '%s\n' 4; }- eval "$retry_query_function"- dns_retry_query_observed_since_schedule-); then- pass "a post-schedule DNS query proves the dedicated retry executed"-else- fail "a fresh DNS query after scheduling does not prove retry execution"-fi-if (- DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3- dns_hold_count() { printf '%s\n' 3; }- eval "$retry_query_function"- ! dns_retry_query_observed_since_schedule-); then- pass "a pre-schedule DNS query cannot satisfy retry execution"-else- fail "stale DNS evidence can masquerade as retry execution"-fi-if grep -Fq '{name:"timeout",value:"30"}' "$MCP_READINESS_GATE" &&- grep -Fq '{name:"attempts",value:"1"}' "$MCP_READINESS_GATE"; then- pass "the DNS fixture disables resolver retransmits that could counterfeit retry execution"-else- fail "the DNS fixture can mistake a resolver retransmit for a dedicated retry"-fi--# Full marker inventory: every runtime log marker any HCC gate consumes, paired-# with the producer statement that emits it. Both sides are matched as fixed-# source text, never as a rendered string: a template literal is pinned by its-# fixed prefix plus the variable that supplies the dynamic identity, so a-# marker that only ever exists rendered (`${config.hostNamespace}`,-# `${reason}`, `${type}`, `${snapshot.channels.length}`) still resolves without-# reporting false drift. Matching the consumer side too means a rename on-# either side breaks the pair instead of silently disarming a live gate: the-# marker no longer matches any log line, the count stays zero, and the gate-# passes on evidence it never observed.-# Entries are flat 4-tuples: consumer file, consumer text, producer file,-# producer text.-# shellcheck disable=SC2016-marker_bindings=(- "$WATCH_GATE" 'CommunicationChannel watch ended;'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'-- "$WATCH_GATE" 'Starting CommunicationChannel watch'- "$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'-- "$WATCH_GATE" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'- "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'-- "$WATCH_GATE" 'cache recovery failed;'- "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'-- "$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'- "$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'-- "$WATCH_GATE" 'Listing all Hosts in namespace ${HOST_NS}'- "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'-- "$WATCH_GATE" 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'-- "$WATCH_GATE" 'CommunicationChannel watch event: DELETED for ${FIXTURE_CHANNEL}'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'-- "$WATCH_GATE" 'CommunicationChannel watch event: ADDED for ${FIXTURE_CHANNEL}'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'-- "$LOG_HELPER" 'CommunicationChannel watch ended;'- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'-- "$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'- "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'-- "$LOG_HELPER" 'Listing all Hosts in namespace '- "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'-- "$LOG_HELPER" 'Host(s) for lifecycle after CommunicationChannel recovery'- "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) for lifecycle after ${reason}'-- "$LOG_HELPER" 'Host(s) after Host watch recovery convergence'- "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) after ${reason}'-- "$LOG_HELPER" 'Completed Host reconciliation after CommunicationChannel recovery$'- "$HCC_K8S_CLIENT" "requestHostFleetReconcile('CommunicationChannel recovery'"-- "$LOG_HELPER" 'Completed Host reconciliation after Host watch recovery convergence$'- "$HCC_K8S_CLIENT" "convergenceReason = 'Host watch recovery convergence'"-- "$LOG_HELPER" 'Periodic resync'- "$HCC_K8S_CLIENT" "requestHostFleetReconcile('Periodic resync'"-- "$READINESS_GATE" "START_MARKER='Starting initial Host background convergence'"- "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'-- "$READINESS_GATE" "COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'"- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'-- "$READINESS_GATE" "FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'"- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'-- "$READINESS_GATE" 'after initial Host reconciliation'- "$HCC_K8S_CLIENT" "'initial Host reconciliation',"-- "$HOST_BUNDLE_MEASURE" "HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'"- "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'-- "$HOST_BUNDLE_MEASURE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'-- "$HOST_BUNDLE_MEASURE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'-- "$HOST_STORM_GATE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'-- "$HOST_STORM_GATE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'-- "$MCP_READINESS_GATE" 'Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};'- "$HCC_EGRESS_COORDINATOR" 'Initial external egress reconciliation failed for ${key};'-- "$MCP_READINESS_GATE" 'Scheduling external egress retry '- "$HCC_EGRESS_COORDINATOR" 'Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}'-- "$MCP_READINESS_GATE" 'for McpServer \"${MCP_NAME}\"'- "$HCC_EGRESS_COORDINATOR" 'for McpServer "${this.retryIntents.get(key)!.server.name}"'-- "$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'- "$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'-- "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'- "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('-- "$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'"- "$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'-- "$MCP_READINESS_GATE" "NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'"- "$HCC_K8S_CLIENT" 'Initial NetworkPolicy post-certification additive reconciliation failed:'-- "$MCP_READINESS_GATE" "NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'"- "$HCC_K8S_CLIENT" 'Scheduling initial ${lane} background convergence retry'-- "$MCP_READINESS_GATE" "NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '"- "$HCC_K8S_CLIENT" 'NetworkPolicy reconciliation failed for context ${context.name}:'-)-marker_drift=""-marker_binding_count=0-for ((marker_index = 0; marker_index < ${#marker_bindings[@]}; marker_index += 4)); do- consumer_file="${marker_bindings[marker_index]}"- consumer_marker="${marker_bindings[marker_index + 1]}"- producer_file="${marker_bindings[marker_index + 2]}"- producer_marker="${marker_bindings[marker_index + 3]}"- marker_binding_count=$((marker_binding_count + 1))- # `grep -Fq -- ""` matches every file, so an empty marker would disarm this- # pair while still reporting a pass. Refuse it instead of searching for it.- if [ -z "$consumer_marker" ] || [ -z "$producer_marker" ]; then- marker_drift+=" marker pair at index ${marker_index} has an empty marker"$'\n'- continue- fi- if ! grep -Fq -- "$consumer_marker" "$consumer_file"; then- marker_drift+=" consumer $(basename "$consumer_file") no longer reads: ${consumer_marker}"$'\n'- fi- if ! grep -Fq -- "$producer_marker" "$producer_file"; then- marker_drift+=" producer $(basename "$producer_file") no longer emits: ${producer_marker}"$'\n'- fi-done-# The stride loop consumes 4 elements per pair, so an array whose length is not-# a multiple of 4 silently shifts every entry after the edit and leaves the tail-# unset. Assert the arity explicitly rather than relying on the pair count to-# happen to notice.-if [ $((${#marker_bindings[@]} % 4)) -eq 0 ] &&- [ "$marker_binding_count" -eq 35 ] &&- [ -z "$marker_drift" ]; then- pass "all ${marker_binding_count} consumed HCC log markers stay bound to their runtime producers"-else- printf '%s' "$marker_drift" >&2- fail "a consumed HCC log marker drifted from its runtime producer (${marker_binding_count} pairs checked)"-fi--# Completeness fence: every static single-quoted *_MARKER defined in the e2e-# gates MUST be inventoried above as a consumer_marker, so a NEW marker cannot be-# born outside the drift fence (the pinned count only catches removals). The scan-# covers both the gates themselves and the _lib/ helpers they source — a marker-# born in a sourced helper is the same escape class. Scoped to single-quoted-# literals — a dynamic id like C_MARKER="refusal-c-${RUN_ID}" (double-quoted, a-# per-run test id, not a cross-file HCC log marker) is not the class this fence-# covers and is excluded by the quote style, not a hard-coded allow-list.-marker_is_inventoried() {- local needle="$1" i- for ((i = 1; i < ${#marker_bindings[@]}; i += 4)); do- [ "${marker_bindings[i]}" = "$needle" ] && return 0- done- return 1-}-uninventoried_markers=""-while IFS= read -r marker_def; do- [ -n "$marker_def" ] || continue- marker_is_inventoried "$marker_def" ||- uninventoried_markers+=" ${marker_def}"$'\n'-done < <(- grep -rhoE "^[[:space:]]*(readonly[[:space:]]+)?[A-Z_]+_MARKER='[^']*'" \- "${SCRIPT_DIR}"/*.sh "${SCRIPT_DIR}"/_lib/*.sh |- sed -E "s/^[[:space:]]*(readonly[[:space:]]+)?//"-)-if [ -z "$uninventoried_markers" ]; then- pass "every single-quoted *_MARKER defined in the e2e gates is inventoried in the drift fence"-else- printf '%s' "$uninventoried_markers" >&2- fail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"-fi--fixture_mcp_runtime_absent_function="$(- sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-hcc_kubernetes_readiness_function="$(- sed -n '/^hcc_kubernetes_readiness_is_exact() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-external_policy_protocol_function="$(- sed -n '/^external_egress_policy_converged_with_protocol() {$/,/^}$/p' \- "$MCP_READINESS_GATE"-)"-hcc_ready_after_revoke_function="$(- sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-ready_probe_function="$(- sed -n '/^probe_new_hcc_ready_endpoint() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"--run_ready_probe_failure_case() (- local expected=$1- MOCK_READY_PROBE_OUTPUT=$2- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- HCC_PORT=8080- HCC_READY_PROBE_DIAGNOSTIC=""- kctl() {- printf '%s\n' "$MOCK_READY_PROBE_OUTPUT"- return 1- }- eval "$ready_probe_function"- ! probe_new_hcc_ready_endpoint &&- [ "$HCC_READY_PROBE_DIAGNOSTIC" = "HCC_READY_PROBE category=${expected}" ]-)--if run_ready_probe_failure_case timeout 'HCC_READY_PROBE category=timeout' &&- run_ready_probe_failure_case transport 'HCC_READY_PROBE category=transport' &&- run_ready_probe_failure_case 'http-status status=503' \- 'HCC_READY_PROBE category=http-status status=503' &&- run_ready_probe_failure_case json-ready-false \- 'HCC_READY_PROBE category=json-ready-false'; then- pass "ready probe preserves distinct timeout, transport, HTTP, and ready=false diagnostics"-else- fail "ready probe collapses a failure boundary into an indiagnostic boolean"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- HCC_PORT=8080- HCC_READY_PROBE_DIAGNOSTIC="stale"- kctl() { return 0; }- eval "$ready_probe_function"- probe_new_hcc_ready_endpoint && [ -z "$HCC_READY_PROBE_DIAGNOSTIC" ]-); then- pass "successful ready probe clears stale failure diagnostics"-else- fail "successful ready probe retains a misleading prior diagnostic"-fi--# H6 gate-fix mutation (E4): the clean-window NetworkPolicy negative must FAIL-# against an injection-window log (moving it inside the injected window would-# false-fail — the scoping is load-bearing), PASS against a clean log, and fail-# closed when kubectl errors. The gate itself can't run (minikube churn), so-# this statically pins the predicate.-clean_window_netpol_negatives_function="$(- sed -n '/^clean_window_netpol_log_negatives_hold() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-clean_window_marker_defs="$(grep -E "^NETPOL_[A-Z_]+_MARKER='" "$MCP_READINESS_GATE")"-run_clean_window_negative_case() (- local mode=$1- CONTEXT_NAME=alpha- HCC_NS=control-plane- eval "$clean_window_marker_defs"- case $mode in- injected)- kctl() {- printf '%s\n' \- '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed: dns timeout' \- '[K8s] Scheduling initial NetworkPolicy background convergence retry 1 in 5000ms'- } ;;- clean) kctl() { printf '%s\n' '[K8s] NetworkPolicy convergence certified'; } ;;- api-error) kctl() { return 1; } ;;- esac- eval "$clean_window_netpol_negatives_function"- clean_window_netpol_log_negatives_hold hcc-pod-)-if ! run_clean_window_negative_case injected &&- run_clean_window_negative_case clean &&- ! run_clean_window_negative_case api-error; then- pass "clean-window NetworkPolicy negative fails inside the injected window, passes clean, and fails closed"-else- fail "clean-window NetworkPolicy negative does not distinguish injected-window failures or fails open"-fi--# Static ordering fence: a clean-window negative invocation must exist in the-# MAIN BODY (after `trap cleanup EXIT`) and before the DNS mutation. cleanup()-# is defined earlier in the file than the mutation but runs on EXIT, so a plain-# "first line < mutation line" test would be satisfied by the cleanup-window-# invocation alone; excluding everything at or before the trap line pins the-# pre-injection (window A) invocation specifically.-trap_line="$(grep -n '^trap cleanup EXIT' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"-dns_mutation_line="$(grep -n '^HCC_MUTATED=1' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"-body_clean_window_negative_line="$(- grep -n 'clean_window_netpol_log_negatives_hold "' "$MCP_READINESS_GATE" |- awk -F: -v t="${trap_line:-0}" -v m="${dns_mutation_line:-0}" \- '$1 > t && $1 < m { print $1; exit }'-)"-if [ -n "$body_clean_window_negative_line" ]; then- pass "clean-window NetworkPolicy negative runs in the body before the DNS fault injection"-else- fail "clean-window NetworkPolicy negative is missing from the pre-injection body window"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_NS=control-plane- HCC_PORT=8080- HCC_READY_PROBE_DIAGNOSTIC=""- kctl() {- printf '%s\n' 'untrusted kubectl output containing super-secret material'- return 1- }- eval "$ready_probe_function"- ! probe_new_hcc_ready_endpoint &&- [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=kubectl-exec' ]-); then- pass "ready probe replaces untrusted exec output with a bounded sanitized category"-else- fail "ready probe can leak untrusted exec output into diagnostics"-fi-# Literal source-code assertions guarantee one request and bounded categories,-# without logging the response body, headers, URL, or transport error text.-if [ "$(grep -Fc 'kctl exec' <<<"$ready_probe_function")" = 1 ] &&- [[ "$ready_probe_function" == *'finish(4, "timeout")'* ]] &&- [[ "$ready_probe_function" == *'finish(5, "transport")'* ]] &&- [[ "$ready_probe_function" == *'finish(3, "http-status"'* ]] &&- [[ "$ready_probe_function" == *'finish(3, "json-ready-false")'* ]] &&- [[ "$ready_probe_function" == *"printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s'"* ]] &&- [[ "$ready_probe_function" != *'console.error(body'* ]] &&- [[ "$ready_probe_function" != *'error.message'* ]]; then- pass "ready probe uses one request and emits only bounded non-secret failure categories"-else- fail "ready probe duplicates requests or exposes an unbounded diagnostic surface"-fi--if (- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- MCP_SERVER_LABEL=clerum.io/mcp-server- POLICY_TYPE_LABEL=clerum.io/policy-type- fixture_runtime_absent() { return 0; }- kctl() { return 0; }- eval "$fixture_mcp_runtime_absent_function"- fixture_mcp_runtime_absent-); then- pass "fixture absence predicate accepts an absent runtime and external-egress policy"-else- fail "fixture absence predicate rejects an actually absent runtime"-fi-if (- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- MCP_SERVER_LABEL=clerum.io/mcp-server- POLICY_TYPE_LABEL=clerum.io/policy-type- fixture_runtime_absent() { return 0; }- kctl() { printf '%s\n' 'networkpolicy.networking.k8s.io/stale-policy'; }- eval "$fixture_mcp_runtime_absent_function"- ! fixture_mcp_runtime_absent-); then- pass "fixture absence predicate rejects a surviving external-egress policy"-else- fail "fixture absence predicate can hide a surviving external-egress policy"-fi-if (- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- MCP_SERVER_LABEL=clerum.io/mcp-server- POLICY_TYPE_LABEL=clerum.io/policy-type- fixture_runtime_absent() { return 1; }- kctl() { return 0; }- eval "$fixture_mcp_runtime_absent_function"- ! fixture_mcp_runtime_absent-); then- pass "fixture absence predicate rejects a surviving MCP runtime"-else- fail "fixture absence predicate can hide a surviving MCP runtime"-fi--if (- NEW_HCC_POD=hcc-pod- HCC_UID=pod-uid- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- case "$2" in- pod)- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'- ;;- deployment)- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'- ;;- *) return 1 ;;- esac- }- eval "$hcc_kubernetes_readiness_function"- hcc_kubernetes_readiness_is_exact-); then- pass "exact Kubernetes readiness accepts one current Ready HCC identity"-else- fail "exact Kubernetes readiness rejects current Ready HCC evidence"-fi-# Kubelet reports an indeterminate pod as Ready=Unknown, not Ready=False. Only-# an Unknown fixture separates the exact `== "True"` readiness contract from a-# weakened `!= "False"` one, so this red case pins the exact comparison instead-# of the single status value both spellings already reject. Every other field is-# healthy, which leaves the Ready condition as the only reason to refuse.-if (- NEW_HCC_POD=hcc-pod- HCC_UID=pod-uid- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- case "$2" in- pod)- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'- ;;- deployment)- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'- ;;- *) return 1 ;;- esac- }- eval "$hcc_kubernetes_readiness_function"- ! hcc_kubernetes_readiness_is_exact-); then- pass "exact Kubernetes readiness rejects a current pod whose Ready condition is Unknown"-else- fail "exact Kubernetes readiness can accept an indeterminate HCC pod"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_UID=pod-uid- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- case "$2" in- pod)- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"False"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'- ;;- deployment)- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'- ;;- *) return 1 ;;- esac- }- eval "$hcc_kubernetes_readiness_function"- ! hcc_kubernetes_readiness_is_exact-); then- pass "exact Kubernetes readiness rejects a current pod whose Ready condition is False"-else- fail "exact Kubernetes readiness can accept a non-Ready HCC pod"-fi-# The container-level readiness clause has its own red case: a pod whose Ready-# condition is True while the host-context-controller container is not ready-# must still be refused, so deleting or weakening that clause cannot pass.-if (- NEW_HCC_POD=hcc-pod- HCC_UID=pod-uid- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- case "$2" in- pod)- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":false}]}}'- ;;- deployment)- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'- ;;- *) return 1 ;;- esac- }- eval "$hcc_kubernetes_readiness_function"- ! hcc_kubernetes_readiness_is_exact-); then- pass "exact Kubernetes readiness rejects a Ready pod whose HCC container is not ready"-else- fail "exact Kubernetes readiness can accept an unready host-context-controller container"-fi-if (- NEW_HCC_POD=hcc-pod- HCC_UID=pod-uid- HCC_NS=control-plane- HCC_DEPLOY='host-context-controller'- kctl() {- case "$2" in- pod)- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'- ;;- deployment)- printf '%s\n' '{"metadata":{"generation":5},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'- ;;- *) return 1 ;;- esac- }- eval "$hcc_kubernetes_readiness_function"- ! hcc_kubernetes_readiness_is_exact-); then- pass "exact Kubernetes readiness rejects a stale Deployment generation"-else- fail "exact Kubernetes readiness can accept stale Deployment status"-fi--if (- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- MCP_SERVER_LABEL=clerum.io/mcp-server- POLICY_TYPE_LABEL=clerum.io/policy-type- MANAGED_BY_LABEL=app.kubernetes.io/managed-by- MANAGED_BY_VALUE='host-context-controller'- EXTERNAL_EGRESS_CIDR=203.0.113.10/32- EXTERNAL_EGRESS_PORT=443- kctl() {- printf '%s\n' '{"items":[{"metadata":{"labels":{"app.kubernetes.io/managed-by":"host-context-controller","clerum.io/policy-type":"external-egress","clerum.io/mcp-server":"e2e-held-server"}},"spec":{"podSelector":{"matchLabels":{"clerum.io/mcp-server":"e2e-held-server"}},"policyTypes":["Egress"],"egress":[{"ports":[{"port":443,"protocol":"UDP"}],"to":[{"ipBlock":{"cidr":"203.0.113.10/32"}}]}]}}]}'- }- eval "$external_policy_protocol_function"- external_egress_policy_converged_with_protocol UDP &&- ! external_egress_policy_converged_with_protocol TCP-); then- pass "external-egress convergence binds the exact current protocol"-else- fail "external-egress convergence can accept a stale protocol"-fi--mcpserver_current_ready_function="$(- sed -n '/^mcpserver_current_ready() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-mcpserver_current_ready_absent_function="$(- sed -n '/^mcpserver_current_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-fixture_runtime_and_ready_absent_function="$(- sed -n '/^fixture_runtime_and_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-fixture_runtime_converged_function="$(- sed -n '/^fixture_runtime_converged_with_protocol() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-peer_fleet_converged_function="$(- sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-fixture_converged_function="$(- sed -n '/^fixture_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-if (- MCP_NS=mcp-server- GENERATION=7- OBSERVED_GENERATION=7- kctl() {- printf '{"metadata":{"generation":%s},"status":{"conditions":[{"type":"Ready","status":"True","observedGeneration":%s}]}}\n' \- "$GENERATION" "$OBSERVED_GENERATION"- }- eval "$mcpserver_current_ready_function"- eval "$mcpserver_current_ready_absent_function"- mcpserver_current_ready current &&- ! mcpserver_current_ready_absent current &&- OBSERVED_GENERATION=6 &&- ! mcpserver_current_ready stale &&- mcpserver_current_ready_absent stale-) &&- [[ "$fixture_runtime_and_ready_absent_function" == *'mcpserver_current_ready_absent "$MCP_NAME"'* ]] &&- [[ "$fixture_runtime_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]] &&- [[ "$peer_fleet_converged_function" == *'mcpserver_current_ready "$server"'* ]] &&- [[ "$fixture_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]]; then- pass "McpServer runtime predicates accept only Ready status observed for the current generation"-else- fail "McpServer runtime predicates can accept stale Ready status or bypass the canonical check"-fi--if (- MCP_NS=mcp-server- MCP_NAME=e2e-held-server- fixture_runtime_absent() { return 0; }- kctl() { return 1; }- eval "$mcpserver_current_ready_absent_function"- eval "$fixture_runtime_and_ready_absent_function"- ! fixture_runtime_and_ready_absent-); then- pass "McpServer Ready absence fails closed when the API read is unavailable"-else- fail "an unavailable McpServer API read can masquerade as Ready absence"-fi--if (- probe_new_hcc_ready_endpoint() { return 0; }- external_egress_policy_absent() { return 0; }- eval "$hcc_ready_after_revoke_function"- hcc_ready_after_stale_policy_revoked-); then- pass "divergent DNS hold permits global readiness only after stale-policy revocation"-else- fail "divergent DNS hold rejects globally safe readiness evidence"-fi-if (- HCC_READY_PROBE_DIAGNOSTIC=""- probe_new_hcc_ready_endpoint() { return 0; }- external_egress_policy_absent() { return 1; }- eval "$hcc_ready_after_revoke_function"- rc=0- hcc_ready_after_stale_policy_revoked || rc=$?- # A 200 observed WITH the stale policy present is a fatal ordering violation- # (rc=2), not a benign retryable "not ready" (rc=1).- [ "$rc" -eq 2 ] &&- [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=ready-200-with-stale-policy-present' ]-); then- pass "global readiness evidence flags a surviving divergent policy as a fatal rc=2 ordering violation"-else- fail "global readiness can self-confirm (or only soft-fail) while a divergent policy survives"-fi-# The loop must PROPAGATE the fatal rc=2 immediately, not retry until a later-# clean poll turns the violation into a false PASS (the round-2 bug).-wait_until_fast_function="$(- sed -n '/^wait_until_fast() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-if (- HCC_READY_PROBE_DIAGNOSTIC=""- POLL=0- probe_new_hcc_ready_endpoint() { return 0; }- # Stale policy present for the first polls, revoked only later. The buggy- # retry loop would poll past the fatal first observation and PASS here.- external_egress_policy_absent() { POLL=$((POLL + 1)); [ "$POLL" -gt 4 ]; }- eval "$hcc_ready_after_revoke_function"- eval "$wait_until_fast_function"- rc=0- wait_until_fast 10 "rc=2 propagation" hcc_ready_after_stale_policy_revoked || rc=$?- # rc must be the fatal 2, and it must have stopped inside the first predicate- # poll (which calls external_egress_policy_absent twice) — POLL<3 proves no- # retry occurred.- [ "$rc" -eq 2 ] && [ "$POLL" -lt 3 ]-); then- pass "wait_until_fast propagates a fatal rc=2 on the first poll instead of retrying to a false PASS"-else- fail "wait_until_fast retried past a fatal ordering violation to a later clean poll"-fi--# M6: the third contract signal adjudicates a real /metrics scrape. Extract the-# PURE predicate (the wrapper does kctl exec, untestable here) and feed it-# synthetic scrapes. The family-name literals are exactly what a metric rename-# would break — a rename leaves the vi.fn() unit tests green but must turn this-# and the real gate red.-safety_pass_metrics_predicate="$(- sed -n '/^safety_pass_metrics_are_certified() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-[ -n "$safety_pass_metrics_predicate" ] ||- fail "could not extract safety_pass_metrics_are_certified from the readiness gate"-metrics_with_revocation='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="listed"} 5-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 1'-metrics_without_family='clerum_hcc_some_other_family_total 3'-metrics_zero_revoked='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 0'-if (- eval "$safety_pass_metrics_predicate"- safety_pass_metrics_are_certified "$metrics_with_revocation"-); then- pass "safety_pass metrics predicate certifies a scrape with >=1 revoked policy"-else- fail "safety_pass metrics predicate rejected a valid certifying scrape"-fi-if (- eval "$safety_pass_metrics_predicate"- ! safety_pass_metrics_are_certified "$metrics_without_family"-); then- pass "safety_pass metrics predicate rejects a scrape missing the duration family (rename guard)"-else- fail "safety_pass metrics predicate passed with the safety_pass family absent"-fi-if (- eval "$safety_pass_metrics_predicate"- ! safety_pass_metrics_are_certified "$metrics_zero_revoked"-); then- pass "safety_pass metrics predicate rejects a scrape with zero revoked policies"-else- fail "safety_pass metrics predicate passed with zero policies revoked"-fi-if (- probe_new_hcc_ready_endpoint() { return 1; }- external_egress_policy_absent() { return 0; }- eval "$hcc_ready_after_revoke_function"- ! hcc_ready_after_stale_policy_revoked-); then- pass "stale-policy revocation alone cannot substitute for global readiness"-else- fail "divergent hold evidence can pass without HCC readiness"-fi--baseline_policy_snapshot_function="$(- sed -n '/^baseline_policy_snapshot() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-baseline_policies_unchanged_function="$(- sed -n '/^baseline_policies_unchanged() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-valid_baseline_policies='{"items":[- {"metadata":{"name":"deny-all-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"default-deny"}},"spec":{"podSelector":{},"policyTypes":["Ingress","Egress"]}},- {"metadata":{"name":"allow-dns-egress-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"infrastructure"}},"spec":{"podSelector":{},"policyTypes":["Egress"],"egress":[{"ports":[{"port":53,"protocol":"UDP"},{"port":53,"protocol":"TCP"}]}]}},- {"metadata":{"name":"allow-host-context-controller-api","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"allow-api"}},"spec":{"podSelector":{"matchLabels":{"app":"host-context-controller"}},"policyTypes":["Ingress"]}}-]}'-if (- MCP_NS=mcp-server- MANAGED_BY_VALUE='host-context-controller'- MOCK_POLICY_JSON="$valid_baseline_policies"- kctl() { printf '%s\n' "$MOCK_POLICY_JSON"; }- eval "$baseline_policy_snapshot_function"- eval "$baseline_policies_unchanged_function"- BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)"- baseline_policies_unchanged &&- MOCK_POLICY_JSON="$(jq -c '- .items[0].metadata.labels["clerum.io/managed-by"] = "foreign-controller"- ' <<<"$valid_baseline_policies")" &&- ! baseline_policies_unchanged &&- MOCK_POLICY_JSON="$(jq -c '.items[1].spec.egress[0].ports[0].port = 5353' \- <<<"$valid_baseline_policies")" &&- ! baseline_policies_unchanged-); then- pass "baseline safety proof rejects ownership and spec drift under stable policy names"-else- fail "baseline safety proof can pass after ownership or spec drift"-fi--peer_fleet_policies_absent_function="$(- sed -n '/^peer_fleet_policies_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-fixture_resources_absent_function="$(- sed -n '/^fixture_resources_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-delete_peer_fleet_runtime_function="$(- sed -n '/^delete_peer_fleet_runtime() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-if (- MCP_NS=mcp-server- HOST_NS=mcp-host- RPC_PROXY_NS=rpc-proxy- MCP_SERVER_LABEL=clerum.io/mcpserver- POLICY_TYPE_LABEL=clerum.io/policy-type- MCP_FLEET_NAMES=(primary peer)- CONTEXT_FLEET_NAMES=(primary-context peer-context)- SURVIVOR_NS=""- resource_absent() { [ "$3" != "$SURVIVOR_NS" ]; }- kctl() { return 0; }- eval "$peer_fleet_policies_absent_function"- peer_fleet_policies_absent &&- SURVIVOR_NS=mcp-host &&- ! peer_fleet_policies_absent &&- SURVIVOR_NS=rpc-proxy &&- ! peer_fleet_policies_absent-) &&- [[ "$fixture_resources_absent_function" == *'peer_fleet_policies_absent'* ]] &&- [[ "$delete_peer_fleet_runtime_function" == *'"${context_policy}-egress"'* ]] &&- [[ "$delete_peer_fleet_runtime_function" == *'"rpc-egress-${context}-${server}"'* ]]; then- pass "peer fleet cleanup rejects surviving mcp-host or rpc-proxy policies"-else- fail "peer fleet cleanup can release ownership with residual derived policies"-fi-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.shsource "$LOG_HELPER"HCC_LOG_BUFFER="$MOCK_LOG_FILE"START_TIME=2026-07-14T00:00:00ZHOST_NS=mcp-host-blocker_holds_function="$(sed -n '/^blocker_holds_token_request() {$/,/^}$/p' "$READINESS_GATE")"-blocker_count_function="$(sed -n '/^blocker_fixture_request_count() {$/,/^}$/p' "$READINESS_GATE")"-if (- BLOCKER_NAME=readiness-blocker- HCC_NS=control-plane- FIXTURE_HOST_PREFIX=e2e-hcc-ready-test- TOKEN_REQUEST_LOG_PREFIX='holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host='- kctl() { cat "$MOCK_LOG_FILE"; }- eval "$blocker_holds_function"- eval "$blocker_count_function"- printf '%s\n' \- 'holding-token GET /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-01' \- 'holding-token POST /wrong/path host=e2e-hcc-ready-test-02' \- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-04' \- >"$MOCK_LOG_FILE"- [ "$(blocker_fixture_request_count)" = 2 ] &&- blocker_holds_token_request-); then- pass "blocker evidence counts unique fixtures only on the canonical token issuance method and path"-else- fail "wrong-method, wrong-path, or duplicate blocker traffic can satisfy the fleet evidence"-fi--create_fixtures_function="$(sed -n '/^create_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"-if (- RUN_ID=fixture-failure- SUITE_NAME=hcc-readiness-bootstrap- HOST_NS=mcp-host- MCP_NS=mcp-server- FIXTURE_SECRET=e2e-secret- FIXTURE_CONTEXT=e2e-context- FIXTURE_HOST_PREFIX=e2e-host- FIXTURE_HOST_COUNT=4- FIXTURE_HOST_NAMES=()- FIXTURES_CREATED=0- KCTL_APPLY_COUNT=0- kctl() {- KCTL_APPLY_COUNT=$((KCTL_APPLY_COUNT + 1))- cat >/dev/null- [ "$KCTL_APPLY_COUNT" -ne 3 ]- }- eval "$create_fixtures_function"- if create_host_fixtures; then- exit 1- fi- [ "$KCTL_APPLY_COUNT" = 3 ]-); then- pass "an intermediate Host apply failure aborts fixture creation even from an OR-list"-else- fail "fixture creation can continue or report success after an intermediate apply failure"-fi--bootstrap_runtime_absent_function="$(- sed -n '/^fixture_runtime_absent() {$/,/^}$/p' "$READINESS_GATE"-)"-bootstrap_delete_fixtures_function="$(- sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE"-)"-# Literal source-code assertions for macOS bash 3.2, where expanding an empty-# array under set -u terminates the shell before cleanup can release its lock.-# shellcheck disable=SC2016-if [[ "$bootstrap_runtime_absent_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0'* ]] &&- [[ "$bootstrap_delete_fixtures_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"'* ]]; then- pass "bootstrap cleanup guards empty fixture arrays before bash 3.2 expansion"-else- fail "bootstrap cleanup can abort on an empty fixture array before lock finalization"-fi--create_peer_fleet_function="$(- sed -n '/^create_peer_fleet() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-peer_flag_line="$(grep -nF ' PEER_FLEET_CREATED=1' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"-peer_loop_line="$(grep -nF ' for ((index = 1;' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"-if [ -n "$peer_flag_line" ] && [ -n "$peer_loop_line" ] &&- [ "$peer_flag_line" -lt "$peer_loop_line" ]; then- pass "peer fleet cleanup is armed before the first partial apply can fail"-else- fail "peer fleet cleanup is armed only after all applies complete"-fi--divergent_ready_function="$(- sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-divergent_ready_line="$(- grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-divergent_context_line="$(- grep -nF 'wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-divergent_release_line="$(- grep -nF 'release_dns_blocker ||' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1-)"-udp_policy_line="$(- grep -nF 'wait_until 120 "current UDP external-egress policy after DNS release"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-# shellcheck disable=SC2016-if [[ "$divergent_ready_function" == *'probe_new_hcc_ready_endpoint || return 1'* ]] &&- [[ "$divergent_ready_function" == *'external_egress_policy_absent'* ]] &&- [ -n "$divergent_ready_line" ] && [ -n "$divergent_context_line" ] &&- [ -n "$divergent_release_line" ] && [ -n "$udp_policy_line" ] &&- [ "$divergent_ready_line" -lt "$divergent_context_line" ] &&- [ "$divergent_context_line" -lt "$divergent_release_line" ] &&- [ "$divergent_release_line" -lt "$udp_policy_line" ] &&- ! grep -Fq 'probe_new_hcc_unready_endpoint' "$MCP_READINESS_GATE" &&- ! grep -Fq '/ready is 503' "$MCP_READINESS_GATE"; then- pass "divergent DNS hold proves global readiness, fail-closed runtime isolation, and post-release UDP convergence"-else- fail "the MCP gate still couples global readiness to divergent DNS replacement"-fi--initial_tcp_policy_line="$(- grep -nF 'wait_until 180 "HCC to create the real initial TCP external-egress policy"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-# B3 retain-identity-stable contract: a same-intent restart RETAINS the-# identity-stable DNS-derived allow (no deny window), certifies /ready with it-# still present, keeps the affected runtime blocked, and converges the additive-# TCP lane only AFTER the DNS hold releases — the inverse of the pre-B3-# revoke-before-ready ordering this gate previously asserted.-same_intent_snapshot_line="$(- grep -nF 'SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)"' \- "$MCP_READINESS_GATE" | head -1 | cut -d: -f1-)"-same_intent_retained_ready_line="$(- grep -nF 'wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-same_intent_retain_guard_line="$(- grep -nF 'the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-same_intent_runtime_blocked_line="$(- grep -nF 'the affected runtime or Ready status escaped the held same-intent DNS boundary' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-same_intent_release_line="$(- grep -nF 'release_dns_blocker || die "could not release the same-intent DNS query"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-same_intent_tcp_policy_line="$(- grep -nF 'wait_until 120 "current TCP external-egress policy after same-intent DNS release"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-divergent_revoke_line="$(- grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \- "$MCP_READINESS_GATE" | cut -d: -f1-)"-if [ -n "$initial_tcp_policy_line" ] && [ -n "$same_intent_snapshot_line" ] &&- [ -n "$same_intent_retained_ready_line" ] && [ -n "$same_intent_retain_guard_line" ] &&- [ -n "$same_intent_runtime_blocked_line" ] && [ -n "$same_intent_release_line" ] &&- [ -n "$same_intent_tcp_policy_line" ] && [ -n "$divergent_revoke_line" ] &&- [ "$initial_tcp_policy_line" -lt "$same_intent_snapshot_line" ] &&- [ "$same_intent_snapshot_line" -lt "$same_intent_retained_ready_line" ] &&- [ "$same_intent_retained_ready_line" -lt "$same_intent_retain_guard_line" ] &&- [ "$same_intent_retain_guard_line" -lt "$same_intent_runtime_blocked_line" ] &&- [ "$same_intent_runtime_blocked_line" -lt "$same_intent_release_line" ] &&- [ "$same_intent_release_line" -lt "$same_intent_tcp_policy_line" ] &&- [ "$same_intent_tcp_policy_line" -lt "$divergent_revoke_line" ] &&- grep -Fq 'hcc_ready_with_identity_stable_policy_retained' "$MCP_READINESS_GATE" &&- ! grep -Fq 'HCC readiness after revoking the same-intent DNS policy' "$MCP_READINESS_GATE" &&- ! grep -Fq 'a real same-intent external-egress retry to be scheduled' "$MCP_READINESS_GATE" &&- ! grep -Fq 'CERTIFIED_POLICY_IDENTITY' "$MCP_READINESS_GATE" &&- ! grep -Fq 'external_egress_policy_identity' "$MCP_READINESS_GATE" &&- ! grep -Fq 'UID/spec' "$MCP_READINESS_GATE"; then- pass "same-intent safety retains the identity-stable DNS policy at readiness; additive TCP convergence follows DNS release"-else- fail "the MCP readiness gate no longer proves B3 retain-identity-stable (or regressed to revoke-same-intent before readiness)"-fi-printf '%s\n' \'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \'[K8s] Listing all Hosts in namespace mcp-host' \@@ -1329,30 +41,6 @@ elsefail "valid LIST-to-recovery ordering was rejected"fi-printf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \- '[K8s] Listing all Hosts in namespace mcp-host' \- '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \- '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \- '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"-if recovery_cycle_used_fresh_host_inventory 1 5; then- pass "a causally ordered Host-watch pass may cover the recovered CC and fresh Host inventories"-else- fail "a valid covering Host-watch recovery pass was rejected"-fi--printf '%s\n' \- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \- '[K8s] Listing all Hosts in namespace mcp-host' \- '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \- '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \- '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"-if recovery_cycle_used_fresh_host_inventory 1 5; then- fail "a Host-watch pass selected before CC recovery satisfied the covering assertion"-else- pass "covering Host-watch recovery requires recovered CC state before pass selection"-fi-printf '%s\n' \'[K8s] Listing all Hosts in namespace mcp-host' \'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \@@ -1394,272 +82,19 @@ elsefail "the second recovery cycle was not isolated from the first"fi-large_log_tail="$(awk 'BEGIN {- for (i = 1; i <= 20000; i++) {- printf "trailing-log-line-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n", i- }-}')"-large_active_snapshot="${START_MARKER:-Starting initial Host background convergence}-${large_log_tail}"-if hcc_log_snapshot_contains \- "$large_active_snapshot" 'Starting initial Host background convergence'; then- pass "buffered marker lookup survives a log tail larger than a pipe buffer"-else- fail "large trailing logs can hide a marker from buffered lookup"-fi--if hcc_initial_pass_snapshot_is_active \- "$large_active_snapshot" \- 'Starting initial Host background convergence' \- 'Completed Host reconciliation after initial Host reconciliation' \- 'Host reconciliation after initial Host reconciliation failed'; then- pass "Host pass snapshot is active only with START and no terminal marker"-else- fail "valid active Host pass snapshot was rejected"-fi--for terminal_marker in \- 'Completed Host reconciliation after initial Host reconciliation' \- 'Host reconciliation after initial Host reconciliation failed'; do- if hcc_initial_pass_snapshot_is_active \- "${large_active_snapshot}-${terminal_marker}" \- 'Starting initial Host background convergence' \- 'Completed Host reconciliation after initial Host reconciliation' \- 'Host reconciliation after initial Host reconciliation failed'; then- fail "terminal Host marker '${terminal_marker}' can still satisfy the active-pass guard"- else- pass "terminal Host marker '${terminal_marker}' invalidates the active-pass guard"- fi-done--readiness_log_function="$(- sed -n '/^hcc_log_contains() {$/,/^}$/p' "$READINESS_GATE"-)"-running_pod_function="$(- sed -n '/^running_hcc_pod() {$/,/^}$/p' "$READINESS_GATE"-)"-# Literal source-code assertions.-# shellcheck disable=SC2016-if [[ "$readiness_log_function" == *'logs="$(hcc_logs "$pod")"'* ]] &&- [[ "$readiness_log_function" == *'hcc_log_snapshot_contains "$logs" "$marker"'* ]] &&- [[ "$running_pod_function" == *'rows="$(kctl get pods'* ]] &&- [[ "$running_pod_function" == *'<<<"$rows"'* ]]; then- pass "readiness log and Running-pod selection buffer producers before early-exit consumers"-else- fail "readiness log or Running-pod selection reintroduced a pipefail/SIGPIPE path"-fi--active_guard_function="$(- sed -n '/^initial_host_pass_is_active() {$/,/^}$/p' "$READINESS_GATE"-)"-initial_host_startup_producer="$(- sed -n \- '/Starting initial Host background convergence/,/const resyncSec = config.hostResyncIntervalSec/p' \- "$HCC_K8S_CLIENT"-)"-# The bootstrap guard's negative COMPLETE/FAIL checks are safe only while all-# three consumed markers remain bound to the runtime templates and the startup-# call supplies the exact reason that renders the terminal pair.-if grep -Fq 'Starting initial Host background convergence...' \- "$HCC_K8S_CLIENT" &&- grep -Fq 'Completed Host reconciliation after ${reason}' \- "$HCC_K8S_CLIENT" &&- grep -Fq 'Host reconciliation after ${reason} failed:' \- "$HCC_K8S_CLIENT" &&- [[ "$initial_host_startup_producer" == *'await this.recoverHostInventoryAndWatch('* ]] &&- [[ "$initial_host_startup_producer" == *"'initial Host reconciliation',"* ]]; then- pass "bootstrap START/COMPLETE/FAIL markers remain bound to the initial Host producer reason"-else- fail "bootstrap Host pass markers drifted from their runtime producer or reason binding"-fi--# The destructive Host-bundle measurement uses the same cold-start transition-# as the bootstrap readiness gate. Keep its active-pass marker tied to the-# emitted producer rather than an obsolete reason-string log line.-if grep -Fq "readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'" \- "$HOST_BUNDLE_MEASURE" &&- grep -Fq 'Starting initial Host background convergence...' "$HCC_K8S_CLIENT"; then- pass "Host-bundle measurement START marker remains bound to the cold-start producer"-else- fail "Host-bundle measurement can wait for a stale HCC START marker"-fi-# shellcheck disable=SC2016-probe_result_line="$(grep -nF 'probe_result="$(kctl exec' "$READINESS_GATE" | cut -d: -f1)"-# shellcheck disable=SC2016-final_active_guard_line="$(- grep -nF 'initial_host_pass_is_active "$new_hcc_pod"' "$READINESS_GATE" |- tail -1 |- cut -d: -f1-)"-# Literal source-code assertions.-# shellcheck disable=SC2016-if [[ "$active_guard_function" == *'"$COMPLETE_MARKER"'* ]] &&- [[ "$active_guard_function" == *'"$FAIL_MARKER"'* ]] &&- [ -n "$probe_result_line" ] &&- [ -n "$final_active_guard_line" ] &&- [ "$final_active_guard_line" -gt "$probe_result_line" ]; then- pass "final readiness assertion rejects both failed and completed Host passes"-else- fail "final readiness assertion does not enforce both terminal Host markers"-fi--cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$READINESS_GATE")"-fixture_cleanup_body="$(sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"-# shellcheck disable=SC2016-cleanup_stop_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0' \- <<<"$cleanup_body" | head -1 | cut -d: -f1)"-cleanup_fixture_line="$(grep -nF 'delete_host_fixtures' <<<"$cleanup_body" | head -1 | cut -d: -f1)"-# shellcheck disable=SC2016-cleanup_restore_line="$(grep -nF 'kctl set env deployment/"$HCC_DEPLOY"' \- <<<"$cleanup_body" | head -1 | cut -d: -f1)"+acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock || die' "$GATE" | cut -d: -f1)"# Literal source-code assertion.# shellcheck disable=SC2016-if grep -Fq 'FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))' "$READINESS_GATE" &&- grep -Fq 'kind: Host' "$READINESS_GATE" &&- grep -Fq "host='+host" "$READINESS_GATE" &&- grep -Fq 'blocker_fixture_request_count' "$READINESS_GATE" &&- [[ "$fixture_cleanup_body" == *'clerum.io/managed-by=host-context-controller,clerum.io/host=${host}'* ]] &&- [[ "$fixture_cleanup_body" == *'fixture_resources_absent'* ]] &&- ! grep -Fq 'RUNTIME_SECRET' "$READINESS_GATE" &&- ! grep -Fq 'HOST_REF' "$READINESS_GATE" &&- [ -n "$cleanup_stop_line" ] &&- [ -n "$cleanup_fixture_line" ] &&- [ -n "$cleanup_restore_line" ] &&- [ "$cleanup_stop_line" -lt "$cleanup_fixture_line" ] &&- [ "$cleanup_fixture_line" -lt "$cleanup_restore_line" ]; then- pass "readiness gate uses a multi-wave owned Host fleet and deletes it while HCC is stopped"-else- fail "readiness gate can mutate a real Host Secret or restore HCC before fixture deletion"-fi--if [[ "$cleanup_body" == *'header "HCC readiness bootstrap gate passed"'* ]] &&- grep -Fq 'header "HCC readiness assertions passed; restoring branch-owned runtime"' \- "$READINESS_GATE" &&- [ "$(grep -Fc 'header "HCC readiness bootstrap gate passed"' "$READINESS_GATE")" = 1 ]; then- pass "readiness gate emits its final pass banner only after verified restoration and lock finalization"-else- fail "readiness gate can announce a final pass before cleanup and ownership release finish"-fi--mcp_cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$MCP_READINESS_GATE")"-# shellcheck disable=SC2016-mcp_input_absence_line="$(- grep -nF 'fixture_inputs_absent' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1-)"-# shellcheck disable=SC2016-mcp_restore_line="$(- grep -nF 'restore_patch="$(jq -cn' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1-)"-if [[ "$mcp_cleanup_body" == *'fixture_inputs_removed=0'* ]] &&- [[ "$mcp_cleanup_body" == *'if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then'* ]] &&- [ -n "$mcp_input_absence_line" ] &&- [ -n "$mcp_restore_line" ] &&- [ "$mcp_input_absence_line" -lt "$mcp_restore_line" ]; then- pass "MCP readiness cleanup keeps HCC stopped until fixture input deletion is verified"-else- fail "MCP readiness cleanup can restart HCC after an unverified Context or McpServer deletion"-fi--mcp_finalize_line="$(- grep -nF 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' \- <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1-)"-mcp_pass_line="$(- grep -nF 'header "HCC McpServer/Context readiness gate passed"' \- <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1-)"-if [ "$(grep -Fc 'header "HCC McpServer/Context readiness gate passed"' \- "$MCP_READINESS_GATE")" = 1 ] &&- [ -n "$mcp_finalize_line" ] &&- [ -n "$mcp_pass_line" ] &&- [ "$mcp_finalize_line" -lt "$mcp_pass_line" ]; then- pass "MCP readiness gate announces PASS only after cleanup and lock finalization"-else- fail "MCP readiness gate can announce PASS before cleanup and lock finalization"-fi--wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$READINESS_GATE")"+mutation_line="$(grep -nF 'kctl get deployment "$HCC_DEPLOY"' "$GATE" | head -1 | cut -d: -f1)"# Literal source-code assertion.# shellcheck disable=SC2016-if [[ "$wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&- ! grep -Fq -- '--for=delete' "$READINESS_GATE" &&- ! grep -Fq 'port: 8081' "$READINESS_GATE" &&- grep -Fq 'HCC_E2E_PORT' "$READINESS_GATE"; then- pass "readiness gate uses wall-clock deadlines, absence waits, and the deployed HCC port"-else- fail "readiness gate retains a command-latency, zero-pod wait, or hard-coded-port edge"-fi--mcp_wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$MCP_READINESS_GATE")"-mcp_port_probe_count="$(grep -Fc 'env "HCC_E2E_PORT=${HCC_PORT}"' "$MCP_READINESS_GATE")"-mcp_pod_absence_count="$(grep -Fc 'hcc_pods_absent' "$MCP_READINESS_GATE")"-# Literal source-code assertions.-# shellcheck disable=SC2016-if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \- "$MCP_READINESS_GATE" &&- [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$MCP_READINESS_GATE")" = 1 ] &&- ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$MCP_READINESS_GATE" &&- ! grep -Fq 'cluster_fingerprint_file=' "$MCP_READINESS_GATE" &&- ! grep -Fq 'profile_env=' "$MCP_READINESS_GATE" &&- [[ "$mcp_wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&- grep -Fq 'hcc_pods_absent' "$MCP_READINESS_GATE" &&- ! grep -Fq -- '--for=delete' "$MCP_READINESS_GATE" &&- ! grep -Fq 'port: 8081' "$MCP_READINESS_GATE" &&- [ "$mcp_port_probe_count" = 4 ] &&- [ "$mcp_pod_absence_count" = 4 ]; then- pass "MCP readiness gate shares exact-head ownership and uses bounded portable runtime probes"-else- fail "MCP readiness gate duplicates ownership or retains a deadline, pod-absence, or port edge"-fi--# A single held McpServer proves the minimal failure, but this gate must also-# cross the controller's bounded worker width with independent, gate-owned-# McpServer/Context pairs. Keep this as a static contract so future edits do-# not silently reduce the fleet scenario back to one fixture. The fixture-# must exist while HCC is stopped, otherwise it would not exercise startup.-mcp_peer_create_line="$(grep -nF 'create_peer_fleet || die' "$MCP_READINESS_GATE" | cut -d: -f1)"-mcp_restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1)"-peer_fleet_converged_body="$(- sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"-)"-if grep -Fq 'MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"' "$MCP_READINESS_GATE" &&- grep -Fq '[ "$MCP_FLEET_SIZE" -gt 10 ]' "$MCP_READINESS_GATE" &&- grep -Fq 'create_peer_fleet' "$MCP_READINESS_GATE" &&- grep -Fq 'peer_fleet_converged' "$MCP_READINESS_GATE" &&- [[ "$peer_fleet_converged_body" == *'mcpserver_current_ready "$server"'* ]] &&- [[ "$peer_fleet_converged_body" == *'context_policies_converged "$context" "$context" "$server"'* ]] &&- grep -Fq 'all peer MCP/Context runtimes to converge after releasing the held primary egress' "$MCP_READINESS_GATE" &&- [ -n "$mcp_peer_create_line" ] && [ -n "$mcp_restart_line" ] &&- [ "$mcp_peer_create_line" -lt "$mcp_restart_line" ]; then- pass "MCP readiness gate crosses the bounded worker width with gate-owned MCP and Context peers"+if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] && [ "$acquire_line" -lt "$mutation_line" ] &&+ grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1' "$GATE"; then+ pass "gate acquires before HCC access and finalizes ownership from cleanup"else- fail "MCP readiness gate does not prove a fleet larger than the worker width"+ fail "gate lock is not held across the full HCC fault-injection window"fi-for gate in "${GATES[@]}"; do- acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock' "$gate" | tail -1 | cut -d: -f1)"- case "$gate" in- "$WATCH_GATE")- # Literal source-code assertion.- # shellcheck disable=SC2016- mutation_line="$(grep -nF 'kctl patch deployment "$HCC_DEPLOY"' "$gate" | tail -1 | cut -d: -f1)"- ;;- *)- mutation_line="$(grep -nF 'HCC_MUTATED=1' "$gate" | tail -1 | cut -d: -f1)"- ;;- esac- # The grep pattern intentionally matches literal shell variable references.- # shellcheck disable=SC2016- if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] &&- [ "$acquire_line" -lt "$mutation_line" ] &&- grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' "$gate"; then- pass "$(basename "$gate") acquires before mutation and finalizes ownership from cleanup"- else- fail "$(basename "$gate") does not hold the shared lock across its full fault-injection window"- fi-done-kctl() {local action=$1 body expected_rv current_rv current_uid next_rvcase "$action" in@@ -1703,7 +138,7 @@ truncate_rfc1123() { printf '%.63s' "$1"; }source "$LOCK_HELPER"HCC_NS=control-plane-HCC_DEPLOY="host-context-controller"+HCC_DEPLOY=host-context-controllerE2E_KUBECONTEXT=clerum-codex-lock-test-1234abcdRUN_ID=owner-1HCC_GATE_LOCK_ACQUIRED=0@@ -1818,62 +253,12 @@ elsefiunset MOCK_CREATE_WITHOUT_UID-cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$WATCH_GATE" | cut -d: -f1)"-results_line="$(grep -nF ' print_results' "$WATCH_GATE" | head -1 | cut -d: -f1)"+cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$GATE" | cut -d: -f1)"+results_line="$(grep -nF ' print_results' "$GATE" | head -1 | cut -d: -f1)"if [ -n "$cleanup_fail_line" ] && [ -n "$results_line" ] && [ "$cleanup_fail_line" -lt "$results_line" ]; thenpass "cleanup failure is counted before the final result summary"elsefail "cleanup can still print a false all-passed summary"fi-# --- H3: SIGPIPE/pipefail sweep on HCC pass-marker log probes --------------# A `printf '%s[\n]' "$logs" | grep -Fq "$..._MARKER"` probe is a false-green-# trap: on a large log with the marker near the top, `grep -Fq` exits early,-# `printf` takes SIGPIPE, and under `set -o pipefail` the whole pipeline reports-# failure — so a PRESENT marker reads as MISSING (deterministic 3/3 repro in the-# round-2 storm gate). The safe idiom is a here-string: no producer to signal.-# grep -Fq -- "$MARKER" <<<"$logs"-#-# Scope: HCC pass-marker gates only (storm + measure). Explicitly OUT OF SCOPE-# (tracked separately — a different failure class, or a non-HCC gate outside-# this PR's blast radius):-# - e2e-gke-secrets.sh:291-294 grep -oE|wc leak COUNT: reads the whole log,-# no early-exit, so it is NOT the SIGPIPE class.-# - e2e-plugin-workload-sdk.sh SDK markers (non-HCC).-# - e2e-lib.sh / e2e-prod-lib.sh / e2e-channel-reader-per-host.sh (non-HCC).-HCC_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"-# shellcheck disable=SC2016-sigpipe_offenders="$(- grep -nE 'printf[^|]*\| *grep -Fq[^<]*_MARKER' \- "$HCC_STORM_GATE" "$HOST_BUNDLE_MEASURE" 2>/dev/null || true-)"-if [ -z "$sigpipe_offenders" ]; then- pass "HCC pass-marker probes use here-strings, not pipefail/SIGPIPE pipes"-else- fail "HCC pass-marker probe reintroduced a printf|grep -Fq SIGPIPE path:-${sigpipe_offenders}"-fi--# Corroborating repro of the mechanism the sweep guards against. The here-string-# form MUST find a marker on line 1 of a 1.6 MB log (hard invariant). The pipe-# form is expected to miss it under pipefail (SIGPIPE) — that miss is the bug,-# demonstrated, not a standing assertion, so a platform that does not reproduce-# it is not a suite failure: the static sweep above is the real enforcement.-h3_marker='HCC_PASS_COMPLETED: h3 repro sentinel'-h3_bulk="$(yes 'xxxxxxxxxxxxxxxx' | head -c 1600000)"-h3_log="${h3_marker}-${h3_bulk}"-if grep -Fq -- "$h3_marker" <<<"$h3_log"; then- pass "here-string probe finds a marker on line 1 of a 1.6 MB log"-else- fail "here-string probe MISSED a present marker on a large log (regression)"-fi-h3_pipe_found=0-( set -o pipefail; printf '%s\n' "$h3_log" | grep -Fq -- "$h3_marker" ) && h3_pipe_found=1-if [ "$h3_pipe_found" -eq 0 ]; then- pass "pipe+pipefail probe misses the same marker (SIGPIPE repro confirmed)"-else- pass "pipe probe did not SIGPIPE on this platform; static sweep still enforces the idiom"-fi-exit "$FAIL"diff --git a/scripts/tests/test-host-storm-metric-gate.sh b/scripts/tests/test-host-storm-metric-gate.shdeleted file mode 100755index e5d994bb5..000000000--- a/scripts/tests/test-host-storm-metric-gate.sh+++ /dev/null@@ -1,296 +0,0 @@-#!/usr/bin/env bash-# shellcheck disable=SC2016 # literal source-contract assertions below-set -euo pipefail--ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-GATE="${ROOT}/scripts/e2e/e2e-host-storm-gate.sh"-WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/host-storm-metric-test.XXXXXX")"-ADJUDICATOR="${WORK_DIR}/adjudicator.py"-trap 'rm -rf "$WORK_DIR"' EXIT--fail() {- echo "FAIL: $*" >&2- exit 1-}--extract_adjudicator() {- awk '- /python3 >"\$METRIC_VERDICTS" <<'\''PY'\''/ { capture = 1; next }- capture && /^PY$/ { exit }- capture { print }- ' "$GATE" >"$ADJUDICATOR"- [ -s "$ADJUDICATOR" ] || fail "could not extract host-storm metric adjudicator"-}--write_metrics() {- local path=$1 le_05=$2 le_5=$3 le_15=$4 total=$5 fleet_started=${6:-0}- local fleet_coalesced=${7:-0} fleet_trailing=${8:-0} fleet_failed=${9:-0}- cat >"$path" <<EOF-# TYPE clerum_hcc_host_delete_cleanup_total counter-clerum_hcc_host_delete_cleanup_total{outcome="queued"} 0-clerum_hcc_host_delete_cleanup_total{outcome="confirmed"} 0-clerum_hcc_host_delete_cleanup_total{outcome="completed"} 0-clerum_hcc_host_delete_cleanup_total{outcome="retried"} 0-clerum_hcc_host_delete_cleanup_total{outcome="superseded"} 0-# TYPE clerum_hcc_host_fleet_requests_total counter-clerum_hcc_host_fleet_requests_total{result="started"} ${fleet_started}-clerum_hcc_host_fleet_requests_total{result="coalesced"} ${fleet_coalesced}-clerum_hcc_host_fleet_requests_total{result="trailing"} ${fleet_trailing}-clerum_hcc_host_fleet_requests_total{result="failed"} ${fleet_failed}-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="0.5"} ${le_05}-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="5"} ${le_5}-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="15"} ${le_15}-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="+Inf"} ${total}-EOF-}--extract_adjudicator--for scrape in a b c1 c2 d wake_pre wake_post; do- write_metrics "${WORK_DIR}/${scrape}.prom" 0 0 0 0-done--cat >>"${WORK_DIR}/c1.prom" <<'EOF'-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="15"} 1-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="+Inf"} 1-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="failure",le="+Inf"} 0-EOF--write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 1--# The post-recovery direct-watch window is healthy.-write_metrics "${WORK_DIR}/d.prom" 10 10 10 10--# The wake window contains one sample over the 5s hard gate plus nineteen-# fast samples. A combined p95 would be green (19/20 == 95%), but the-# supported wake journey violated its binding urgent queue budget.-write_metrics "${WORK_DIR}/wake_post.prom" 19 19 20 20--run_adjudicator() {- local wake_post=$1 fleet_completion_proven=${2:-1}- SCRAPE_A="${WORK_DIR}/a.prom" \- SCRAPE_B="${WORK_DIR}/b.prom" \- SCRAPE_C1="${WORK_DIR}/c1.prom" \- SCRAPE_C2="${WORK_DIR}/c2.prom" \- SCRAPE_D="${WORK_DIR}/d.prom" \- SCRAPE_WAKE_PRE="${WORK_DIR}/wake_pre.prom" \- SCRAPE_WAKE_POST="$wake_post" \- QUEUE_WAIT_METRIC=clerum_hcc_host_reconcile_queue_wait_seconds \- WATCH_RECOVERY_METRIC=clerum_hcc_host_watch_recovery_seconds \- FLEET_METRIC=clerum_hcc_host_fleet_requests_total \- DELETE_CLEANUP_METRIC=clerum_hcc_host_delete_cleanup_total \- P95_BUDGET_S=5 \- RECOVERY_BUDGET_S=15 \- MIN_URGENT_SAMPLES=10 \- FLEET_COMPLETION_PROVEN="$fleet_completion_proven" \- python3 "$ADJUDICATOR"-}--OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_post.prom")"--if ! grep -Fq 'VERDICT|A2-direct-watch-urgent-p95|PASS|' <<<"$OUTPUT" &&- ! grep -Fq 'VERDICT|A2-urgent-queue-wait-p95|PASS|' <<<"$OUTPUT"; then- fail "healthy post-recovery direct-watch window did not pass"-fi-grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|FAIL|' <<<"$OUTPUT" ||- fail "slow wake window was diluted into a false PASS"--write_metrics "${WORK_DIR}/wake_healthy.prom" 20 20 20 20-HEALTHY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|PASS|' <<<"$HEALTHY_OUTPUT" ||- fail "healthy wake window did not pass its hard gate"--# A7 is fenced to the replacement pod's explicit initial-fleet COMPLETED log.-# A second non-overlapping request can therefore be legitimate activity after-# recovery; an absolute started==1 assertion must not false-red it.-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2-TWO_STARTS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$TWO_STARTS_OUTPUT" ||- fail "healthy replacement-pod recovery false-reds when a second sequential fleet request starts"--write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0-NO_START_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$NO_START_OUTPUT" ||- fail "A7 still treats a scheduled counter as stronger than causal completion evidence"--NO_COMPLETION_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom" 0)"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$NO_COMPLETION_OUTPUT" ||- fail "A7 false-passed without replacement-pod fleet completion evidence"--write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 0 0 1-FAILED_PASS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$FAILED_PASS_OUTPUT" ||- fail "A7 false-passed a failed replacement-pod fleet pass"--write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 1 2-UNBOUNDED_TRAILING_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$UNBOUNDED_TRAILING_OUTPUT" ||- fail "A7 false-passed trailing fleet work without a matching coalesced request"--# M8: without the FLEET_METRIC family registered on the running image, counter()-# returns 0 for every result label and A7 would pass vacuously (all-zero looks-# like "no failures"). Strip the family's # TYPE line so it drops out of-# c2_types (mirrors a metric rename / an image that never registered it) and-# assert A7 FAILs on the presence check, not vacuously passes.-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0-grep -v '^# TYPE clerum_hcc_host_fleet_requests_total ' "${WORK_DIR}/c2.prom" \- >"${WORK_DIR}/c2.nofleet.prom"-mv "${WORK_DIR}/c2.nofleet.prom" "${WORK_DIR}/c2.prom"-MISSING_FAMILY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$MISSING_FAMILY_OUTPUT" ||- fail "A7 vacuously passed with the FLEET_METRIC family absent from the scrape"-grep -Fq 'not registered on the running image' <<<"$MISSING_FAMILY_OUTPUT" ||- fail "A7 failed for the wrong reason when the FLEET_METRIC family was absent"--WAKE_EVIDENCE_FUNCTION="$(- sed -n '/^_wake_urgent_evidence_recorded() {$/,/^}$/p' "$GATE"-)"-WAKE_GENERATION_FUNCTION="$(- sed -n '/^wake_generation_is_handled() {$/,/^}$/p' "$GATE"-)"-POST_WAKE_FUNCTION="$(- sed -n '/^post_wake() {$/,/^}$/p' "$GATE"-)"--[[ -n "$WAKE_GENERATION_FUNCTION" ]] ||- fail "runtime wake evidence has no generation-correlation predicate"--# Exercise the exact pure predicate embedded in the E2E gate. The requested-# generation is the business identifier returned by the public wake route;-# an older handled generation must never satisfy that request.-eval "$WAKE_GENERATION_FUNCTION"-if wake_generation_is_handled 7 6; then- fail "wake generation predicate accepted an older handled generation"-fi-wake_generation_is_handled 7 7 ||- fail "wake generation predicate rejected the exact handled generation"-wake_generation_is_handled 7 8 ||- fail "wake generation predicate rejected a newer handled generation"-if wake_generation_is_handled 0 8 || wake_generation_is_handled invalid 8 ||- wake_generation_is_handled 7 invalid; then- fail "wake generation predicate accepted malformed generation evidence"-fi--# Literal source assertions intentionally match unexpanded shell variables.-# shellcheck disable=SC2016-if ! grep -Fq '"$current_pod" == "$HCC_POD_WAKE"' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq '"$lifecycle_state" != '\''suspended'\''' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq 'handled_generation="$(wake_handled_generation)"' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq 'metric="${IN_FLIGHT_METRIC}"' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq '(after - before) >= 1 && active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then- fail "runtime wake evidence is not fenced to the same pod, requested generation, metric delta, and idle urgent lane"-fi-generation_check_line="$(- grep -nF 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' \- <<<"$WAKE_EVIDENCE_FUNCTION" | cut -d: -f1-)"-metrics_scrape_line="$(- grep -nF 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \- <<<"$WAKE_EVIDENCE_FUNCTION" | head -n1 | cut -d: -f1-)"-if [[ -z "$generation_check_line" || -z "$metrics_scrape_line" ]] ||- (( generation_check_line >= metrics_scrape_line )); then- fail "wake generation is not correlated before accepting shared urgent metrics"-fi-wake_scrape_count="$(- grep -Fc 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \- <<<"$WAKE_EVIDENCE_FUNCTION"-)"-if (( wake_scrape_count < 2 )) ||- ! grep -Fq 'fence_in_flight=' <<<"$WAKE_EVIDENCE_FUNCTION" ||- ! grep -Fq 'active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then- fail "wake evidence lacks a completed-lane fence followed by a fresh metric scrape"-fi--# shellcheck disable=SC2016-if ! grep -Fq 'WAKE_GENERATION=' <<<"$POST_WAKE_FUNCTION" ||- ! grep -Fq '"$WAKE_STATUS" == '\''202'\''' <<<"$POST_WAKE_FUNCTION" ||- ! grep -Fq '.status // empty' <<<"$POST_WAKE_FUNCTION" ||- ! grep -Fq '.wakeGeneration // empty' <<<"$POST_WAKE_FUNCTION"; then- fail "public wake response is not validated as an accepted wake-generation contract"-fi--# A7 must start from a fresh scrape immediately before the kill. Reusing the-# earlier wake-evidence scrape creates an unobserved interval in which a fleet-# failure could occur and escape the pre/post recovery counter delta.-# shellcheck disable=SC2016-if ! grep -Fq 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" ||- grep -Fq 'cp "$SCRAPE_WAKE_POST" "$SCRAPE_B"' "$GATE"; then- fail "pre-kill scrape B is not freshly captured after wake adjudication"-fi-fresh_scrape_line="$(- grep -nF 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" | cut -d: -f1-)"-kill_line="$(grep -nF 'T_KILL_MS="$(now_ms)"' "$GATE" | cut -d: -f1)"-if [[ -z "$fresh_scrape_line" || -z "$kill_line" ]] ||- (( fresh_scrape_line >= kill_line )); then- fail "fresh scrape B does not precede the HCC kill"-fi--# The destructive Host storm must use the same exact-HEAD/profile/fingerprint-# ownership contract and exclusive compare-and-swap lock as every HCC gate.-# shellcheck disable=SC2016-for required in \- 'source "${SCRIPT_DIR}/e2e-lib.sh"' \- 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \- 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"' \- 'require_branch_owned_hcc_gate "$CONTROL_NS"' \- 'acquire_hcc_watch_gate_lock' \- 'finalize_hcc_watch_gate_lock'; do- grep -Fq "$required" "$GATE" ||- fail "Host storm omits shared HCC gate contract: ${required}"-done-grep -Fq '.metadata.labels["minikube.k8s.io/name"] == $context' "$GATE" ||- fail "Host storm does not bind the node's exact minikube profile label"-grep -Fq 'MINIKUBE_PROFILE must explicitly select context' \- "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" ||- fail "shared profile ownership helper no longer binds MINIKUBE_PROFILE"--# Branch-owned URLs must come from the selected profile's ports.env. Fixed-# machine-global service or metrics ports are a cross-worktree false-green risk.-grep -Fq 'CLERUM_PROFILE_PORTS_ENV' "$GATE" ||- fail "Host storm does not load branch-profile port evidence"-grep -Fq 'profile_env_value EXTERNAL_REST_API_URL' "$GATE" ||- fail "Host storm does not derive External REST from branch ports.env"-grep -Fq 'profile_env_value RPC_PROXY_URL' "$GATE" ||- fail "Host storm does not derive RPC proxy from branch ports.env"-if grep -Eq '127\.0\.0\.1:(8091|8094|18082)' "$GATE"; then- fail "Host storm retains a fixed machine-global service or metrics port"-fi-grep -Fq '"pod/${pod}" ":${HCC_PORT}"' "$GATE" ||- fail "HCC metric scrape no longer asks kubectl for a process-owned random port"--# KEEP_FIXTURES may be useful interactively, but it cannot produce PASS evidence-# or leave a lock-free mutated profile that could later be mistaken for T2.-keep_guard_line="$(grep -nF 'KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result' "$GATE" | cut -d: -f1)"-branch_gate_line="$(grep -nF 'require_branch_owned_hcc_gate "$CONTROL_NS"' "$GATE" | cut -d: -f1)"-if [[ -z "$keep_guard_line" || -z "$branch_gate_line" ]] ||- (( keep_guard_line >= branch_gate_line )); then- fail "KEEP_FIXTURES is not rejected before branch verification and mutation"-fi-if grep -Fq 'leaving fixtures in place' "$GATE"; then- fail "Host storm can still report through the fixture-retention path"-fi--# Completion must be read from the exact replacement pod and established-# before C1/C2 metric adjudication. Merely observing `started` is not proof.-FLEET_COMPLETION_FUNCTION="$(- sed -n '/^_replacement_fleet_completed_probe() {$/,/^}$/p' "$GATE"-)"-grep -Fq '"$current_pod" == "$HCC_POD_C"' <<<"$FLEET_COMPLETION_FUNCTION" ||- fail "fleet completion probe is not fenced to the replacement pod"-grep -Fq '"pod/${HCC_POD_C}"' <<<"$FLEET_COMPLETION_FUNCTION" ||- fail "fleet completion probe reads a deployment aggregate instead of the replacement pod"-grep -Fq '$HCC_PASS_COMPLETED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||- fail "fleet completion probe does not require the explicit COMPLETED marker"-grep -Fq '$HCC_PASS_FAILED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||- fail "fleet completion probe does not fail loud on the explicit FAILED marker"-completion_wait_line="$(grep -nF '"replacement HCC initial Host fleet reconciliation completion"' "$GATE" | cut -d: -f1)"-c1_line="$(grep -nF 'scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"' "$GATE" | head -n1 | cut -d: -f1)"-if [[ -z "$completion_wait_line" || -z "$c1_line" ]] ||- (( completion_wait_line >= c1_line )); then- fail "replacement fleet completion is not causally established before post-recovery metrics"-fi--echo "PASS: host-storm oracle binds exact branch/profile ownership, shared fault locking, random endpoints, wake generation, and causal fleet completion"diff --git a/scripts/tests/test-lint-networkpolicies.sh b/scripts/tests/test-lint-networkpolicies.shindex 3153dd5dd..2400d8517 100755--- a/scripts/tests/test-lint-networkpolicies.sh+++ b/scripts/tests/test-lint-networkpolicies.sh@@ -3,23 +3,7 @@ set -euo pipefailROOT="$(cd "$(dirname "$0")/../.." && pwd)"TMP_DIR="$(mktemp -d)"--# The minikube k8s-api-ip patch is a machine-specific, gitignored file rendered-# at setup time. On a developer box with a live minikube it may already exist;-# this test shadows it with a dummy render below. Save the real one (if any) and-# restore it on EXIT so the test leaves the working tree exactly as it found it.-MINIKUBE_API_IP_PATCH=""-MINIKUBE_API_IP_PATCH_BACKUP=""-MINIKUBE_API_IP_PATCH_PREEXISTING=0-cleanup() {- if [[ "${MINIKUBE_API_IP_PATCH_PREEXISTING}" -eq 1 ]]; then- cp "${MINIKUBE_API_IP_PATCH_BACKUP}" "${MINIKUBE_API_IP_PATCH}"- elif [[ -n "${MINIKUBE_API_IP_PATCH:-}" && -f "${MINIKUBE_API_IP_PATCH}" ]]; then- rm -f "${MINIKUBE_API_IP_PATCH}"- fi- rm -rf "${TMP_DIR}"-}-trap cleanup EXIT+trap 'rm -rf "${TMP_DIR}"' EXITwrite_policy() {local path="$1"@@ -78,7 +62,7 @@ write_policy "${TMP_DIR}/safe-specific.yaml" ' - ports:protocol: TCPto:- ipBlock:- cidr: 198.51.100.10/32'+ cidr: 203.0.113.10/32'run_lint "${TMP_DIR}/safe-specific.yaml" >/dev/nullecho "PASS: lint-networkpolicies allows explicit port plus specific destination"@@ -151,30 +135,8 @@ ${PUBLIC_EXCEPT_BLOCK}"run_lint "${TMP_DIR}/public-with-excepts.yaml" >/dev/nullecho "PASS: lint-networkpolicies allows public-only exception set with explicit ports"-# The minikube overlay references patches/k8s-api-ip.yaml, which is rendered-# from a gitignored template at setup time (deploy/scripts/minikube-detect-k8s-api-ip.sh)-# and absent on a fresh checkout — so `kubectl kustomize` would abort with a-# "no such file" evalsymlink error before ever reaching a lint verdict. Render-# it inline here from the template with a placeholder CIDR, exactly as the-# single-writer CI step does (.github/workflows/ci-public.yml), so this test-# exercises the real rendered overlay instead of failing on a missing env file.-MINIKUBE_API_IP_TEMPLATE="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template"-MINIKUBE_API_IP_PATCH="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml"-if [[ ! -f "${MINIKUBE_API_IP_TEMPLATE}" ]]; then- echo "FAIL: missing ${MINIKUBE_API_IP_TEMPLATE}; cannot render the minikube k8s-api-ip patch" >&2- exit 1-fi-if [[ -f "${MINIKUBE_API_IP_PATCH}" ]]; then- # A developer's real rendered patch is present — preserve it so the dummy- # render below doesn't clobber it; the EXIT trap restores it verbatim.- MINIKUBE_API_IP_PATCH_BACKUP="${TMP_DIR}/k8s-api-ip.yaml.orig"- cp "${MINIKUBE_API_IP_PATCH}" "${MINIKUBE_API_IP_PATCH_BACKUP}"- MINIKUBE_API_IP_PATCH_PREEXISTING=1-fi-sed 's#__K8S_API_IP__#10.96.0.1#g' "${MINIKUBE_API_IP_TEMPLATE}" >"${MINIKUBE_API_IP_PATCH}"--MINIKUBE_RENDERED="${TMP_DIR}/minikube-rendered.yaml"-kubectl kustomize "${ROOT}/deploy/overlays/minikube" >"${MINIKUBE_RENDERED}"+GCP_DEV_RENDERED="${TMP_DIR}/gcp-dev-rendered.yaml"+kubectl kustomize "${ROOT}/deploy/overlays/gcp-dev" >"${GCP_DEV_RENDERED}"RUBYOPT=--disable=gems ruby -ryaml -e 'forbidden = %w[allow-dns-egress-sandbox-ui deny-all-sandbox-ui]required = %w[sandbox-ui-static-deny-all sandbox-ui-static-dns-egress]@@ -189,6 +151,7 @@ RUBYOPT=--disable=gems ruby -ryaml -e 'found << name if forbidden.include?(name)if name == "sandbox-ui-static-dns-egress"saw_dns_peer = false+ saw_dns_ipblock = falseArray(doc.dig("spec", "egress")).each_with_index do |rule, rule_idx|ports = Array(rule["ports"]).map { |port| [port.fetch("protocol", "TCP"), port["port"]] }.sortexpected_ports = [["TCP", 53], ["UDP", 53]]@@ -201,20 +164,27 @@ RUBYOPT=--disable=gems ruby -ryaml -e 'peers.each_with_index do |peer, peer_idx|ns_name = peer.dig("namespaceSelector", "matchLabels", "kubernetes.io/metadata.name")dns_selector = peer.dig("podSelector", "matchLabels", "k8s-app")+ dns_ipblock = peer.dig("ipBlock", "cidr")if ns_name == "kube-system" && dns_selector == "kube-dns"saw_dns_peer = truenextend- abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods")+ if dns_ipblock == "203.0.113.10/32"+ saw_dns_ipblock = true+ next+ end++ abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods or the example-dev kube-dns ClusterIP")endendabort("sandbox-ui static DNS egress must include a kube-system kube-dns peer") unless saw_dns_peer+ abort("sandbox-ui static DNS egress must include the example-dev kube-dns ClusterIP") unless saw_dns_ipblockendendabort("sandbox-ui static slice must not render HCC-owned policies: #{found.join(", ")}") unless found.empty?missing = required - namesabort("sandbox-ui static slice must render renamed baseline policies: #{missing.join(", ")}") unless missing.empty?-' "${MINIKUBE_RENDERED}"-echo "PASS: minikube sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and retain kube-dns-only DNS egress"+' "${GCP_DEV_RENDERED}"+echo "PASS: gcp-dev sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and include kube-dns ClusterIP"diff --git a/scripts/tests/test-postgres-single-writer-strategy.sh b/scripts/tests/test-postgres-single-writer-strategy.shdeleted file mode 100755index 40174bf46..000000000--- a/scripts/tests/test-postgres-single-writer-strategy.sh+++ /dev/null@@ -1,160 +0,0 @@-#!/usr/bin/env bash-# Regression test for deploy/scripts/validate-postgres-single-writer-strategy.sh-#-# Captures the [R4-D] fix: a Deployment this repo DEFINES (deploy/base/<ns>/-# exists) that is ABSENT from a rendered overlay must FAIL LOUDLY, not be-# NOTICE-and-skipped. Before the fix the validator skipped any absent deploy,-# so a local overlay that stopped rendering control-plane/host-context-controller-# validated the single-writer rollout invariant over an empty set and still-# printed "validation passed" (a fail-open / false green).-#-# The CI step "Validate single-writer rollout invariant on rendered overlays"-# already runs the validator against the real minikube overlay (a happy-path-# check). This test is complementary: it fabricates the absent-deploy condition-# in a hermetic fixture so the fail-loud behavior itself is under test.-#-# The test builds a hermetic fixture ROOT_DIR, copies the validator into it so-# its ROOT_DIR ($(dirname)/../..) resolves to the fixture, and renders overlays-# OFFLINE with `kubectl kustomize` — the same path the validator uses. No cluster.-#-# Fail-loud / fail-fast: every assertion fails the suite immediately, and the-# reporter refuses to pass unless a non-zero number of assertions actually ran.-set -uo pipefail--REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-VALIDATOR="${REPO_ROOT}/deploy/scripts/validate-postgres-single-writer-strategy.sh"--for tool in kubectl yq jq; do- command -v "${tool}" >/dev/null 2>&1 ||- { echo "FAIL: required tool '${tool}' is missing" >&2; exit 1; }-done-[ -f "${VALIDATOR}" ] ||- { echo "FAIL: validator not found at ${VALIDATOR}" >&2; exit 1; }--TESTS_RUN=0-FAILURES=0-pass() { TESTS_RUN=$((TESTS_RUN + 1)); echo "ok - $1"; }-fail() { TESTS_RUN=$((TESTS_RUN + 1)); FAILURES=$((FAILURES + 1)); echo "FAIL - $1" >&2; }--WORK="$(mktemp -d "${TMPDIR:-/tmp}/single-writer-test.XXXXXX")"-trap 'rm -rf "${WORK}"' EXIT--ROOT="${WORK}/repo"-mkdir -p "${ROOT}/deploy/scripts"-cp "${VALIDATOR}" "${ROOT}/deploy/scripts/"-FIXTURE_VALIDATOR="${ROOT}/deploy/scripts/$(basename "${VALIDATOR}")"--# control-plane is a namespace this fixture repo DEFINES (base tree present).-# registry is intentionally NOT defined here -> it is FOREIGN (NOTICE-and-skip).-mkdir -p "${ROOT}/deploy/base/control-plane"-: >"${ROOT}/deploy/base/control-plane/.keep"--write_kustomization() { # dir, resource-file...- local dir="$1"; shift- {- echo 'apiVersion: kustomize.config.k8s.io/v1beta1'- echo 'kind: Kustomization'- echo 'resources:'- local r- for r in "$@"; do echo " - ${r}"; done- } >"${dir}/kustomization.yaml"-}--write_control_postgres() { # dir (valid stateful DB deploy: PVC + Recreate + replicas 1)- cat >"$1/control-postgres.yaml" <<'YAML'-apiVersion: apps/v1-kind: Deployment-metadata:- name: control-postgres- namespace: control-plane-spec:- replicas: 1- strategy:- type: Recreate- template:- spec:- volumes:- - name: data- persistentVolumeClaim:- claimName: control-postgres-data-YAML-}--write_hcc() { # dir (valid single-writer deploy: replicas 1 + Recreate)- cat >"$1/host-context-controller.yaml" <<'YAML'-apiVersion: apps/v1-kind: Deployment-metadata:- name: host-context-controller- namespace: control-plane-spec:- replicas: 1- strategy:- type: Recreate- template:- spec:- containers:- - name: host-context-controller- image: example/hcc:test-YAML-}--# --- Overlay A: LOCAL single-writer deploy ABSENT (the fail-open trigger) ----# Renders a valid control-postgres but OMITS host-context-controller.-OV_MISSING="${ROOT}/deploy/overlays/local-missing"-mkdir -p "${OV_MISSING}"-write_control_postgres "${OV_MISSING}"-write_kustomization "${OV_MISSING}" control-postgres.yaml--# --- Overlay B: every LOCAL deploy PRESENT and valid (foreign still skipped) ----OV_OK="${ROOT}/deploy/overlays/all-local-present"-mkdir -p "${OV_OK}"-write_control_postgres "${OV_OK}"-write_hcc "${OV_OK}"-write_kustomization "${OV_OK}" control-postgres.yaml host-context-controller.yaml--run_validator() { # overlays-value ; prints combined output, returns exit code- SINGLE_WRITER_OVERLAYS="$1" bash "${FIXTURE_VALIDATOR}" 2>&1-}--# 1) A locally-defined single-writer deploy absent from a rendered overlay MUST-# fail loudly (this is exactly what the pre-fix NOTICE-and-skip let pass).-out="$(run_validator "deploy/overlays/local-missing")"; rc=$?-if [ "${rc}" -ne 0 ] && printf '%s' "${out}" |- grep -q "renders no Deployment control-plane/host-context-controller"; then- pass "absent locally-defined single-writer deploy fails loudly (exit ${rc})"-else- fail "absent local single-writer deploy did NOT fail loudly (exit ${rc}); output: ${out}"-fi--# 2) With every locally-defined deploy present and valid, a still-absent FOREIGN-# deploy (registry/registry-postgres) is NOTICE-and-skipped, and the script-# passes — the fix must not over-fail on foreign deploys.-out="$(run_validator "deploy/overlays/all-local-present")"; rc=$?-if [ "${rc}" -eq 0 ] &&- printf '%s' "${out}" | grep -q "Single-writer rollout strategy validation passed" &&- printf '%s' "${out}" | grep -q "does not render foreign Deployment registry/registry-postgres"; then- pass "all local deploys present -> passes; absent foreign deploy is skipped"-else- fail "expected pass with foreign-skip (exit ${rc}); output: ${out}"-fi--# 3) Zero overlays must not pass vacuously.-out="$(run_validator " ")"; rc=$?-if [ "${rc}" -ne 0 ] && printf '%s' "${out}" | grep -q "zero overlays"; then- pass "whitespace-only overlay set fails loudly (no vacuous pass)"-else- fail "empty overlay set did not fail loudly (exit ${rc}); output: ${out}"-fi--# --- Reporter (fail-loud / zero-tests-is-never-success) ----echo "1..${TESTS_RUN}"-if [ "${TESTS_RUN}" -lt 3 ]; then- echo "FAIL: expected at least 3 assertions, ran ${TESTS_RUN}" >&2- exit 1-fi-if [ "${FAILURES}" -ne 0 ]; then- echo "FAIL: ${FAILURES}/${TESTS_RUN} assertions failed" >&2- exit 1-fi-echo "All ${TESTS_RUN} single-writer validator assertions passed"diff --git a/scripts/tests/test-wrc-hcc-contracts-gate.sh b/scripts/tests/test-wrc-hcc-contracts-gate.shdeleted file mode 100755index a0d6180bc..000000000--- a/scripts/tests/test-wrc-hcc-contracts-gate.sh+++ /dev/null@@ -1,28 +0,0 @@-#!/usr/bin/env bash-set -u--ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-GATE="${ROOT}/scripts/e2e/e2e-wrc-hcc-contracts.sh"-FAIL=0--pass() { echo "PASS: $1"; }-fail() { echo "FAIL: $1"; FAIL=1; }--if bash -n "$GATE"; then- pass "WRC-HCC contracts gate has valid bash syntax"-else- fail "WRC-HCC contracts gate has invalid bash syntax"-fi--# The Phase 2 fixture has steps, so the admission policy requires an explicit-# trigger. Keep this immediately alongside the fixture's agent/steps contract-# so the E2E cannot regress into an admission-only false negative.-phase_two_fixture="$(sed -n '/^header "Phase 2 - WRC toolsCalled args status contract"$/,/^YAML$/p' "$GATE")"-if [[ "$phase_two_fixture" == *$' triggers:\n onDemand: {}'* ]] &&- [[ "$phase_two_fixture" == *$' steps:'* ]]; then- pass "WRC-HCC Phase 2 fixture declares its required on-demand trigger"-else- fail "WRC-HCC Phase 2 fixture can be rejected before its status contract runs"-fi--exit "$FAIL"diff --git a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.shdeleted file mode 100755index 98f9a9177..000000000--- a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh+++ /dev/null@@ -1,33 +0,0 @@-#!/usr/bin/env bash-set -u--ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-GATE="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"-FAIL=0--pass() { echo "PASS: $1"; }-fail() { echo "FAIL: $1"; FAIL=1; }--if bash -n "$GATE"; then- pass "WRC internal-dependency gate has valid bash syntax"-else- fail "WRC internal-dependency gate has invalid bash syntax"-fi--# WRC persists its generated workload resource names in status before-# materializing workloads. The gate must use that public CRD contract rather-# than assuming a workload ID is also a Deployment or Service name.-# shellcheck disable=SC2016-if grep -Fq 'wait_for_workload_instance() {' "$GATE" &&- grep -Fq 'SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID"' "$GATE" &&- grep -Fq 'BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID"' "$GATE" &&- ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID"' "$GATE" &&- ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID"' "$GATE" &&- grep -Fq 'deploy/${SOURCE_DEPLOYMENT}' "$GATE" &&- grep -Fq '${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local' "$GATE"; then- pass "WRC gate resolves persisted workload instances before runtime assertions"-else- fail "WRC gate assumes raw workload IDs are runtime resource names"-fi--exit "$FAIL"diff --git a/tests/e2e/integration/network-policies.test.ts b/tests/e2e/integration/network-policies.test.tsindex 07925e9db..aba9302b2 100644--- a/tests/e2e/integration/network-policies.test.ts+++ b/tests/e2e/integration/network-policies.test.ts@@ -8,50 +8,12 @@* the NetworkPolicy is blocking traffic as expected. A successful response or* connection refused (port not listening) would indicate a policy gap.*/-import { beforeAll, describe, expect, it } from 'vitest'+import { describe, expect, it } from 'vitest'import { execFileSync } from 'child_process'// Short timeout for negative tests — we expect these to time outconst NP_TIMEOUT = 3-// ─── Cluster reachability gate ───────────────────────────────────────────────-// This suite drives kubectl against a live minikube cluster (Calico CNI + all-// services deployed). Mirrors the opt-in-skip convention already used by the-// sibling integration suites — channel-reader-via-api.test.ts and-// mcpCredentialRotation.helpers.ts (E2E_SKIP_IF_CLUSTER_UNREACHABLE) plus-// registry-pull-secret-runtime.test.ts (E2E_REQUIRE_CLUSTER / CI):-// - Default (no flag): fail-fast when the cluster is unreachable. A silent-// pass would hide a broken E2E setup, so the suite must NOT be neutralized-// unconditionally.-// - E2E_SKIP_IF_CLUSTER_UNREACHABLE=1: opt in to a clean skip for the tier-// that runs integration WITHOUT a cluster (T1).-// - E2E_REQUIRE_CLUSTER=1 or CI=true: force a hard failure even if the skip-// flag is also set — CI must never skip.-const SKIP_IF_UNREACHABLE = process.env.E2E_SKIP_IF_CLUSTER_UNREACHABLE === '1'-const REQUIRE_CLUSTER = process.env.E2E_REQUIRE_CLUSTER === '1' || process.env.CI === 'true'--let clusterReachable = false--/**- * One-shot reachability probe: succeeds only when kubectl can talk to the API- * server. This is a precondition probe (the established sibling idiom —- * registry-pull-secret-runtime.test.ts's `kubectlOrNull`), NOT an error- * swallow: its boolean result drives the fail-hard-or-skip decision in- * `beforeAll`, which stays loud in the default and CI paths.- */-function probeClusterReachable(): boolean {- try {- execFileSync('kubectl', ['get', 'ns', 'default', '-o', 'name'], {- encoding: 'utf-8',- timeout: 15_000,- stdio: ['ignore', 'pipe', 'pipe'],- })- return true- } catch {- return false- }-}-interface ConnectResult {reachable: booleanstatusCode?: number@@ -179,32 +141,8 @@ function testConnectivity(}describe('NetworkPolicy Enforcement (Calico)', () => {- beforeAll(() => {- clusterReachable = probeClusterReachable()- if (clusterReachable) return-- const msg =- '[network-policies] kubernetes cluster not reachable (kubectl get ns default failed)'- if (REQUIRE_CLUSTER) {- throw new Error(- `${msg}. Refusing to skip: E2E_REQUIRE_CLUSTER/CI is set, and a silent skip here ` +- 'would report success while asserting nothing about NetworkPolicy enforcement. ' +- 'Bring up the cluster (e.g. `make minikube-setup`) or unset the flag.'- )- }- if (SKIP_IF_UNREACHABLE) {- console.log(`${msg} — tests will be skipped (E2E_SKIP_IF_CLUSTER_UNREACHABLE=1)`)- return- }- throw new Error(- `${msg}. This E2E suite requires a minikube cluster with Calico CNI and all services ` +- 'deployed. Run `make minikube-setup` first, or set E2E_SKIP_IF_CLUSTER_UNREACHABLE=1 to skip.'- )- })-// ─── Prerequisite Check ──────────────────────────────────────────────- it('Calico CNI is active', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('Calico CNI is active', () => {const pods = execFileSync('kubectl',['get', 'pods', '-n', 'kube-system', '-l', 'k8s-app=calico-node', '--no-headers'],@@ -213,8 +151,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(pods).toContain('Running')})- it('deny-all policies exist in secured namespaces', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('deny-all policies exist in secured namespaces', () => {const namespaces = ['mcp-host', 'mcp-server', 'rpc-proxy', 'sandbox-recipes']for (const ns of namespaces) {let nps: string@@ -247,8 +184,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {// ─── POSITIVE TESTS (Allowed Traffic) ────────────────────────────────describe('Allowed traffic (positive)', () => {- it('control-api → registry-api:8085 (cross-namespace, registry access)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('control-api → registry-api:8085 (cross-namespace, registry access)', () => {const result = testConnectivity('control-plane','control-api',@@ -257,8 +193,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('control-ui → control-api:8090 (UI backend path)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('control-ui → control-api:8090 (UI backend path)', () => {const result = testConnectivity('control-plane','control-ui',@@ -267,8 +202,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('external-rest-api → profile-control-funnel:8080 (internal control path)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('external-rest-api → profile-control-funnel:8080 (internal control path)', () => {const result = testConnectivity('profiles','external-rest-api',@@ -277,14 +211,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.reachable).toBe(true)})- it('rpc-proxy → mcp-host:8080 (message relay)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('rpc-proxy → mcp-host:8080 (message relay)', () => {const result = testConnectivity('rpc-proxy', 'rpc-proxy', getMcpHostRuntimeUrl())expect(result.reachable).toBe(true)})- it('mcp-host can reach the HCC gateway readiness endpoint', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('mcp-host can reach the HCC gateway readiness endpoint', () => {const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -298,8 +230,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {// ─── NEGATIVE TESTS (Blocked by Design) ──────────────────────────────describe('Blocked traffic (negative — deny-all enforced)', () => {- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', () => {const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -309,8 +240,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('mcp-host → control-api:8090 (no direct access to control-plane)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('mcp-host → control-api:8090 (no direct access to control-plane)', () => {const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -320,12 +250,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', () => {const podName = getChannelReaderPodName()if (!podName) {console.warn('SKIPPED: no running channel-reader pod in channels')- ctx.skip()+ return}const result = testConnectivity(@@ -337,8 +266,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('mcp-host → registry-api:8085 (no direct registry access)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('mcp-host → registry-api:8085 (no direct registry access)', () => {const result = testConnectivity('mcp-host',getMcpHostRuntimeName(),@@ -348,8 +276,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('control-api → rpc-proxy:8094 (no direct control-plane hop)', () => {const result = testConnectivity('control-plane','control-api',@@ -359,8 +286,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', () => {const result = testConnectivity('profiles','external-rest-api',@@ -370,22 +296,19 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {const result = testConnectivity('control-plane', 'control-api', getMcpHostRuntimeUrl())expect(result.reachable).toBe(false)expect(result.error).toContain('timeout')})- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {const result = testConnectivity('profiles', 'external-rest-api', getMcpHostRuntimeUrl())expect(result.reachable).toBe(false)expect(result.error).toContain('timeout')})- it('mcp-proxy → external-rest-api:3000 (no reverse path)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('mcp-proxy → external-rest-api:3000 (no reverse path)', () => {const result = testConnectivity('mcp-server','mcp-proxy',@@ -425,12 +348,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {}}- it('sandbox-recipes → control-api:8090 (isolated sandbox)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('sandbox-recipes → control-api:8090 (isolated sandbox)', () => {const podName = findExecPod('sandbox-recipes', 'app=db')if (!podName) {console.warn('SKIPPED: no running db pod in sandbox-recipes')- ctx.skip()+ return}// sandbox-recipes has deny-all; only DNS + HCC API egress allowedconst result = testConnectivity(@@ -442,12 +364,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {expect(result.error).toContain('timeout')})- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', (ctx) => {- if (!clusterReachable) ctx.skip()+ it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', () => {const podName = findExecPod('sandbox-recipes', 'app=db')if (!podName) {console.warn('SKIPPED: no running db pod in sandbox-recipes')- ctx.skip()+ return}const result = testConnectivity('sandbox-recipes',diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex 9981d0aa7..558c31760 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.65",+ "version": "0.1.66","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.65",+ "version": "0.1.66","devDependencies": {"@types/jsonwebtoken": "^9.0.10","@types/node": "^22.0.0",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex 63c62d203..19145d619 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.65",+ "version": "0.1.66","private": true,"type": "module","scripts": {